From 8f72d28c042d5f2f66865da885b716b46017d987 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Tue, 1 Dec 2020 14:38:03 -0800 Subject: [PATCH 01/16] Make sure DNS discovery URLs are never present if using a genesis file (#1635) * Make sure DNS discovery URLs are never present if using a genesis file Signed-off-by: Antoine Toulme * Add tests to make sure we cover extensively the genesis file vs named network use cases Signed-off-by: Antoine Toulme --- .../org/hyperledger/besu/cli/BesuCommand.java | 1 + .../besu/cli/config/EthNetworkConfig.java | 7 +- .../hyperledger/besu/cli/BesuCommandTest.java | 71 +++++++++++++++++ .../besu/cli/config/EthNetworkConfigTest.java | 76 +++++++++++++++++++ 4 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 besu/src/test/java/org/hyperledger/besu/cli/config/EthNetworkConfigTest.java diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index f3460f8d089..85dd95331d1 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -2257,6 +2257,7 @@ private EthNetworkConfig updateNetworkConfig(final NetworkName network) { // than a useless one that may make user think that it can work when it can't. builder.setBootNodes(new ArrayList<>()); } + builder.setDnsDiscoveryUrl(null); } if (networkId != null) { diff --git a/besu/src/main/java/org/hyperledger/besu/cli/config/EthNetworkConfig.java b/besu/src/main/java/org/hyperledger/besu/cli/config/EthNetworkConfig.java index d427ddcce9e..656f0ba66a8 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/config/EthNetworkConfig.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/config/EthNetworkConfig.java @@ -205,7 +205,7 @@ public static String jsonConfig(final NetworkName network) { public static class Builder { - private final String dnsDiscoveryUrl; + private String dnsDiscoveryUrl; private String genesisConfig; private BigInteger networkId; private List bootNodes; @@ -232,6 +232,11 @@ public Builder setBootNodes(final List bootNodes) { return this; } + public Builder setDnsDiscoveryUrl(final String dnsDiscoveryUrl) { + this.dnsDiscoveryUrl = dnsDiscoveryUrl; + return this; + } + public EthNetworkConfig build() { return new EthNetworkConfig(genesisConfig, networkId, bootNodes, dnsDiscoveryUrl); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 6dd478c38ee..26d634b956c 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -30,8 +30,12 @@ import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.NET; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.PERM; import static org.hyperledger.besu.ethereum.api.jsonrpc.RpcApis.WEB3; +import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.GOERLI_BOOTSTRAP_NODES; +import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.GOERLI_DISCOVERY_URL; import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.MAINNET_BOOTSTRAP_NODES; import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.MAINNET_DISCOVERY_URL; +import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.RINKEBY_BOOTSTRAP_NODES; +import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.RINKEBY_DISCOVERY_URL; import static org.hyperledger.besu.nat.kubernetes.KubernetesNatManager.DEFAULT_BESU_SERVICE_NAME_FILTER; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -345,6 +349,7 @@ public void overrideDefaultValuesIfKeyIsPresentInConfigFile() throws IOException .setNetworkId(BigInteger.valueOf(42)) .setGenesisConfig(encodeJsonGenesis(GENESIS_VALID_JSON)) .setBootNodes(nodes) + .setDnsDiscoveryUrl(null) .build(); verify(mockControllerBuilder).dataDirectory(eq(dataFolder.toPath())); verify(mockControllerBuilderFactory).fromEthNetworkConfig(eq(networkConfig), any()); @@ -891,6 +896,72 @@ public void genesisPathOptionMustBeUsed() throws Exception { assertThat(commandErrorOutput.toString()).isEmpty(); } + @Test + public void testGenesisPathEthOptions() throws Exception { + final Path genesisFile = createFakeGenesisFile(GENESIS_VALID_JSON); + + final ArgumentCaptor networkArg = + ArgumentCaptor.forClass(EthNetworkConfig.class); + + parseCommand("--genesis-file", genesisFile.toString()); + + verify(mockControllerBuilderFactory).fromEthNetworkConfig(networkArg.capture(), any()); + verify(mockControllerBuilder).build(); + + EthNetworkConfig config = networkArg.getValue(); + assertThat(config.getBootNodes()).isEmpty(); + assertThat(config.getDnsDiscoveryUrl()).isNull(); + assertThat(config.getNetworkId()).isEqualTo(BigInteger.valueOf(3141592)); + } + + @Test + public void testGenesisPathMainnetEthConfig() throws Exception { + final ArgumentCaptor networkArg = + ArgumentCaptor.forClass(EthNetworkConfig.class); + + parseCommand("--network", "mainnet"); + + verify(mockControllerBuilderFactory).fromEthNetworkConfig(networkArg.capture(), any()); + verify(mockControllerBuilder).build(); + + EthNetworkConfig config = networkArg.getValue(); + assertThat(config.getBootNodes()).isEqualTo(MAINNET_BOOTSTRAP_NODES); + assertThat(config.getDnsDiscoveryUrl()).isEqualTo(MAINNET_DISCOVERY_URL); + assertThat(config.getNetworkId()).isEqualTo(BigInteger.valueOf(1)); + } + + @Test + public void testGenesisPathGoerliEthConfig() throws Exception { + final ArgumentCaptor networkArg = + ArgumentCaptor.forClass(EthNetworkConfig.class); + + parseCommand("--network", "goerli"); + + verify(mockControllerBuilderFactory).fromEthNetworkConfig(networkArg.capture(), any()); + verify(mockControllerBuilder).build(); + + EthNetworkConfig config = networkArg.getValue(); + assertThat(config.getBootNodes()).isEqualTo(GOERLI_BOOTSTRAP_NODES); + assertThat(config.getDnsDiscoveryUrl()).isEqualTo(GOERLI_DISCOVERY_URL); + assertThat(config.getNetworkId()).isEqualTo(BigInteger.valueOf(5)); + } + + @Test + public void testGenesisPathRinkebyEthConfig() throws Exception { + final ArgumentCaptor networkArg = + ArgumentCaptor.forClass(EthNetworkConfig.class); + + parseCommand("--network", "rinkeby"); + + verify(mockControllerBuilderFactory).fromEthNetworkConfig(networkArg.capture(), any()); + verify(mockControllerBuilder).build(); + + EthNetworkConfig config = networkArg.getValue(); + assertThat(config.getBootNodes()).isEqualTo(RINKEBY_BOOTSTRAP_NODES); + assertThat(config.getDnsDiscoveryUrl()).isEqualTo(RINKEBY_DISCOVERY_URL); + assertThat(config.getNetworkId()).isEqualTo(BigInteger.valueOf(4)); + } + @Test public void genesisAndNetworkMustNotBeUsedTogether() throws Exception { final Path genesisFile = createFakeGenesisFile(GENESIS_VALID_JSON); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/config/EthNetworkConfigTest.java b/besu/src/test/java/org/hyperledger/besu/cli/config/EthNetworkConfigTest.java new file mode 100644 index 00000000000..bab4fd68a19 --- /dev/null +++ b/besu/src/test/java/org/hyperledger/besu/cli/config/EthNetworkConfigTest.java @@ -0,0 +1,76 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hyperledger.besu.cli.config.NetworkName.MAINNET; +import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.GOERLI_BOOTSTRAP_NODES; +import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.GOERLI_DISCOVERY_URL; +import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.MAINNET_BOOTSTRAP_NODES; +import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.MAINNET_DISCOVERY_URL; +import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.RINKEBY_BOOTSTRAP_NODES; +import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.RINKEBY_DISCOVERY_URL; + +import java.math.BigInteger; + +import org.junit.Test; + +public class EthNetworkConfigTest { + + @Test + public void testDefaultMainnetConfig() { + EthNetworkConfig config = EthNetworkConfig.getNetworkConfig(NetworkName.MAINNET); + assertThat(config.getDnsDiscoveryUrl()).isEqualTo(MAINNET_DISCOVERY_URL); + assertThat(config.getBootNodes()).isEqualTo(MAINNET_BOOTSTRAP_NODES); + assertThat(config.getNetworkId()).isEqualTo(BigInteger.ONE); + } + + @Test + public void testDefaultRinkebyConfig() { + EthNetworkConfig config = EthNetworkConfig.getNetworkConfig(NetworkName.RINKEBY); + assertThat(config.getDnsDiscoveryUrl()).isEqualTo(RINKEBY_DISCOVERY_URL); + assertThat(config.getBootNodes()).isEqualTo(RINKEBY_BOOTSTRAP_NODES); + assertThat(config.getNetworkId()).isEqualTo(BigInteger.valueOf(4)); + } + + @Test + public void testDefaultGoerliConfig() { + EthNetworkConfig config = EthNetworkConfig.getNetworkConfig(NetworkName.GOERLI); + assertThat(config.getDnsDiscoveryUrl()).isEqualTo(GOERLI_DISCOVERY_URL); + assertThat(config.getBootNodes()).isEqualTo(GOERLI_BOOTSTRAP_NODES); + assertThat(config.getNetworkId()).isEqualTo(BigInteger.valueOf(5)); + } + + @Test + public void testDefaultDevConfig() { + EthNetworkConfig config = EthNetworkConfig.getNetworkConfig(NetworkName.DEV); + assertThat(config.getDnsDiscoveryUrl()).isNull(); + assertThat(config.getBootNodes()).isEmpty(); + assertThat(config.getNetworkId()).isEqualTo(BigInteger.valueOf(2018)); + } + + @Test + public void testBuilderWithNetworkId() { + EthNetworkConfig config = + new EthNetworkConfig.Builder(EthNetworkConfig.getNetworkConfig(MAINNET)) + .setNetworkId(BigInteger.valueOf(42)) + .setGenesisConfig("{\"config\":{\"chainId\":\"1234567\"}") + .build(); + assertThat(config.getGenesisConfig()).isEqualTo("{\"config\":{\"chainId\":\"1234567\"}"); + assertThat(config.getDnsDiscoveryUrl()).isNotNull(); + assertThat(config.getBootNodes()).isNotEmpty(); + assertThat(config.getNetworkId()).isEqualTo(BigInteger.valueOf(42)); + } +} From 44c4d75016bb57531323256c9df848373febda5f Mon Sep 17 00:00:00 2001 From: Stefan Pingel <16143240+pinges@users.noreply.github.com> Date: Wed, 2 Dec 2020 09:48:00 +1000 Subject: [PATCH 02/16] add sendSignedTransaction method to QuorumEnclave (#1642) * add sendSignedTransaction method to QuorumEnclave Signed-off-by: Stefan Pingel --- .../besu/enclave/GoQuorumEnclaveTest.java | 20 +++++++-- .../besu/enclave/GoQuorumEnclave.java | 11 +++++ .../types/GoQuorumSendSignedRequest.java | 41 +++++++++++++++++++ 3 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 enclave/src/main/java/org/hyperledger/besu/enclave/types/GoQuorumSendSignedRequest.java diff --git a/enclave/src/integration-test/java/org/hyperledger/besu/enclave/GoQuorumEnclaveTest.java b/enclave/src/integration-test/java/org/hyperledger/besu/enclave/GoQuorumEnclaveTest.java index 3ca102e0e98..b07657a5516 100644 --- a/enclave/src/integration-test/java/org/hyperledger/besu/enclave/GoQuorumEnclaveTest.java +++ b/enclave/src/integration-test/java/org/hyperledger/besu/enclave/GoQuorumEnclaveTest.java @@ -38,7 +38,8 @@ public class GoQuorumEnclaveTest { private static final byte[] PAYLOAD = Base64.getDecoder().decode("EAAAAAAA"); private static final String MOCK_KEY = "iOCzoGo5kwtZU0J41Z9xnGXHN6ZNukIa9MspvHtu3Jk="; - private static final String KEY = "key"; + private static final String KEY = + "tQEmN0d/xXJZs5OMgl8QVBIyYxu1XubAKehsSYbcOjbxai+QJQpEOs6ghrYAZizLtnM4EJdMyVeVrxO3cA9JJA=="; private static GoQuorumEnclave enclave; private RequestTransmitter vertxTransmitter; @@ -49,7 +50,7 @@ public void setUp() { } @Test - public void testUpCheck() { + public void upCheck() { when(vertxTransmitter.get(any(), any(), ArgumentMatchers.contains("/upcheck"), any())) .thenReturn("I'm up!"); @@ -57,7 +58,7 @@ public void testUpCheck() { } @Test - public void testReceiveThrowsWhenPayloadDoesNotExist() { + public void receiveThrowsWhenPayloadDoesNotExist() { when(vertxTransmitter.get(any(), any(), ArgumentMatchers.contains("/receive"), any())) .thenThrow( new EnclaveClientException(404, "Message with hash " + MOCK_KEY + " was not found")); @@ -68,7 +69,7 @@ public void testReceiveThrowsWhenPayloadDoesNotExist() { } @Test - public void testSendAndReceive() { + public void sendAndReceive() { when(vertxTransmitter.post(any(), any(), any(), any())).thenReturn(new SendResponse(KEY)); when(vertxTransmitter.get(any(), any(), ArgumentMatchers.contains("/receive"), any())) .thenReturn(new GoQuorumReceiveResponse(PAYLOAD, 0, null, null)); @@ -83,6 +84,17 @@ public void testSendAndReceive() { assertThat(rr.getPayload()).isEqualTo(PAYLOAD); } + @Test + public void sendSignedTransaction() { + when(vertxTransmitter.post(any(), any(), ArgumentMatchers.contains("/sendsignedtx"), any())) + .thenReturn(new SendResponse(KEY)); + + final List publicKeys = Arrays.asList("/+UuD63zItL1EbjxkKUljMgG8Z1w0AJ8pNOR4iq2yQc="); + + final SendResponse sr = enclave.sendSignedTransaction(PAYLOAD, publicKeys); + assertThat(sr.getKey()).isEqualTo(KEY); + } + @Test public void upcheckReturnsFalseIfNoResponseReceived() throws URISyntaxException { final Vertx vertx = Vertx.vertx(); diff --git a/enclave/src/main/java/org/hyperledger/besu/enclave/GoQuorumEnclave.java b/enclave/src/main/java/org/hyperledger/besu/enclave/GoQuorumEnclave.java index c654194577c..8705d445068 100644 --- a/enclave/src/main/java/org/hyperledger/besu/enclave/GoQuorumEnclave.java +++ b/enclave/src/main/java/org/hyperledger/besu/enclave/GoQuorumEnclave.java @@ -17,6 +17,7 @@ import org.hyperledger.besu.enclave.RequestTransmitter.ResponseBodyHandler; import org.hyperledger.besu.enclave.types.GoQuorumReceiveResponse; import org.hyperledger.besu.enclave.types.GoQuorumSendRequest; +import org.hyperledger.besu.enclave.types.GoQuorumSendSignedRequest; import org.hyperledger.besu.enclave.types.ReceiveRequest; import org.hyperledger.besu.enclave.types.SendResponse; @@ -58,6 +59,16 @@ public SendResponse send( (statusCode, body) -> handleJsonResponse(statusCode, body, SendResponse.class, 201)); } + public SendResponse sendSignedTransaction( + final byte[] txLookupId, final List privateFor) { + final GoQuorumSendSignedRequest request = new GoQuorumSendSignedRequest(txLookupId, privateFor); + return post( + JSON, + request, + "/sendsignedtx", + (statusCode, body) -> handleJsonResponse(statusCode, body, SendResponse.class, 201)); + } + public GoQuorumReceiveResponse receive(final String payloadKey) { final ReceiveRequest request = new ReceiveRequest(payloadKey); return get( diff --git a/enclave/src/main/java/org/hyperledger/besu/enclave/types/GoQuorumSendSignedRequest.java b/enclave/src/main/java/org/hyperledger/besu/enclave/types/GoQuorumSendSignedRequest.java new file mode 100644 index 00000000000..cbe38c3055b --- /dev/null +++ b/enclave/src/main/java/org/hyperledger/besu/enclave/types/GoQuorumSendSignedRequest.java @@ -0,0 +1,41 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.enclave.types; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +@JsonPropertyOrder({"hash", "to"}) +public class GoQuorumSendSignedRequest { + private final byte[] hash; + private final List to; + + public GoQuorumSendSignedRequest( + @JsonProperty(value = "hash") final byte[] hash, + @JsonProperty(value = "to") final List privateFor) { + this.hash = hash; + this.to = privateFor; + } + + public byte[] getHash() { + return hash; + } + + public List getTo() { + return to; + } +} From 67efaa914519271f1c1ddc9679bfb5d51b4f8515 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Wed, 2 Dec 2020 12:10:20 +1000 Subject: [PATCH 03/16] transaction RLP encode/decode handle GoQuorum private tx (#1605) * modifying transaction RLP encode/decode to handle GoQuorum private tx value * added static flag to be read by TransactionRLPDecoder so that we can still allow chainId=1 * ensure chainId and v not both provided Signed-off-by: Sally MacFarlane --- .../org/hyperledger/besu/cli/BesuCommand.java | 3 + .../besu/config/GoQuorumOptions.java | 26 ++++++ .../besu/ethereum/core/Transaction.java | 90 ++++++++++++++++--- .../core/encoding/TransactionRLPDecoder.java | 55 +++++++++++- .../encoding/TransactionRLPDecoderTest.java | 19 ++++ 5 files changed, 182 insertions(+), 11 deletions(-) create mode 100644 config/src/main/java/org/hyperledger/besu/config/GoQuorumOptions.java diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 85dd95331d1..81ee77dfd44 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -73,6 +73,7 @@ import org.hyperledger.besu.cli.util.VersionProvider; import org.hyperledger.besu.config.GenesisConfigFile; import org.hyperledger.besu.config.GenesisConfigOptions; +import org.hyperledger.besu.config.GoQuorumOptions; import org.hyperledger.besu.config.experimental.ExperimentalEIPs; import org.hyperledger.besu.controller.BesuController; import org.hyperledger.besu.controller.BesuControllerBuilder; @@ -1399,6 +1400,8 @@ private void validateDnsOptionsParams() { private void validateGoQuorumCompatibilityModeParam() { if (isGoQuorumCompatibilityMode) { final GenesisConfigOptions genesisConfigOptions = readGenesisConfigOptions(); + // this static flag is read by the RLP decoder + GoQuorumOptions.goquorumCompatibilityMode = true; if (!genesisConfigOptions.isQuorum()) { throw new IllegalStateException( diff --git a/config/src/main/java/org/hyperledger/besu/config/GoQuorumOptions.java b/config/src/main/java/org/hyperledger/besu/config/GoQuorumOptions.java new file mode 100644 index 00000000000..772e634875a --- /dev/null +++ b/config/src/main/java/org/hyperledger/besu/config/GoQuorumOptions.java @@ -0,0 +1,26 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.config; + +/** + * Flag to determine whether we are processing in GoQuorum mode. Note that this mode is incompatible + * with MainNet. + */ +public class GoQuorumOptions { + // To make it easier for tests to reset the value to default + public static final boolean GOQUORUM_COMPATIBILITY_MODE_DEFAULT_VALUE = false; + + public static boolean goquorumCompatibilityMode = GOQUORUM_COMPATIBILITY_MODE_DEFAULT_VALUE; +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java index 1b8298b7da0..10a0d0c56b8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java @@ -45,6 +45,9 @@ public class Transaction implements org.hyperledger.besu.plugin.data.Transaction public static final BigInteger REPLAY_PROTECTED_V_BASE = BigInteger.valueOf(35); + public static final BigInteger GO_QUORUM_PRIVATE_TRANSACTION_V_VALUE_MIN = BigInteger.valueOf(37); + public static final BigInteger GO_QUORUM_PRIVATE_TRANSACTION_V_VALUE_MAX = BigInteger.valueOf(38); + // The v signature parameter starts at 36 because 1 is the first valid chainId so: // chainId > 1 implies that 2 * chainId + V_BASE > 36. public static final BigInteger REPLAY_PROTECTED_V_MIN = BigInteger.valueOf(36); @@ -71,6 +74,8 @@ public class Transaction implements org.hyperledger.besu.plugin.data.Transaction private final Optional chainId; + private final Optional v; + // Caches a "hash" of a portion of the transaction used for sender recovery. // Note that this hash does not include the transaction signature so it does not // fully identify the transaction (use the result of the {@code hash()} for that). @@ -105,6 +110,9 @@ public static Transaction readFrom(final RLPInput input) throws RLPException { * @param payload the payload * @param sender the transaction sender * @param chainId the chain id to apply the transaction to + * @param v the v value. This is only passed in directly for GoQuorum private transactions + * (v=37|38). For all other transactions, the v value is derived from the signature. If v is + * provided here, the chain id must be empty. *

The {@code to} will be an {@code Optional.empty()} for a contract creation transaction; * otherwise it should contain an address. *

The {@code chainId} must be greater than 0 to be applied to a specific chain; otherwise @@ -121,7 +129,12 @@ public Transaction( final SECP256K1.Signature signature, final Bytes payload, final Address sender, - final Optional chainId) { + final Optional chainId, + final Optional v) { + if (v.isPresent() && chainId.isPresent()) { + throw new IllegalStateException( + String.format("chainId '%s' and v '%s' cannot both be provided", chainId.get(), v.get())); + } this.nonce = nonce; this.gasPrice = gasPrice; this.gasPremium = gasPremium; @@ -133,6 +146,7 @@ public Transaction( this.payload = payload; this.sender = sender; this.chainId = chainId; + this.v = v; } /** @@ -150,6 +164,7 @@ public Transaction( *

The {@code to} will be an {@code Optional.empty()} for a contract creation transaction; * otherwise it should contain an address. *

The {@code chainId} must be greater than 0 to be applied to a specific chain; otherwise + * it will default to any chain. */ public Transaction( final long nonce, @@ -161,7 +176,50 @@ public Transaction( final Bytes payload, final Address sender, final Optional chainId) { - this(nonce, gasPrice, null, null, gasLimit, to, value, signature, payload, sender, chainId); + this( + nonce, + gasPrice, + null, + null, + gasLimit, + to, + value, + signature, + payload, + sender, + chainId, + Optional.empty()); + } + /** + * Instantiates a transaction instance. + * + * @param nonce the nonce + * @param gasPrice the gas price + * @param gasLimit the gas limit + * @param to the transaction recipient + * @param value the value being transferred to the recipient + * @param signature the signature + * @param payload the payload + * @param sender the transaction sender + * @param chainId the chain id to apply the transaction to + * @param v the v value (only passed in directly for GoQuorum private transactions) + *

The {@code to} will be an {@code Optional.empty()} for a contract creation transaction; + * otherwise it should contain an address. + *

The {@code chainId} must be greater than 0 to be applied to a specific chain; otherwise + * it will default to any chain. + */ + public Transaction( + final long nonce, + final Wei gasPrice, + final long gasLimit, + final Optional

to, + final Wei value, + final SECP256K1.Signature signature, + final Bytes payload, + final Address sender, + final Optional chainId, + final Optional v) { + this(nonce, gasPrice, null, null, gasLimit, to, value, signature, payload, sender, chainId, v); } /** @@ -302,7 +360,7 @@ public Address getSender() { .orElseThrow( () -> new IllegalStateException( - "Cannot recover public key from " + "signature for " + this)); + "Cannot recover public key from signature for " + this)); sender = Address.extract(Hash.hash(publicKey.getEncodedBytes())); } return sender; @@ -346,14 +404,16 @@ public BigInteger getS() { @Override public BigInteger getV() { - final BigInteger v; + if (this.v.isPresent()) { + return this.v.get(); + } + final BigInteger recId = BigInteger.valueOf(signature.getRecId()); if (chainId.isEmpty()) { - v = recId.add(REPLAY_UNPROTECTED_V_BASE); + return recId.add(REPLAY_UNPROTECTED_V_BASE); } else { - v = recId.add(REPLAY_PROTECTED_V_BASE).add(TWO.multiply(chainId.get())); + return recId.add(REPLAY_PROTECTED_V_BASE).add(TWO.multiply(chainId.get())); } - return v; } /** @@ -479,13 +539,14 @@ public boolean equals(final Object other) { && this.payload.equals(that.payload) && this.signature.equals(that.signature) && this.to.equals(that.to) - && this.value.equals(that.value); + && this.value.equals(that.value) + && this.v.equals(that.v); } @Override public int hashCode() { return Objects.hash( - nonce, gasPrice, gasPremium, feeCap, gasLimit, to, value, payload, signature, chainId); + nonce, gasPrice, gasPremium, feeCap, gasLimit, to, value, payload, signature, chainId, v); } @Override @@ -503,6 +564,7 @@ public String toString() { sb.append("value=").append(getValue()).append(", "); sb.append("sig=").append(getSignature()).append(", "); if (chainId.isPresent()) sb.append("chainId=").append(getChainId().get()).append(", "); + if (v.isPresent()) sb.append("v=").append(v.get()).append(", "); sb.append("payload=").append(getPayload()); return sb.append("}").toString(); } @@ -538,11 +600,18 @@ public static class Builder { protected Optional chainId = Optional.empty(); + protected Optional v = Optional.empty(); + public Builder chainId(final BigInteger chainId) { this.chainId = Optional.of(chainId); return this; } + public Builder v(final BigInteger v) { + this.v = Optional.of(v); + return this; + } + public Builder gasPrice(final Wei gasPrice) { this.gasPrice = gasPrice; return this; @@ -605,7 +674,8 @@ public Transaction build() { signature, payload, sender, - chainId); + chainId, + v); } public Transaction signAndBuild(final SECP256K1.KeyPair keys) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPDecoder.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPDecoder.java index b4d4b7debfe..321eec1bce3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPDecoder.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPDecoder.java @@ -14,12 +14,15 @@ */ package org.hyperledger.besu.ethereum.core.encoding; +import static org.hyperledger.besu.ethereum.core.Transaction.GO_QUORUM_PRIVATE_TRANSACTION_V_VALUE_MAX; +import static org.hyperledger.besu.ethereum.core.Transaction.GO_QUORUM_PRIVATE_TRANSACTION_V_VALUE_MIN; import static org.hyperledger.besu.ethereum.core.Transaction.REPLAY_PROTECTED_V_BASE; import static org.hyperledger.besu.ethereum.core.Transaction.REPLAY_PROTECTED_V_MIN; import static org.hyperledger.besu.ethereum.core.Transaction.REPLAY_UNPROTECTED_V_BASE; import static org.hyperledger.besu.ethereum.core.Transaction.REPLAY_UNPROTECTED_V_BASE_PLUS_1; import static org.hyperledger.besu.ethereum.core.Transaction.TWO; +import org.hyperledger.besu.config.GoQuorumOptions; import org.hyperledger.besu.config.experimental.ExperimentalEIPs; import org.hyperledger.besu.crypto.SECP256K1; import org.hyperledger.besu.ethereum.core.Address; @@ -37,8 +40,12 @@ public interface TransactionRLPDecoder { TransactionRLPDecoder FRONTIER = frontierDecoder(); TransactionRLPDecoder EIP1559 = eip1559Decoder(); + TransactionRLPDecoder GOQUORUM_PRIVATE_TRANSACTION_DECODER = goQuorumPrivateTransactionDecoder(); static Transaction decodeTransaction(final RLPInput input) { + if (GoQuorumOptions.goquorumCompatibilityMode) { + return GOQUORUM_PRIVATE_TRANSACTION_DECODER.decode(input); + } return (ExperimentalEIPs.eip1559Enabled ? EIP1559 : FRONTIER).decode(input); } @@ -47,6 +54,7 @@ static Transaction decodeTransaction(final RLPInput input) { static TransactionRLPDecoder frontierDecoder() { return input -> { input.enterList(); + final Transaction.Builder builder = Transaction.builder() .nonce(input.readLongScalar()) @@ -74,7 +82,6 @@ static TransactionRLPDecoder frontierDecoder() { final SECP256K1.Signature signature = SECP256K1.Signature.create(r, s, recId); input.leaveList(); - chainId.ifPresent(builder::chainId); return builder.signature(signature).build(); }; @@ -129,4 +136,50 @@ static TransactionRLPDecoder eip1559Decoder() { return builder.signature(signature).build(); }; } + + static TransactionRLPDecoder goQuorumPrivateTransactionDecoder() { + return input -> { + input.enterList(); + + final Transaction.Builder builder = + Transaction.builder() + .nonce(input.readLongScalar()) + .gasPrice(Wei.of(input.readUInt256Scalar())) + .gasLimit(input.readLongScalar()) + .to(input.readBytes(v -> v.size() == 0 ? null : Address.wrap(v))) + .value(Wei.of(input.readUInt256Scalar())) + .payload(input.readBytes()); + + final BigInteger v = input.readBigIntegerScalar(); + final byte recId; + Optional chainId = Optional.empty(); + if (isGoQuorumPrivateTransaction(v)) { + // GoQuorum private TX. No chain ID. Preserve the v value as provided. + builder.v(v); + recId = v.subtract(GO_QUORUM_PRIVATE_TRANSACTION_V_VALUE_MIN).byteValueExact(); + } else if (v.equals(REPLAY_UNPROTECTED_V_BASE) + || v.equals(REPLAY_UNPROTECTED_V_BASE_PLUS_1)) { + recId = v.subtract(REPLAY_UNPROTECTED_V_BASE).byteValueExact(); + } else if (v.compareTo(REPLAY_PROTECTED_V_MIN) > 0) { + chainId = Optional.of(v.subtract(REPLAY_PROTECTED_V_BASE).divide(TWO)); + recId = + v.subtract(TWO.multiply(chainId.get()).add(REPLAY_PROTECTED_V_BASE)).byteValueExact(); + } else { + throw new RuntimeException( + String.format("An unsupported encoded `v` value of %s was found", v)); + } + final BigInteger r = input.readUInt256Scalar().toBytes().toUnsignedBigInteger(); + final BigInteger s = input.readUInt256Scalar().toBytes().toUnsignedBigInteger(); + final SECP256K1.Signature signature = SECP256K1.Signature.create(r, s, recId); + + input.leaveList(); + chainId.ifPresent(builder::chainId); + return builder.signature(signature).build(); + }; + } + + private static boolean isGoQuorumPrivateTransaction(final BigInteger v) { + return v.equals(GO_QUORUM_PRIVATE_TRANSACTION_V_VALUE_MAX) + || v.equals(GO_QUORUM_PRIVATE_TRANSACTION_V_VALUE_MIN); + } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPDecoderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPDecoderTest.java index f48a36f8c12..3f61c63b1bf 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPDecoderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPDecoderTest.java @@ -17,11 +17,14 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import org.hyperledger.besu.config.GoQuorumOptions; import org.hyperledger.besu.config.experimental.ExperimentalEIPs; +import org.hyperledger.besu.ethereum.core.Address; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.Wei; import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLPException; +import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.apache.tuweni.bytes.Bytes; import org.junit.Test; @@ -32,6 +35,22 @@ public class TransactionRLPDecoderTest { "0xf901fc8032830138808080b901ae60056013565b6101918061001d6000396000f35b3360008190555056006001600060e060020a6000350480630a874df61461003a57806341c0e1b514610058578063a02b161e14610066578063dbbdf0831461007757005b610045600435610149565b80600160a060020a031660005260206000f35b610060610161565b60006000f35b6100716004356100d4565b60006000f35b61008560043560243561008b565b60006000f35b600054600160a060020a031632600160a060020a031614156100ac576100b1565b6100d0565b8060018360005260205260406000208190555081600060005260206000a15b5050565b600054600160a060020a031633600160a060020a031614158015610118575033600160a060020a0316600182600052602052604060002054600160a060020a031614155b61012157610126565b610146565b600060018260005260205260406000208190555080600060005260206000a15b50565b60006001826000526020526040600020549050919050565b600054600160a060020a031633600160a060020a0316146101815761018f565b600054600160a060020a0316ff5b561ca0c5689ed1ad124753d54576dfb4b571465a41900a1dff4058d8adf16f752013d0a01221cbd70ec28c94a3b55ec771bcbc70778d6ee0b51ca7ea9514594c861b1884"; private static final String EIP1559_TX_RLP = "0xf902028032830138808080b901ae60056013565b6101918061001d6000396000f35b3360008190555056006001600060e060020a6000350480630a874df61461003a57806341c0e1b514610058578063a02b161e14610066578063dbbdf0831461007757005b610045600435610149565b80600160a060020a031660005260206000f35b610060610161565b60006000f35b6100716004356100d4565b60006000f35b61008560043560243561008b565b60006000f35b600054600160a060020a031632600160a060020a031614156100ac576100b1565b6100d0565b8060018360005260205260406000208190555081600060005260206000a15b5050565b600054600160a060020a031633600160a060020a031614158015610118575033600160a060020a0316600182600052602052604060002054600160a060020a031614155b61012157610126565b610146565b600060018260005260205260406000208190555080600060005260206000a15b50565b60006001826000526020526040600020549050919050565b600054600160a060020a031633600160a060020a0316146101815761018f565b600054600160a060020a0316ff5b5682020f8201711ca0c5689ed1ad124753d54576dfb4b571465a41900a1dff4058d8adf16f752013d0a01221cbd70ec28c94a3b55ec771bcbc70778d6ee0b51ca7ea9514594c861b1884"; + private static final String GOQUORUM_PRIVATE_TX_RLP = + "0xf88d0b808347b7608080b840290a80a37d198ff06abe189b638ff53ac8a8dc51a0aff07609d2aa75342783ae493b3e3c6b564c0eebe49284b05a0726fb33087b9e0231d349ea0c7b5661c8c526a07144db7045a395e608cda6ab051c86cc4fb42e319960b82087f3b26f0cbc3c2da00223ac129b22aec7a6c2ace3c3ef39c5eaaa54070fd82d8ee2140b0e70b1dca9"; + + @Test + public void decodeGoQuorumPrivateTransactionRlp() { + GoQuorumOptions.goquorumCompatibilityMode = true; + RLPInput input = RLP.input(Bytes.fromHexString(GOQUORUM_PRIVATE_TX_RLP)); + + final Transaction transaction = TransactionRLPDecoder.decodeTransaction(input); + assertThat(transaction).isNotNull(); + assertThat(transaction.getV()).isEqualTo(38); + assertThat(transaction.getSender()) + .isEqualByComparingTo(Address.fromHexString("0xed9d02e382b34818e88b88a309c7fe71e65f419d")); + GoQuorumOptions.goquorumCompatibilityMode = + GoQuorumOptions.GOQUORUM_COMPATIBILITY_MODE_DEFAULT_VALUE; + } @Test public void decodeFrontierNominalCase() { From 809b801add61103024ccbbc22ce590c7a2c0aac0 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Wed, 2 Dec 2020 17:58:33 +1000 Subject: [PATCH 04/16] prevent Besu starting in GoQuorum mode on Mainnet (#1647) * prevent startup on mainnet if GoQuorum mode enabled Signed-off-by: Sally MacFarlane --- CHANGELOG.md | 4 +- .../org/hyperledger/besu/cli/BesuCommand.java | 65 +++++++++++-------- .../hyperledger/besu/cli/BesuCommandTest.java | 46 +++++++++++-- 3 files changed, 82 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dce18aaeaeb..85d26eefad2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,8 @@ ### Additions and Improvements * Added support for batched requests in WebSockets. [#1583](https://github.com/hyperledger/besu/pull/1583) -* Added a protocols section to `admin_peers` to provide info about peer health. [\#1582](https://github.com/hyperledger/besu/pull/1582) -* Added CLI option `--goquorum-compatibility-enabled` to enable GoQuorum compatibility mode. [#1598](https://github.com/hyperledger/besu/pull/1598) +* Added protocols section to `admin_peers` to provide info about peer health. [\#1582](https://github.com/hyperledger/besu/pull/1582) +* Added CLI option `--goquorum-compatibility-enabled` to enable GoQuorum compatibility mode. [#1598](https://github.com/hyperledger/besu/pull/1598). Note that this mode is incompatible with Mainnet. ### Bug Fixes diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 81ee77dfd44..600715d747e 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -1327,7 +1327,6 @@ private BesuCommand validateOptions() { validateNatParams(); validateNetStatsParams(); validateDnsOptionsParams(); - validateGoQuorumCompatibilityModeParam(); return this; } @@ -1397,19 +1396,6 @@ private void validateDnsOptionsParams() { } } - private void validateGoQuorumCompatibilityModeParam() { - if (isGoQuorumCompatibilityMode) { - final GenesisConfigOptions genesisConfigOptions = readGenesisConfigOptions(); - // this static flag is read by the RLP decoder - GoQuorumOptions.goquorumCompatibilityMode = true; - - if (!genesisConfigOptions.isQuorum()) { - throw new IllegalStateException( - "GoQuorum compatibility mode (enabled) can only be used if genesis file has 'isQuorum' flag set to true."); - } - } - } - private GenesisConfigOptions readGenesisConfigOptions() { final GenesisConfigOptions genesisConfigOptions; try { @@ -1471,10 +1457,6 @@ private void issueOptionWarnings() { private BesuCommand configure() throws Exception { checkPortClash(); - if (isGoQuorumCompatibilityMode) { - checkGoQuorumCompatibilityConfig(); - } - syncMode = Optional.ofNullable(syncMode) .orElse( @@ -1483,6 +1465,9 @@ private BesuCommand configure() throws Exception { : SyncMode.FULL); ethNetworkConfig = updateNetworkConfig(getNetwork()); + if (isGoQuorumCompatibilityMode) { + checkGoQuorumCompatibilityConfig(ethNetworkConfig); + } jsonRpcConfiguration = jsonRpcConfiguration(); graphQLConfiguration = graphQLConfiguration(); webSocketConfiguration = webSocketConfiguration(); @@ -2281,7 +2266,6 @@ private EthNetworkConfig updateNetworkConfig(final NetworkName network) { throw new ParameterException(commandLine, e.getMessage()); } } - return builder.build(); } @@ -2400,13 +2384,42 @@ private void checkPortClash() { }); } - private void checkGoQuorumCompatibilityConfig() { - if (genesisFile != null - && getGenesisConfigFile().getConfigOptions().isQuorum() - && !minTransactionGasPrice.isZero()) { - throw new ParameterException( - this.commandLine, - "--min-gas-price must be set to zero if GoQuorum compatibility is enabled in the genesis config."); + private void checkGoQuorumCompatibilityConfig(final EthNetworkConfig ethNetworkConfig) { + if (isGoQuorumCompatibilityMode) { + final GenesisConfigOptions genesisConfigOptions = readGenesisConfigOptions(); + // this static flag is read by the RLP decoder + GoQuorumOptions.goquorumCompatibilityMode = true; + + if (!genesisConfigOptions.isQuorum()) { + throw new IllegalStateException( + "GoQuorum compatibility mode (enabled) can only be used if genesis file has 'isQuorum' flag set to true."); + } + genesisConfigOptions + .getChainId() + .ifPresent( + chainId -> + ensureGoQuorumCompatibilityModeNotUsedOnMainnet( + chainId, isGoQuorumCompatibilityMode)); + + if (genesisFile != null + && getGenesisConfigFile().getConfigOptions().isQuorum() + && !minTransactionGasPrice.isZero()) { + throw new ParameterException( + this.commandLine, + "--min-gas-price must be set to zero if GoQuorum compatibility is enabled in the genesis config."); + } + if (ethNetworkConfig.getNetworkId().equals(EthNetworkConfig.MAINNET_NETWORK_ID)) { + throw new ParameterException( + this.commandLine, "GoQuorum compatibility mode (enabled) cannot be used on Mainnet."); + } + } + } + + private void ensureGoQuorumCompatibilityModeNotUsedOnMainnet( + final BigInteger chainId, final boolean isGoQuorumCompatibilityMode) { + if (isGoQuorumCompatibilityMode && chainId.equals(EthNetworkConfig.MAINNET_NETWORK_ID)) { + throw new IllegalStateException( + "GoQuorum compatibility mode (enabled) cannot be used on Mainnet."); } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 26d634b956c..4562b4253bc 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -125,7 +125,12 @@ public class BesuCommandTest extends CommandTestAbstract { .put("config", (new JsonObject()).put("chainId", GENESIS_CONFIG_TEST_CHAINID)); private static final JsonObject GENESIS_INVALID_DATA = (new JsonObject()).put("config", new JsonObject()); - private static final JsonObject GENESIS_QUORUM_INTEROP_ENABLED = + private static final JsonObject VALID_GENESIS_QUORUM_INTEROP_ENABLED_WITH_CHAINID = + (new JsonObject()) + .put( + "config", + new JsonObject().put("isquorum", true).put("chainId", GENESIS_CONFIG_TEST_CHAINID)); + private static final JsonObject INVALID_GENESIS_QUORUM_INTEROP_ENABLED_MAINNET = (new JsonObject()).put("config", new JsonObject().put("isquorum", true)); private static final String ENCLAVE_PUBLIC_KEY_PATH = BesuCommand.class.getResource("/orion_publickey.pub").getPath(); @@ -3923,7 +3928,8 @@ public void assertThatCompatibilityEth64ForkIdIsPresentInHelpMessage() { @Test public void quorumInteropDisabledDoesNotEnforceZeroGasPrice() throws IOException { - final Path genesisFile = createFakeGenesisFile(GENESIS_QUORUM_INTEROP_ENABLED); + final Path genesisFile = + createFakeGenesisFile(VALID_GENESIS_QUORUM_INTEROP_ENABLED_WITH_CHAINID); parseCommand( "--goquorum-compatibility-enabled=false", "--genesis-file", genesisFile.toString()); assertThat(commandErrorOutput.toString()).isEmpty(); @@ -3931,7 +3937,8 @@ public void quorumInteropDisabledDoesNotEnforceZeroGasPrice() throws IOException @Test public void quorumInteropEnabledFailsWithoutGasPriceSet() throws IOException { - final Path genesisFile = createFakeGenesisFile(GENESIS_QUORUM_INTEROP_ENABLED); + final Path genesisFile = + createFakeGenesisFile(VALID_GENESIS_QUORUM_INTEROP_ENABLED_WITH_CHAINID); parseCommand("--goquorum-compatibility-enabled", "--genesis-file", genesisFile.toString()); assertThat(commandErrorOutput.toString()) .contains( @@ -3940,7 +3947,8 @@ public void quorumInteropEnabledFailsWithoutGasPriceSet() throws IOException { @Test public void quorumInteropEnabledFailsWithoutGasPriceSetToZero() throws IOException { - final Path genesisFile = createFakeGenesisFile(GENESIS_QUORUM_INTEROP_ENABLED); + final Path genesisFile = + createFakeGenesisFile(VALID_GENESIS_QUORUM_INTEROP_ENABLED_WITH_CHAINID); parseCommand( "--goquorum-compatibility-enabled", "--genesis-file", @@ -3954,7 +3962,8 @@ public void quorumInteropEnabledFailsWithoutGasPriceSetToZero() throws IOExcepti @Test public void quorumInteropEnabledSucceedsWithGasPriceSetToZero() throws IOException { - final Path genesisFile = createFakeGenesisFile(GENESIS_QUORUM_INTEROP_ENABLED); + final Path genesisFile = + createFakeGenesisFile(VALID_GENESIS_QUORUM_INTEROP_ENABLED_WITH_CHAINID); parseCommand( "--goquorum-compatibility-enabled", "--genesis-file", @@ -3963,4 +3972,31 @@ public void quorumInteropEnabledSucceedsWithGasPriceSetToZero() throws IOExcepti "0"); assertThat(commandErrorOutput.toString()).isEmpty(); } + + @Test + public void quorumInteropEnabledFailsWithMainnetDefaultNetwork() throws IOException { + final Path genesisFile = createFakeGenesisFile(INVALID_GENESIS_QUORUM_INTEROP_ENABLED_MAINNET); + parseCommand( + "--goquorum-compatibility-enabled", + "--genesis-file", + genesisFile.toString(), + "--min-gas-price", + "0"); + assertThat(commandErrorOutput.toString()) + .contains("GoQuorum compatibility mode (enabled) cannot be used on Mainnet"); + } + + @Test + public void quorumInteropEnabledFailsWithMainnetChainId() throws IOException { + final Path genesisFile = + createFakeGenesisFile(INVALID_GENESIS_QUORUM_INTEROP_ENABLED_MAINNET.put("chainId", "1")); + parseCommand( + "--goquorum-compatibility-enabled", + "--genesis-file", + genesisFile.toString(), + "--min-gas-price", + "0"); + assertThat(commandErrorOutput.toString()) + .contains("GoQuorum compatibility mode (enabled) cannot be used on Mainnet"); + } } From 021ce8ee040427d804c69bd7ecb3ed893a948124 Mon Sep 17 00:00:00 2001 From: "Ratan (Rai) Sur" Date: Wed, 2 Dec 2020 09:02:53 -0500 Subject: [PATCH 05/16] EIP-2718 Typed Transaction Envelopes (#1645) This implements the encoding/decoding logic and because it's backwards compatible we can introduce it immediately. You can see that some of the typed-transaction specific encoding/decoding logic is tested where there are EIP-1559 encoding/decoding tests but there'll be more tests included in the EIP-2930 PR. Signed-off-by: Ratan Rai Sur --- .../BlockTransactionSelector.java | 3 +- .../BlockTransactionSelectorTest.java | 6 +- .../besu/ethereum/core/BlockBody.java | 2 + .../besu/ethereum/core/Transaction.java | 5 +- .../ethereum/core/TransactionReceipt.java | 56 +++- .../core/encoding/TransactionRLPDecoder.java | 261 +++++++++--------- .../core/encoding/TransactionRLPEncoder.java | 96 ++++--- .../mainnet/AbstractBlockProcessor.java | 9 +- .../mainnet/ClassicProtocolSpecs.java | 13 +- .../mainnet/MainnetProtocolSpecs.java | 49 +++- ...PrivateGroupRehydrationBlockProcessor.java | 2 +- .../PrivateMigrationBlockProcessor.java | 2 +- .../encoding/TransactionRLPDecoderTest.java | 17 +- .../encoding/TransactionRLPEncoderTest.java | 15 +- .../besu/ethereum/core/fees/EIP1559Test.java | 2 +- .../mainnet/PrivacyBlockProcessorTest.java | 2 +- .../eth/messages/BlockBodiesMessage.java | 4 +- .../messages/LimitedTransactionsMessages.java | 2 +- .../eth/messages/TransactionsMessage.java | 9 +- .../TransactionsMessageProcessor.java | 5 +- .../eth/messages/TransactionsMessageTest.java | 10 +- .../TransactionsMessageSenderTest.java | 2 +- ethereum/eth/src/test/resources/50.blocks | Bin 30779 -> 30668 bytes .../besu/ethereum/rlp/AbstractRLPOutput.java | 2 +- .../hyperledger/besu/ethereum/rlp/RLP.java | 6 +- .../besu/ethereum/rlp/RLPOutput.java | 20 +- .../besu/ethereum/trie/BranchNode.java | 2 +- .../besu/ethereum/trie/ExtensionNode.java | 2 +- plugin-api/build.gradle | 2 +- .../besu/plugin/data/TransactionType.java | 27 +- 30 files changed, 380 insertions(+), 253 deletions(-) diff --git a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockTransactionSelector.java b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockTransactionSelector.java index 4c1e50c159e..e436e787643 100644 --- a/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockTransactionSelector.java +++ b/ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockTransactionSelector.java @@ -257,7 +257,8 @@ private void updateTransactionResultTracking( transactionSelectionResult.update( transaction, - transactionReceiptFactory.create(result, worldState, cumulativeGasUsed), + transactionReceiptFactory.create( + transaction.getType(), result, worldState, cumulativeGasUsed), gasUsedByTransaction); } diff --git a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/BlockTransactionSelectorTest.java b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/BlockTransactionSelectorTest.java index 5af5542a20d..381beb79a2f 100644 --- a/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/BlockTransactionSelectorTest.java +++ b/ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/BlockTransactionSelectorTest.java @@ -50,6 +50,7 @@ import org.hyperledger.besu.ethereum.referencetests.ReferenceTestBlockchain; import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason; import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +import org.hyperledger.besu.plugin.data.TransactionType; import org.hyperledger.besu.plugin.services.MetricsSystem; import org.hyperledger.besu.testutil.TestClock; @@ -566,7 +567,10 @@ private Transaction createTransaction(final int transactionNumber) { // This is a duplicate of the MainnetProtocolSpec::frontierTransactionReceiptFactory private TransactionReceipt createReceipt( - final TransactionProcessingResult result, final WorldState worldState, final long gasUsed) { + final TransactionType __, + final TransactionProcessingResult result, + final WorldState worldState, + final long gasUsed) { return new TransactionReceipt( worldState.rootHash(), gasUsed, Lists.newArrayList(), Optional.empty()); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/BlockBody.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/BlockBody.java index 4abdeb0c99e..8ae7024b32d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/BlockBody.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/BlockBody.java @@ -57,8 +57,10 @@ public List getOmmers() { */ public void writeTo(final RLPOutput output) { output.startList(); + output.writeList(getTransactions(), Transaction::writeTo); output.writeList(getOmmers(), BlockHeader::writeTo); + output.endList(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java index 10a0d0c56b8..e20f82e9267 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java @@ -22,7 +22,6 @@ import org.hyperledger.besu.ethereum.core.encoding.TransactionRLPDecoder; import org.hyperledger.besu.ethereum.core.encoding.TransactionRLPEncoder; import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.rlp.RLPException; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.rlp.RLPOutput; import org.hyperledger.besu.plugin.data.Quantity; @@ -92,8 +91,8 @@ public static Builder builder() { return new Builder(); } - public static Transaction readFrom(final RLPInput input) throws RLPException { - return TransactionRLPDecoder.decodeTransaction(input); + public static Transaction readFrom(final RLPInput rlpInput) { + return TransactionRLPDecoder.decode(rlpInput); } /** diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/TransactionReceipt.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/TransactionReceipt.java index b568515fe7f..c537fec9913 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/TransactionReceipt.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/TransactionReceipt.java @@ -18,6 +18,7 @@ import org.hyperledger.besu.ethereum.rlp.RLPException; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.rlp.RLPOutput; +import org.hyperledger.besu.plugin.data.TransactionType; import java.util.List; import java.util.Objects; @@ -41,6 +42,7 @@ public class TransactionReceipt implements org.hyperledger.besu.plugin.data.Tran private static final int NONEXISTENT = -1; + private final TransactionType transactionType; private final Hash stateRoot; private final long cumulativeGasUsed; private final List logs; @@ -63,6 +65,7 @@ public TransactionReceipt( final List logs, final Optional revertReason) { this( + TransactionType.FRONTIER, stateRoot, NONEXISTENT, cumulativeGasUsed, @@ -72,12 +75,20 @@ public TransactionReceipt( } private TransactionReceipt( + final TransactionType transactionType, final Hash stateRoot, final long cumulativeGasUsed, final List logs, final LogsBloomFilter bloomFilter, final Optional revertReason) { - this(stateRoot, NONEXISTENT, cumulativeGasUsed, logs, bloomFilter, revertReason); + this( + transactionType, + stateRoot, + NONEXISTENT, + cumulativeGasUsed, + logs, + bloomFilter, + revertReason); } /** @@ -94,6 +105,7 @@ public TransactionReceipt( final List logs, final Optional revertReason) { this( + TransactionType.FRONTIER, null, status, cumulativeGasUsed, @@ -103,21 +115,39 @@ public TransactionReceipt( } private TransactionReceipt( + final TransactionType transactionType, final int status, final long cumulativeGasUsed, final List logs, final LogsBloomFilter bloomFilter, final Optional revertReason) { - this(null, status, cumulativeGasUsed, logs, bloomFilter, revertReason); + this(transactionType, null, status, cumulativeGasUsed, logs, bloomFilter, revertReason); + } + + public TransactionReceipt( + final TransactionType transactionType, + final int status, + final long cumulativeGasUsed, + final List logs, + final Optional maybeRevertReason) { + this( + transactionType, + status, + cumulativeGasUsed, + logs, + LogsBloomFilter.builder().insertLogs(logs).build(), + maybeRevertReason); } private TransactionReceipt( + final TransactionType transactionType, final Hash stateRoot, final int status, final long cumulativeGasUsed, final List logs, final LogsBloomFilter bloomFilter, final Optional revertReason) { + this.transactionType = transactionType; this.stateRoot = stateRoot; this.cumulativeGasUsed = cumulativeGasUsed; this.status = status; @@ -142,6 +172,9 @@ public void writeToWithRevertReason(final RLPOutput out) { } private void writeTo(final RLPOutput out, final boolean withRevertReason) { + if (!transactionType.equals(TransactionType.FRONTIER)) { + out.writeRaw(Bytes.of((byte) transactionType.getSerializedType())); + } out.startList(); // Determine whether it's a state root-encoded transaction receipt @@ -179,7 +212,18 @@ public static TransactionReceipt readFrom(final RLPInput input) { */ public static TransactionReceipt readFrom( final RLPInput input, final boolean revertReasonAllowed) { - input.enterList(); + TransactionType transactionType = TransactionType.FRONTIER; + try { + input.enterList(); + } catch (RLPException rlpe) { + if (rlpe.getMessage().contains("Expected current item to be a list")) { + // This is an EIP-2718 receipt + transactionType = TransactionType.of(input.readByte()); + input.enterList(); + } else { + throw rlpe; + } + } try { // Get the first element to check later to determine the @@ -204,10 +248,12 @@ public static TransactionReceipt readFrom( // byte for success (0x01) or failure (0x80). if (firstElement.raw().size() == 1) { final int status = firstElement.readIntScalar(); - return new TransactionReceipt(status, cumulativeGas, logs, bloomFilter, revertReason); + return new TransactionReceipt( + transactionType, status, cumulativeGas, logs, bloomFilter, revertReason); } else { final Hash stateRoot = Hash.wrap(firstElement.readBytes32()); - return new TransactionReceipt(stateRoot, cumulativeGas, logs, bloomFilter, revertReason); + return new TransactionReceipt( + transactionType, stateRoot, cumulativeGas, logs, bloomFilter, revertReason); } } finally { input.leaveList(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPDecoder.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPDecoder.java index 321eec1bce3..2c036684af8 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPDecoder.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPDecoder.java @@ -29,153 +29,152 @@ import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.Wei; import org.hyperledger.besu.ethereum.rlp.RLPInput; +import org.hyperledger.besu.plugin.data.TransactionType; import java.math.BigInteger; import java.util.Optional; +import com.google.common.collect.ImmutableMap; import org.apache.tuweni.bytes.Bytes; -@FunctionalInterface -public interface TransactionRLPDecoder { +public class TransactionRLPDecoder { - TransactionRLPDecoder FRONTIER = frontierDecoder(); - TransactionRLPDecoder EIP1559 = eip1559Decoder(); - TransactionRLPDecoder GOQUORUM_PRIVATE_TRANSACTION_DECODER = goQuorumPrivateTransactionDecoder(); + @FunctionalInterface + interface Decoder { + Transaction decode(RLPInput input); + } + + private static final ImmutableMap + TYPED_TRANSACTION_DECODERS = + ImmutableMap.of(TransactionType.EIP1559, TransactionRLPDecoder::decodeEIP1559); - static Transaction decodeTransaction(final RLPInput input) { + public static Transaction decode(final RLPInput rlpInput) { if (GoQuorumOptions.goquorumCompatibilityMode) { - return GOQUORUM_PRIVATE_TRANSACTION_DECODER.decode(input); + return decodeGoQuorum(rlpInput); + } + final Bytes typedTransactionBytes = rlpInput.raw(); + final int firstByte = typedTransactionBytes.get(0) & 0xff; + final TransactionType transactionType = TransactionType.of(firstByte); + if (transactionType.equals(TransactionType.FRONTIER)) { + return decodeFrontier(rlpInput); + } else { + rlpInput.skipNext(); // throw away the type byte + final Decoder decoder = + Optional.ofNullable(TYPED_TRANSACTION_DECODERS.get(transactionType)) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Developer Error. A supported transaction type %s has no associated" + + " decoding logic", + transactionType))); + return decoder.decode(rlpInput); } - return (ExperimentalEIPs.eip1559Enabled ? EIP1559 : FRONTIER).decode(input); } - Transaction decode(RLPInput input); - - static TransactionRLPDecoder frontierDecoder() { - return input -> { - input.enterList(); - - final Transaction.Builder builder = - Transaction.builder() - .nonce(input.readLongScalar()) - .gasPrice(Wei.of(input.readUInt256Scalar())) - .gasLimit(input.readLongScalar()) - .to(input.readBytes(v -> v.size() == 0 ? null : Address.wrap(v))) - .value(Wei.of(input.readUInt256Scalar())) - .payload(input.readBytes()); - - final BigInteger v = input.readBigIntegerScalar(); - final byte recId; - Optional chainId = Optional.empty(); - if (v.equals(REPLAY_UNPROTECTED_V_BASE) || v.equals(REPLAY_UNPROTECTED_V_BASE_PLUS_1)) { - recId = v.subtract(REPLAY_UNPROTECTED_V_BASE).byteValueExact(); - } else if (v.compareTo(REPLAY_PROTECTED_V_MIN) > 0) { - chainId = Optional.of(v.subtract(REPLAY_PROTECTED_V_BASE).divide(TWO)); - recId = - v.subtract(TWO.multiply(chainId.get()).add(REPLAY_PROTECTED_V_BASE)).byteValueExact(); - } else { - throw new RuntimeException( - String.format("An unsupported encoded `v` value of %s was found", v)); - } - final BigInteger r = input.readUInt256Scalar().toBytes().toUnsignedBigInteger(); - final BigInteger s = input.readUInt256Scalar().toBytes().toUnsignedBigInteger(); - final SECP256K1.Signature signature = SECP256K1.Signature.create(r, s, recId); - - input.leaveList(); - chainId.ifPresent(builder::chainId); - return builder.signature(signature).build(); - }; + static Transaction decodeFrontier(final RLPInput input) { + input.enterList(); + final Transaction.Builder builder = + Transaction.builder() + .nonce(input.readLongScalar()) + .gasPrice(Wei.of(input.readUInt256Scalar())) + .gasLimit(input.readLongScalar()) + .to(input.readBytes(v -> v.size() == 0 ? null : Address.wrap(v))) + .value(Wei.of(input.readUInt256Scalar())) + .payload(input.readBytes()); + + final BigInteger v = input.readBigIntegerScalar(); + final byte recId; + Optional chainId = Optional.empty(); + if (v.equals(REPLAY_UNPROTECTED_V_BASE) || v.equals(REPLAY_UNPROTECTED_V_BASE_PLUS_1)) { + recId = v.subtract(REPLAY_UNPROTECTED_V_BASE).byteValueExact(); + } else if (v.compareTo(REPLAY_PROTECTED_V_MIN) > 0) { + chainId = Optional.of(v.subtract(REPLAY_PROTECTED_V_BASE).divide(TWO)); + recId = v.subtract(TWO.multiply(chainId.get()).add(REPLAY_PROTECTED_V_BASE)).byteValueExact(); + } else { + throw new RuntimeException( + String.format("An unsupported encoded `v` value of %s was found", v)); + } + final BigInteger r = input.readUInt256Scalar().toBytes().toUnsignedBigInteger(); + final BigInteger s = input.readUInt256Scalar().toBytes().toUnsignedBigInteger(); + final SECP256K1.Signature signature = SECP256K1.Signature.create(r, s, recId); + + input.leaveList(); + + chainId.ifPresent(builder::chainId); + return builder.signature(signature).build(); } - static TransactionRLPDecoder eip1559Decoder() { - return input -> { - input.enterList(); - - final Transaction.Builder builder = - Transaction.builder() - .nonce(input.readLongScalar()) - .gasPrice(Wei.of(input.readUInt256Scalar())) - .gasLimit(input.readLongScalar()) - .to(input.readBytes(v -> v.size() == 0 ? null : Address.wrap(v))) - .value(Wei.of(input.readUInt256Scalar())) - .payload(input.readBytes()); - - final Bytes maybeGasPremiumOrV = input.readBytes(); - final Bytes maybeFeeCapOrR = input.readBytes(); - final Bytes maybeVOrS = input.readBytes(); - final BigInteger v, r, s; - // if this is the end of the list we are processing a legacy transaction - if (input.isEndOfCurrentList()) { - v = maybeGasPremiumOrV.toUnsignedBigInteger(); - r = maybeFeeCapOrR.toUnsignedBigInteger(); - s = maybeVOrS.toUnsignedBigInteger(); - } else { - // otherwise this is an EIP-1559 transaction - builder - .gasPremium(Wei.of(maybeGasPremiumOrV.toBigInteger())) - .feeCap(Wei.of(maybeFeeCapOrR.toBigInteger())); - v = maybeVOrS.toBigInteger(); - r = input.readUInt256Scalar().toBytes().toUnsignedBigInteger(); - s = input.readUInt256Scalar().toBytes().toUnsignedBigInteger(); - } - final byte recId; - Optional chainId = Optional.empty(); - if (v.equals(REPLAY_UNPROTECTED_V_BASE) || v.equals(REPLAY_UNPROTECTED_V_BASE_PLUS_1)) { - recId = v.subtract(REPLAY_UNPROTECTED_V_BASE).byteValueExact(); - } else if (v.compareTo(REPLAY_PROTECTED_V_MIN) > 0) { - chainId = Optional.of(v.subtract(REPLAY_PROTECTED_V_BASE).divide(TWO)); - recId = - v.subtract(TWO.multiply(chainId.get()).add(REPLAY_PROTECTED_V_BASE)).byteValueExact(); - } else { - throw new RuntimeException( - String.format("An unsupported encoded `v` value of %s was found", v)); - } - final SECP256K1.Signature signature = SECP256K1.Signature.create(r, s, recId); - input.leaveList(); - chainId.ifPresent(builder::chainId); - return builder.signature(signature).build(); - }; + static Transaction decodeEIP1559(final RLPInput input) { + ExperimentalEIPs.eip1559MustBeEnabled(); + input.enterList(); + + final Transaction.Builder builder = + Transaction.builder() + .nonce(input.readLongScalar()) + .gasPrice(Wei.of(input.readUInt256Scalar())) + .gasLimit(input.readLongScalar()) + .to(input.readBytes(v -> v.size() == 0 ? null : Address.wrap(v))) + .value(Wei.of(input.readUInt256Scalar())) + .payload(input.readBytes()) + .gasPremium(Wei.of(input.readBytes().toBigInteger())) + .feeCap(Wei.of(input.readBytes().toBigInteger())); + + final BigInteger v = input.readBytes().toBigInteger(); + final BigInteger r = input.readUInt256Scalar().toBytes().toUnsignedBigInteger(); + final BigInteger s = input.readUInt256Scalar().toBytes().toUnsignedBigInteger(); + final byte recId; + Optional chainId = Optional.empty(); + if (v.equals(REPLAY_UNPROTECTED_V_BASE) || v.equals(REPLAY_UNPROTECTED_V_BASE_PLUS_1)) { + recId = v.subtract(REPLAY_UNPROTECTED_V_BASE).byteValueExact(); + } else if (v.compareTo(REPLAY_PROTECTED_V_MIN) > 0) { + chainId = Optional.of(v.subtract(REPLAY_PROTECTED_V_BASE).divide(TWO)); + recId = v.subtract(TWO.multiply(chainId.get()).add(REPLAY_PROTECTED_V_BASE)).byteValueExact(); + } else { + throw new RuntimeException( + String.format("An unsupported encoded `v` value of %s was found", v)); + } + final SECP256K1.Signature signature = SECP256K1.Signature.create(r, s, recId); + input.leaveList(); + chainId.ifPresent(builder::chainId); + return builder.signature(signature).build(); } - static TransactionRLPDecoder goQuorumPrivateTransactionDecoder() { - return input -> { - input.enterList(); - - final Transaction.Builder builder = - Transaction.builder() - .nonce(input.readLongScalar()) - .gasPrice(Wei.of(input.readUInt256Scalar())) - .gasLimit(input.readLongScalar()) - .to(input.readBytes(v -> v.size() == 0 ? null : Address.wrap(v))) - .value(Wei.of(input.readUInt256Scalar())) - .payload(input.readBytes()); - - final BigInteger v = input.readBigIntegerScalar(); - final byte recId; - Optional chainId = Optional.empty(); - if (isGoQuorumPrivateTransaction(v)) { - // GoQuorum private TX. No chain ID. Preserve the v value as provided. - builder.v(v); - recId = v.subtract(GO_QUORUM_PRIVATE_TRANSACTION_V_VALUE_MIN).byteValueExact(); - } else if (v.equals(REPLAY_UNPROTECTED_V_BASE) - || v.equals(REPLAY_UNPROTECTED_V_BASE_PLUS_1)) { - recId = v.subtract(REPLAY_UNPROTECTED_V_BASE).byteValueExact(); - } else if (v.compareTo(REPLAY_PROTECTED_V_MIN) > 0) { - chainId = Optional.of(v.subtract(REPLAY_PROTECTED_V_BASE).divide(TWO)); - recId = - v.subtract(TWO.multiply(chainId.get()).add(REPLAY_PROTECTED_V_BASE)).byteValueExact(); - } else { - throw new RuntimeException( - String.format("An unsupported encoded `v` value of %s was found", v)); - } - final BigInteger r = input.readUInt256Scalar().toBytes().toUnsignedBigInteger(); - final BigInteger s = input.readUInt256Scalar().toBytes().toUnsignedBigInteger(); - final SECP256K1.Signature signature = SECP256K1.Signature.create(r, s, recId); - - input.leaveList(); - chainId.ifPresent(builder::chainId); - return builder.signature(signature).build(); - }; + static Transaction decodeGoQuorum(final RLPInput input) { + input.enterList(); + + final Transaction.Builder builder = + Transaction.builder() + .nonce(input.readLongScalar()) + .gasPrice(Wei.of(input.readUInt256Scalar())) + .gasLimit(input.readLongScalar()) + .to(input.readBytes(v -> v.size() == 0 ? null : Address.wrap(v))) + .value(Wei.of(input.readUInt256Scalar())) + .payload(input.readBytes()); + + final BigInteger v = input.readBigIntegerScalar(); + final byte recId; + Optional chainId = Optional.empty(); + if (isGoQuorumPrivateTransaction(v)) { + // GoQuorum private TX. No chain ID. Preserve the v value as provided. + builder.v(v); + recId = v.subtract(GO_QUORUM_PRIVATE_TRANSACTION_V_VALUE_MIN).byteValueExact(); + } else if (v.equals(REPLAY_UNPROTECTED_V_BASE) || v.equals(REPLAY_UNPROTECTED_V_BASE_PLUS_1)) { + recId = v.subtract(REPLAY_UNPROTECTED_V_BASE).byteValueExact(); + } else if (v.compareTo(REPLAY_PROTECTED_V_MIN) > 0) { + chainId = Optional.of(v.subtract(REPLAY_PROTECTED_V_BASE).divide(TWO)); + recId = v.subtract(TWO.multiply(chainId.get()).add(REPLAY_PROTECTED_V_BASE)).byteValueExact(); + } else { + throw new RuntimeException( + String.format("An unsupported encoded `v` value of %s was found", v)); + } + final BigInteger r = input.readUInt256Scalar().toBytes().toUnsignedBigInteger(); + final BigInteger s = input.readUInt256Scalar().toBytes().toUnsignedBigInteger(); + final SECP256K1.Signature signature = SECP256K1.Signature.create(r, s, recId); + + input.leaveList(); + chainId.ifPresent(builder::chainId); + return builder.signature(signature).build(); } private static boolean isGoQuorumPrivateTransaction(final BigInteger v) { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPEncoder.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPEncoder.java index 8e8c56c0b36..24bfb8a5955 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPEncoder.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPEncoder.java @@ -17,60 +17,75 @@ import org.hyperledger.besu.config.experimental.ExperimentalEIPs; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.Wei; +import org.hyperledger.besu.ethereum.rlp.RLP; import org.hyperledger.besu.ethereum.rlp.RLPOutput; import org.hyperledger.besu.plugin.data.Quantity; import org.hyperledger.besu.plugin.data.TransactionType; +import java.util.Optional; + import com.google.common.collect.ImmutableMap; import org.apache.tuweni.bytes.Bytes; public class TransactionRLPEncoder { - private static final Encoder FRONTIER = frontierEncoder(); - private static final Encoder EIP1559 = eip1559Encoder(); + @FunctionalInterface + interface Encoder { + void encode(Transaction transaction, RLPOutput output); + } - private static final ImmutableMap ENCODERS = - ImmutableMap.of(TransactionType.FRONTIER, FRONTIER, TransactionType.EIP1559, EIP1559); + private static final ImmutableMap TYPED_TRANSACTION_ENCODERS = + ImmutableMap.of(TransactionType.EIP1559, TransactionRLPEncoder::encodeEIP1559); - public static void encode(final Transaction transaction, final RLPOutput output) { - ENCODERS.getOrDefault(transaction.getType(), FRONTIER).encode(transaction, output); + public static void encode(final Transaction transaction, final RLPOutput rlpOutput) { + if (transaction.getType().equals(TransactionType.FRONTIER)) { + encodeFrontier(transaction, rlpOutput); + } else { + final TransactionType type = transaction.getType(); + final Encoder encoder = + Optional.ofNullable(TYPED_TRANSACTION_ENCODERS.get(type)) + .orElseThrow( + () -> + new IllegalStateException( + String.format( + "Developer Error. A supported transaction type %s has no associated" + + " encoding logic", + type))); + rlpOutput.writeRaw( + Bytes.concatenate( + Bytes.of((byte) type.getSerializedType()), + RLP.encode(output -> encoder.encode(transaction, output)))); + } } - static Encoder frontierEncoder() { - return (transaction, out) -> { - out.startList(); - out.writeLongScalar(transaction.getNonce()); - out.writeUInt256Scalar(transaction.getGasPrice()); - out.writeLongScalar(transaction.getGasLimit()); - out.writeBytes(transaction.getTo().map(Bytes::copy).orElse(Bytes.EMPTY)); - out.writeUInt256Scalar(transaction.getValue()); - out.writeBytes(transaction.getPayload()); - writeSignature(transaction, out); - out.endList(); - }; + static void encodeFrontier(final Transaction transaction, final RLPOutput out) { + out.startList(); + out.writeLongScalar(transaction.getNonce()); + out.writeUInt256Scalar(transaction.getGasPrice()); + out.writeLongScalar(transaction.getGasLimit()); + out.writeBytes(transaction.getTo().map(Bytes::copy).orElse(Bytes.EMPTY)); + out.writeUInt256Scalar(transaction.getValue()); + out.writeBytes(transaction.getPayload()); + writeSignature(transaction, out); + out.endList(); } - static Encoder eip1559Encoder() { - return (transaction, out) -> { - if (!ExperimentalEIPs.eip1559Enabled - || !TransactionType.EIP1559.equals(transaction.getType())) { - throw new RuntimeException("Invalid transaction format"); - } + static void encodeEIP1559(final Transaction transaction, final RLPOutput out) { + ExperimentalEIPs.eip1559MustBeEnabled(); - out.startList(); - out.writeLongScalar(transaction.getNonce()); - out.writeNull(); - out.writeLongScalar(transaction.getGasLimit()); - out.writeBytes(transaction.getTo().map(Bytes::copy).orElse(Bytes.EMPTY)); - out.writeUInt256Scalar(transaction.getValue()); - out.writeBytes(transaction.getPayload()); - out.writeUInt256Scalar( - transaction.getGasPremium().map(Quantity::getValue).map(Wei::ofNumber).orElseThrow()); - out.writeUInt256Scalar( - transaction.getFeeCap().map(Quantity::getValue).map(Wei::ofNumber).orElseThrow()); - writeSignature(transaction, out); - out.endList(); - }; + out.startList(); + out.writeLongScalar(transaction.getNonce()); + out.writeNull(); + out.writeLongScalar(transaction.getGasLimit()); + out.writeBytes(transaction.getTo().map(Bytes::copy).orElse(Bytes.EMPTY)); + out.writeUInt256Scalar(transaction.getValue()); + out.writeBytes(transaction.getPayload()); + out.writeUInt256Scalar( + transaction.getGasPremium().map(Quantity::getValue).map(Wei::ofNumber).orElseThrow()); + out.writeUInt256Scalar( + transaction.getFeeCap().map(Quantity::getValue).map(Wei::ofNumber).orElseThrow()); + writeSignature(transaction, out); + out.endList(); } private static void writeSignature(final Transaction transaction, final RLPOutput out) { @@ -78,9 +93,4 @@ private static void writeSignature(final Transaction transaction, final RLPOutpu out.writeBigIntegerScalar(transaction.getSignature().getR()); out.writeBigIntegerScalar(transaction.getSignature().getS()); } - - @FunctionalInterface - interface Encoder { - void encode(Transaction transaction, RLPOutput output); - } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java index 223fe010361..c6f55b8f4af 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java @@ -28,6 +28,7 @@ import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.vm.BlockHashLookup; import org.hyperledger.besu.ethereum.vm.OperationTracer; +import org.hyperledger.besu.plugin.data.TransactionType; import java.util.ArrayList; import java.util.List; @@ -41,7 +42,10 @@ public abstract class AbstractBlockProcessor implements BlockProcessor { public interface TransactionReceiptFactory { TransactionReceipt create( - TransactionProcessingResult result, WorldState worldState, long gasUsed); + TransactionType transactionType, + TransactionProcessingResult result, + WorldState worldState, + long gasUsed); } private static final Logger LOG = LogManager.getLogger(); @@ -165,7 +169,8 @@ public AbstractBlockProcessor.Result processBlock( currentGasUsed += transaction.getGasLimit() - result.getGasRemaining(); final TransactionReceipt transactionReceipt = - transactionReceiptFactory.create(result, worldState, currentGasUsed); + transactionReceiptFactory.create( + transaction.getType(), result, worldState, currentGasUsed); receipts.add(transactionReceipt); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ClassicProtocolSpecs.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ClassicProtocolSpecs.java index 079d27d84cd..a859ecc61fd 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ClassicProtocolSpecs.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/ClassicProtocolSpecs.java @@ -23,6 +23,7 @@ import org.hyperledger.besu.ethereum.mainnet.contractvalidation.MaxCodeSizeRule; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.vm.MessageFrame; +import org.hyperledger.besu.plugin.data.TransactionType; import java.math.BigInteger; import java.util.Collections; @@ -238,13 +239,21 @@ public static ProtocolSpecBuilder thanosDefinition( } private static TransactionReceipt byzantiumTransactionReceiptFactory( - final TransactionProcessingResult result, final WorldState worldState, final long gasUsed) { + // ignored because it's always FRONTIER for byzantium + final TransactionType __, + final TransactionProcessingResult result, + final WorldState worldState, + final long gasUsed) { return new TransactionReceipt( result.isSuccessful() ? 1 : 0, gasUsed, result.getLogs(), Optional.empty()); } private static TransactionReceipt byzantiumTransactionReceiptFactoryWithReasonEnabled( - final TransactionProcessingResult result, final WorldState worldState, final long gasUsed) { + // ignored because it's always FRONTIER for byzantium + final TransactionType __, + final TransactionProcessingResult result, + final WorldState worldState, + final long gasUsed) { return new TransactionReceipt( result.isSuccessful() ? 1 : 0, gasUsed, result.getLogs(), result.getRevertReason()); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java index c61b2d190d2..e062eeb06fa 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetProtocolSpecs.java @@ -39,6 +39,7 @@ import org.hyperledger.besu.ethereum.privacy.storage.PrivateMetadataUpdater; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.vm.MessageFrame; +import org.hyperledger.besu.plugin.data.TransactionType; import java.io.IOException; import java.math.BigInteger; @@ -403,6 +404,10 @@ static ProtocolSpecBuilder berlinDefinition( gasCalculator -> MainnetEvmRegistries.berlin(gasCalculator, chainId.orElse(BigInteger.ZERO))) .precompileContractRegistryBuilder(MainnetPrecompiledContractRegistries::berlin) + .transactionReceiptFactory( + enableRevertReason + ? MainnetProtocolSpecs::berlinTransactionReceiptFactoryWithReasonEnabled + : MainnetProtocolSpecs::berlinTransactionReceiptFactory) .name("Berlin"); } @@ -459,7 +464,11 @@ static ProtocolSpecBuilder eip1559Definition( } private static TransactionReceipt frontierTransactionReceiptFactory( - final TransactionProcessingResult result, final WorldState worldState, final long gasUsed) { + // ignored because it's always FRONTIER + final TransactionType __, + final TransactionProcessingResult result, + final WorldState worldState, + final long gasUsed) { return new TransactionReceipt( worldState.rootHash(), gasUsed, @@ -468,17 +477,51 @@ private static TransactionReceipt frontierTransactionReceiptFactory( } private static TransactionReceipt byzantiumTransactionReceiptFactory( - final TransactionProcessingResult result, final WorldState worldState, final long gasUsed) { + // ignored because it's always FRONTIER + final TransactionType __, + final TransactionProcessingResult result, + final WorldState worldState, + final long gasUsed) { return new TransactionReceipt( result.isSuccessful() ? 1 : 0, gasUsed, result.getLogs(), Optional.empty()); } private static TransactionReceipt byzantiumTransactionReceiptFactoryWithReasonEnabled( - final TransactionProcessingResult result, final WorldState worldState, final long gasUsed) { + // ignored because it's always FRONTIER + final TransactionType __, + final TransactionProcessingResult result, + final WorldState worldState, + final long gasUsed) { return new TransactionReceipt( result.isSuccessful() ? 1 : 0, gasUsed, result.getLogs(), result.getRevertReason()); } + private static TransactionReceipt berlinTransactionReceiptFactory( + final TransactionType transactionType, + final TransactionProcessingResult transactionProcessingResult, + final WorldState worldState, + final long gasUsed) { + return new TransactionReceipt( + transactionType, + transactionProcessingResult.isSuccessful() ? 1 : 0, + gasUsed, + transactionProcessingResult.getLogs(), + Optional.empty()); + } + + private static TransactionReceipt berlinTransactionReceiptFactoryWithReasonEnabled( + final TransactionType transactionType, + final TransactionProcessingResult transactionProcessingResult, + final WorldState worldState, + final long gasUsed) { + return new TransactionReceipt( + transactionType, + transactionProcessingResult.isSuccessful() ? 1 : 0, + gasUsed, + transactionProcessingResult.getLogs(), + transactionProcessingResult.getRevertReason()); + } + private static class DaoBlockProcessor implements BlockProcessor { private final BlockProcessor wrapped; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateGroupRehydrationBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateGroupRehydrationBlockProcessor.java index 13c68655206..0f67b70bbfe 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateGroupRehydrationBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateGroupRehydrationBlockProcessor.java @@ -174,7 +174,7 @@ public AbstractBlockProcessor.Result processBlock( gasUsed = transaction.getGasLimit() - result.getGasRemaining() + gasUsed; final TransactionReceipt transactionReceipt = - transactionReceiptFactory.create(result, worldState, gasUsed); + transactionReceiptFactory.create(transaction.getType(), result, worldState, gasUsed); receipts.add(transactionReceipt); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateMigrationBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateMigrationBlockProcessor.java index 6981af364fb..8a9da8b5f0b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateMigrationBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/storage/migration/PrivateMigrationBlockProcessor.java @@ -114,7 +114,7 @@ public AbstractBlockProcessor.Result processBlock( worldStateUpdater.commit(); gasUsed = transaction.getGasLimit() - result.getGasRemaining() + gasUsed; final TransactionReceipt transactionReceipt = - transactionReceiptFactory.create(result, worldState, gasUsed); + transactionReceiptFactory.create(transaction.getType(), result, worldState, gasUsed); receipts.add(transactionReceipt); } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPDecoderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPDecoderTest.java index 3f61c63b1bf..9a46e6d8c82 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPDecoderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPDecoderTest.java @@ -23,7 +23,6 @@ import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.Wei; import org.hyperledger.besu.ethereum.rlp.RLP; -import org.hyperledger.besu.ethereum.rlp.RLPException; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.apache.tuweni.bytes.Bytes; @@ -34,7 +33,7 @@ public class TransactionRLPDecoderTest { private static final String FRONTIER_TX_RLP = "0xf901fc8032830138808080b901ae60056013565b6101918061001d6000396000f35b3360008190555056006001600060e060020a6000350480630a874df61461003a57806341c0e1b514610058578063a02b161e14610066578063dbbdf0831461007757005b610045600435610149565b80600160a060020a031660005260206000f35b610060610161565b60006000f35b6100716004356100d4565b60006000f35b61008560043560243561008b565b60006000f35b600054600160a060020a031632600160a060020a031614156100ac576100b1565b6100d0565b8060018360005260205260406000208190555081600060005260206000a15b5050565b600054600160a060020a031633600160a060020a031614158015610118575033600160a060020a0316600182600052602052604060002054600160a060020a031614155b61012157610126565b610146565b600060018260005260205260406000208190555080600060005260206000a15b50565b60006001826000526020526040600020549050919050565b600054600160a060020a031633600160a060020a0316146101815761018f565b600054600160a060020a0316ff5b561ca0c5689ed1ad124753d54576dfb4b571465a41900a1dff4058d8adf16f752013d0a01221cbd70ec28c94a3b55ec771bcbc70778d6ee0b51ca7ea9514594c861b1884"; private static final String EIP1559_TX_RLP = - "0xf902028032830138808080b901ae60056013565b6101918061001d6000396000f35b3360008190555056006001600060e060020a6000350480630a874df61461003a57806341c0e1b514610058578063a02b161e14610066578063dbbdf0831461007757005b610045600435610149565b80600160a060020a031660005260206000f35b610060610161565b60006000f35b6100716004356100d4565b60006000f35b61008560043560243561008b565b60006000f35b600054600160a060020a031632600160a060020a031614156100ac576100b1565b6100d0565b8060018360005260205260406000208190555081600060005260206000a15b5050565b600054600160a060020a031633600160a060020a031614158015610118575033600160a060020a0316600182600052602052604060002054600160a060020a031614155b61012157610126565b610146565b600060018260005260205260406000208190555080600060005260206000a15b50565b60006001826000526020526040600020549050919050565b600054600160a060020a031633600160a060020a0316146101815761018f565b600054600160a060020a0316ff5b5682020f8201711ca0c5689ed1ad124753d54576dfb4b571465a41900a1dff4058d8adf16f752013d0a01221cbd70ec28c94a3b55ec771bcbc70778d6ee0b51ca7ea9514594c861b1884"; + "0x03f902028032830138808080b901ae60056013565b6101918061001d6000396000f35b3360008190555056006001600060e060020a6000350480630a874df61461003a57806341c0e1b514610058578063a02b161e14610066578063dbbdf0831461007757005b610045600435610149565b80600160a060020a031660005260206000f35b610060610161565b60006000f35b6100716004356100d4565b60006000f35b61008560043560243561008b565b60006000f35b600054600160a060020a031632600160a060020a031614156100ac576100b1565b6100d0565b8060018360005260205260406000208190555081600060005260206000a15b5050565b600054600160a060020a031633600160a060020a031614158015610118575033600160a060020a0316600182600052602052604060002054600160a060020a031614155b61012157610126565b610146565b600060018260005260205260406000208190555080600060005260206000a15b50565b60006001826000526020526040600020549050919050565b600054600160a060020a031633600160a060020a0316146101815761018f565b600054600160a060020a0316ff5b5682020f8201711ca0c5689ed1ad124753d54576dfb4b571465a41900a1dff4058d8adf16f752013d0a01221cbd70ec28c94a3b55ec771bcbc70778d6ee0b51ca7ea9514594c861b1884"; private static final String GOQUORUM_PRIVATE_TX_RLP = "0xf88d0b808347b7608080b840290a80a37d198ff06abe189b638ff53ac8a8dc51a0aff07609d2aa75342783ae493b3e3c6b564c0eebe49284b05a0726fb33087b9e0231d349ea0c7b5661c8c526a07144db7045a395e608cda6ab051c86cc4fb42e319960b82087f3b26f0cbc3c2da00223ac129b22aec7a6c2ace3c3ef39c5eaaa54070fd82d8ee2140b0e70b1dca9"; @@ -43,7 +42,7 @@ public void decodeGoQuorumPrivateTransactionRlp() { GoQuorumOptions.goquorumCompatibilityMode = true; RLPInput input = RLP.input(Bytes.fromHexString(GOQUORUM_PRIVATE_TX_RLP)); - final Transaction transaction = TransactionRLPDecoder.decodeTransaction(input); + final Transaction transaction = TransactionRLPDecoder.decode(input); assertThat(transaction).isNotNull(); assertThat(transaction.getV()).isEqualTo(38); assertThat(transaction.getSender()) @@ -55,7 +54,7 @@ public void decodeGoQuorumPrivateTransactionRlp() { @Test public void decodeFrontierNominalCase() { final Transaction transaction = - TransactionRLPDecoder.decodeTransaction(RLP.input(Bytes.fromHexString(FRONTIER_TX_RLP))); + TransactionRLPDecoder.decode(RLP.input(Bytes.fromHexString(FRONTIER_TX_RLP))); assertThat(transaction).isNotNull(); assertThat(transaction.getGasPrice()).isEqualByComparingTo(Wei.of(50L)); assertThat(transaction.getGasPremium()).isEmpty(); @@ -66,7 +65,7 @@ public void decodeFrontierNominalCase() { public void decodeEIP1559NominalCase() { ExperimentalEIPs.eip1559Enabled = true; final Transaction transaction = - TransactionRLPDecoder.decodeTransaction(RLP.input(Bytes.fromHexString(EIP1559_TX_RLP))); + TransactionRLPDecoder.decode(RLP.input(Bytes.fromHexString(EIP1559_TX_RLP))); assertThat(transaction).isNotNull(); assertThat(transaction.getGasPremium()).hasValue(Wei.of(527L)); assertThat(transaction.getFeeCap()).hasValue(Wei.of(369L)); @@ -77,11 +76,9 @@ public void decodeEIP1559NominalCase() { public void decodeEIP1559FailureWhenNotEnabled() { ExperimentalEIPs.eip1559Enabled = false; assertThatThrownBy( - () -> - TransactionRLPDecoder.decodeTransaction( - RLP.input(Bytes.fromHexString(EIP1559_TX_RLP)))) - .isInstanceOf(RLPException.class) - .hasMessageContaining("Not at the end of the current list"); + () -> TransactionRLPDecoder.decode(RLP.input(Bytes.fromHexString(EIP1559_TX_RLP)))) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("EIP-1559 feature flag"); ExperimentalEIPs.eip1559Enabled = ExperimentalEIPs.EIP1559_ENABLED_DEFAULT_VALUE; } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPEncoderTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPEncoderTest.java index b2e8a158fea..43659ad7eef 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPEncoderTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/encoding/TransactionRLPEncoderTest.java @@ -43,13 +43,12 @@ public class TransactionRLPEncoderTest { private static final String FRONTIER_TX_RLP = "0xf901fc8032830138808080b901ae60056013565b6101918061001d6000396000f35b3360008190555056006001600060e060020a6000350480630a874df61461003a57806341c0e1b514610058578063a02b161e14610066578063dbbdf0831461007757005b610045600435610149565b80600160a060020a031660005260206000f35b610060610161565b60006000f35b6100716004356100d4565b60006000f35b61008560043560243561008b565b60006000f35b600054600160a060020a031632600160a060020a031614156100ac576100b1565b6100d0565b8060018360005260205260406000208190555081600060005260206000a15b5050565b600054600160a060020a031633600160a060020a031614158015610118575033600160a060020a0316600182600052602052604060002054600160a060020a031614155b61012157610126565b610146565b600060018260005260205260406000208190555080600060005260206000a15b50565b60006001826000526020526040600020549050919050565b600054600160a060020a031633600160a060020a0316146101815761018f565b600054600160a060020a0316ff5b561ca0c5689ed1ad124753d54576dfb4b571465a41900a1dff4058d8adf16f752013d0a01221cbd70ec28c94a3b55ec771bcbc70778d6ee0b51ca7ea9514594c861b1884"; private static final String EIP1559_TX_RLP = - "0xf902028032830138808080b901ae60056013565b6101918061001d6000396000f35b3360008190555056006001600060e060020a6000350480630a874df61461003a57806341c0e1b514610058578063a02b161e14610066578063dbbdf0831461007757005b610045600435610149565b80600160a060020a031660005260206000f35b610060610161565b60006000f35b6100716004356100d4565b60006000f35b61008560043560243561008b565b60006000f35b600054600160a060020a031632600160a060020a031614156100ac576100b1565b6100d0565b8060018360005260205260406000208190555081600060005260206000a15b5050565b600054600160a060020a031633600160a060020a031614158015610118575033600160a060020a0316600182600052602052604060002054600160a060020a031614155b61012157610126565b610146565b600060018260005260205260406000208190555080600060005260206000a15b50565b60006001826000526020526040600020549050919050565b600054600160a060020a031633600160a060020a0316146101815761018f565b600054600160a060020a0316ff5b5682020f8201711ca0c5689ed1ad124753d54576dfb4b571465a41900a1dff4058d8adf16f752013d0a01221cbd70ec28c94a3b55ec771bcbc70778d6ee0b51ca7ea9514594c861b1884"; + "0x03f902028032830138808080b901ae60056013565b6101918061001d6000396000f35b3360008190555056006001600060e060020a6000350480630a874df61461003a57806341c0e1b514610058578063a02b161e14610066578063dbbdf0831461007757005b610045600435610149565b80600160a060020a031660005260206000f35b610060610161565b60006000f35b6100716004356100d4565b60006000f35b61008560043560243561008b565b60006000f35b600054600160a060020a031632600160a060020a031614156100ac576100b1565b6100d0565b8060018360005260205260406000208190555081600060005260206000a15b5050565b600054600160a060020a031633600160a060020a031614158015610118575033600160a060020a0316600182600052602052604060002054600160a060020a031614155b61012157610126565b610146565b600060018260005260205260406000208190555080600060005260206000a15b50565b60006001826000526020526040600020549050919050565b600054600160a060020a031633600160a060020a0316146101815761018f565b600054600160a060020a0316ff5b5682020f8201711ca0c5689ed1ad124753d54576dfb4b571465a41900a1dff4058d8adf16f752013d0a01221cbd70ec28c94a3b55ec771bcbc70778d6ee0b51ca7ea9514594c861b1884"; @Test public void encodeFrontierTxNominalCase() { final Transaction transaction = - TransactionRLPDecoder.frontierDecoder() - .decode(RLP.input(Bytes.fromHexString(FRONTIER_TX_RLP))); + TransactionRLPDecoder.decode(RLP.input(Bytes.fromHexString(FRONTIER_TX_RLP))); final BytesValueRLPOutput output = new BytesValueRLPOutput(); TransactionRLPEncoder.encode(transaction, output); assertThat(FRONTIER_TX_RLP).isEqualTo(output.encoded().toHexString()); @@ -59,12 +58,11 @@ public void encodeFrontierTxNominalCase() { public void encodeEIP1559TxNominalCase() { ExperimentalEIPs.eip1559Enabled = true; final Transaction transaction = - TransactionRLPDecoder.eip1559Decoder() - .decode(RLP.input(Bytes.fromHexString(EIP1559_TX_RLP))); + TransactionRLPDecoder.decode(RLP.input(Bytes.fromHexString(EIP1559_TX_RLP))); final BytesValueRLPOutput output = new BytesValueRLPOutput(); TransactionRLPEncoder.encode(transaction, output); assertThat( - "0xf902028080830138808080b901ae60056013565b6101918061001d6000396000f35b3360008190555056006001600060e060020a6000350480630a874df61461003a57806341c0e1b514610058578063a02b161e14610066578063dbbdf0831461007757005b610045600435610149565b80600160a060020a031660005260206000f35b610060610161565b60006000f35b6100716004356100d4565b60006000f35b61008560043560243561008b565b60006000f35b600054600160a060020a031632600160a060020a031614156100ac576100b1565b6100d0565b8060018360005260205260406000208190555081600060005260206000a15b5050565b600054600160a060020a031633600160a060020a031614158015610118575033600160a060020a0316600182600052602052604060002054600160a060020a031614155b61012157610126565b610146565b600060018260005260205260406000208190555080600060005260206000a15b50565b60006001826000526020526040600020549050919050565b600054600160a060020a031633600160a060020a0316146101815761018f565b600054600160a060020a0316ff5b5682020f8201711ca0c5689ed1ad124753d54576dfb4b571465a41900a1dff4058d8adf16f752013d0a01221cbd70ec28c94a3b55ec771bcbc70778d6ee0b51ca7ea9514594c861b1884") + "0x03f902028080830138808080b901ae60056013565b6101918061001d6000396000f35b3360008190555056006001600060e060020a6000350480630a874df61461003a57806341c0e1b514610058578063a02b161e14610066578063dbbdf0831461007757005b610045600435610149565b80600160a060020a031660005260206000f35b610060610161565b60006000f35b6100716004356100d4565b60006000f35b61008560043560243561008b565b60006000f35b600054600160a060020a031632600160a060020a031614156100ac576100b1565b6100d0565b8060018360005260205260406000208190555081600060005260206000a15b5050565b600054600160a060020a031633600160a060020a031614158015610118575033600160a060020a0316600182600052602052604060002054600160a060020a031614155b61012157610126565b610146565b600060018260005260205260406000208190555080600060005260206000a15b50565b60006001826000526020526040600020549050919050565b600054600160a060020a031633600160a060020a0316146101815761018f565b600054600160a060020a0316ff5b5682020f8201711ca0c5689ed1ad124753d54576dfb4b571465a41900a1dff4058d8adf16f752013d0a01221cbd70ec28c94a3b55ec771bcbc70778d6ee0b51ca7ea9514594c861b1884") .isEqualTo(output.encoded().toHexString()); ExperimentalEIPs.eip1559Enabled = ExperimentalEIPs.EIP1559_ENABLED_DEFAULT_VALUE; } @@ -75,11 +73,10 @@ public void encodeEIP1559TxFailureNotEnabled() { assertThatThrownBy( () -> TransactionRLPEncoder.encode( - TransactionRLPDecoder.eip1559Decoder() - .decode(RLP.input(Bytes.fromHexString(EIP1559_TX_RLP))), + TransactionRLPDecoder.decode(RLP.input(Bytes.fromHexString(EIP1559_TX_RLP))), new BytesValueRLPOutput())) .isInstanceOf(RuntimeException.class) - .hasMessageContaining("Invalid transaction format"); + .hasMessageContaining("EIP-1559 feature flag"); ExperimentalEIPs.eip1559Enabled = ExperimentalEIPs.EIP1559_ENABLED_DEFAULT_VALUE; } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/fees/EIP1559Test.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/fees/EIP1559Test.java index ca4ef5c5195..fc56038bd55 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/fees/EIP1559Test.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/fees/EIP1559Test.java @@ -139,6 +139,6 @@ private static class TransactionFixture { Transaction.readFrom( RLP.input( Bytes.fromHexString( - "0xf902028032830138808080b901ae60056013565b6101918061001d6000396000f35b3360008190555056006001600060e060020a6000350480630a874df61461003a57806341c0e1b514610058578063a02b161e14610066578063dbbdf0831461007757005b610045600435610149565b80600160a060020a031660005260206000f35b610060610161565b60006000f35b6100716004356100d4565b60006000f35b61008560043560243561008b565b60006000f35b600054600160a060020a031632600160a060020a031614156100ac576100b1565b6100d0565b8060018360005260205260406000208190555081600060005260206000a15b5050565b600054600160a060020a031633600160a060020a031614158015610118575033600160a060020a0316600182600052602052604060002054600160a060020a031614155b61012157610126565b610146565b600060018260005260205260406000208190555080600060005260206000a15b50565b60006001826000526020526040600020549050919050565b600054600160a060020a031633600160a060020a0316146101815761018f565b600054600160a060020a0316ff5b5682020f8201711ca0c5689ed1ad124753d54576dfb4b571465a41900a1dff4058d8adf16f752013d0a01221cbd70ec28c94a3b55ec771bcbc70778d6ee0b51ca7ea9514594c861b1884"))); + "0x03f902028032830138808080b901ae60056013565b6101918061001d6000396000f35b3360008190555056006001600060e060020a6000350480630a874df61461003a57806341c0e1b514610058578063a02b161e14610066578063dbbdf0831461007757005b610045600435610149565b80600160a060020a031660005260206000f35b610060610161565b60006000f35b6100716004356100d4565b60006000f35b61008560043560243561008b565b60006000f35b600054600160a060020a031632600160a060020a031614156100ac576100b1565b6100d0565b8060018360005260205260406000208190555081600060005260206000a15b5050565b600054600160a060020a031633600160a060020a031614158015610118575033600160a060020a0316600182600052602052604060002054600160a060020a031614155b61012157610126565b610146565b600060018260005260205260406000208190555080600060005260206000a15b50565b60006001826000526020526040600020549050919050565b600054600160a060020a031633600160a060020a0316146101815761018f565b600054600160a060020a0316ff5b5682020f8201711ca0c5689ed1ad124753d54576dfb4b571465a41900a1dff4058d8adf16f752013d0a01221cbd70ec28c94a3b55ec771bcbc70778d6ee0b51ca7ea9514594c861b1884"))); } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessorTest.java index 3b168e2c293..c7dcf105760 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/PrivacyBlockProcessorTest.java @@ -209,7 +209,7 @@ private ProtocolSpec mockProtocolSpec() { when(protocolSpec.getPrivateTransactionProcessor()).thenReturn(mockPrivateTransactionProcessor); final AbstractBlockProcessor.TransactionReceiptFactory mockTransactionReceiptFactory = mock(AbstractBlockProcessor.TransactionReceiptFactory.class); - when(mockTransactionReceiptFactory.create(any(), any(), anyLong())) + when(mockTransactionReceiptFactory.create(any(), any(), any(), anyLong())) .thenReturn(new TransactionReceipt(0, 0, Collections.emptyList(), Optional.empty())); when(protocolSpec.getTransactionReceiptFactory()).thenReturn(mockTransactionReceiptFactory); when(protocolSpec.getBlockReward()).thenReturn(Wei.ZERO); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/BlockBodiesMessage.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/BlockBodiesMessage.java index e93667ef5b8..8d4dd31ec73 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/BlockBodiesMessage.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/BlockBodiesMessage.java @@ -43,9 +43,7 @@ public static BlockBodiesMessage readFrom(final MessageData message) { public static BlockBodiesMessage create(final Iterable bodies) { final BytesValueRLPOutput tmp = new BytesValueRLPOutput(); - tmp.startList(); - bodies.forEach(body -> body.writeTo(tmp)); - tmp.endList(); + tmp.writeList(bodies, BlockBody::writeTo); return new BlockBodiesMessage(tmp.encoded()); } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/LimitedTransactionsMessages.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/LimitedTransactionsMessages.java index 97bc81c22c8..9673b1f87a9 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/LimitedTransactionsMessages.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/LimitedTransactionsMessages.java @@ -49,7 +49,7 @@ public static LimitedTransactionsMessages createLimited( if (encodedBytes.size() > LIMIT && (messageSize != 0)) { break; } - message.writeRLPUnsafe(encodedBytes); + message.writeRaw(encodedBytes); includedTransactions.add(transaction); // Check if last transaction to add to the message messageSize += encodedBytes.size(); diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/TransactionsMessage.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/TransactionsMessage.java index 18fa23e4beb..2865551a10d 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/TransactionsMessage.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/messages/TransactionsMessage.java @@ -19,10 +19,8 @@ import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; -import org.hyperledger.besu.ethereum.rlp.RLPInput; -import java.util.Iterator; -import java.util.function.Function; +import java.util.List; import org.apache.tuweni.bytes.Bytes; @@ -59,8 +57,7 @@ public int getCode() { return EthPV62.TRANSACTIONS; } - public Iterator transactions( - final Function transactionReader) { - return new BytesValueRLPInput(data, false).readList(transactionReader).iterator(); + public List transactions() { + return new BytesValueRLPInput(data, false).readList(Transaction::readFrom); } } diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionsMessageProcessor.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionsMessageProcessor.java index 0a514cc4aaf..5fdb5d40886 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionsMessageProcessor.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionsMessageProcessor.java @@ -27,7 +27,7 @@ import java.time.Duration; import java.time.Instant; -import java.util.Iterator; +import java.util.List; import java.util.Set; import com.google.common.collect.Sets; @@ -75,8 +75,7 @@ private void processTransactionsMessage( try { LOG.trace("Received transactions message from {}", peer); - final Iterator readTransactions = - transactionsMessage.transactions(Transaction::readFrom); + final List readTransactions = transactionsMessage.transactions(); final Set transactions = Sets.newHashSet(readTransactions); transactionTracker.markTransactionsAsSeen(peer, transactions); transactionPool.addRemoteTransactions(transactions); diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/TransactionsMessageTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/TransactionsMessageTest.java index 1a26c832ff1..f59710ec2ae 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/TransactionsMessageTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/messages/TransactionsMessageTest.java @@ -14,16 +14,16 @@ */ package org.hyperledger.besu.ethereum.eth.messages; +import static org.assertj.core.api.Assertions.assertThat; + import org.hyperledger.besu.ethereum.core.BlockDataGenerator; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.MessageData; import org.hyperledger.besu.ethereum.p2p.rlpx.wire.RawMessage; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; -import org.assertj.core.api.Assertions; import org.junit.Test; public class TransactionsMessageTest { @@ -46,10 +46,6 @@ public void transactionRoundTrip() { final TransactionsMessage message = TransactionsMessage.readFrom(raw); // Check that transactions match original inputs after transformations - final Iterator readTransactions = message.transactions(Transaction::readFrom); - for (int i = 0; i < txCount; ++i) { - Assertions.assertThat(readTransactions.next()).isEqualTo(transactions.get(i)); - } - Assertions.assertThat(readTransactions.hasNext()).isFalse(); + assertThat(message.transactions()).isEqualTo(transactions); } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionsMessageSenderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionsMessageSenderTest.java index 48655d12267..b7b2582ce7e 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionsMessageSenderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/transactions/TransactionsMessageSenderTest.java @@ -104,6 +104,6 @@ private MessageData transactionsMessageContaining(final Transaction... transacti private Set getTransactionsFromMessage(final MessageData message) { final TransactionsMessage transactionsMessage = TransactionsMessage.readFrom(message); - return newHashSet(transactionsMessage.transactions(Transaction::readFrom)); + return newHashSet(transactionsMessage.transactions()); } } diff --git a/ethereum/eth/src/test/resources/50.blocks b/ethereum/eth/src/test/resources/50.blocks index 0802ad0a6035b543b71e0d4c435e95b0db0cac6d..daa9ab2c2cb4199090f0e72eed1d8bca99f5dd27 100644 GIT binary patch literal 30668 zcmeIbWmuJM*S1RuDAFY$r8LsrCDPpu(vp&bu(CNs`zw7u@X(@iWn(S5mR-xYV8F zgKLp4#n^jqFiGxPTA307zEyYj4Be2zMbK(2(q#;NsFEp1y874_A6z_;NQN`Z0x&F7^Gf6N z?vdn|Ba)`U<4onZ@NoaP|AOJwbvu6Wf*=?meKU|MYM@3N$9#&Exr-Hx8yhn#GaHM$ zwY`~xyEBWW0~<3ZGdu7w;I6+H_BZgkZHo5~kayy1Few zy`J}@)oMN zYpCS&fyu(WcnAc@tL3zS-E%@EF_>o$GvFGID>!sNJ<0+>ZiDoBp>RNwewHd*YkSvc ze+iXaThWc#aij|=-)T0ul7wY{o)({xgqs}&G=md8p-;%bwQ)o!!J+}~M+jk;Z~#^| zXFP1LGv8`9@UAN9ZJ5Xg1@e!MuSQKeZrupznDhzH5pJc zTC+ZZAXtB=v6EzRYZp@oYkLZ7X9{&`NeXdW2U8nUD-&ybW`Jua61UjR4z-xZ>9t`P z<%VhWdlDh^;4_1u7Yika>oZ`;gi@Lt=K@i<-3Q z9Sbsv#cFB3S;+n1U56HE4x4P6Kv~fnEYJ8-7t<9d65m@?o5z=eryIk|tDOb!R1Byn z(@kzEG~PO6%@x@pHCJg4qq$G;LQt{do2=^Gy$g>^nIgK9sW$DLr)@p2grtW3HBi3v z$)D5PT}US0A`4y^pGl1_zXuz}9E|3{*9EvsEF`6lE=%8S3!et*0Tct11tab?6(8gc z155iE4*RotO<3}y9%QW$;e1oH>xQX9Faxe(yn-=E;gk{t!TvLcUHl|dtStYODL4~r zYsY@E%{QH0OdfSSt=d8wBfZMOk|<6~YM_`WRgMCow-;pby4I7*08{@!beRDJ4X98xs6GZka4v=FDur?0 z2L4WAtV|ZBJbY#x>}Eeh0#Mk%MJKG$u$#_KS>1i&#e;qyxo%kD_?)>lc5lP_b z>=>k3A`{>W5t6GCIM)v^|E1j9oZ(X%vl7a&l}A66xG2^%GF2Fkp80vJ%fUwEaJE}` z3~gBKt=z_E>hy?RJmR?fKo0s-AzB1XVNqK?NY~I#eOhZ~6ip4puj6d<{G&Vh3QPiE z23#Wol@IB{+aL(;AM(M8{@W7ReQp*#%4JN8hOO1XuEQ~JIodNcvqy#;l4e(86rgVb z&T)GSsJ-r5`il-yu{3)bNp)V&b+&Aj@+do@c8lgx4-4UB`So0pa-U<^bZoEkA#^^V z=u&&9)gQOin&@Mb!&f20vf4t7bT8escV;z?CO8!4;lm8L zrsIl^vT?P`f_AUuPcMM*XV>CimZ@sl=h2If&=)3KD7 zG32{TVoFKCu;yi3r8O*N(+#xn1tPT@B$z78ZL%CC*d*U1nd_w7UwVQddqWR#N9zpc z;lm8LrUR-S5?Ti!2tG(Z7fJ_H_@$TQ=-^<>?BHYxP;oWGSxM*9j@w6twz5sSl~6b6 zsUJi=q<8*Z-D)>0G%i7GepqyS z&l5#FON}cGHVwZt_r9$NVXU;Jh}e*Ja6ZsYcNKyA@w9hs;+ATB@{FD3DGLyd^(YiO zOnYyC#q8<(klYD{GxXUU67|D?L>_~p>UB6EKNe4xSeKW8is4P;(7FWPMwm;!tj_05 z?pOV7bb$|hu4#Z48x>`dAP50Szy3ew9;e)I)nbPFNo&7SzBmQQ)`ww6M!#^LR4CqC~a$JgyQ z7`!7cX^g%qeNDbGlM?o{%cN?dK2~}hfw=cBl``k4u54d3RbFq@mIiDZ>z;!`zHb+B zNXdGvEk39(aQDD|3OpdomGxG0I7du~=CP6W@)K@5)K+)8pIQOp%3t9uR0M*)RG5sm z;l~-dB3QjaCb@fog4Nk?9MKnVlvimnhHXPr34%QEDIbRY5N?5JMw6Nkvg3^v#wsldLp8> zaZR3l61W8I$23N5mvvLI&b}!x%wz~N;F?INK9(5X2SM(D^jDzS(lq(+%NQGrnTgY7 z_v0@^(f|1a%dZz*{6R>xf9+yLp3sa9~LZhaODUy=PP-pRD^=p=|)7 zxp9g);TedNRriJ`f0CGR_rvJ-AXAMQyw<+2dqS|`aQskZG@FBMpHPldi=1cAqB8#w z7cg}vYwDiTC?``!3mJ#^WqPi(bE@s{JfH|~YCBLm2*5mhm;u*tKuam!%3ctJ=u)nr zaG3v;I$8hK7b6HPyP-8O@rF}R&Gh{S&iS+(SG8_KeC^w`x|bw3Th73ytU46z(mSz# z(XpGSXH9r#zm8Wl&)Bd$JVI8rn10PU@?Bo;XIJX{_F&j_P;$;^*4&vYF!%db%(kSq zMn0`Q0Z`BA_sWh#;-8D5Gt_?*wil21v81CY%1$X+|4?M;VIj=JhZ%592UI%pZ`Xn# z#D7YM*H3i(yKAZ$dkfpd8kL)OSD%w;x!QV`)@;d8a(}9E;Y=`oyv`L|O z;$_7yPE=45DER696K-hv<`ApP?{N}ZseJYcq;?Y+yMI`(qMuu>Ky_AZf4|B5LVeR#io z&tag*nzN*t9J$f}K6n8%d8=T(Ti~k7@e2yC=yHUX-R+N_*1SHQ;%!%MYTe`SRw|2h zFaw90PDea2gtjBH>9nZKgO8r+Sq-x1j9AC0Ww@#VGq=A^$2NUV3?yd-q~3Dz*FqzpZM*@nj5B9IiP_`VvN{pU5^XCS=JPoWb{yoVz+fs)=v?qwM52y=4TK4SLJ30KAx`p!EA3tZ> zqmR(UXmpKfPJukcUJ#}d!3?;D@e0PWHdJa5NI z!%nt06=?bu-{Ib`_~CK{-nxMYhU_-fSiTVo|0|L8Ma64ab{&Cc>atU%-bt)KlZ$ZH zL)%85wpp_c^Pew;O~)_YX7=n|pVYNC_jiIU7s#eOd)YUMx2d)Q>y!2M&~%k1KEphG zm;u*xK&6TqGaCdU1sO0v>8SlJ9nFLl`7|N-j9enoP!#)aQ?Un{5)+g$?99Bb|FEw-jfrYVEFEnnYFU3#_k*|dIM_h<3=eD}Wb}R$Gbz97|ya0Y&n!>&sDbM2f*^!^x z>Qu-_j}px=)YhMw%W&dYzXZCC40?y9m1c|9lTiFwUjaAMKoPe-A$!Pu+zKwJ;0^O3dC*ic+Dq?8V+Q1caF3K7hJYZjqOMb{7mf{=p@ zG@(k>|68RZ6ieLtF#lBc$)L2gi65>cUtt|zrLd=}qeKg>+6C!dFhgJfQQ7VK#4DvD zyPUfKTzw$-e6Z4``2;l=naW&BtwS{Nv*(K8;NdQ43|U1SVbk$TcX39-q}ayf-daL5 z66IR+K|x-jQ;~>>F2j(yjA7;Qc9Z;o8yyF3aP=E|

%A*@)m1D?pv0| z7@OXtQ<{Yc-;m~)W#8H*Ied1$>T&WayeJTi^8o2>?D~)0D?nZyA&>%MQ2|F^%u&pR zz)PzsnC`_du(;?+R9<32D9agb8m#kS1HzRnnr0am*_>bwYPq#KnaSwI(E=pC;?$Dk zjwVa)W!y9PsP~%DDkfB0s1f@a!ZHqS0+R(GncVyIeZU?-STAKg<@)@I*j5h9?C_yl zl0eqc*-W^#Q3NPxylvu5ZggDMce+|eLAgs6B-9Qk^ zOZoUuD&yk($Ls~GshKGcHz&`}mop#({b}sH=K(ywftQ2+!W75otYVAfn`DlWz*Eh< zNJu|e@%^5Q?3t9>q2r-Fd@wwMf4*$+<=_8u+&T1y!BU<_T`|;x?oVUg0r5`vKjBxk z93-7wtqg|?Kcjm%pJJP+sa2DQ0q_mt`}rrCY#*){eo+!s)iSADt~D~OZKG$nSNoL zh%Cr=;13k3aRkj@V0Jb%DTlm{dhdw{=e*#7C7N~mnT?7#p)?p?~( zA5`-E&V;5Mrf%KiweiA$PpcHk3t}ErLPyq-pmL&bx5j^N9QqV|_K9IV&D>J-s=`A2 zyF=&Ir$c*wE2~8;MPTJn5kPs8VszdLjl_{wj2M{m`9wv3~opv1}Qe2Pn{&ta|0 z2?{J;tv6T(sMzL&raq?nXx3G9P@bzk$w(lff7RWZ{VY{$jRo@s@+FMkM+m4utYO^K z`GXD_f@00QW;P1z<>?0A$eT(HeA;yo%m;*)uwne=^uYTj?p;s}yTAw5;%*6qm}g4; zL+|ERbSHMYb`kn9Wd?wlsyy0aPe@hS-IT`yo!RqfO`B{O5BG*naElY3Zy6~AcoX5J zJS2@o`=dltzAsf6Te)5x-J>?INP5R#LMu>qaSbC>B2iqbL6G|(1JEBZ8vbK`0EbHx zTo@j3`=w{IJX`k8umZi$;N(EYCdOwhS;Gd9|(Vs&Hkz#7YYw( zSfi)aLY*qS(PiOQgN66Tf;es%)OEMD_-~dUuMw;lSCLKlqPgH65x4NUl;REBa zst~=cg>=8Cn?)?dHmA1;`A)QEU-AcaswOmhl=(`rnkdqzefkPTH@VhEoTNtK9WjY| zi{2U@8^Y$#_OT=}Rj;oP6LOT(GBaAiXa;o&ptQ;pj7($J7Ha*NJ_#qIG(kMeee?E8 z?BVb-UY*b47|aq4X23OnP%(^jz4RieKn7t~H5`}4PZj>ZTQ15JHh7-TE^H#Iv4#1b z-M`s8&6*v&sQV(7{Sk>D!$ldmf4-5f^2O-HRRISbtB&6tS>j=FplK0xq%`^z*{&XS?LBNfs^xOHAfU%CA^|XCDSd}rK=I*B@MtJlD zn1>HD;F^vrI)d(WsDdEWAOiqOhvjda2*qq-?M!U^9Q4E_r%=?F`8SPj40QM{q6wKf zKnjq^^TBqR?Kjl0Vll7SfX=GkPiwchg(-fLFnx3t;5gL&c% zn~h&)uFnaqJ6+n6VW*weVIgBazHQ!UEMK0Hus0Hw*Q;CguERWfm;u*pK$Qbbb@^-AbLnn`nONZmv(~!ECW7wGW$yZ$&Agb+y`g3)2f5 zXt=TV6YClr0C%Y+HqC*wX#BUY!;9pc_vk)wrN}>~NG3{yv$~bK#tieYVFp~&096WH zS78u@7G&@dN<+|37dXpql>OIx7wS(LEif6{CGI_}oNXX}gfo)5A5F{R$$ zYkIHW==^WI*}iIuWfK;IAao#uc_<f&WO=6>1Rf)9Y;$}I!~#&k@~*g{2m=NpF^N*P27#=?tm4=baz z_n7lxssqe`Yx%g6j|u+U%peFo$lwslMl5Xk*e|$S?6R_?!+WT0noL4ilT7i=m;M|B z<9kWMlZ|=7esIb1hz*~_r1lje(8)o!cAg8!!4p$+Ng*)0Ci(C@ujjIMg39_GKYfEjR&$R#2g zNW*FlAPB>s@{#hp*QSr65Isj1QagvAf{dHEm++4OMWX1nS`SZG+_(pz4nHfwEW(f@Y|G&lPy?VOU`~ z%~~q;*`d_@D))Ic=BvyEy}sGImq zve$u&Gc3q_I)uMgvat;}o`O#$u>m`q{IYG?aG)pcyA#}p|J{b;O@iM<1@}tOV=Emi z@B1}X+9ocvFb^AMz%>n5H0Y8{jesBzK!zC5Y?AqtsqybgLI3$~1$KY+ME~*+W)Ypw zAzI(+QXY@x9^_TkPl#2HRDyfL>Eov2y#)TMHPXm53@1oEc<+`l8d6pu>wFloq2l&G zT967q~eLt!(|09;(9TzI4G!Ls7IN0R$VNwx?~ z^cIy8ON?%lNI{m75ik!QX23NaP_1xG(gQ)5K!#LMIvQZt8U=6sR6_Q;AEq!xTXz)1 zeZsdfQfU}EuyozUKdNuj8v)B5U`FXl&r@C@0-eE8J#=F?WKqA@JxH0=Sr&Xt$vGyj zxN1(A=ybd@WSZnVY(##!P9UYvT8xg#3u{_jL+oU#YPS$ox%mM2coRziIZFeHRui~S zK387&En4RF!-=-R$ic1m7Y5?+C%wWP`)DRxx*S`;TI|Hz{nT;>5Qc#KL)p`ZIDr; zx<#(`sX)Ibo{T-?kFWeQHJ2=gTsbSmcNF-HTkO?*8;xl1WO4<^6?j_q8?NT{oaeYs zD_5n%X7QI(7VeED?zf|b?m#?kSPn-ZiY4WZ1w}fui@>z5PEyR8FCd4vhnhNkf%;*a zWW$s&4swIt$(Tr6xv;_*OCy(4yAFs6fY`*3GV6(RoHQdX4f~v?1^boVTHC*gNiler z&hFhci%_|;d6*7@uz(De|6p+f)|^E;_^c`1`C%Db+63mZ4&GgsP0T`LYC$Hqpob6? zp1>}!0QIU2p899rLyJQ*zrTU>Bzpf|guxGuCClWDiq@;>)?Lzazsp5ozDh>6grlhiS1N4dD+z&2ZX?F*LSq8+S>)fAC>={a}`e7X< zHOcLLSh^(f@@*>tz;`}Fw?e8;9@oBIWI`&9qSNd;6K?=kcav_EruWZic(x+vFMtE# zXO&rt{X~;GOWsN13e|ZMNkx@%trtro^Xw=c=$G>z0=Vq*pX}GGkHwVtxW<(WkSpf{ zgKN{h&K(#ZM6<9dJhW*8=gZ#Wy(6-?0bN@cWc7g%4?1EmxY~*mg8Pk&^Pc`Ugb%f4>bz{u|9AiI=SkBXU zDb1#ZNXxeZSqi-;;>8v_&rok=j<@0(U(!f%rHj6B5ngX>E!()vIxLPv7Mz!Ivx7p! z6k-JK(lDW;e>8?oV{PdO-f3N1yt3`<=t8manEP`KcEGaqTR8>G+n?8csFfI(kKH?& zbZTQW9rp+6LEiQU8Q1Ub|99*}jlg=B2OtO=$j}&C#!dZH#<4P)@vyV8nVFmZ!*Mnu z3o^$*D{Ua)vqUibu1Yrkl{>H(cdlM0lhpFFQr{prwa{we*~ejqt1=GSzVsz1!aYn2 z-IrytGw^S*Uko5;R}^N&sZ#Oxg@H82Z#m;u*<1r;o`sms*K4l;DRLWCRR zC%O9X7eQ%0@fY>cJhkg)*`fRARNkR8ezqwpWFa;Q>=pGKDa2rj+~*eH$I^^{vBC6; z(2=>8tbxI^tbzwc_$KtTrK-L4$Py68F09q=LjaY+?pyE+9Wa;od8_asfqtp2f(IVPA^37!2d_FVB58xdDa zJhT~wJ0pdW9?Zj`M7`}a`97<5ql`2##@k z3G7LuKqguVk|6^kPSVcv*^eV7Y{qVG*6Xh35+XBSTFbB~>Vhu9X1 zRVur%8%tadq@=xDo#I7~@Oy!TUETv?GDEB~9pj%jac&kzeByp%GscY)VB>pVJ!@(sTpNG;T zlZQVofVJ=t*F~)l$^L6+!msM>pBMB-90fQICN~D=W84LJ<;+yI?s|Hq!uLC-yw~gv zpD`%~v@BK<4;WRv?%#_II<1Tmlb%<0(943fE=tOEmR6R(yTlQ?FdDrW()(Zu$y4q=x$y+{1?_FWz$ZG%Ca$M#K#NcRRMh(Jm@lWtN7Wq`1$+Se%l5bY{+v^VZhTU+C%O_mFLFCgFP4(J z@Ador$W-$Hz7>zHk+C0hA4x@w8ftN;6(*ChyGMEynYkOBC}=+Dy~gnhM;5(D79a@M zrBwZaqrp!&{@c*R&?^`gTt@}HYu(KKUb$`cykzOI=doiUYPaO6L)7a!FxIlZ9Sa+H z|38)Ye^zb&4{w4(Nr)Fm=C_ogmG{1yg+)jY&i0(1{@eXKLL1kzY&!BKB_C`eZ;%|jhM!*^47S;v8%G$mx~8emkXm~j}Q=n_m7%|kU@z^ zE^OU3ntS|XJD*TQ$$P~oXq*I25v-<6fmdW{n-tlNbZmPZUB(2E@t@B&7u6-x~K5nc4Pa`oZD*-)Rrx``=ti=TpdfXP$ zKlbq3((a`L7++-~z8*L#hK~#x`BQ8OWaf1j_0iUh#XHY^#*J6r1oRcZc7>6(R^5;E zya=MVAx8!~lAuR;n+N=WBX#FIw7ljKDv$7NmunihL57)s@MsjwPRC|W&jA$akUym9 z2=t~=B_s9WstWFrgNJv}K{UJL?*9E9d&>;i$6Q#yxr0aOKkusl+Vc8CerQc&fMc0Z zudZ`gQ?kUkdCeINKXLICAWDA>ecjr= zx-A9JM2YV)>{5%p$5cz?W#b0K!rDv{i1i2Kaft?Oo_~#k_I3U@@-q9qcOt?bYZz>g zd^X+qARhNMB0d2=Bks?{hX{;6(B=9vu9h} zLYg8(;2#}1ij2hLIo8tc3(I>O=R|_fiukX=`$t~>6Fjb;gz(>adCO_fz?5K55>b0{ zj=qI>RflXc&0SSW_+q|7H&ipI4J=!N!$SyGO_uvl^jbD57Sddm+J2 zSu54~I<(mbH4?{8(`F|s=T+BJ1rh|Q93(ZqxQA`XQjh3F^?aKz&jZ0`Bev#jn_SVr zt>Qh|dZ5FLJz8ViL_nfMXiLbuRAl7vdwTPGKAC=|C5O1av{8fUFrHUWqz+*oJ zHm=wR%p;itLHIz1i%>SAe=-vPhtX9m&JMgCGXBiOkKGm&l!j`YV^|N*Hz?w8X4|bn zT~oSX6$bIcoMr>am3%-4N`mKTxI9YNgGe}X>pFmdZo-)*xePl#o++1k)F4mqQ`m_7 z(h&IKj9Y~2^;|rr_-Vkz;?^w%XAZ%~U^1+v5xkpkd|T*Y5&$#c8j&kR%Cz3Jfgt>U z%E!y!mK{1o_ZjL#{8wr`^)|i7H%douHOFnWalB&7hm` z`0aMhWAcQbzY0D%g2Z6W!90AJ0oQau1tg>W@;)T+r+~cviH?8Y&e2=iV~9>kyi_Jz z_^|d@pJpH4A2tUNmOwgb9LOIAIKZ#52lrn}6{`P5N2V7A>TW|>^2+(N8{b^``i)y`fEkM~5(YmMR;uM}`l&^3vnVOJUvcv}k=@nuX6AuLn*#UMg%9 zFs=?S`-EPRitC6*Apx6?UwSD=DjSa20{APZ{^meS<5R9$Pwm3B;^K2!lggg{1Ee|RZbe|ahBOK!Bbf^W8B6SM7X0+}Ag zBM}gS1mbV=~c#8T6Mo{x9aJN2L`%I&1dEH$~1t1;6M+9cnLCK zgM=bVCwZO*%`-5#W!oV%NuW*F)Ip znBWeXmXxRJfZ7GZRB>$`HQuuXbGpKQTy^D4ICbF{Y#Tk+`?bPeG}r7ws~QDF84yGm zWJL1^d-bp;i)^WAdyfxoFJ4T1*mmhUvV46~sD0|PYv*K4b&I=BA0KRsVu7VKvG7x>|lzloxqC8vGhSqYV%vt9~@|fTK<_ zVGvW9pkJW3GNMetFb7pv*nr~lQ*)OV4gv6go-1*9CDe@|)SYvjT{%0t6ISLQd8Gj- zba4gi!w4thyO6kl%7`U&c$faDX7#SZ*uL6Y&O6-;G%y> zeS;uV{pc$~NFnGg#t1_Jkh`q5(Ne%t*21OFG^Bhu@{vw&PowyAiW5ShlhY>4k83ud zRdxbV0SF=jGGhKitu}wUsQ(Wia&lRBgd+>s?vPe>R5}-USCo%a*kvtRACFp_3XbAs zzXvY#7+Yfzey{s;E9`QYP_2O5cm4eeycEvOn~Wk|0?#3`k)bziIt-aiv7oImyKR%4 zm6DrisySLL4@8E!zf7o2ggy_{Z{c;P3}3C00Pcqn*Z#1aFA>?2j^F!o+&EV#;v)W% QvjPdR890Aq{^wfHde%NbAU;6A1H%D)fp2vwEKRoQy65_rJ}K+! zC+WkHB8?7q!re-hI2P-Vl8J`)4}gdEW0gwto{i+)NQfVRf3@ED71d$%o3A_JjZtfzXN^m!00ous(0=j%tMBaXvV^cQCo6o_xb$~zw1}#68;;T&`sn^9(jk{hb_FHgX#|UxcNnVX6`zG)=wJw}hF&a*`wB#!lE+PSK;u?bS#; zNsgq)DCJBF7~?hay=%qdfW89?U|^hs0fz;L00BcB{5LRI*o{q$`FOec{!R#(?Rt1i z4o7Y@u^xqqzvSYp)W9>qPxEFIk4ncvl78C(wj!+LZ=KsDDzY3VO?>xApW!%B(Kn~Xl45wJ z2=q)(Jo+JMkfsW=8=hl2$UVaGzJi~oB^d0Idow86FHq9g-32*HncG@w6>;Xp_FnmH+4|7u(31*?{_Gi7PtNmK(AlLAcGg zJ2Mik2{JFYXptguU%7}49I%KXwHaDFEuSs+#!EK8{Ba(-nTaQlE33CY99m|D?1_W@ zvPOeU&Ctqfwwk<1u?e5e#i}$@3GFq_c>nI64Ek9b4V8<9k=RkZbfXV*@j1qSK)Jo)d7DAaipZyH-5-~Xc7-<>B}i_IT0^t3-7%VTEULTf z6FIK=l9hK)6*T5c;;MZx`t`463QhjbyIM?To?;8}>Y@arhZ4`$v=gV&Tq`51?z;>3 zr2)55sLPalcH84*?Kd2w(+m9BS@XEerJ01u4wdiz&X#DHknyDiG#^j^qvN7&P$D5i zV#Du>WeQ=1wq6rfbAYehAsb(aU&%K6%y8^HpP)}nw6*=WZ^@ve?4Is}ah>hU_3O5$ zF$*s_YSh-Nv;-1nyU@fO(eFv*OTA!voVpXat!!X0;G;&wOVJ~0O`o)FkYO%XX%#KAY@ASNM2X%Nd~^5Ox1wqWn}oS1wlRIPAdgDkq|SnR;`8+h*ZDl5AX$IS z8x4LUXHEOsFfeg6uFx+2)=^$hS(G#SmKCQBxQo6UUA6{n@#-zv_1rI#!|ERB)8(~? z8l8{I`TMJgzDGk{zn+H1j(mpS&}@rp9%V}Z8p6@1QNl)LS@eVC*QXmEmP~-Y0}5bZ zTqFh->U9Vh$$!ifsO#5f^`>zKX*iB+3b7cT4%XrLNlUTvZ*`wC6(Cc5I1aW3!LdKg zR{C9%e^xi}{%`iZpytM^!Q{u>e!1P{?c$`T@}A6y_i>F2X1}*3lrZatZ{r~Xx(_IT z;qmWvZCGW72M(Yjt0rhM6QXuKool~`4!m7V| z#&lbKi4C?@prq>f!tMiqpBkEofWo`%+0SOj`Auyd1pmWv;;R$;7^9)~XZ%C)uwS#T z-@;A;Z*1bR7YlhR$8F1Vr~|qVD1c#co*JjqXef}>0AGOx7TJ5Vn7)h;6so#TN7ri@y6TJrI+t0#(L-c4}#nH8}4X7u%uC%aRgPL{_=trj6#!f|b5~1|2d= z5-DqHrun~hDd=7Sg8_A%%%IfF6;xmQG$i-r!Nvod%vH4TqM8^CrdBWP60@YJDL~%= z1u!tq^TMkh=BG;?85I@;PQG0cq1Y zZ*K?VVqNyoxEC^Y@_7#vja*ggjs5ci%*5AhL44e&Zui8o;KtQ$fcbzrPYQpb-?-)U z(@yk3>bPya7cLzpf8!%JL*y{5T1K-_S0h0C0R=EVE`p;63l+jg;T6FVpt$-4Bj(ra zW)`MM-P;8uPfr% zO9o^g?@>kmT-qCbM*ZWa0x%fFqbI~a$lqYdceiHbfWlXYWIvOG5f9kw1IsbF{Kw2s ztbW|KPiBdKfQl=sQp{3xNY^KI4Cp(c00zcIa1^56gn&^2930vdi5;w_OwqD0(Jbds zjeEZ|ZX0Y2PE`_SDT(@LVXlH`?zw-XtWx8+434{c&eo#8E7{|CNuN)k=bq$f(h5kP zJ>IpsC*0|?%qaoP2h`y3B@^F9WgF4BhRnUFGm& zFh0)tI0eI@LHMY@A~R%_B7Awf&7l{(6Yu_^bX}47b(O-lsf{v7-%_Wg z$;OP~GB{fAni=CCk{1xpsveed7Ho8@CM=8c1oHKmggqRv^8*8eF*J)hi`DSW8&lha zG{^lwo0VcD1pH!L57C%;xloSS?ZX%W4-%G%b^*}?R$C-f|Tf6xZLHl89n@pTv9{KQGm_Lnm?s*Wtoe@>fb)|6zk+u2PEfkeaB48qCRp&`9(}08$E~(%tv1P zx`P@$+S)b$z_6r6b}Tbz8CKi6mjkLQXnXIxrtl2a9o8l<)7g zVHbB|er~m(3R)ZsMY?+B58fl@b89jYh+%;C0}5b#z@44?@(TNj3E^Y-iortUUo@pw zIP;yKSOJ++j55+h3*2g@^CDUGB%WgvKieD>v2MqT+|G`785}HJ1vQF|~AqH^doJ6)Z zv)@()^c_$D1LGWw)09~(2pE&V!C~aKnJPLsXB%8jjvp@(up`;&VCA#N%Jk{scon>_ zEh?xmmN$c;=5_pKaJ*5k{opUbAmSvu+w1ztuwZKsE}N7lxV=~&0n@9(OdgmIsKFt# zwRWwbDF zu%WbYq*RraqLi?9 zN)A-?hP^_=opbt<3|Y+W{q&e}lQ~`zYgR^E?j;AZA8V>7I?}#wd^|!wi2g$|`U$x) z5b(u+yOe24izmM)6f=HXS?;8yDW(uT!SY=0A)xVq0vH)*WFUC6ea3~5v3eC5DWBcP zw~9l`9*Oi(%eCz9VZ0mb?IU+#H9!aBwfenr1a-`j9^MQu&bb^WYoO5OL+MU;nerN+ zJ8TBGqCtQZ0Ew~o~ zYv3ZJ*+{>i2~^oT>>z{(n`3pzQ@PhBV5r?hsjs~_ldBq5@v9@V^U%A ziVLo-Kq}k3l_vObS6Hkk<|Ux-fC3m8=U{kGP2xkq_;FQa@HY*$>ix9v6^oI9 z{f{D3C2AA`J>}LL^H11ONOudGaI^lw5kRQ>p#=sBBp!#K9josJKG&A+6Ff>L%9-~G zHDw==HxO-bF$T09PyplNj0+@qr2_&;VEhK&4>fJ*)Z3Uz(ZMPvF4il1klLi~_;j>{ zm2RMs$$xAwtq>&iE*+WSHb>oM)5tm?Pui{fCUHNxqYy6zK8K@gqpyfG&(UQB-3x6P z?bmkOiFBRBzD2ls|RYzRn z50_RDKxWmK{)5p|^mYo!GDIrOxb6AQIT>f~91wu>C#WjinE9UjW#LSaPN*cjW^hue zzD>m9OwVquQ}U`Fn2b*zzJc3Y*<3RB-B#w9tFv9*uNX z;V0B-xoCKm`nRfkIUd8R2GDED|K#UqWca~l5ktthmVEJ!qx@sJ$I#z^jTw0$IA>f= zq;WDGxnnZeTEOaVBbL!xwAcS3)OOX>9<+%_!YHc!rRSWEiz8n(+a-l?IxUa~FPyC^Gm*_D4zf^w;+&lD1YpeoAQ})b8>MS z{qct5hW2gEHkE;ajw${TgK*t(?p#N*?&e^$g`rIkO@ZZVko;sm9R{kz|2{`)4&)Yl zw%+Gtj%P}>xuNR45@^QFllBOs{&P|Tj^mm6PS-Nlj_FXQRYjGNz;u4PrqrMeC)CP zjburuc>hRxmKUCQpw8J67z|F9{bPC}$|fW3{c9if zAEi+WhJnCMX4+wO@%Z;+Dm^l0=z(0m0Y(E&KhMoIy4l{3y-1zWK$N&FTrC|g-fx+(wkcw1OAdNLp)Q#d z_dV;fmM)Vv(kocpwpZ^Jk;35Kj-&fpYkF5`i{-vg7JDc{eiASrP;04$FLrG$58jH( z;#3Q{W_!=-NNkx-S8^b8Md=-Lw9~hM_5%uFe4O!tavk>>IfM@?pjsO7t8%6(g{*{~;QU0>Lp+kSt> zTMNe>vvT262x<( z!~;?oceQ*wpT2KqAo;CG$hDzTEPz&AMyP=W{l*@wA~TW5xFOEi6)vm$X)=AQb@x9m zl2n&XB#o}QP}9S1?D;3V89;Yo|Pyow~|6s0Y*j*2C|7h%J z_a2cRQ;2S0>)Q3_KMyejLT}iuaLO4tgLoa+MSMJB>i#RtI z|7HZxd_Vz=jx#z?Z@RWqL&Agks!ep6q4Z^o_Vg`I;h9F2{?~q4{H6RT1X>(c>R>R& z!1ITo2&d2JGW$dw=Y8YiRN(t0?kt>iJzZHwe~e&RCr>?zIY6XRCoVF zGBVvu$3(-KQ};ZQHev}Ec(?VU3OuWGHOGCOWur&a%K(iB6u`(hCnKbimj*%x+f`(I zd(d{H(UtToT12NhEq@sv2FA9f7bUG8yQ;I9dn3~nXf>*oEBYjU_PlU#oIhR>c!-?g z*;d-PxD*rLvpBCa_7LEp9T zOolHO!o_c_kfWilW4Z4Rjv>B>lb*Sbn?iyO*KTTb1oo~(U2Drwf84ID0GlX4W#XUw z+#l&C2I?1D2pIxE>sYxw9;Jk(&D`n_M#4F7Wkp=ak}?KT)>2*ew^=FKgcL!dNgC>o zgT0w&v2k$$2MDN*0$xf?&x8Ix)Z#w#y3SB4dqU1LMl-Mg*x>%%APhGet|YYJ_kuZ? zMJc8?XR2M8Dai`PL$_<)N&r&;z9k>f*sbRAFt!{Q8!n_OAD;%PC*W`+Vrh7vf(DdO1Kwq?0ADc$KM~5s={`;B%_4wD8=F2kO)!Zud zMTtUQ1ZlT!Gl8^*5$4ZST520b2oZ{az5@ziVElW{VE@6}Gf-nNtCq-u8 zG#l^;p;&bYrDSA(>(#^D@y^%M2Njoy;X<$Eu)$8a%H4Tz4zRi=6yV74ptd~5WzbO;@@>Ew{Ddw zft-}ym`Z;yessP%xp>yLo6Mb(lVD5SC*Cnd{VC}4igq~(r~^GB@K!o*8SP{bFc?r{ z!{VB8VCB%$6HD%h{^eJl@U*ujbvKpwjc4~L3;giSs{wrn6u`hZ2P4wp4Fd!WG2qxh zQ%*a%epiGmxILv{h~hKdBSFr5m3v6-B06uzRE66OK^qf&%emPv;LrKEI09xuSR<-& zRB~IQmdAtiUN*_Sl5me1`%i&!n1k>-arJw^d_Y~JXxI_*s_}UrN4=(s>HI}#Jw+F# zUy%6PX@K(+KTi%{Euj5?0vI1>e4yk0oVg9*LmX(0(!L=7*idTAcg=08`MPHDT#=B0 zyFBt-rS(AFDh~CcDu`HRxZhDq+Tb!h8r0))yL)TGA7&V8ntxyJYN~Q3<&tzZf3&vL zpQYUM5||FCuS(Mi$Q}wY6YrC_sd%~4ZEHqG^I_N27OFp(oaqWMplt;-A5Z|Jn3M7oQ(8g^@ zpm@&5#om{hP&nvyzn0huW3RKlUYm^Cj}N=I7woljA9L%OIvQXC^8s~NblpJ;f%{sU z(9g$x?_Eku#C5@Sf_}w`(v@v9eo44)7=ZQz3SfMk@qs}oi*pCUhtyT8mHWf5@}38j zlaUA)$h;d8Rt$1mcp%d~kz#R~98N=0D$)K@{m%D{dPDD~ z?{AiE>!GNm^gp$8l`%+({63|1f7S#zIiU6q z!%UZ@!}+aprj$agWwl#K>4v6ouO`0L5I*@1-d6Smp$Pj<$EG-q@qHn_4t=y-V5Tv0a z>z%0TY!qTd%m5ntMt94=fc)Y*NumP;T<}|lJ;9pFpxH3{UOKL$#AxDeq+8Q0>IJb8 zkKPu)&m09NV;J#fCHikoQA5S8g8s*Kvl97IJ7B|;m79;X0<&5l*B8DEtyAT4Cp=Q& z|5YwW1Bz)ctU3aW1wa8TH_mbci@X$*6+(t`!rwlYi_O)-#>Cdu@$wVfa)&EBcW|`& z^zmi}MYtOr1VSFvRi}X*Ucb#+Z_RY!1%0Wn4p}q2d4a}786|JB@z5VNay+mp$7GvZ z{G@5n80nI>Alz^-qVGj{zb-HuP(wqlZfW6qSK{uVb!O^uJ!L}gj3~a$Ex+k(W5uKB zi6Txw=K%#UG|tfgSIo0P&``N5G>RfP+a~S!(#LMuv8#iB9n6UOe3FHMal(F0gLf+oD}W0mXkS&yc|!9wF&h4INO54qVgOQBb_hRweCl<*gv?O7%CaO zQcud>wQ%~4JX~6Nu1G+j@f$m# 1 || values.isEmpty(), "Terminated RLP output, cannot add more elements"); values.add(v); diff --git a/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLP.java b/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLP.java index 03cdb1bd015..145b01bc003 100644 --- a/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLP.java +++ b/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLP.java @@ -52,7 +52,11 @@ private RLP() {} * read. */ public static RLPInput input(final Bytes encoded) { - return new BytesValueRLPInput(encoded, false); + return input(encoded, false); + } + + public static RLPInput input(final Bytes encoded, final boolean lenient) { + return new BytesValueRLPInput(encoded, lenient); } /** diff --git a/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPOutput.java b/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPOutput.java index 43ef548996b..dd1b9c8bd88 100644 --- a/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPOutput.java +++ b/ethereum/rlp/src/main/java/org/hyperledger/besu/ethereum/rlp/RLPOutput.java @@ -18,7 +18,6 @@ import java.math.BigInteger; import java.net.InetAddress; -import java.util.Collection; import java.util.function.BiConsumer; import org.apache.tuweni.bytes.Bytes; @@ -243,8 +242,7 @@ default void writeInetAddress(final InetAddress address) { * writes this value to the output. * @param The type of values to write. */ - default void writeList( - final Collection values, final BiConsumer valueWriter) { + default void writeList(final Iterable values, final BiConsumer valueWriter) { startList(); for (final T v : values) { valueWriter.accept(v, this); @@ -260,23 +258,23 @@ default void writeList( * more efficient in that it saves most of that decoding/re-encoding work. Please note however * that this method does validate that the input is a valid RLP encoding. If you can * guaranteed that the input is valid and do not want this validation step, please have a look at - * {@link #writeRLPUnsafe(Bytes)}. + * {@link #writeRaw(Bytes)}. * * @param rlpEncodedValue An already RLP encoded value to write as next item of this output. */ - default void writeRLP(final Bytes rlpEncodedValue) { + default void writeRLPBytes(final Bytes rlpEncodedValue) { RLP.validate(rlpEncodedValue); - writeRLPUnsafe(rlpEncodedValue); + writeRaw(rlpEncodedValue); } /** * Writes an already RLP encoded item to the output. * - *

This method is equivalent to {@link #writeRLP(Bytes)}, but is unsafe in that it does not do - * any validation of the its input. As such, it is faster but can silently yield invalid RLP - * output if misused. + *

This method is equivalent to {@link #writeRLPBytes(Bytes)}, but is unsafe in that it does + * not do any validation of the its input. As such, it is faster but can silently yield invalid + * RLP output if misused. * - * @param rlpEncodedValue An already RLP encoded value to write as next item of this output. + * @param bytes An already RLP encoded value to write as next item of this output. */ - void writeRLPUnsafe(Bytes rlpEncodedValue); + void writeRaw(Bytes bytes); } diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/BranchNode.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/BranchNode.java index 107ccac9816..b16a3342f11 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/BranchNode.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/BranchNode.java @@ -98,7 +98,7 @@ public Bytes getRlp() { final BytesValueRLPOutput out = new BytesValueRLPOutput(); out.startList(); for (int i = 0; i < RADIX; ++i) { - out.writeRLPUnsafe(children.get(i).getRlpRef()); + out.writeRaw(children.get(i).getRlpRef()); } if (value.isPresent()) { out.writeBytes(valueSerializer.apply(value.get())); diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/ExtensionNode.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/ExtensionNode.java index a52b68c109e..04ee6f0d2c6 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/ExtensionNode.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/ExtensionNode.java @@ -85,7 +85,7 @@ public Bytes getRlp() { final BytesValueRLPOutput out = new BytesValueRLPOutput(); out.startList(); out.writeBytes(CompactEncoding.encode(path)); - out.writeRLPUnsafe(child.getRlpRef()); + out.writeRaw(child.getRlpRef()); out.endList(); final Bytes encoded = out.encoded(); rlp = new WeakReference<>(encoded); diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index bf484ff8fd2..c87bd2b01a8 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -64,7 +64,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = 'gQ91kSJ/YS1eUWUrEbuei1qGi0uq5yGe/5pVmY5MwU4=' + knownHash = 'Ev0Y22aT+tcysFVaGcdaJON9Mf+aYqk8+5h+aHKUzuk=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/data/TransactionType.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/data/TransactionType.java index 706dc53c536..b5c428c5fb7 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/data/TransactionType.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/data/TransactionType.java @@ -15,6 +15,29 @@ package org.hyperledger.besu.plugin.data; public enum TransactionType { - FRONTIER, - EIP1559 + FRONTIER(0xf8 /* doesn't end up being used as we don't serialize legacy txs with their type */), + EIP1559(0x3 /* placeholder value until we know what the real type byte will be */); + + private final int typeValue; + + TransactionType(final int typeValue) { + this.typeValue = typeValue; + } + + public int getSerializedType() { + return this.typeValue; + } + + public static TransactionType of(final int serializedTypeValue) { + if (serializedTypeValue >= 0xc0 && serializedTypeValue <= 0xfe) { + return FRONTIER; + } + for (TransactionType transactionType : TransactionType.values()) { + if (transactionType.typeValue == serializedTypeValue) { + return transactionType; + } + } + throw new IllegalArgumentException( + String.format("Unsupported transaction type %x", serializedTypeValue)); + } } From e8facd0f1c1eada2fbbaf37176052a09f6b75b28 Mon Sep 17 00:00:00 2001 From: "Ratan (Rai) Sur" Date: Wed, 2 Dec 2020 12:19:17 -0500 Subject: [PATCH 06/16] [REFACTOR] Clearly Separate Common MessageFrame Fields (#1652) Signed-off-by: Ratan Rai Sur --- .../mainnet/MainnetTransactionProcessor.java | 75 +++++++------------ .../privacy/PrivateTransactionProcessor.java | 74 +++++++----------- 2 files changed, 56 insertions(+), 93 deletions(-) diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java index 76633b1c46e..372808f9b93 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java @@ -16,6 +16,7 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.Account; +import org.hyperledger.besu.ethereum.core.AccountState; import org.hyperledger.besu.ethereum.core.Address; import org.hyperledger.besu.ethereum.core.EvmAccount; import org.hyperledger.besu.ethereum.core.Gas; @@ -295,75 +296,55 @@ public TransactionProcessingResult processTransaction( intrinsicGas); final WorldUpdater worldUpdater = worldState.updater(); - final MessageFrame initialFrame; final Deque messageFrameStack = new ArrayDeque<>(); - final ReturnStack returnStack = new ReturnStack(); + final MessageFrame.Builder commonMessageFrameBuilder = + MessageFrame.builder() + .messageFrameStack(messageFrameStack) + .maxStackSize(maxStackSize) + .returnStack(new ReturnStack()) + .blockchain(blockchain) + .worldState(worldUpdater.updater()) + .initialGas(gasAvailable) + .originator(senderAddress) + .gasPrice(transactionGasPrice) + .sender(senderAddress) + .value(transaction.getValue()) + .apparentValue(transaction.getValue()) + .blockHeader(blockHeader) + .depth(0) + .completer(__ -> {}) + .miningBeneficiary(miningBeneficiary) + .blockHashLookup(blockHashLookup) + .isPersistingPrivateState(isPersistingPrivateState) + .transactionHash(transaction.getHash()) + .privateMetadataUpdater(privateMetadataUpdater); + final MessageFrame initialFrame; if (transaction.isContractCreation()) { final Address contractAddress = Address.contractAddress(senderAddress, sender.getNonce() - 1L); initialFrame = - MessageFrame.builder() + commonMessageFrameBuilder .type(MessageFrame.Type.CONTRACT_CREATION) - .messageFrameStack(messageFrameStack) - .returnStack(returnStack) - .blockchain(blockchain) - .worldState(worldUpdater.updater()) - .initialGas(gasAvailable) .address(contractAddress) - .originator(senderAddress) .contract(contractAddress) .contractAccountVersion(createContractAccountVersion) - .gasPrice(transactionGasPrice) .inputData(Bytes.EMPTY) - .sender(senderAddress) - .value(transaction.getValue()) - .apparentValue(transaction.getValue()) .code(new Code(transaction.getPayload())) - .blockHeader(blockHeader) - .depth(0) - .completer(c -> {}) - .miningBeneficiary(miningBeneficiary) - .blockHashLookup(blockHashLookup) - .isPersistingPrivateState(isPersistingPrivateState) - .maxStackSize(maxStackSize) - .transactionHash(transaction.getHash()) - .privateMetadataUpdater(privateMetadataUpdater) .build(); - } else { final Address to = transaction.getTo().get(); - final Account contract = worldState.get(to); - + final Optional maybeContract = Optional.ofNullable(worldState.get(to)); initialFrame = - MessageFrame.builder() + commonMessageFrameBuilder .type(MessageFrame.Type.MESSAGE_CALL) - .messageFrameStack(messageFrameStack) - .returnStack(returnStack) - .blockchain(blockchain) - .worldState(worldUpdater.updater()) - .initialGas(gasAvailable) .address(to) - .originator(senderAddress) .contract(to) .contractAccountVersion( - contract != null ? contract.getVersion() : Account.DEFAULT_VERSION) - .gasPrice(transactionGasPrice) + maybeContract.map(AccountState::getVersion).orElse(Account.DEFAULT_VERSION)) .inputData(transaction.getPayload()) - .sender(senderAddress) - .value(transaction.getValue()) - .apparentValue(transaction.getValue()) - .code(new Code(contract != null ? contract.getCode() : Bytes.EMPTY)) - .blockHeader(blockHeader) - .depth(0) - .completer(c -> {}) - .miningBeneficiary(miningBeneficiary) - .blockHashLookup(blockHashLookup) - .maxStackSize(maxStackSize) - .isPersistingPrivateState(isPersistingPrivateState) - .transactionHash(transaction.getHash()) - .privateMetadataUpdater(privateMetadataUpdater) + .code(new Code(maybeContract.map(AccountState::getCode).orElse(Bytes.EMPTY))) .build(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionProcessor.java index 831a1eeb208..ca1e9905afd 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateTransactionProcessor.java @@ -16,6 +16,7 @@ import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.core.Account; +import org.hyperledger.besu.ethereum.core.AccountState; import org.hyperledger.besu.ethereum.core.Address; import org.hyperledger.besu.ethereum.core.EvmAccount; import org.hyperledger.besu.ethereum.core.Gas; @@ -40,6 +41,7 @@ import java.util.ArrayDeque; import java.util.Deque; +import java.util.Optional; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -122,14 +124,30 @@ public TransactionProcessingResult processTransaction( previousNonce, sender.getNonce()); - final MessageFrame initialFrame; - final Deque messageFrameStack = new ArrayDeque<>(); - final WorldUpdater mutablePrivateWorldStateUpdater = new DefaultMutablePrivateWorldStateUpdater(publicWorldState, privateWorldState); + final Deque messageFrameStack = new ArrayDeque<>(); + final MessageFrame.Builder commonMessageFrameBuilder = + MessageFrame.builder() + .messageFrameStack(messageFrameStack) + .maxStackSize(maxStackSize) + .returnStack(new ReturnStack()) + .blockchain(blockchain) + .worldState(mutablePrivateWorldStateUpdater) + .initialGas(Gas.MAX_VALUE) + .originator(senderAddress) + .gasPrice(transaction.getGasPrice()) + .sender(senderAddress) + .value(transaction.getValue()) + .apparentValue(transaction.getValue()) + .blockHeader(blockHeader) + .depth(0) + .completer(__ -> {}) + .miningBeneficiary(miningBeneficiary) + .blockHashLookup(blockHashLookup) + .transactionHash(pmtHash); - final ReturnStack returnStack = new ReturnStack(); - + final MessageFrame initialFrame; if (transaction.isContractCreation()) { final Address privateContractAddress = Address.privateContractAddress(senderAddress, previousNonce, privacyGroupId); @@ -142,62 +160,26 @@ public TransactionProcessingResult processTransaction( privacyGroupId.toString()); initialFrame = - MessageFrame.builder() + commonMessageFrameBuilder .type(MessageFrame.Type.CONTRACT_CREATION) - .messageFrameStack(messageFrameStack) - .returnStack(returnStack) - .blockchain(blockchain) - .worldState(mutablePrivateWorldStateUpdater) .address(privateContractAddress) - .originator(senderAddress) .contract(privateContractAddress) .contractAccountVersion(createContractAccountVersion) - .initialGas(Gas.MAX_VALUE) - .gasPrice(transaction.getGasPrice()) .inputData(Bytes.EMPTY) - .sender(senderAddress) - .value(transaction.getValue()) - .apparentValue(transaction.getValue()) .code(new Code(transaction.getPayload())) - .blockHeader(blockHeader) - .depth(0) - .completer(c -> {}) - .miningBeneficiary(miningBeneficiary) - .blockHashLookup(blockHashLookup) - .maxStackSize(maxStackSize) - .transactionHash(pmtHash) .build(); - } else { final Address to = transaction.getTo().get(); - final Account contract = privateWorldState.get(to); - + final Optional maybeContract = Optional.ofNullable(privateWorldState.get(to)); initialFrame = - MessageFrame.builder() + commonMessageFrameBuilder .type(MessageFrame.Type.MESSAGE_CALL) - .messageFrameStack(messageFrameStack) - .returnStack(returnStack) - .blockchain(blockchain) - .worldState(mutablePrivateWorldStateUpdater) .address(to) - .originator(senderAddress) .contract(to) .contractAccountVersion( - contract != null ? contract.getVersion() : Account.DEFAULT_VERSION) - .initialGas(Gas.MAX_VALUE) - .gasPrice(transaction.getGasPrice()) + maybeContract.map(AccountState::getVersion).orElse(Account.DEFAULT_VERSION)) .inputData(transaction.getPayload()) - .sender(senderAddress) - .value(transaction.getValue()) - .apparentValue(transaction.getValue()) - .code(new Code(contract != null ? contract.getCode() : Bytes.EMPTY)) - .blockHeader(blockHeader) - .depth(0) - .completer(c -> {}) - .miningBeneficiary(miningBeneficiary) - .blockHashLookup(blockHashLookup) - .maxStackSize(maxStackSize) - .transactionHash(pmtHash) + .code(new Code(maybeContract.map(AccountState::getCode).orElse(Bytes.EMPTY))) .build(); } From 250d4fe4f646eaab717536cba04809fd1bf33b78 Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Wed, 2 Dec 2020 11:10:17 -0700 Subject: [PATCH 07/16] 20.10.2 release (#1654) Signed-off-by: Danno Ferrin --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index c255b15bf89..b9c7312f866 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -version=20.10.2-SNAPSHOT +version=20.10.2 From aa8d2b588512b842708de73f9bda85848d19b1bc Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Wed, 2 Dec 2020 11:43:46 -0700 Subject: [PATCH 08/16] Prepare for version 20.10.3-SNAPSHOT (#1658) Signed-off-by: Danno Ferrin --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index b9c7312f866..0ad42f75a5e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -version=20.10.2 +version=20.10.3-SNAPSHOT From 92d98fc612325a4678b87c762d74baefed66a0ab Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Wed, 2 Dec 2020 13:18:54 -0700 Subject: [PATCH 09/16] Set up changelog for 20.10.3 (#1659) Set up changelog for 20.10.3 Signed-off-by: Danno Ferrin --- CHANGELOG.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85d26eefad2..37e259d5947 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## 20.10.3 + +### Additions and Improvements + +### Bug Fixes + +#### Previously identified known issues + +- [Fast sync when running Besu on cloud providers](KNOWN_ISSUES.md#fast-sync-when-running-besu-on-cloud-providers) +- [Privacy users with private transactions created using v1.3.4 or earlier](KNOWN_ISSUES.md#privacy-users-with-private-transactions-created-using-v134-or-earlier) + ## 20.10.2 ### Additions and Improvements @@ -19,6 +30,11 @@ - [Fast sync when running Besu on cloud providers](KNOWN_ISSUES.md#fast-sync-when-running-besu-on-cloud-providers) - [Privacy users with private transactions created using v1.3.4 or earlier](KNOWN_ISSUES.md#privacy-users-with-private-transactions-created-using-v134-or-earlier) +### Download Link + +https://dl.bintray.com/hyperledger-org/besu-repo/besu-20.10.2.zip +sha256: `710aed228dcbe9b8103aef39e4431b0c63e73c3a708ce88bcd1ecfa1722ad307` + ## 20.10.1 ### Additions and Improvements From fa2d53859c45c26e40a9265092308bcdba5b7b8f Mon Sep 17 00:00:00 2001 From: Lucas Saldanha Date: Thu, 3 Dec 2020 13:44:05 +1300 Subject: [PATCH 10/16] Adding Transaction.isGoQuorumPrivateTransaction() method (#1661) Signed-off-by: Lucas Saldanha --- .../besu/ethereum/core/Transaction.java | 15 ++++ .../core/TransactionGoQuorumTest.java | 76 +++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/TransactionGoQuorumTest.java diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java index e20f82e9267..6c5533cb639 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java @@ -490,6 +490,21 @@ public boolean isEIP1559Transaction() { return getGasPremium().isPresent() && getFeeCap().isPresent(); } + /** + * Returns whether or not the transaction is a GoQuorum private transaction.
+ *
+ * A GoQuorum private transaction has its v value equal to 37 or 38. + * + * @return true if GoQuorum private transaction, false otherwise + */ + public boolean isGoQuorumPrivateTransaction() { + return v.map( + value -> + GO_QUORUM_PRIVATE_TRANSACTION_V_VALUE_MIN.equals(value) + || GO_QUORUM_PRIVATE_TRANSACTION_V_VALUE_MAX.equals(value)) + .orElse(false); + } + private static Bytes32 computeSenderRecoveryHash( final long nonce, final Wei gasPrice, diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/TransactionGoQuorumTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/TransactionGoQuorumTest.java new file mode 100644 index 00000000000..37f4219e7e3 --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/core/TransactionGoQuorumTest.java @@ -0,0 +1,76 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.core; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.config.GoQuorumOptions; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.rlp.RLPInput; + +import org.apache.tuweni.bytes.Bytes; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TransactionGoQuorumTest { + + private static final RLPInput ETHEREUM_PUBLIC_TX_RLP = + toRLP( + "0xf901fc8032830138808080b901ae60056013565b6101918061001d6000396000f35b3360008190555056006001600060e060020a6000350480630a874df61461003a57806341c0e1b514610058578063a02b161e14610066578063dbbdf0831461007757005b610045600435610149565b80600160a060020a031660005260206000f35b610060610161565b60006000f35b6100716004356100d4565b60006000f35b61008560043560243561008b565b60006000f35b600054600160a060020a031632600160a060020a031614156100ac576100b1565b6100d0565b8060018360005260205260406000208190555081600060005260206000a15b5050565b600054600160a060020a031633600160a060020a031614158015610118575033600160a060020a0316600182600052602052604060002054600160a060020a031614155b61012157610126565b610146565b600060018260005260205260406000208190555080600060005260206000a15b50565b60006001826000526020526040600020549050919050565b600054600160a060020a031633600160a060020a0316146101815761018f565b600054600160a060020a0316ff5b561ca0c5689ed1ad124753d54576dfb4b571465a41900a1dff4058d8adf16f752013d0a01221cbd70ec28c94a3b55ec771bcbc70778d6ee0b51ca7ea9514594c861b1884"); + private static final RLPInput GOQUORUM_PRIVATE_TX_RLP_V37 = + toRLP( + "0xf88d0b808347b7608080b840290a80a37d198ff06abe189b638ff53ac8a8dc51a0aff07609d2aa75342783ae493b3e3c6b564c0eebe49284b05a0726fb33087b9e0231d349ea0c7b5661c8c525a07144db7045a395e608cda6ab051c86cc4fb42e319960b82087f3b26f0cbc3c2da00223ac129b22aec7a6c2ace3c3ef39c5eaaa54070fd82d8ee2140b0e70b1dca9"); + private static final RLPInput GOQUORUM_PRIVATE_TX_RLP_V38 = + toRLP( + "0xf88d0b808347b7608080b840290a80a37d198ff06abe189b638ff53ac8a8dc51a0aff07609d2aa75342783ae493b3e3c6b564c0eebe49284b05a0726fb33087b9e0231d349ea0c7b5661c8c526a07144db7045a395e608cda6ab051c86cc4fb42e319960b82087f3b26f0cbc3c2da00223ac129b22aec7a6c2ace3c3ef39c5eaaa54070fd82d8ee2140b0e70b1dca9"); + + @BeforeClass + public static void beforeClass() { + GoQuorumOptions.goquorumCompatibilityMode = true; + } + + @AfterClass + public static void afterClass() { + GoQuorumOptions.goquorumCompatibilityMode = + GoQuorumOptions.GOQUORUM_COMPATIBILITY_MODE_DEFAULT_VALUE; + } + + @Test + public void givenPublicTransaction_assertThatIsGoQuorumFlagIsFalse() { + Transaction transaction = Transaction.readFrom(ETHEREUM_PUBLIC_TX_RLP); + assertThat(transaction.isGoQuorumPrivateTransaction()).isFalse(); + } + + @Test + public void givenGoQuorumTransactionV37_assertThatIsGoQuorumFlagIsTrue() { + Transaction transaction = Transaction.readFrom(GOQUORUM_PRIVATE_TX_RLP_V37); + + assertThat(transaction.getV()).isEqualTo(37); + assertThat(transaction.isGoQuorumPrivateTransaction()).isTrue(); + } + + @Test + public void givenGoQuorumTransactionV38_assertThatIsGoQuorumFlagIsTrue() { + Transaction transaction = Transaction.readFrom(GOQUORUM_PRIVATE_TX_RLP_V38); + + assertThat(transaction.getV()).isEqualTo(38); + assertThat(transaction.isGoQuorumPrivateTransaction()).isTrue(); + } + + private static RLPInput toRLP(final String bytes) { + return RLP.input(Bytes.fromHexString(bytes)); + } +} From a430ea1bba6e807de9e10074ef34e3a47b951455 Mon Sep 17 00:00:00 2001 From: Sally MacFarlane Date: Thu, 3 Dec 2020 15:16:53 +1000 Subject: [PATCH 11/16] Add extra info in log messages to help with flaky test (#1649) * extra logging including pid Signed-off-by: Sally MacFarlane --- .../acceptance/dsl/node/ProcessBesuNodeRunner.java | 9 +++++---- .../tests/acceptance/dsl/node/cluster/Cluster.java | 11 +++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java index 523537a5ea2..f9a02c87b13 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java @@ -414,14 +414,15 @@ public boolean isActive(final String nodeName) { private void killBesuProcess(final String name) { final Process process = besuProcesses.remove(name); if (process == null) { - LOG.error("Process {} wasn't in our list", name); + LOG.error("Process {} wasn't in our list, pid {}", name, process.pid()); + return; } if (!process.isAlive()) { - LOG.info("Process {} already exited", name); + LOG.info("Process {} already exited, pid {}", name, process.pid()); return; } - LOG.info("Killing {} process", name); + LOG.info("Killing {} process, pid {}", name, process.pid()); process.destroy(); try { @@ -431,7 +432,7 @@ private void killBesuProcess(final String name) { } if (process.isAlive()) { - LOG.warn("Process {} still alive, destroying forcibly now", name); + LOG.warn("Process {} still alive, destroying forcibly now, pid {}", name, process.pid()); try { process.destroyForcibly().waitFor(30, TimeUnit.SECONDS); } catch (final Exception e) { diff --git a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/cluster/Cluster.java b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/cluster/Cluster.java index 8db6abf5c52..ed1f38d6850 100644 --- a/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/cluster/Cluster.java +++ b/acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/cluster/Cluster.java @@ -87,12 +87,19 @@ public void start(final List nodes) { nodes .parallelStream() - .filter(node -> bootnode.map(boot -> boot != node).orElse(true)) + .filter( + node -> { + LOG.info("starting non-bootnode {}", node.getName()); + return bootnode.map(boot -> boot != node).orElse(true); + }) .forEach(this::startNode); if (clusterConfiguration.isAwaitPeerDiscovery()) { for (final RunnableNode node : nodes) { - LOG.info("Awaiting peer discovery for node {}", node.getName()); + LOG.info( + "Awaiting peer discovery for node {}, expecting {} peers", + node.getName(), + nodes.size() - 1); node.awaitPeerDiscovery(net.awaitPeerCount(nodes.size() - 1)); } } From 9c22fbc61c7e3edf110252a035750044d04f4cf2 Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Thu, 3 Dec 2020 14:44:13 -0700 Subject: [PATCH 12/16] Add memory as a key value storage option (#1617) Add `memory` as an option for `--key-value-storage`. This is useful in small network synchronization tests as memory is faster and is easier to inspect via a debugger. Signed-off-by: Danno Ferrin --- CHANGELOG.md | 1 + .../org/hyperledger/besu/cli/BesuCommand.java | 2 + .../kvstore/InMemoryStoragePlugin.java | 123 ++++++++++++++++++ 3 files changed, 126 insertions(+) create mode 100644 services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryStoragePlugin.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 37e259d5947..2e75f7fdb08 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 20.10.3 ### Additions and Improvements +* Added `memory` as an option to `--key-value-storage`. This ephemeral storage is intended for sync testing and debugging. [\#1617](https://github.com/hyperledger/besu/pull/1617) ### Bug Fixes diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 600715d747e..4572bfa9b2e 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -144,6 +144,7 @@ import org.hyperledger.besu.services.PicoCLIOptionsImpl; import org.hyperledger.besu.services.SecurityModuleServiceImpl; import org.hyperledger.besu.services.StorageServiceImpl; +import org.hyperledger.besu.services.kvstore.InMemoryStoragePlugin; import org.hyperledger.besu.util.NetworkUtility; import org.hyperledger.besu.util.PermissioningConfigurationValidator; import org.hyperledger.besu.util.number.Fraction; @@ -1226,6 +1227,7 @@ private void preparePlugins() { // register built-in plugins new RocksDBPlugin().register(besuPluginContext); + new InMemoryStoragePlugin().register(besuPluginContext); besuPluginContext.registerPlugins(pluginsDir()); diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryStoragePlugin.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryStoragePlugin.java new file mode 100644 index 00000000000..6d23db335a1 --- /dev/null +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryStoragePlugin.java @@ -0,0 +1,123 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.services.kvstore; + +import org.hyperledger.besu.plugin.BesuContext; +import org.hyperledger.besu.plugin.BesuPlugin; +import org.hyperledger.besu.plugin.services.BesuConfiguration; +import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.hyperledger.besu.plugin.services.StorageService; +import org.hyperledger.besu.plugin.services.exception.StorageException; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorageFactory; +import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class InMemoryStoragePlugin implements BesuPlugin { + + private static final Logger LOG = LogManager.getLogger(); + private BesuContext context; + private MemoryKeyValueStorageFactory factory; + private MemoryKeyValueStorageFactory privacyFactory; + + @Override + public void register(final BesuContext context) { + LOG.debug("Registering plugin"); + this.context = context; + + createFactoriesAndRegisterWithStorageService(); + + LOG.debug("Plugin registered."); + } + + @Override + public void start() { + LOG.debug("Starting plugin."); + if (factory == null) { + createFactoriesAndRegisterWithStorageService(); + } + } + + @Override + public void stop() { + LOG.debug("Stopping plugin."); + + if (factory != null) { + factory.close(); + factory = null; + } + + if (privacyFactory != null) { + privacyFactory.close(); + privacyFactory = null; + } + } + + private void createAndRegister(final StorageService service) { + + factory = new MemoryKeyValueStorageFactory("memory"); + privacyFactory = new MemoryKeyValueStorageFactory("memory-privacy"); + + service.registerKeyValueStorage(factory); + service.registerKeyValueStorage(privacyFactory); + } + + private void createFactoriesAndRegisterWithStorageService() { + context + .getService(StorageService.class) + .ifPresentOrElse( + this::createAndRegister, + () -> LOG.error("Failed to register KeyValueFactory due to missing StorageService.")); + } + + public static class MemoryKeyValueStorageFactory implements KeyValueStorageFactory { + + private final String name; + private final Map storageMap = new HashMap<>(); + + public MemoryKeyValueStorageFactory(final String name) { + this.name = name; + } + + @Override + public String getName() { + return name; + } + + @Override + public KeyValueStorage create( + final SegmentIdentifier segment, + final BesuConfiguration configuration, + final MetricsSystem metricsSystem) + throws StorageException { + return storageMap.computeIfAbsent(segment, __ -> new InMemoryKeyValueStorage()); + } + + @Override + public boolean isSegmentIsolationSupported() { + return true; + } + + @Override + public void close() { + storageMap.clear(); + } + } +} From 4b692ff102a1362183912d8600b0c8e59d021877 Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Fri, 4 Dec 2020 07:08:00 -0700 Subject: [PATCH 13/16] Add Ropsten to DNS Discovery (#1629) DNS Discovery had options for mainnet, rinkeby, and goerli, but Ropsten was left out. Signed-off-by: Danno Ferrin --- .../org/hyperledger/besu/cli/config/EthNetworkConfig.java | 6 +++++- .../besu/ethereum/p2p/config/DiscoveryConfiguration.java | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/config/EthNetworkConfig.java b/besu/src/main/java/org/hyperledger/besu/cli/config/EthNetworkConfig.java index 656f0ba66a8..d59db1ff232 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/config/EthNetworkConfig.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/config/EthNetworkConfig.java @@ -25,6 +25,7 @@ import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.RINKEBY_BOOTSTRAP_NODES; import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.RINKEBY_DISCOVERY_URL; import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.ROPSTEN_BOOTSTRAP_NODES; +import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.ROPSTEN_DISCOVERY_URL; import static org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration.YOLO_V2_BOOTSTRAP_NODES; import org.hyperledger.besu.ethereum.p2p.peers.EnodeURL; @@ -131,7 +132,10 @@ public static EthNetworkConfig getNetworkConfig(final NetworkName networkName) { switch (networkName) { case ROPSTEN: return new EthNetworkConfig( - jsonConfig(ROPSTEN_GENESIS), ROPSTEN_NETWORK_ID, ROPSTEN_BOOTSTRAP_NODES, null); + jsonConfig(ROPSTEN_GENESIS), + ROPSTEN_NETWORK_ID, + ROPSTEN_BOOTSTRAP_NODES, + ROPSTEN_DISCOVERY_URL); case RINKEBY: return new EthNetworkConfig( jsonConfig(RINKEBY_GENESIS), diff --git a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/DiscoveryConfiguration.java b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/DiscoveryConfiguration.java index a67ad04d9fd..39fc116c708 100644 --- a/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/DiscoveryConfiguration.java +++ b/ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/config/DiscoveryConfiguration.java @@ -33,6 +33,8 @@ public class DiscoveryConfiguration { "enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@all.mainnet.ethdisco.net"; public static final String RINKEBY_DISCOVERY_URL = "enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@all.rinkeby.ethdisco.net"; + public static final String ROPSTEN_DISCOVERY_URL = + "enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@all.ropsten.ethdisco.net"; public static final List MAINNET_BOOTSTRAP_NODES = Collections.unmodifiableList( From 5241747ba4a62e08953b9584ae03732196b69795 Mon Sep 17 00:00:00 2001 From: Danno Ferrin Date: Mon, 7 Dec 2020 08:59:44 -0700 Subject: [PATCH 14/16] Bonsai Tries early prototype (#1664) This is a not-fully-functional prototype of Bonsai Tries. Bonsai tries is a flat leaf storage, branch-by-location, and diff based reorgs refactoring of the existing forest based trie storage mechanism aimed at creating sustainable performance at mainnet loads. * Since it is experimental a feature flag of --Xdata-storage-format=BONSAI controls activation. Some required changes have a long reach: * To accommodate location based storage many Trie operations accept both a location and hash value. Each data storage format is keyed off of only one of the fields, so many tests will pass in null to the other field. * MutableWorldStateUpdater.persist now takes an argument of a block hash. If this is a natural progression of blocks the hash of the new block is passed in. Otherwise null should be passed in. Signed-off-by: Danno Ferrin --- .../org/hyperledger/besu/cli/BesuCommand.java | 14 +- .../options/unstable/DataStorageOptions.java | 54 ++ .../subcommands/blocks/BlocksSubCommand.java | 7 +- .../subcommands/operator/RestoreState.java | 6 +- .../controller/BesuControllerBuilder.java | 22 +- .../besu/cli/CommandTestAbstract.java | 1 + .../api/query/StateBackupService.java | 2 +- ethereum/core/build.gradle | 2 + .../besu/ethereum/bonsai/BonsaiAccount.java | 358 ++++++++++ .../bonsai/BonsaiLayeredWorldState.java | 104 +++ .../bonsai/BonsaiPersistedWorldState.java | 345 ++++++++++ .../besu/ethereum/bonsai/BonsaiValue.java | 75 +++ .../ethereum/bonsai/BonsaiWorldState.java | 49 ++ .../bonsai/BonsaiWorldStateArchive.java | 102 +++ .../bonsai/BonsaiWorldStateUpdater.java | 620 ++++++++++++++++++ .../besu/ethereum/bonsai/TrieLogLayer.java | 236 +++++++ .../besu/ethereum/chain/GenesisState.java | 10 +- .../ethereum/core/AbstractWorldUpdater.java | 5 + .../besu/ethereum/core/MutableWorldState.java | 9 +- .../ethereum/core/UpdateTrackingAccount.java | 14 +- .../mainnet/AbstractBlockProcessor.java | 2 +- .../OnChainPrivacyPrecompiledContract.java | 4 +- .../privacy/PrivacyPrecompiledContract.java | 2 +- ...PrivateGroupRehydrationBlockProcessor.java | 4 +- .../keyvalue/KeyValueSegmentIdentifier.java | 30 +- .../keyvalue/WorldStateKeyValueStorage.java | 14 +- .../worldstate/DataStorageConfiguration.java | 30 + .../worldstate/DataStorageFormat.java | 32 + .../worldstate/DefaultMutableWorldState.java | 16 +- .../worldstate/DefaultWorldStateArchive.java | 8 +- .../worldstate/StateTrieAccountValue.java | 29 +- .../worldstate/WorldStateArchive.java | 2 - .../worldstate/WorldStateStorage.java | 13 +- .../ethereum/core/BlockDataGenerator.java | 4 +- .../besu/ethereum/core/TestCodeExecutor.java | 2 +- .../besu/ethereum/bonsai/LogRollingTests.java | 305 +++++++++ .../besu/ethereum/bonsai/RollingImport.java | 127 ++++ .../KeyValueStorageWorldStateStorageTest.java | 48 +- .../DefaultMutableWorldStateTest.java | 22 +- .../WorldStateDownloaderBenchmark.java | 5 +- .../AccountTrieNodeDataRequest.java | 4 +- .../StorageTrieNodeDataRequest.java | 4 +- .../sync/worldstate/TrieNodeDataRequest.java | 2 +- .../sync/worldstate/WorldDownloadState.java | 2 +- .../sync/worldstate/PersistDataStepTest.java | 2 +- .../worldstate/WorldDownloadStateTest.java | 4 +- .../worldstate/WorldStateDownloaderTest.java | 8 +- .../BlockchainReferenceTestCaseSpec.java | 2 +- .../GeneralStateTestCaseSpec.java | 2 +- .../VMReferenceTestCaseSpec.java | 2 +- .../besu/ethereum/trie/BranchNode.java | 7 +- .../besu/ethereum/trie/CommitVisitor.java | 24 +- .../ethereum/trie/DefaultNodeFactory.java | 4 +- .../besu/ethereum/trie/ExtensionNode.java | 17 +- .../ethereum/trie/KeyValueMerkleStorage.java | 12 +- .../besu/ethereum/trie/LeafNode.java | 5 + .../ethereum/trie/LocationNodeVisitor.java | 28 + .../besu/ethereum/trie/MerkleStorage.java | 6 +- .../hyperledger/besu/ethereum/trie/Node.java | 2 + .../besu/ethereum/trie/NodeFactory.java | 2 +- .../besu/ethereum/trie/NodeLoader.java | 2 +- .../besu/ethereum/trie/NodeUpdater.java | 2 +- .../besu/ethereum/trie/NullNode.java | 9 +- .../besu/ethereum/trie/PutVisitor.java | 4 +- .../besu/ethereum/trie/RestoreVisitor.java | 5 + .../trie/StoredMerklePatriciaTrie.java | 16 +- .../besu/ethereum/trie/StoredNode.java | 12 +- .../besu/ethereum/trie/StoredNodeFactory.java | 49 +- .../besu/ethereum/trie/TrieNodeDecoder.java | 29 +- .../trie/AbstractMerklePatriciaTrieTest.java | 8 +- .../ethereum/trie/TrieNodeDecoderTest.java | 48 +- plugin-api/build.gradle | 2 +- .../plugin/services/BesuConfiguration.java | 12 + .../services/storage/SegmentIdentifier.java | 11 + .../RocksDBKeyValueStorageFactory.java | 12 +- ...ksDBKeyValuePrivacyStorageFactoryTest.java | 2 + .../RocksDBKeyValueStorageFactoryTest.java | 2 + .../kvstore/InMemoryKeyValueStorage.java | 12 + 78 files changed, 2889 insertions(+), 221 deletions(-) create mode 100644 besu/src/main/java/org/hyperledger/besu/cli/options/unstable/DataStorageOptions.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiValue.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldState.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateUpdater.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/TrieLogLayer.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java create mode 100644 ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java create mode 100644 ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java create mode 100644 ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/LocationNodeVisitor.java diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 4572bfa9b2e..766b40d6ebf 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -48,6 +48,7 @@ import org.hyperledger.besu.cli.custom.JsonRPCAllowlistHostsProperty; import org.hyperledger.besu.cli.custom.RpcAuthFileValidator; import org.hyperledger.besu.cli.error.BesuExceptionHandler; +import org.hyperledger.besu.cli.options.unstable.DataStorageOptions; import org.hyperledger.besu.cli.options.unstable.DnsOptions; import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions; import org.hyperledger.besu.cli.options.unstable.EthstatsOptions; @@ -230,6 +231,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable { final MetricsCLIOptions unstableMetricsCLIOptions = MetricsCLIOptions.create(); final TransactionPoolOptions unstableTransactionPoolOptions = TransactionPoolOptions.create(); private final EthstatsOptions unstableEthstatsOptions = EthstatsOptions.create(); + private final DataStorageOptions unstableDataStorageOptions = DataStorageOptions.create(); private final DnsOptions unstableDnsOptions = DnsOptions.create(); private final MiningOptions unstableMiningOptions = MiningOptions.create(); private final NatOptions unstableNatOptions = NatOptions.create(); @@ -1214,6 +1216,7 @@ private void handleUnstableOptions() { .put("Ethstats", unstableEthstatsOptions) .put("Mining", unstableMiningOptions) .put("Native Library", unstableNativeLibraryOptions) + .put("Data Storage Options", unstableDataStorageOptions) .build(); UnstableOptionsSubCommand.createUnstableOptions(commandLine, unstableOptions); @@ -1569,7 +1572,8 @@ public BesuControllerBuilder getControllerBuilder() { .map(TargetingGasLimitCalculator::new) .orElse(GasLimitCalculator.constant())) .requiredBlocks(requiredBlocks) - .reorgLoggingThreshold(reorgLoggingThreshold); + .reorgLoggingThreshold(reorgLoggingThreshold) + .dataStorageConfiguration(unstableDataStorageOptions.toDomainObject()); } private GraphQLConfiguration graphQLConfiguration() { @@ -2441,5 +2445,13 @@ public Path getStoragePath() { public Path getDataPath() { return dataDir(); } + + @Override + public int getDatabaseVersion() { + return unstableDataStorageOptions + .toDomainObject() + .getDataStorageFormat() + .getDatabaseVersion(); + } } } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/DataStorageOptions.java b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/DataStorageOptions.java new file mode 100644 index 00000000000..c35ed4407a7 --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/options/unstable/DataStorageOptions.java @@ -0,0 +1,54 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.cli.options.unstable; + +import org.hyperledger.besu.cli.options.CLIOptions; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; +import org.hyperledger.besu.ethereum.worldstate.DataStorageFormat; +import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration; + +import java.util.List; + +import picocli.CommandLine.Option; + +public class DataStorageOptions implements CLIOptions { + + private static final String DATA_STORAGE_FORMAT = "--Xdata-storage-format"; + + // Use Bonsai DB + @Option( + names = {DATA_STORAGE_FORMAT}, + hidden = true, + description = + "Format to store trie data in. Either FOREST or BONSAI (default: ${DEFAULT-VALUE}).", + arity = "1") + private final DataStorageFormat dataStorageFormat = DataStorageFormat.FOREST; + + public static DataStorageOptions create() { + return new DataStorageOptions(); + } + + @Override + public DataStorageConfiguration toDomainObject() { + return ImmutableDataStorageConfiguration.builder().dataStorageFormat(dataStorageFormat).build(); + } + + @Override + public List getCLIOptions() { + return List.of(DATA_STORAGE_FORMAT, dataStorageFormat.toString()); + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/blocks/BlocksSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/blocks/BlocksSubCommand.java index d57c9537555..2034631d27e 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/blocks/BlocksSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/blocks/BlocksSubCommand.java @@ -278,9 +278,10 @@ private void importJsonBlocks(final BesuController controller, final Path path) private void importRlpBlocks(final BesuController controller, final Path path) throws IOException { - try (final RlpBlockImporter rlpBlockImporter = parentCommand.rlpBlockImporter.get()) { - rlpBlockImporter.importBlockchain(path, controller, skipPow, startBlock, endBlock); - } + parentCommand + .rlpBlockImporter + .get() + .importBlockchain(path, controller, skipPow, startBlock, endBlock); } } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/RestoreState.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/RestoreState.java index 9f1a2f72ac2..cac9d58ee03 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/RestoreState.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/operator/RestoreState.java @@ -265,13 +265,15 @@ private void updateCode(final Bytes code) { private void updateAccountState(final Bytes32 key, final Bytes value) { maybeCommitUpdater(); - updater.putAccountStateTrieNode(key, value); + // restore by path not supported + updater.putAccountStateTrieNode(null, key, value); trieNodeCount++; } private void updateAccountStorage(final Bytes32 key, final Bytes value) { maybeCommitUpdater(); - updater.putAccountStorageTrieNode(key, value); + // restore by path not supported + updater.putAccountStorageTrieNode(null, key, value); trieNodeCount++; } diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java index fd2364513eb..44df5792052 100644 --- a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java +++ b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java @@ -24,6 +24,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.methods.JsonRpcMethods; import org.hyperledger.besu.ethereum.blockcreation.GasLimitCalculator; import org.hyperledger.besu.ethereum.blockcreation.MiningCoordinator; +import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateArchive; import org.hyperledger.besu.ethereum.chain.Blockchain; import org.hyperledger.besu.ethereum.chain.GenesisState; import org.hyperledger.besu.ethereum.chain.MutableBlockchain; @@ -54,6 +55,7 @@ import org.hyperledger.besu.ethereum.p2p.config.SubProtocolConfiguration; import org.hyperledger.besu.ethereum.storage.StorageProvider; import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration; import org.hyperledger.besu.ethereum.worldstate.DefaultWorldStateArchive; import org.hyperledger.besu.ethereum.worldstate.MarkSweepPruner; import org.hyperledger.besu.ethereum.worldstate.Pruner; @@ -99,6 +101,8 @@ public abstract class BesuControllerBuilder { Map genesisConfigOverrides; private Map requiredBlocks = Collections.emptyMap(); private long reorgLoggingThreshold; + private DataStorageConfiguration dataStorageConfiguration = + DataStorageConfiguration.DEFAULT_CONFIG; public BesuControllerBuilder storageProvider(final StorageProvider storageProvider) { this.storageProvider = storageProvider; @@ -199,6 +203,12 @@ public BesuControllerBuilder reorgLoggingThreshold(final long reorgLoggingThresh return this; } + public BesuControllerBuilder dataStorageConfiguration( + final DataStorageConfiguration dataStorageConfiguration) { + this.dataStorageConfiguration = dataStorageConfiguration; + return this; + } + public BesuController build() { checkNotNull(genesisConfig, "Missing genesis config"); checkNotNull(syncConfig, "Missing sync config"); @@ -414,9 +424,15 @@ protected EthProtocolManager createEthProtocolManager( } public WorldStateArchive createWorldStateArchive(final WorldStateStorage worldStateStorage) { - final WorldStatePreimageStorage preimageStorage = - storageProvider.createWorldStatePreimageStorage(); - return new DefaultWorldStateArchive(worldStateStorage, preimageStorage); + switch (dataStorageConfiguration.getDataStorageFormat()) { + case BONSAI: + return new BonsaiWorldStateArchive(storageProvider); + case FOREST: + default: + final WorldStatePreimageStorage preimageStorage = + storageProvider.createWorldStatePreimageStorage(); + return new DefaultWorldStateArchive(worldStateStorage, preimageStorage); + } } private List createPeerValidators(final ProtocolSchedule protocolSchedule) { diff --git a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java index 7a1f07fba1a..3fd32004f8e 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java @@ -189,6 +189,7 @@ public void initMocks() throws Exception { when(mockControllerBuilder.gasLimitCalculator(any())).thenReturn(mockControllerBuilder); when(mockControllerBuilder.requiredBlocks(any())).thenReturn(mockControllerBuilder); when(mockControllerBuilder.reorgLoggingThreshold(anyLong())).thenReturn(mockControllerBuilder); + when(mockControllerBuilder.dataStorageConfiguration(any())).thenReturn(mockControllerBuilder); // doReturn used because of generic BesuController doReturn(mockController).when(mockControllerBuilder).build(); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java index b7644357556..080229cdfa8 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/StateBackupService.java @@ -219,7 +219,7 @@ private void backupLeaves() throws IOException { return; } final Optional worldStateRoot = - worldStateStorage.getAccountStateTrieNode(header.get().getStateRoot()); + worldStateStorage.getAccountStateTrieNode(Bytes.EMPTY, header.get().getStateRoot()); if (worldStateRoot.isEmpty()) { backupStatus.currentAccount = null; return; diff --git a/ethereum/core/build.gradle b/ethereum/core/build.gradle index 269c49ebafb..a64a893c214 100644 --- a/ethereum/core/build.gradle +++ b/ethereum/core/build.gradle @@ -49,6 +49,8 @@ dependencies { implementation 'org.hyperledger.besu:bls12-381' implementation 'org.immutables:value-annotations' + implementation 'org.xerial.snappy:snappy-java' + annotationProcessor 'org.immutables:value' runtimeOnly 'org.apache.logging.log4j:log4j-core' diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java new file mode 100644 index 00000000000..99e71076a62 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java @@ -0,0 +1,358 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.bonsai; + +import org.hyperledger.besu.ethereum.core.Account; +import org.hyperledger.besu.ethereum.core.AccountStorageEntry; +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.EvmAccount; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.ModificationNotAllowedException; +import org.hyperledger.besu.ethereum.core.MutableAccount; +import org.hyperledger.besu.ethereum.core.UpdateTrackingAccount; +import org.hyperledger.besu.ethereum.core.Wei; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.rlp.RLPException; +import org.hyperledger.besu.ethereum.rlp.RLPInput; +import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; + +import java.util.HashMap; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Objects; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +public class BonsaiAccount implements MutableAccount, EvmAccount { + private final BonsaiWorldState context; + private final boolean mutable; + + private final Address address; + private final Hash addressHash; + private Hash codeHash; + private long nonce; + private Wei balance; + private Hash storageRoot; + private Bytes code; + private int version; + + private final Map updatedStorage = new HashMap<>(); + + BonsaiAccount( + final BonsaiWorldState context, + final Address address, + final Hash addressHash, + final long nonce, + final Wei balance, + final Hash storageRoot, + final Hash codeHash, + final int version, + final boolean mutable) { + this.context = context; + this.address = address; + this.addressHash = addressHash; + this.nonce = nonce; + this.balance = balance; + this.storageRoot = storageRoot; + this.codeHash = codeHash; + this.version = version; + + this.mutable = mutable; + } + + BonsaiAccount( + final BonsaiWorldState context, + final Address address, + final StateTrieAccountValue stateTrieAccount, + final boolean mutable) { + this( + context, + address, + Hash.hash(address), + stateTrieAccount.getNonce(), + stateTrieAccount.getBalance(), + stateTrieAccount.getStorageRoot(), + stateTrieAccount.getCodeHash(), + stateTrieAccount.getVersion(), + mutable); + } + + BonsaiAccount(final BonsaiAccount toCopy) { + this(toCopy, toCopy.context, false); + } + + BonsaiAccount(final BonsaiAccount toCopy, final BonsaiWorldState context, final boolean mutable) { + this.context = context; + this.address = toCopy.address; + this.addressHash = toCopy.addressHash; + this.nonce = toCopy.nonce; + this.balance = toCopy.balance; + this.storageRoot = toCopy.storageRoot; + this.codeHash = toCopy.codeHash; + this.code = toCopy.code; + this.version = toCopy.version; + updatedStorage.putAll(toCopy.updatedStorage); + + this.mutable = mutable; + } + + BonsaiAccount( + final BonsaiWorldState context, final UpdateTrackingAccount tracked) { + this.context = context; + this.address = tracked.getAddress(); + this.addressHash = tracked.getAddressHash(); + this.nonce = tracked.getNonce(); + this.balance = tracked.getBalance(); + this.storageRoot = Hash.EMPTY_TRIE_HASH; + this.codeHash = tracked.getCodeHash(); + this.code = tracked.getCode(); + this.version = tracked.getVersion(); + updatedStorage.putAll(tracked.getUpdatedStorage()); + + this.mutable = true; + } + + static BonsaiAccount fromRLP( + final BonsaiWorldState context, + final Address address, + final Bytes encoded, + final boolean mutable) + throws RLPException { + final RLPInput in = RLP.input(encoded); + in.enterList(); + + final long nonce = in.readLongScalar(); + final Wei balance = Wei.of(in.readUInt256Scalar()); + final Hash storageRoot = Hash.wrap(in.readBytes32()); + final Hash codeHash = Hash.wrap(in.readBytes32()); + final int version; + if (!in.isEndOfCurrentList()) { + version = in.readIntScalar(); + } else { + version = Account.DEFAULT_VERSION; + } + + in.leaveList(); + + return new BonsaiAccount( + context, + address, + Hash.hash(address), + nonce, + balance, + storageRoot, + codeHash, + version, + mutable); + } + + @Override + public Address getAddress() { + return address; + } + + @Override + public Hash getAddressHash() { + return addressHash; + } + + @Override + public long getNonce() { + return nonce; + } + + @Override + public void setNonce(final long value) { + if (!mutable) { + throw new UnsupportedOperationException("Account is immutable"); + } + nonce = value; + } + + @Override + public Wei getBalance() { + return balance; + } + + @Override + public void setBalance(final Wei value) { + if (!mutable) { + throw new UnsupportedOperationException("Account is immutable"); + } + balance = value; + } + + @Override + public Bytes getCode() { + if (code == null) { + code = context.getCode(address); + } + return code; + } + + @Override + public void setCode(final Bytes code) { + if (!mutable) { + throw new UnsupportedOperationException("Account is immutable"); + } + this.code = code; + if (code == null || code.isEmpty()) { + this.codeHash = Hash.EMPTY; + } else { + this.codeHash = Hash.hash(code); + } + } + + @Override + public Hash getCodeHash() { + return codeHash; + } + + @Override + public int getVersion() { + return version; + } + + @Override + public void setVersion(final int version) { + if (!mutable) { + throw new UnsupportedOperationException("Account is immutable"); + } + this.version = version; + } + + @Override + public UInt256 getStorageValue(final UInt256 key) { + return context.getStorageValue(address, key); + } + + @Override + public UInt256 getOriginalStorageValue(final UInt256 key) { + return context.getOriginalStorageValue(address, key); + } + + @Override + public NavigableMap storageEntriesFrom( + final Bytes32 startKeyHash, final int limit) { + throw new RuntimeException("Bonsai Tries does not currently support enumerating storage"); + } + + Bytes serializeAccount() { + final BytesValueRLPOutput out = new BytesValueRLPOutput(); + out.startList(); + + out.writeLongScalar(nonce); + out.writeUInt256Scalar(balance); + out.writeBytes(storageRoot); + out.writeBytes(codeHash); + + if (version != Account.DEFAULT_VERSION) { + // version of zero is never written out. + out.writeIntScalar(version); + } + + out.endList(); + return out.encoded(); + } + + @Override + public void setStorageValue(final UInt256 key, final UInt256 value) { + if (!mutable) { + throw new UnsupportedOperationException("Account is immutable"); + } + updatedStorage.put(key, value); + } + + @Override + public void clearStorage() { + updatedStorage.clear(); + } + + @Override + public Map getUpdatedStorage() { + return updatedStorage; + } + + @Override + public MutableAccount getMutable() throws ModificationNotAllowedException { + if (mutable) { + return this; + } else { + throw new ModificationNotAllowedException(); + } + } + + public Hash getStorageRoot() { + return storageRoot; + } + + public void setStorageRoot(final Hash storageRoot) { + if (!mutable) { + throw new UnsupportedOperationException("Account is immutable"); + } + this.storageRoot = storageRoot; + } + + @Override + public String toString() { + return "AccountState{" + + "address=" + + address + + ", nonce=" + + nonce + + ", balance=" + + balance + + ", storageRoot=" + + storageRoot + + ", codeHash=" + + codeHash + + ", version=" + + version + + '}'; + } + + /** + * Throws an exception if the two accounts represent different stored states + * + * @param source The bonsai account to compare + * @param account The State Trie account to compare + * @param context a description to be added to the thrown exceptions + * @throws IllegalStateException if the stored values differ + */ + static void assertCloseEnoughForDiffing( + final BonsaiAccount source, final StateTrieAccountValue account, final String context) { + if (source == null) { + throw new IllegalStateException(context + ": source is null but target isn't"); + } else { + if (source.nonce != account.getNonce()) { + throw new IllegalStateException(context + ": nonces differ"); + } + if (!Objects.equals(source.balance, account.getBalance())) { + throw new IllegalStateException(context + ": balances differ"); + } + if (!Objects.equals(source.storageRoot, account.getStorageRoot())) { + throw new IllegalStateException(context + ": Storage Roots differ"); + } + if (source.version != account.getVersion()) { + throw new IllegalStateException(context + ": versions differ"); + } + } + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java new file mode 100644 index 00000000000..d0a489eee16 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java @@ -0,0 +1,104 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.bonsai; + +import static com.google.common.base.Preconditions.checkArgument; + +import org.hyperledger.besu.ethereum.core.Account; +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.WorldState; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +/** A World State backed first by trie log layer and then by another world state. */ +public class BonsaiLayeredWorldState implements BonsaiWorldState, WorldState { + + private final BonsaiWorldState parent; + private final TrieLogLayer trieLog; + private final Map cachedAccounts = new HashMap<>(); + + public BonsaiLayeredWorldState(final BonsaiWorldState parent, final TrieLogLayer trieLog) { + checkArgument(trieLog.isFrozen(), "TrieLogs must be frozen to be used as world state."); + this.parent = parent; + this.trieLog = trieLog; + } + + @Override + public Bytes getCode(final Address address) { + return trieLog.getCode(address).orElseGet(() -> parent.getCode(address)); + } + + @Override + public UInt256 getStorageValue(final Address address, final UInt256 key) { + return getStorageValueBySlotHash(address, Hash.hash(key.toBytes())).orElse(UInt256.ZERO); + } + + @Override + public Optional getStorageValueBySlotHash(final Address address, final Hash slotHash) { + return trieLog + .getStorageBySlotHash(address, slotHash) + .or(() -> parent.getStorageValueBySlotHash(address, slotHash)); + } + + @Override + public UInt256 getOriginalStorageValue(final Address address, final UInt256 key) { + // This is the base layer for a block, all values are original. + return getStorageValue(address, key); + } + + @Override + public Map getAllAccountStorage(final Address address, final Hash rootHash) { + final Map results = parent.getAllAccountStorage(address, rootHash); + trieLog + .streamStorageChanges(address) + .forEach(entry -> results.put(entry.getKey(), entry.getValue().getUpdated().toBytes())); + return results; + } + + @Override + public Account get(final Address address) { + return cachedAccounts.computeIfAbsent( + address, + addr -> + trieLog + .getAccount(addr) + .map( + stateTrieAccountValue -> + (Account) + new BonsaiAccount( + BonsaiLayeredWorldState.this, addr, stateTrieAccountValue, false)) + .orElseGet(() -> parent.get(address))); + } + + @Override + public Hash rootHash() { + return trieLog.getBlockHash(); + } + + @Override + public Stream streamAccounts(final Bytes32 startKeyHash, final int limit) { + throw new UnsupportedOperationException("Bonsai does not support pruning and debug RPCs"); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java new file mode 100644 index 00000000000..d4f7cd3d337 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiPersistedWorldState.java @@ -0,0 +1,345 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.bonsai; + +import static org.hyperledger.besu.ethereum.bonsai.BonsaiAccount.fromRLP; + +import org.hyperledger.besu.ethereum.core.Account; +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.core.WorldUpdater; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput; +import org.hyperledger.besu.ethereum.trie.MerklePatriciaTrie; +import org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; + +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Stream; +import javax.annotation.Nonnull; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +public class BonsaiPersistedWorldState implements MutableWorldState, BonsaiWorldState { + + private static final byte[] WORLD_ROOT_KEY = "worldRoot".getBytes(StandardCharsets.UTF_8); + + private final KeyValueStorage accountStorage; + private final KeyValueStorage codeStorage; + private final KeyValueStorage storageStorage; + private final KeyValueStorage trieBranchStorage; + private final KeyValueStorage trieLogStorage; + + private Bytes32 worldStateRootHash; + + private final BonsaiWorldStateArchive archive; + private BonsaiWorldStateUpdater updater; + + public BonsaiPersistedWorldState( + final BonsaiWorldStateArchive archive, + final KeyValueStorage accountStorage, + final KeyValueStorage codeStorage, + final KeyValueStorage storageStorage, + final KeyValueStorage trieBranchStorage, + final KeyValueStorage trieLogStorage) { + this.archive = archive; + this.accountStorage = accountStorage; + this.codeStorage = codeStorage; + this.storageStorage = storageStorage; + this.trieBranchStorage = trieBranchStorage; + this.trieLogStorage = trieLogStorage; + worldStateRootHash = + Bytes32.wrap( + trieBranchStorage.get(WORLD_ROOT_KEY).map(Bytes::wrap).orElse(Hash.EMPTY_TRIE_HASH)); + } + + public BonsaiWorldStateArchive getArchive() { + return archive; + } + + @Override + public MutableWorldState copy() { + throw new UnsupportedOperationException( + "Bonsai Tries does not support direct duplication of the persisted tries."); + } + + @Override + public Bytes getCode(@Nonnull final Address address) { + return codeStorage.get(address.toArrayUnsafe()).map(Bytes::wrap).orElse(Bytes.EMPTY); + } + + @Override + public void persist(final Hash blockHash) { + boolean success = false; + final KeyValueStorageTransaction accountTx = accountStorage.startTransaction(); + final KeyValueStorageTransaction codeTx = codeStorage.startTransaction(); + final KeyValueStorageTransaction storageTx = storageStorage.startTransaction(); + final KeyValueStorageTransaction trieBranchTx = trieBranchStorage.startTransaction(); + final KeyValueStorageTransaction trieLogTx = trieLogStorage.startTransaction(); + + try { + // first clear storage + for (final Address address : updater.getStorageToClear()) { + // because we are clearing persisted values we need the account root as persisted + final BonsaiAccount oldAccount = + accountStorage + .get(address.toArrayUnsafe()) + .map( + bytes -> + fromRLP(BonsaiPersistedWorldState.this, address, Bytes.wrap(bytes), true)) + .orElse(null); + if (oldAccount == null) { + // This is when an account is both created and deleted within the scope of the same + // block. A not-uncommon DeFi bot pattern. + continue; + } + final StoredMerklePatriciaTrie storageTrie = + new StoredMerklePatriciaTrie<>( + (location, key) -> getStorageTrieNode(address, location, key), + oldAccount.getStorageRoot(), + Function.identity(), + Function.identity()); + Map entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256); + while (!entriesToDelete.isEmpty()) { + entriesToDelete + .keySet() + .forEach(k -> storageTx.remove(Bytes.concatenate(address, k).toArrayUnsafe())); + if (entriesToDelete.size() == 256) { + entriesToDelete.keySet().forEach(storageTrie::remove); + entriesToDelete = storageTrie.entriesFrom(Bytes32.ZERO, 256); + } else { + break; + } + } + } + + // second update account storage state. This must be done before updating the accounts so + // that we can get the storage state hash + for (final Map.Entry>> storageAccountUpdate : + updater.getStorageToUpdate().entrySet()) { + final Address updatedAddress = storageAccountUpdate.getKey(); + final BonsaiValue accountValue = + updater.getAccountsToUpdate().get(updatedAddress); + final BonsaiAccount accountOriginal = accountValue.getOriginal(); + final Hash storageRoot = + (accountOriginal == null) ? Hash.EMPTY_TRIE_HASH : accountOriginal.getStorageRoot(); + final StoredMerklePatriciaTrie storageTrie = + new StoredMerklePatriciaTrie<>( + (location, key) -> getStorageTrieNode(updatedAddress, location, key), + storageRoot, + Function.identity(), + Function.identity()); + + // for manicured tries and composting, collect branches here (not implemented) + + for (final Map.Entry> storageUpdate : + storageAccountUpdate.getValue().entrySet()) { + final Hash keyHash = storageUpdate.getKey(); + final byte[] writeAddress = Bytes.concatenate(updatedAddress, keyHash).toArrayUnsafe(); + final UInt256 updatedStorage = storageUpdate.getValue().getUpdated(); + if (updatedStorage == null || updatedStorage.equals(UInt256.ZERO)) { + storageTx.remove(writeAddress); + storageTrie.remove(keyHash); + } else { + final Bytes32 updatedStorageBytes = updatedStorage.toBytes(); + storageTx.put(writeAddress, updatedStorageBytes.toArrayUnsafe()); + storageTrie.put(keyHash, rlpEncode(updatedStorageBytes)); + } + } + + final BonsaiAccount accountUpdated = accountValue.getUpdated(); + if (accountUpdated != null) { + storageTrie.commit( + (location, key, value) -> + writeStorageTrieNode(trieBranchTx, updatedAddress, location, value)); + final Hash newStorageRoot = Hash.wrap(storageTrie.getRootHash()); + accountUpdated.setStorageRoot(newStorageRoot); + } + // for manicured tries and composting, trim and compost here + } + + // Third update the code. This has the side effect of ensuring a code hash is calculated. + for (final Map.Entry> codeUpdate : + updater.getCodeToUpdate().entrySet()) { + final Bytes updatedCode = codeUpdate.getValue().getUpdated(); + if (updatedCode == null || updatedCode.size() == 0) { + codeTx.remove(codeUpdate.getKey().toArrayUnsafe()); + } else { + codeTx.put(codeUpdate.getKey().toArrayUnsafe(), updatedCode.toArrayUnsafe()); + } + } + + // next collect the branches that will be trimmed + final StoredMerklePatriciaTrie accountTrie = + new StoredMerklePatriciaTrie<>( + this::getTrieNode, worldStateRootHash, Function.identity(), Function.identity()); + + // for manicured tries and composting, collect branches here (not implemented) + + // now add the accounts + for (final Map.Entry> accountUpdate : + updater.getAccountsToUpdate().entrySet()) { + final Bytes accountKey = accountUpdate.getKey(); + final BonsaiValue bonsaiValue = accountUpdate.getValue(); + final BonsaiAccount updatedAccount = bonsaiValue.getUpdated(); + if (updatedAccount == null) { + final Hash addressHash = Hash.hash(accountKey); + accountTrie.remove(addressHash); + accountTx.remove(accountKey.toArrayUnsafe()); + } else { + final Hash addressHash = updatedAccount.getAddressHash(); + final Bytes accountValue = updatedAccount.serializeAccount(); + accountTx.put(accountKey.toArrayUnsafe(), accountValue.toArrayUnsafe()); + accountTrie.put(addressHash, accountValue); + } + } + + accountTrie.commit((location, hash, value) -> writeTrieNode(trieBranchTx, location, value)); + worldStateRootHash = accountTrie.getRootHash(); + trieBranchTx.put(WORLD_ROOT_KEY, worldStateRootHash.toArrayUnsafe()); + + // for manicured tries and composting, trim and compost branches here + + if (blockHash != null) { + final TrieLogLayer trieLog = updater.generateTrieLog(blockHash); + trieLog.freeze(); + // TODO add to archive here, but only once we get persisted follow distance implemented + // archive.addLayeredWorldState(new BonsaiLayeredWorldState(this, trieLog)); + + final BytesValueRLPOutput rlpLog = new BytesValueRLPOutput(); + trieLog.writeTo(rlpLog); + trieLogTx.put(blockHash.toArrayUnsafe(), rlpLog.encoded().toArrayUnsafe()); + } + + success = true; + } finally { + if (success) { + accountTx.commit(); + codeTx.commit(); + storageTx.commit(); + trieBranchTx.commit(); + trieLogTx.commit(); + updater.reset(); + } else { + accountTx.rollback(); + codeTx.rollback(); + storageTx.rollback(); + trieBranchTx.rollback(); + trieLogTx.rollback(); + } + } + } + + private static Bytes rlpEncode(final Bytes bytes) { + final BytesValueRLPOutput out = new BytesValueRLPOutput(); + out.writeBytes(bytes.trimLeadingZeros()); + return out.encoded(); + } + + @Override + public WorldUpdater updater() { + if (updater == null) { + updater = new BonsaiWorldStateUpdater(this); + } + return updater; + } + + @Override + public Hash rootHash() { + return Hash.wrap(worldStateRootHash); + } + + @Override + public Stream streamAccounts(final Bytes32 startKeyHash, final int limit) { + throw new RuntimeException("Bonsai Tries do not provide account streaming."); + } + + @Override + public Account get(final Address address) { + return accountStorage + .get(address.toArrayUnsafe()) + .map(bytes -> fromRLP(updater, address, Bytes.wrap(bytes), true)) + .orElse(null); + } + + private Optional getTrieNode(final Bytes location, final Bytes32 nodeHash) { + if (nodeHash.equals(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) { + return Optional.of(MerklePatriciaTrie.EMPTY_TRIE_NODE); + } else { + return trieBranchStorage.get(location.toArrayUnsafe()).map(Bytes::wrap); + } + } + + private void writeTrieNode( + final KeyValueStorageTransaction tx, final Bytes location, final Bytes value) { + tx.put(location.toArrayUnsafe(), value.toArrayUnsafe()); + } + + private Optional getStorageTrieNode( + final Address address, final Bytes location, final Bytes32 nodeHash) { + if (nodeHash.equals(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) { + return Optional.of(MerklePatriciaTrie.EMPTY_TRIE_NODE); + } else { + return trieBranchStorage + .get(Bytes.concatenate(address, location).toArrayUnsafe()) + .map(Bytes::wrap); + } + } + + private void writeStorageTrieNode( + final KeyValueStorageTransaction tx, + final Address address, + final Bytes location, + final Bytes value) { + tx.put(Bytes.concatenate(address, location).toArrayUnsafe(), value.toArrayUnsafe()); + } + + @Override + public UInt256 getStorageValue(final Address address, final UInt256 storageKey) { + return getStorageValueBySlotHash(address, Hash.hash(storageKey.toBytes())).orElse(UInt256.ZERO); + } + + @Override + public Optional getStorageValueBySlotHash(final Address address, final Hash slotHash) { + return storageStorage + .get(Bytes.concatenate(address, slotHash).toArrayUnsafe()) + .map(Bytes::wrap) + .map(UInt256::fromBytes); + } + + @Override + public UInt256 getOriginalStorageValue(final Address address, final UInt256 storageKey) { + return getStorageValue(address, storageKey); + } + + @Override + public Map getAllAccountStorage(final Address address, final Hash rootHash) { + final StoredMerklePatriciaTrie storageTrie = + new StoredMerklePatriciaTrie<>( + (location, key) -> getStorageTrieNode(address, location, key), + rootHash, + Function.identity(), + Function.identity()); + return storageTrie.entriesFrom(Bytes32.ZERO, Integer.MAX_VALUE); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiValue.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiValue.java new file mode 100644 index 00000000000..869221b7858 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiValue.java @@ -0,0 +1,75 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.bonsai; + +import org.hyperledger.besu.ethereum.rlp.RLPOutput; + +import java.util.Objects; +import java.util.function.BiConsumer; + +public class BonsaiValue { + private T original; + private T updated; + + BonsaiValue(final T original, final T updated) { + this.original = original; + this.updated = updated; + } + + public T getOriginal() { + return original; + } + + public T getUpdated() { + return updated; + } + + public void setOriginal(final T original) { + this.original = original; + } + + public void setUpdated(final T updated) { + this.updated = updated; + } + + void writeRlp(final RLPOutput output, final BiConsumer writer) { + output.startList(); + writeInnerRlp(output, writer); + output.endList(); + } + + void writeInnerRlp(final RLPOutput output, final BiConsumer writer) { + if (original == null) { + output.writeNull(); + } else { + writer.accept(output, original); + } + if (updated == null) { + output.writeNull(); + } else { + writer.accept(output, updated); + } + } + + boolean isUnchanged() { + return Objects.equals(updated, original); + } + + T effective() { + return updated == null ? original : updated; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldState.java new file mode 100644 index 00000000000..85904b58dbb --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldState.java @@ -0,0 +1,49 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.bonsai; + +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.WorldView; + +import java.util.Map; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +public interface BonsaiWorldState extends WorldView { + + Bytes getCode(Address address); + + UInt256 getStorageValue(Address address, UInt256 key); + + Optional getStorageValueBySlotHash(Address address, Hash slotHash); + + UInt256 getOriginalStorageValue(Address address, UInt256 key); + + /** + * Stream all the storage values of a account. + * + * @param address the account to stream + * @param rootHash the root hash of the account storage trie + * @return A map that is a copy of the entries. The key is the hashed slot number, and the value + * is the Bytes representation of the storage value. + */ + Map getAllAccountStorage(Address address, Hash rootHash); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java new file mode 100644 index 00000000000..9f268ff3799 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java @@ -0,0 +1,102 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.bonsai; + +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.MutableWorldState; +import org.hyperledger.besu.ethereum.core.WorldState; +import org.hyperledger.besu.ethereum.proof.WorldStateProof; +import org.hyperledger.besu.ethereum.storage.StorageProvider; +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +public class BonsaiWorldStateArchive implements WorldStateArchive { + + private final BonsaiPersistedWorldState persistedState; + private final Map layeredWorldStates; + + public BonsaiWorldStateArchive(final StorageProvider provider) { + persistedState = + new BonsaiPersistedWorldState( + this, + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE), + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE), + provider.getStorageBySegmentIdentifier( + KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE), + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE), + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE)); + layeredWorldStates = new HashMap<>(); + } + + @Override + public Optional get(final Hash rootHash) { + if (layeredWorldStates.containsKey(rootHash)) { + return Optional.of(layeredWorldStates.get(rootHash)); + } else if (rootHash.equals(persistedState.rootHash())) { + return Optional.of(persistedState); + } else { + return Optional.empty(); + } + } + + public void addLayeredWorldState(final BonsaiLayeredWorldState worldState) { + layeredWorldStates.put(worldState.rootHash(), worldState); + } + + @Override + public boolean isWorldStateAvailable(final Hash rootHash) { + return layeredWorldStates.containsKey(rootHash) + || persistedState.rootHash().equals(rootHash) /* || check disk storage */; + } + + @Override + public Optional getMutable(final Hash rootHash) { + if (rootHash.equals(persistedState.rootHash())) { + return Optional.of(persistedState); + } else { + return Optional.empty(); + } + } + + @Override + public MutableWorldState getMutable() { + return persistedState; + } + + @Override + public Optional getNodeData(final Hash hash) { + return Optional.empty(); + } + + @Override + public Optional getAccountProof( + final Hash worldStateRoot, + final Address accountAddress, + final List accountStorageKeys) { + return Optional.empty(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateUpdater.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateUpdater.java new file mode 100644 index 00000000000..b417169a8c7 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateUpdater.java @@ -0,0 +1,620 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.bonsai; + +import org.hyperledger.besu.ethereum.core.AbstractWorldUpdater; +import org.hyperledger.besu.ethereum.core.Account; +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.EvmAccount; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.UpdateTrackingAccount; +import org.hyperledger.besu.ethereum.core.Wei; +import org.hyperledger.besu.ethereum.core.WrappedEvmAccount; +import org.hyperledger.besu.ethereum.rlp.RLP; +import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; + +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Function; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.units.bigints.UInt256; + +public class BonsaiWorldStateUpdater + extends AbstractWorldUpdater + implements BonsaiWorldState { + + private final Map> accountsToUpdate = new HashMap<>(); + private final Map> codeToUpdate = new HashMap<>(); + private final Set

storageToClear = new HashSet<>(); + + // storage sub mapped by _hashed_ key. This is because in self_destruct calls we need to + // enumerate the old storage and delete it. Those are trie stored by hashed key by spec and the + // alternative was to keep a giant pre-image cache of the entire trie. + private final Map>> storageToUpdate = new HashMap<>(); + + BonsaiWorldStateUpdater(final BonsaiPersistedWorldState world) { + super(world); + } + + @Override + public Account get(final Address address) { + return super.get(address); + } + + @Override + protected UpdateTrackingAccount track( + final UpdateTrackingAccount account) { + return super.track(account); + } + + @Override + public EvmAccount getAccount(final Address address) { + return super.getAccount(address); + } + + @Override + public EvmAccount createAccount(final Address address, final long nonce, final Wei balance) { + BonsaiValue bonsaiValue = accountsToUpdate.get(address); + if (bonsaiValue == null) { + bonsaiValue = new BonsaiValue<>(null, null); + accountsToUpdate.put(address, bonsaiValue); + } else if (bonsaiValue.getUpdated() != null) { + throw new IllegalStateException("Cannot create an account when one already exists"); + } + final BonsaiAccount newAccount = + new BonsaiAccount( + this, + address, + Hash.hash(address), + nonce, + balance, + Hash.EMPTY_TRIE_HASH, + Hash.EMPTY, + Account.DEFAULT_VERSION, + true); + bonsaiValue.setUpdated(newAccount); + return new WrappedEvmAccount(track(new UpdateTrackingAccount<>(newAccount))); + } + + Map> getAccountsToUpdate() { + return accountsToUpdate; + } + + Map> getCodeToUpdate() { + return codeToUpdate; + } + + public Set
getStorageToClear() { + return storageToClear; + } + + Map>> getStorageToUpdate() { + return storageToUpdate; + } + + @Override + protected BonsaiAccount getForMutation(final Address address) { + final BonsaiValue bonsaiValue = accountsToUpdate.get(address); + if (bonsaiValue == null) { + final Account account = wrappedWorldView().get(address); + if (account instanceof BonsaiAccount) { + final BonsaiAccount mutableAccount = new BonsaiAccount((BonsaiAccount) account, this, true); + accountsToUpdate.put(address, new BonsaiValue<>((BonsaiAccount) account, mutableAccount)); + return mutableAccount; + } else { + return null; + } + } else { + return bonsaiValue.getUpdated(); + } + } + + @Override + public Collection getTouchedAccounts() { + return getUpdatedAccounts(); + } + + @Override + public Collection
getDeletedAccountAddresses() { + return getDeletedAccounts(); + } + + @Override + public void revert() { + super.reset(); + } + + @Override + public void commit() { + for (final Address deletedAddress : getDeletedAccounts()) { + storageToClear.add(deletedAddress); + final BonsaiValue codeValue = codeToUpdate.get(deletedAddress); + if (codeValue != null) { + codeValue.setUpdated(null); + } else { + final Bytes deletedCode = wrappedWorldView().getCode(deletedAddress); + if (deletedCode != null) { + codeToUpdate.put(deletedAddress, new BonsaiValue<>(deletedCode, null)); + } + } + final BonsaiValue accountValue = + accountsToUpdate.computeIfAbsent( + deletedAddress, + __ -> loadAccountFromParent(deletedAddress, new BonsaiValue<>(null, null))); + + // mark all updated storage as to be cleared + final Map> deletedStorageUpdates = + storageToUpdate.computeIfAbsent(deletedAddress, k -> new HashMap<>()); + final Iterator>> iter = + deletedStorageUpdates.entrySet().iterator(); + while (iter.hasNext()) { + final Map.Entry> updateEntry = iter.next(); + final BonsaiValue updatedSlot = updateEntry.getValue(); + if (updatedSlot.getOriginal() == null || updatedSlot.getOriginal().isZero()) { + iter.remove(); + } else { + updatedSlot.setUpdated(null); + } + } + + final BonsaiAccount effective = accountValue.effective(); + if (effective != null) { + // Enumerate and delete addresses not updated + wrappedWorldView() + .getAllAccountStorage(deletedAddress, effective.getStorageRoot()) + .forEach( + (keyHash, entryValue) -> { + final Hash slotHash = Hash.wrap(keyHash); + if (!deletedStorageUpdates.containsKey(slotHash)) { + final UInt256 value = UInt256.fromBytes(RLP.decodeOne(entryValue)); + deletedStorageUpdates.put(slotHash, new BonsaiValue<>(value, null)); + } + }); + } + if (deletedStorageUpdates.isEmpty()) { + storageToUpdate.remove(deletedAddress); + } + accountValue.setUpdated(null); + } + + for (final UpdateTrackingAccount tracked : getUpdatedAccounts()) { + final Address updatedAddress = tracked.getAddress(); + BonsaiAccount updatedAccount = tracked.getWrappedAccount(); + if (updatedAccount == null) { + final BonsaiValue updatedAccountValue = accountsToUpdate.get(updatedAddress); + updatedAccount = new BonsaiAccount(this, tracked); + tracked.setWrappedAccount(updatedAccount); + if (updatedAccountValue == null) { + accountsToUpdate.put(updatedAddress, new BonsaiValue<>(null, updatedAccount)); + codeToUpdate.put(updatedAddress, new BonsaiValue<>(null, updatedAccount.getCode())); + } else { + updatedAccountValue.setUpdated(updatedAccount); + } + } else { + updatedAccount.setBalance(tracked.getBalance()); + updatedAccount.setNonce(tracked.getNonce()); + updatedAccount.setCode(tracked.getCode()); + if (tracked.getStorageWasCleared()) { + updatedAccount.clearStorage(); + } + tracked.getUpdatedStorage().forEach(updatedAccount::setStorageValue); + } + + if (tracked.codeWasUpdated()) { + final BonsaiValue pendingCode = + codeToUpdate.computeIfAbsent( + updatedAddress, addr -> new BonsaiValue<>(wrappedWorldView().getCode(addr), null)); + pendingCode.setUpdated(updatedAccount.getCode()); + } + + final Map> pendingStorageUpdates = + storageToUpdate.computeIfAbsent(updatedAddress, __ -> new HashMap<>()); + if (tracked.getStorageWasCleared()) { + storageToClear.add(updatedAddress); + pendingStorageUpdates.clear(); + } + + final TreeSet> entries = + new TreeSet<>( + Comparator.comparing( + (Function, UInt256>) Map.Entry::getKey)); + entries.addAll(updatedAccount.getUpdatedStorage().entrySet()); + + for (final Map.Entry storageUpdate : entries) { + final UInt256 keyUInt = storageUpdate.getKey(); + final Hash slotHash = Hash.hash(keyUInt.toBytes()); + final UInt256 value = storageUpdate.getValue(); + final BonsaiValue pendingValue = pendingStorageUpdates.get(slotHash); + if (pendingValue == null) { + pendingStorageUpdates.put( + slotHash, new BonsaiValue<>(updatedAccount.getOriginalStorageValue(keyUInt), value)); + } else { + pendingValue.setUpdated(value); + } + } + updatedAccount.getUpdatedStorage().clear(); + + // TODO maybe add address preimage? + } + } + + @Override + public Bytes getCode(final Address address) { + final BonsaiValue localCode = codeToUpdate.get(address); + if (localCode == null) { + return wrappedWorldView().getCode(address); + } else { + return localCode.getUpdated(); + } + } + + @Override + public UInt256 getStorageValue(final Address address, final UInt256 storageKey) { + // TODO maybe log the read into the trie layer? + final Hash slotHashBytes = Hash.hash(storageKey.toBytes()); + return getStorageValueBySlotHash(address, slotHashBytes).orElse(UInt256.ZERO); + } + + @Override + public Optional getStorageValueBySlotHash(final Address address, final Hash slotHash) { + final Map> localAccountStorage = + storageToUpdate.computeIfAbsent(address, key -> new HashMap<>()); + final BonsaiValue value = localAccountStorage.get(slotHash); + if (value != null) { + return Optional.of(value.getUpdated()); + } else { + final Optional valueUInt = + wrappedWorldView().getStorageValueBySlotHash(address, slotHash); + valueUInt.ifPresent(v -> localAccountStorage.put(slotHash, new BonsaiValue<>(v, v))); + return valueUInt; + } + } + + @Override + public UInt256 getOriginalStorageValue(final Address address, final UInt256 storageKey) { + // TODO maybe log the read into the trie layer? + final Map> localAccountStorage = + storageToUpdate.computeIfAbsent(address, key -> new HashMap<>()); + final Hash slotHashBytes = Hash.hash(storageKey.toBytes()); + final BonsaiValue value = localAccountStorage.get(slotHashBytes); + if (value != null) { + final UInt256 updated = value.getUpdated(); + if (updated != null) { + return updated; + } + final UInt256 original = value.getOriginal(); + if (original != null) { + return original; + } + } + return getStorageValue(address, storageKey); + } + + @Override + public Map getAllAccountStorage(final Address address, final Hash rootHash) { + final Map results = wrappedWorldView().getAllAccountStorage(address, rootHash); + storageToUpdate + .get(address) + .forEach((key, value) -> results.put(key, value.getUpdated().toBytes())); + return results; + } + + public TrieLogLayer generateTrieLog(final Hash blockHash) { + final TrieLogLayer layer = new TrieLogLayer(); + layer.setBlockHash(blockHash); + for (final Map.Entry> updatedAccount : + accountsToUpdate.entrySet()) { + final BonsaiValue bonsaiValue = updatedAccount.getValue(); + final BonsaiAccount oldValue = bonsaiValue.getOriginal(); + final StateTrieAccountValue oldAccount = + oldValue == null + ? null + : new StateTrieAccountValue( + oldValue.getNonce(), + oldValue.getBalance(), + oldValue.getStorageRoot(), + oldValue.getCodeHash(), + oldValue.getVersion()); + final BonsaiAccount newValue = bonsaiValue.getUpdated(); + final StateTrieAccountValue newAccount = + newValue == null + ? null + : new StateTrieAccountValue( + newValue.getNonce(), + newValue.getBalance(), + newValue.getStorageRoot(), + newValue.getCodeHash(), + newValue.getVersion()); + layer.addAccountChange(updatedAccount.getKey(), oldAccount, newAccount); + } + + for (final Map.Entry> updatedCode : codeToUpdate.entrySet()) { + layer.addCodeChange( + updatedCode.getKey(), + updatedCode.getValue().getOriginal(), + updatedCode.getValue().getUpdated()); + } + + for (final Map.Entry>> updatesStorage : + storageToUpdate.entrySet()) { + final Address address = updatesStorage.getKey(); + for (final Map.Entry> slotUpdate : + updatesStorage.getValue().entrySet()) { + layer.addStorageChange( + address, + slotUpdate.getKey(), + slotUpdate.getValue().getOriginal(), + slotUpdate.getValue().getUpdated()); + } + } + + return layer; + } + + public void rollForward(final TrieLogLayer layer) { + layer + .streamAccountChanges() + .forEach( + entry -> + rollAccountChange( + entry.getKey(), entry.getValue().getOriginal(), entry.getValue().getUpdated())); + layer + .streamCodeChanges() + .forEach( + entry -> + rollCodeChange( + entry.getKey(), entry.getValue().getOriginal(), entry.getValue().getUpdated())); + layer + .streamStorageChanges() + .forEach( + entry -> + entry + .getValue() + .forEach( + (key, value) -> + rollStorageChange( + entry.getKey(), key, value.getOriginal(), value.getUpdated()))); + } + + public void rollBack(final TrieLogLayer layer) { + layer + .streamAccountChanges() + .forEach( + entry -> + rollAccountChange( + entry.getKey(), entry.getValue().getUpdated(), entry.getValue().getOriginal())); + layer + .streamCodeChanges() + .forEach( + entry -> + rollCodeChange( + entry.getKey(), entry.getValue().getUpdated(), entry.getValue().getOriginal())); + layer + .streamStorageChanges() + .forEach( + entry -> + entry + .getValue() + .forEach( + (slotHash, value) -> + rollStorageChange( + entry.getKey(), + slotHash, + value.getUpdated(), + value.getOriginal()))); + } + + private void rollAccountChange( + final Address address, + final StateTrieAccountValue expectedValue, + final StateTrieAccountValue replacementValue) { + if (Objects.equals(expectedValue, replacementValue)) { + // non-change, a cached read. + return; + } + BonsaiValue accountValue = accountsToUpdate.get(address); + if (accountValue == null) { + accountValue = loadAccountFromParent(address, accountValue); + } + if (accountValue == null) { + if (expectedValue == null && replacementValue != null) { + accountsToUpdate.put( + address, + new BonsaiValue<>(null, new BonsaiAccount(this, address, replacementValue, true))); + } else { + throw new IllegalStateException( + String.format( + "Expected to update account, but the account does not exist. Address=%s", address)); + } + } else { + if (expectedValue == null) { + throw new IllegalStateException( + String.format( + "Expected to create account, but the account exists. Address=%s", address)); + } + BonsaiAccount.assertCloseEnoughForDiffing( + accountValue.getUpdated(), expectedValue, "Prior Value in Rolling Change"); + if (replacementValue == null) { + if (accountValue.getOriginal() == null) { + accountsToUpdate.remove(address); + } else { + accountValue.setUpdated(null); + } + } else { + final BonsaiAccount existingAccount = accountValue.getUpdated(); + existingAccount.setNonce(replacementValue.getNonce()); + existingAccount.setBalance(replacementValue.getBalance()); + existingAccount.setStorageRoot(replacementValue.getStorageRoot()); + // depend on correctly structured layers to set code hash + existingAccount.setVersion(replacementValue.getVersion()); + } + } + } + + private BonsaiValue loadAccountFromParent( + final Address address, final BonsaiValue defaultValue) { + final Account parentAccount = wrappedWorldView().get(address); + if (parentAccount instanceof BonsaiAccount) { + final BonsaiAccount account = (BonsaiAccount) parentAccount; + final BonsaiValue loadedAccountValue = + new BonsaiValue<>(new BonsaiAccount(account), account); + accountsToUpdate.put(address, loadedAccountValue); + return loadedAccountValue; + } else { + return defaultValue; + } + } + + private void rollCodeChange( + final Address address, final Bytes expectedCode, final Bytes replacementCode) { + if (Objects.equals(expectedCode, replacementCode)) { + // non-change, a cached read. + return; + } + BonsaiValue codeValue = codeToUpdate.get(address); + if (codeValue == null) { + final Bytes storedCode = wrappedWorldView().getCode(address); + if (!storedCode.isEmpty()) { + codeValue = new BonsaiValue<>(storedCode, storedCode); + codeToUpdate.put(address, codeValue); + } + } + + if (codeValue == null) { + if (expectedCode == null && replacementCode != null) { + codeToUpdate.put(address, new BonsaiValue<>(null, replacementCode)); + } else { + throw new IllegalStateException( + String.format( + "Expected to update code, but the code does not exist. Address=%s", address)); + } + } else { + if (expectedCode == null) { + throw new IllegalStateException( + String.format("Expected to create code, but the code exists. Address=%s", address)); + } + if (!codeValue.getUpdated().equals(expectedCode)) { + throw new IllegalStateException( + String.format( + "Old value of code does not match expected value. Address=%s ExpectedHash=%s ActualHash=%s", + address, Hash.hash(expectedCode), Hash.hash(codeValue.getUpdated()))); + } + if (replacementCode == null) { + if (codeValue.getOriginal() == null) { + codeToUpdate.remove(address); + } else { + codeValue.setUpdated(null); + } + } else { + codeValue.setUpdated(replacementCode); + } + } + } + + private Map> maybeCreateStorageMap( + final Map> storageMap, final Address address) { + if (storageMap == null) { + final Map> newMap = new HashMap<>(); + storageToUpdate.put(address, newMap); + return newMap; + } else { + return storageMap; + } + } + + private void rollStorageChange( + final Address address, + final Hash slotHash, + final UInt256 expectedValue, + final UInt256 replacementValue) { + if (Objects.equals(expectedValue, replacementValue)) { + // non-change, a cached read. + return; + } + final Map> storageMap = storageToUpdate.get(address); + BonsaiValue slotValue = storageMap == null ? null : storageMap.get(slotHash); + if (slotValue == null) { + final Optional storageValue = + wrappedWorldView().getStorageValueBySlotHash(address, slotHash); + if (storageValue.isPresent()) { + slotValue = new BonsaiValue<>(storageValue.get(), storageValue.get()); + storageToUpdate.computeIfAbsent(address, k -> new HashMap<>()).put(slotHash, slotValue); + } + } + if (slotValue == null) { + if (expectedValue == null && replacementValue != null) { + maybeCreateStorageMap(storageMap, address) + .put(slotHash, new BonsaiValue<>(null, replacementValue)); + } else { + throw new IllegalStateException( + String.format( + "Expected to update storage value, but the slot does not exist. Account=%s SlotHash=%s", + address, slotHash)); + } + } else { + if (expectedValue == null) { + throw new IllegalStateException( + String.format( + "Expected to create slot, but the slot exists. Account=%s SlotHash=%s", + address, slotHash)); + } + final UInt256 existingSlotValue = slotValue.getUpdated(); + if (!existingSlotValue.equals(expectedValue)) { + throw new IllegalStateException( + String.format( + "Old value of slot does not match expected value. Account=%s SlotHash=%s Expected=%s Actual=%s", + address, + slotHash, + expectedValue.toShortHexString(), + existingSlotValue.toShortHexString())); + } + if (replacementValue == null) { + if (slotValue.getOriginal() == null) { + final Map> thisStorageUpdate = + maybeCreateStorageMap(storageMap, address); + thisStorageUpdate.remove(slotHash); + if (thisStorageUpdate.isEmpty()) { + storageToUpdate.remove(address); + } + } else { + slotValue.setUpdated(null); + } + } else { + slotValue.setUpdated(replacementValue); + } + } + } + + @Override + public void reset() { + storageToClear.clear(); + storageToUpdate.clear(); + codeToUpdate.clear(); + accountsToUpdate.clear(); + super.reset(); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/TrieLogLayer.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/TrieLogLayer.java new file mode 100644 index 00000000000..410c46905b8 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/TrieLogLayer.java @@ -0,0 +1,236 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.bonsai; + +import static com.google.common.base.Preconditions.checkState; + +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.rlp.RLPInput; +import org.hyperledger.besu.ethereum.rlp.RLPOutput; +import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.Function; +import java.util.stream.Stream; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; + +/** + * This class encapsulates the changes that are done to transition one block to the next. This + * includes serialization and deserialization tasks for storing this log to off-memory storage. + * + *

In this particular formulation only the "Leaves" are tracked" Future layers may track patrica + * trie changes as well. + */ +public class TrieLogLayer { + + private Hash blockHash; + private final Map> accounts = new TreeMap<>(); + private final Map> code = new TreeMap<>(); + private final Map>> storage = new TreeMap<>(); + private boolean frozen = false; + + /** Locks the layer so no new changes can be added; */ + void freeze() { + frozen = true; // The code never bothered me anyway + } + + public Hash getBlockHash() { + return blockHash; + } + + public void setBlockHash(final Hash blockHash) { + checkState(!frozen, "Layer is Frozen"); + this.blockHash = blockHash; + } + + public void addAccountChange( + final Address address, + final StateTrieAccountValue oldValue, + final StateTrieAccountValue newValue) { + checkState(!frozen, "Layer is Frozen"); + accounts.put(address, new BonsaiValue<>(oldValue, newValue)); + } + + void addCodeChange(final Address address, final Bytes oldValue, final Bytes newValue) { + checkState(!frozen, "Layer is Frozen"); + code.put( + address, + new BonsaiValue<>( + oldValue == null ? Bytes.EMPTY : oldValue, newValue == null ? Bytes.EMPTY : newValue)); + } + + void addStorageChange( + final Address address, final Hash slotHash, final UInt256 oldValue, final UInt256 newValue) { + checkState(!frozen, "Layer is Frozen"); + storage + .computeIfAbsent(address, a -> new TreeMap<>()) + .put(slotHash, new BonsaiValue<>(oldValue, newValue)); + } + + static TrieLogLayer readFrom(final RLPInput input) { + final TrieLogLayer newLayer = new TrieLogLayer(); + + input.enterList(); + newLayer.blockHash = Hash.wrap(input.readBytes32()); + + while (!input.isEndOfCurrentList()) { + input.enterList(); + final Address address = Address.readFrom(input); + + if (input.nextIsNull()) { + input.skipNext(); + } else { + input.enterList(); + final StateTrieAccountValue oldValue = nullOrValue(input, StateTrieAccountValue::readFrom); + final StateTrieAccountValue newValue = nullOrValue(input, StateTrieAccountValue::readFrom); + input.leaveList(); + newLayer.accounts.put(address, new BonsaiValue<>(oldValue, newValue)); + } + + if (input.nextIsNull()) { + input.skipNext(); + } else { + input.enterList(); + final Bytes oldCode = nullOrValue(input, RLPInput::readBytes); + final Bytes newCode = nullOrValue(input, RLPInput::readBytes); + input.leaveList(); + newLayer.code.put(address, new BonsaiValue<>(oldCode, newCode)); + } + + if (input.nextIsNull()) { + input.skipNext(); + } else { + final Map> storageChanges = new TreeMap<>(); + input.enterList(); + while (!input.isEndOfCurrentList()) { + input.enterList(); + final Hash slotHash = Hash.wrap(input.readBytes32()); + final UInt256 oldValue = nullOrValue(input, RLPInput::readUInt256Scalar); + final UInt256 newValue = nullOrValue(input, RLPInput::readUInt256Scalar); + storageChanges.put(slotHash, new BonsaiValue<>(oldValue, newValue)); + input.leaveList(); + } + input.leaveList(); + newLayer.storage.put(address, storageChanges); + } + // lenient leave list for forward compatible additions. + input.leaveListLenient(); + } + input.leaveListLenient(); + newLayer.freeze(); + + return newLayer; + } + + void writeTo(final RLPOutput output) { + freeze(); + + final Set

addresses = new TreeSet<>(); + addresses.addAll(accounts.keySet()); + addresses.addAll(code.keySet()); + addresses.addAll(storage.keySet()); + + output.startList(); // container + output.writeBytes(blockHash); + + for (final Address address : addresses) { + output.startList(); // this change + output.writeBytes(address); + + final BonsaiValue accountChange = accounts.get(address); + if (accountChange == null || accountChange.isUnchanged()) { + output.writeNull(); + } else { + accountChange.writeRlp(output, (o, sta) -> sta.writeTo(o)); + } + + final BonsaiValue codeChange = code.get(address); + if (codeChange == null || codeChange.isUnchanged()) { + output.writeNull(); + } else { + codeChange.writeRlp(output, RLPOutput::writeBytes); + } + + final Map> storageChanges = storage.get(address); + if (storageChanges == null) { + output.writeNull(); + } else { + output.startList(); + for (final Map.Entry> storageChangeEntry : + storageChanges.entrySet()) { + output.startList(); + output.writeBytes(storageChangeEntry.getKey()); + storageChangeEntry.getValue().writeInnerRlp(output, RLPOutput::writeUInt256Scalar); + output.endList(); + } + output.endList(); + } + output.endList(); // this change + } + output.endList(); // container + } + + Stream>> streamAccountChanges() { + return accounts.entrySet().stream(); + } + + Stream>> streamCodeChanges() { + return code.entrySet().stream(); + } + + Stream>>> streamStorageChanges() { + return storage.entrySet().stream(); + } + + Stream>> streamStorageChanges(final Address address) { + return storage.getOrDefault(address, Map.of()).entrySet().stream(); + } + + private static T nullOrValue(final RLPInput input, final Function reader) { + if (input.nextIsNull()) { + input.skipNext(); + return null; + } else { + return reader.apply(input); + } + } + + boolean isFrozen() { + return frozen; + } + + public Optional getCode(final Address address) { + return Optional.ofNullable(code.get(address)).map(BonsaiValue::getUpdated); + } + + Optional getStorageBySlotHash(final Address address, final Hash slotHash) { + return Optional.ofNullable(storage.get(address)) + .map(i -> i.get(slotHash)) + .map(BonsaiValue::getUpdated); + } + + public Optional getAccount(final Address address) { + return Optional.ofNullable(accounts.get(address)).map(BonsaiValue::getUpdated); + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java index a86cfef881e..5c81ede3c62 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/GenesisState.java @@ -102,11 +102,13 @@ public Block getBlock() { * @param target WorldView to write genesis state to */ public void writeStateTo(final MutableWorldState target) { - writeAccountsTo(target, genesisAccounts); + writeAccountsTo(target, genesisAccounts, (Hash) block.getHeader().getBlockHash()); } private static void writeAccountsTo( - final MutableWorldState target, final List genesisAccounts) { + final MutableWorldState target, + final List genesisAccounts, + final Hash rootHash) { final WorldUpdater updater = target.updater(); genesisAccounts.forEach( genesisAccount -> { @@ -118,7 +120,7 @@ private static void writeAccountsTo( genesisAccount.storage.forEach(account::setStorageValue); }); updater.commit(); - target.persist(); + target.persist(rootHash); } private static Hash calculateGenesisStateHash(final List genesisAccounts) { @@ -128,7 +130,7 @@ private static Hash calculateGenesisStateHash(final List genesis new WorldStatePreimageKeyValueStorage(new InMemoryKeyValueStorage()); final MutableWorldState worldState = new DefaultMutableWorldState(stateStorage, preimageStorage); - writeAccountsTo(worldState, genesisAccounts); + writeAccountsTo(worldState, genesisAccounts, Hash.ZERO); return worldState.rootHash(); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/AbstractWorldUpdater.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/AbstractWorldUpdater.java index 2b89a621ee0..db841d481b2 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/AbstractWorldUpdater.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/AbstractWorldUpdater.java @@ -231,4 +231,9 @@ public void markTransactionBoundary() { getUpdatedAccounts().forEach(UpdateTrackingAccount::markTransactionBoundary); } } + + protected void reset() { + updatedAccounts.clear(); + deletedAccounts.clear(); + } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java index d4740af0747..e5928d55b27 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MutableWorldState.java @@ -23,6 +23,11 @@ public interface MutableWorldState extends WorldState, MutableWorldView { */ MutableWorldState copy(); - /** Persist accumulated changes to underlying storage. */ - void persist(); + /** + * Persist accumulated changes to underlying storage. + * + * @param blockhash the block hash of the world state this represents. If this does not represent + * a forward transition from one block to the next `null` should be passed in. + */ + void persist(Hash blockhash); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/UpdateTrackingAccount.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/UpdateTrackingAccount.java index d84b1cd0e48..5a481616b9e 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/UpdateTrackingAccount.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/UpdateTrackingAccount.java @@ -40,7 +40,7 @@ public class UpdateTrackingAccount implements MutableAccount private final Address address; private final Hash addressHash; - @Nullable private final A account; // null if this is a new account. + @Nullable private A account; // null if this is a new account. private long nonce; private Wei balance; @@ -69,7 +69,7 @@ public class UpdateTrackingAccount implements MutableAccount this.updatedStorage = new TreeMap<>(); } - UpdateTrackingAccount(final A account) { + public UpdateTrackingAccount(final A account) { checkNotNull(account); this.address = account.getAddress(); @@ -96,6 +96,15 @@ public A getWrappedAccount() { return account; } + public void setWrappedAccount(final A account) { + if (this.account == null) { + this.account = account; + storageWasCleared = false; + } else { + throw new IllegalStateException("Already tracking a wrapped account"); + } + } + /** * Whether the code of the account was modified. * @@ -176,6 +185,7 @@ public boolean hasCode() { @Override public void setCode(final Bytes code) { this.updatedCode = code; + this.updatedCodeHash = null; } @Override diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java index c6f55b8f4af..cb92dd3aa2d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java @@ -179,7 +179,7 @@ public AbstractBlockProcessor.Result processBlock( return AbstractBlockProcessor.Result.failed(); } - worldState.persist(); + worldState.persist(blockHeader.getHash()); return AbstractBlockProcessor.Result.successful(receipts); } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/OnChainPrivacyPrecompiledContract.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/OnChainPrivacyPrecompiledContract.java index e95be6b0520..ac5960d5592 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/OnChainPrivacyPrecompiledContract.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/OnChainPrivacyPrecompiledContract.java @@ -191,7 +191,7 @@ public Bytes compute(final Bytes input, final MessageFrame messageFrame) { if (messageFrame.isPersistingPrivateState()) { privateWorldStateUpdater.commit(); - disposablePrivateState.persist(); + disposablePrivateState.persist(null); storePrivateMetadata( pmtHash, privacyGroupId, disposablePrivateState, privateMetadataUpdater, result); @@ -439,7 +439,7 @@ protected void maybeInjectDefaultManagementAndProxy( UInt256.fromBytes(Bytes32.leftPad(Address.DEFAULT_ONCHAIN_PRIVACY_MANAGEMENT))); privateWorldStateUpdater.commit(); - disposablePrivateState.persist(); + disposablePrivateState.persist(null); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContract.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContract.java index 9eb603862e4..9b40048a2fd 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContract.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/precompiles/privacy/PrivacyPrecompiledContract.java @@ -180,7 +180,7 @@ public Bytes compute(final Bytes input, final MessageFrame messageFrame) { if (messageFrame.isPersistingPrivateState()) { privateWorldStateUpdater.commit(); - disposablePrivateState.persist(); + disposablePrivateState.persist(null); storePrivateMetadata( pmtHash, privacyGroupId, disposablePrivateState, privateMetadataUpdater, result); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateGroupRehydrationBlockProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateGroupRehydrationBlockProcessor.java index 0f67b70bbfe..480fda83ebb 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateGroupRehydrationBlockProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/privacy/PrivateGroupRehydrationBlockProcessor.java @@ -144,7 +144,7 @@ public AbstractBlockProcessor.Result processBlock( privateTransaction.getPrivacyGroupId().get()); privateStateUpdater.commit(); - disposablePrivateState.persist(); + disposablePrivateState.persist(null); storePrivateMetadata( transactionHash, @@ -233,7 +233,7 @@ protected void maybeInjectDefaultManagementAndProxy( UInt256.fromBytes(Bytes32.leftPad(Address.DEFAULT_ONCHAIN_PRIVACY_MANAGEMENT))); privateWorldStateUpdater.commit(); - disposablePrivateState.persist(); + disposablePrivateState.persist(null); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java index 9d20e600708..996b6ec8f4d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueSegmentIdentifier.java @@ -16,17 +16,30 @@ import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier; +import org.bouncycastle.util.Arrays; + public enum KeyValueSegmentIdentifier implements SegmentIdentifier { - BLOCKCHAIN((byte) 1), - WORLD_STATE((byte) 2), - PRIVATE_TRANSACTIONS((byte) 3), - PRIVATE_STATE((byte) 4), - PRUNING_STATE((byte) 5); + BLOCKCHAIN(new byte[] {1}), + WORLD_STATE(new byte[] {2}, new int[] {0, 1}), + PRIVATE_TRANSACTIONS(new byte[] {3}), + PRIVATE_STATE(new byte[] {4}), + PRUNING_STATE(new byte[] {5}, new int[] {0, 1}), + ACCOUNT_INFO_STATE(new byte[] {6}, new int[] {2}), + CODE_STORAGE(new byte[] {7}, new int[] {2}), + ACCOUNT_STORAGE_STORAGE(new byte[] {8}, new int[] {2}), + TRIE_BRANCH_STORAGE(new byte[] {9}, new int[] {2}), + TRIE_LOG_STORAGE(new byte[] {10}, new int[] {2}); private final byte[] id; + private final int[] versionList; + + KeyValueSegmentIdentifier(final byte[] id) { + this(id, new int[] {0, 1, 2}); + } - KeyValueSegmentIdentifier(final byte... id) { + KeyValueSegmentIdentifier(final byte[] id, final int[] versionList) { this.id = id; + this.versionList = versionList; } @Override @@ -38,4 +51,9 @@ public String getName() { public byte[] getId() { return id; } + + @Override + public boolean includeInDatabaseVersion(final int version) { + return Arrays.contains(versionList, version); + } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStateKeyValueStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStateKeyValueStorage.java index 4f43853f191..455b1e75e9b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStateKeyValueStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/storage/keyvalue/WorldStateKeyValueStorage.java @@ -53,12 +53,12 @@ public Optional getCode(final Bytes32 codeHash) { } @Override - public Optional getAccountStateTrieNode(final Bytes32 nodeHash) { + public Optional getAccountStateTrieNode(final Bytes location, final Bytes32 nodeHash) { return getTrieNode(nodeHash); } @Override - public Optional getAccountStorageTrieNode(final Bytes32 nodeHash) { + public Optional getAccountStorageTrieNode(final Bytes location, final Bytes32 nodeHash) { return getTrieNode(nodeHash); } @@ -71,7 +71,7 @@ private Optional getTrieNode(final Bytes32 nodeHash) { } @Override - public Optional getNodeData(final Bytes32 hash) { + public Optional getNodeData(final Bytes location, final Bytes32 hash) { if (hash.equals(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) { return Optional.of(MerklePatriciaTrie.EMPTY_TRIE_NODE); } else if (hash.equals(Hash.EMPTY)) { @@ -83,7 +83,7 @@ public Optional getNodeData(final Bytes32 hash) { @Override public boolean isWorldStateAvailable(final Bytes32 rootHash) { - return getAccountStateTrieNode(rootHash).isPresent(); + return getAccountStateTrieNode(Bytes.EMPTY, rootHash).isPresent(); } @Override @@ -156,7 +156,8 @@ public Updater putCode(final Bytes32 codeHash, final Bytes code) { } @Override - public Updater putAccountStateTrieNode(final Bytes32 nodeHash, final Bytes node) { + public Updater putAccountStateTrieNode( + final Bytes location, final Bytes32 nodeHash, final Bytes node) { if (nodeHash.equals(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) { // Don't save empty nodes return this; @@ -167,7 +168,8 @@ public Updater putAccountStateTrieNode(final Bytes32 nodeHash, final Bytes node) } @Override - public Updater putAccountStorageTrieNode(final Bytes32 nodeHash, final Bytes node) { + public Updater putAccountStorageTrieNode( + final Bytes location, final Bytes32 nodeHash, final Bytes node) { if (nodeHash.equals(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) { // Don't save empty nodes return this; diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java new file mode 100644 index 00000000000..165bcc01ee4 --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageConfiguration.java @@ -0,0 +1,30 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.worldstate; + +import org.immutables.value.Value; + +@Value.Immutable +public interface DataStorageConfiguration { + + DataStorageConfiguration DEFAULT_CONFIG = + ImmutableDataStorageConfiguration.builder() + .dataStorageFormat(DataStorageFormat.FOREST) + .build(); + + DataStorageFormat getDataStorageFormat(); +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java new file mode 100644 index 00000000000..b97f79006ae --- /dev/null +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DataStorageFormat.java @@ -0,0 +1,32 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.worldstate; + +public enum DataStorageFormat { + FOREST(1), // Original format. Store all tries + BONSAI(2); // New format. Store one trie, and trie logs to roll forward and backward. + + private final int databaseVersion; + + DataStorageFormat(final int databaseVersion) { + this.databaseVersion = databaseVersion; + } + + public int getDatabaseVersion() { + return databaseVersion; + } +} diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java index c528ba775f8..217267754c2 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldState.java @@ -169,7 +169,7 @@ public final boolean equals(final Object other) { } @Override - public void persist() { + public void persist(final Hash blockhash) { final WorldStateStorage.Updater stateUpdater = worldStateStorage.updater(); // Store updated code for (final Bytes code : updatedAccountCode.values()) { @@ -202,6 +202,13 @@ private Optional getStorageTrieKeyPreimage(final Bytes32 trieKey) { .or(() -> preimageStorage.getStorageTrieKeyPreimage(trieKey)); } + private static UInt256 convertToUInt256(final Bytes value) { + // TODO: we could probably have an optimized method to decode a single scalar since it's used + // pretty often. + final RLPInput in = RLP.input(value); + return in.readUInt256Scalar(); + } + private Optional
getAccountTrieKeyPreimage(final Bytes32 trieKey) { return Optional.ofNullable(newAccountKeyPreimages.get(trieKey)) .or(() -> preimageStorage.getAccountTrieKeyPreimage(trieKey)); @@ -320,13 +327,6 @@ public NavigableMap storageEntriesFrom( return storageEntries; } - private UInt256 convertToUInt256(final Bytes value) { - // TODO: we could probably have an optimized method to decode a single scalar since it's used - // pretty often. - final RLPInput in = RLP.input(value); - return in.readUInt256Scalar(); - } - @Override public String toString() { final StringBuilder builder = new StringBuilder(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java index f677fdc3224..7bbb25efdae 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/DefaultWorldStateArchive.java @@ -61,11 +61,6 @@ public Optional getMutable(final Hash rootHash) { return Optional.of(new DefaultMutableWorldState(rootHash, worldStateStorage, preimageStorage)); } - @Override - public WorldState get() { - return get(EMPTY_ROOT_HASH).get(); - } - @Override public MutableWorldState getMutable() { return getMutable(EMPTY_ROOT_HASH).get(); @@ -73,7 +68,8 @@ public MutableWorldState getMutable() { @Override public Optional getNodeData(final Hash hash) { - return worldStateStorage.getNodeData(hash); + // query by location is not supported, only query by content + return worldStateStorage.getNodeData(null, hash); } public WorldStateStorage getWorldStateStorage() { diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/StateTrieAccountValue.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/StateTrieAccountValue.java index 6e9daf81f50..dd3a99432f1 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/StateTrieAccountValue.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/StateTrieAccountValue.java @@ -14,12 +14,16 @@ */ package org.hyperledger.besu.ethereum.worldstate; +import static com.google.common.base.Preconditions.checkNotNull; + import org.hyperledger.besu.ethereum.core.Account; import org.hyperledger.besu.ethereum.core.Hash; import org.hyperledger.besu.ethereum.core.Wei; import org.hyperledger.besu.ethereum.rlp.RLPInput; import org.hyperledger.besu.ethereum.rlp.RLPOutput; +import java.util.Objects; + /** Represents the raw values associated with an account in the world state trie. */ public class StateTrieAccountValue { @@ -29,17 +33,15 @@ public class StateTrieAccountValue { private final Hash codeHash; private final int version; - private StateTrieAccountValue( - final long nonce, final Wei balance, final Hash storageRoot, final Hash codeHash) { - this(nonce, balance, storageRoot, codeHash, Account.DEFAULT_VERSION); - } - public StateTrieAccountValue( final long nonce, final Wei balance, final Hash storageRoot, final Hash codeHash, final int version) { + checkNotNull(balance, "balance cannot be null"); + checkNotNull(storageRoot, "storageRoot cannot be null"); + checkNotNull(codeHash, "codeHash cannot be null"); this.nonce = nonce; this.balance = balance; this.storageRoot = storageRoot; @@ -92,6 +94,23 @@ public int getVersion() { return version; } + @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + final StateTrieAccountValue that = (StateTrieAccountValue) o; + return nonce == that.nonce + && version == that.version + && balance.equals(that.balance) + && storageRoot.equals(that.storageRoot) + && codeHash.equals(that.codeHash); + } + + @Override + public int hashCode() { + return Objects.hash(nonce, balance, storageRoot, codeHash, version); + } + public void writeTo(final RLPOutput out) { out.startList(); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java index d43108f8962..1e28a9c08f7 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateArchive.java @@ -36,8 +36,6 @@ public interface WorldStateArchive { Optional getMutable(final Hash rootHash); - WorldState get(); - MutableWorldState getMutable(); Optional getNodeData(final Hash hash); diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java index 2a3ed83a598..3efec2fe13d 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/worldstate/WorldStateStorage.java @@ -27,16 +27,17 @@ public interface WorldStateStorage { Optional getCode(Bytes32 codeHash); - Optional getAccountStateTrieNode(Bytes32 nodeHash); + Optional getAccountStateTrieNode(Bytes location, Bytes32 nodeHash); - Optional getAccountStorageTrieNode(Bytes32 nodeHash); + Optional getAccountStorageTrieNode(Bytes location, Bytes32 nodeHash); - Optional getNodeData(Bytes32 hash); + Optional getNodeData(Bytes location, Bytes32 hash); boolean isWorldStateAvailable(Bytes32 rootHash); default boolean contains(final Bytes32 hash) { - return getNodeData(hash).isPresent(); + // we don't have location info + return getNodeData(null, hash).isPresent(); } Updater updater(); @@ -59,9 +60,9 @@ default Updater putCode(final Bytes code) { return putCode(codeHash, code); } - Updater putAccountStateTrieNode(Bytes32 nodeHash, Bytes node); + Updater putAccountStateTrieNode(Bytes location, Bytes32 nodeHash, Bytes node); - Updater putAccountStorageTrieNode(Bytes32 nodeHash, Bytes node); + Updater putAccountStorageTrieNode(Bytes location, Bytes32 nodeHash, Bytes node); void commit(); diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockDataGenerator.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockDataGenerator.java index ed40b3e0e89..780f7a5a31c 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockDataGenerator.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/BlockDataGenerator.java @@ -138,7 +138,7 @@ private List blockSequence( seq.add(next); parentHash = next.getHash(); nextBlockNumber = nextBlockNumber + 1L; - worldState.persist(); + worldState.persist(null); } return seq; @@ -180,7 +180,7 @@ private List createRandomAccounts( accounts.add(account); } updater.commit(); - worldState.persist(); + worldState.persist(null); return accounts; } diff --git a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TestCodeExecutor.java b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TestCodeExecutor.java index 41e371cb531..501629f246f 100644 --- a/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TestCodeExecutor.java +++ b/ethereum/core/src/test-support/java/org/hyperledger/besu/ethereum/core/TestCodeExecutor.java @@ -100,7 +100,7 @@ private WorldUpdater createInitialWorldState( worldState.getOrCreate(TestCodeExecutor.SENDER_ADDRESS).getMutable(); accountSetup.accept(senderAccount); worldState.commit(); - initialWorldState.persist(); + initialWorldState.persist(null); return stateArchive.getMutable(initialWorldState.rootHash()).get().updater(); } } diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java new file mode 100644 index 00000000000..c06b225e096 --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java @@ -0,0 +1,305 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.bonsai; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.InMemoryStorageProvider; +import org.hyperledger.besu.ethereum.core.MutableAccount; +import org.hyperledger.besu.ethereum.core.Wei; +import org.hyperledger.besu.ethereum.core.WorldUpdater; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; +import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; + +import java.util.Optional; +import java.util.stream.Collectors; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.units.bigints.UInt256; +import org.junit.Before; +import org.junit.Test; + +public class LogRollingTests { + + private BonsaiWorldStateArchive archive; + private InMemoryKeyValueStorage accountStorage; + private InMemoryKeyValueStorage codeStorage; + private InMemoryKeyValueStorage storageStorage; + private InMemoryKeyValueStorage trieBranchStorage; + private InMemoryKeyValueStorage trieLogStorage; + + private BonsaiWorldStateArchive secondArchive; + private InMemoryKeyValueStorage secondAccountStorage; + private InMemoryKeyValueStorage secondCodeStorage; + private InMemoryKeyValueStorage secondStorageStorage; + private InMemoryKeyValueStorage secondTrieBranchStorage; + private InMemoryKeyValueStorage secondTrieLogStorage; + + private static final Address addressOne = + Address.fromHexString("0x1111111111111111111111111111111111111111"); + + private static final Hash hashOne = Hash.hash(Bytes.of(1)); + private static final Hash hashTwo = Hash.hash(Bytes.of(2)); + + @Before + public void createStorage() { + final InMemoryStorageProvider provider = new InMemoryStorageProvider(); + archive = new BonsaiWorldStateArchive(provider); + accountStorage = + (InMemoryKeyValueStorage) + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE); + codeStorage = + (InMemoryKeyValueStorage) + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE); + storageStorage = + (InMemoryKeyValueStorage) + provider.getStorageBySegmentIdentifier( + KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE); + trieBranchStorage = + (InMemoryKeyValueStorage) + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE); + trieLogStorage = + (InMemoryKeyValueStorage) + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); + + final InMemoryStorageProvider secondProvider = new InMemoryStorageProvider(); + secondArchive = new BonsaiWorldStateArchive(secondProvider); + secondAccountStorage = + (InMemoryKeyValueStorage) + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE); + secondCodeStorage = + (InMemoryKeyValueStorage) + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE); + secondStorageStorage = + (InMemoryKeyValueStorage) + provider.getStorageBySegmentIdentifier( + KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE); + secondTrieBranchStorage = + (InMemoryKeyValueStorage) + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE); + secondTrieLogStorage = + (InMemoryKeyValueStorage) + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); + } + + @Test + public void simpleRollForwardTest() { + final BonsaiPersistedWorldState worldState = + new BonsaiPersistedWorldState( + archive, + accountStorage, + codeStorage, + storageStorage, + trieBranchStorage, + trieLogStorage); + final WorldUpdater updater = worldState.updater(); + + final MutableAccount mutableAccount = + updater.createAccount(addressOne, 1, Wei.of(1L)).getMutable(); + mutableAccount.setCode(Bytes.of(0, 1, 2)); + mutableAccount.setStorageValue(UInt256.ONE, UInt256.ONE); + updater.commit(); + worldState.persist(hashOne); + + final BonsaiPersistedWorldState secondWorldState = + new BonsaiPersistedWorldState( + secondArchive, + secondAccountStorage, + secondCodeStorage, + secondStorageStorage, + secondTrieBranchStorage, + secondTrieLogStorage); + final BonsaiWorldStateUpdater secondUpdater = + (BonsaiWorldStateUpdater) secondWorldState.updater(); + + final Optional value = trieLogStorage.get(hashOne.toArrayUnsafe()); + + final TrieLogLayer layer = + TrieLogLayer.readFrom(new BytesValueRLPInput(Bytes.wrap(value.get()), false)); + + secondUpdater.rollForward(layer); + secondUpdater.commit(); + secondWorldState.persist(null); + + assertKeyValueStorageEqual(accountStorage, secondAccountStorage); + assertKeyValueStorageEqual(codeStorage, secondCodeStorage); + assertKeyValueStorageEqual(storageStorage, secondStorageStorage); + assertKeyValueStorageEqual(trieBranchStorage, secondTrieBranchStorage); + // trie logs won't be the same, we shouldn't generate logs on rolls. + assertKeyValueSubset(trieLogStorage, secondTrieLogStorage); + assertThat(secondWorldState.rootHash()).isEqualByComparingTo(worldState.rootHash()); + } + + @Test + public void rollForwardTwice() { + final BonsaiPersistedWorldState worldState = + new BonsaiPersistedWorldState( + archive, + accountStorage, + codeStorage, + storageStorage, + trieBranchStorage, + trieLogStorage); + + final WorldUpdater updater = worldState.updater(); + final MutableAccount mutableAccount = + updater.createAccount(addressOne, 1, Wei.of(1L)).getMutable(); + mutableAccount.setCode(Bytes.of(0, 1, 2)); + mutableAccount.setStorageValue(UInt256.ONE, UInt256.ONE); + updater.commit(); + + worldState.persist(hashOne); + + final WorldUpdater updater2 = worldState.updater(); + final MutableAccount mutableAccount2 = updater2.getAccount(addressOne).getMutable(); + mutableAccount2.setStorageValue(UInt256.ONE, UInt256.valueOf(2)); + updater2.commit(); + + worldState.persist(hashTwo); + + final BonsaiPersistedWorldState secondWorldState = + new BonsaiPersistedWorldState( + secondArchive, + secondAccountStorage, + secondCodeStorage, + secondStorageStorage, + secondTrieBranchStorage, + secondTrieLogStorage); + final BonsaiWorldStateUpdater secondUpdater = + (BonsaiWorldStateUpdater) secondWorldState.updater(); + + final TrieLogLayer layerOne = getTrieLogLayer(trieLogStorage, hashOne); + secondUpdater.rollForward(layerOne); + secondUpdater.commit(); + secondWorldState.persist(null); + + final TrieLogLayer layerTwo = getTrieLogLayer(trieLogStorage, hashTwo); + secondUpdater.rollForward(layerTwo); + secondUpdater.commit(); + secondWorldState.persist(null); + + assertKeyValueStorageEqual(accountStorage, secondAccountStorage); + assertKeyValueStorageEqual(codeStorage, secondCodeStorage); + assertKeyValueStorageEqual(storageStorage, secondStorageStorage); + assertKeyValueStorageEqual(trieBranchStorage, secondTrieBranchStorage); + // trie logs won't be the same, we shouldn't generate logs on rolls. + assertKeyValueSubset(trieLogStorage, secondTrieLogStorage); + assertThat(secondWorldState.rootHash()).isEqualByComparingTo(worldState.rootHash()); + } + + @Test + public void rollBackOnce() { + final BonsaiPersistedWorldState worldState = + new BonsaiPersistedWorldState( + archive, + accountStorage, + codeStorage, + storageStorage, + trieBranchStorage, + trieLogStorage); + + final WorldUpdater updater = worldState.updater(); + final MutableAccount mutableAccount = + updater.createAccount(addressOne, 1, Wei.of(1L)).getMutable(); + mutableAccount.setCode(Bytes.of(0, 1, 2)); + mutableAccount.setStorageValue(UInt256.ONE, UInt256.ONE); + updater.commit(); + + worldState.persist(hashOne); + + final WorldUpdater updater2 = worldState.updater(); + final MutableAccount mutableAccount2 = updater2.getAccount(addressOne).getMutable(); + mutableAccount2.setStorageValue(UInt256.ONE, UInt256.valueOf(2)); + updater2.commit(); + + worldState.persist(hashTwo); + final BonsaiWorldStateUpdater firstRollbackUpdater = + (BonsaiWorldStateUpdater) worldState.updater(); + + final TrieLogLayer layerTwo = getTrieLogLayer(trieLogStorage, hashTwo); + firstRollbackUpdater.rollBack(layerTwo); + + worldState.persist(hashTwo); + + final BonsaiPersistedWorldState secondWorldState = + new BonsaiPersistedWorldState( + secondArchive, + secondAccountStorage, + secondCodeStorage, + secondStorageStorage, + secondTrieBranchStorage, + secondTrieLogStorage); + + final WorldUpdater secondUpdater = secondWorldState.updater(); + final MutableAccount secondMutableAccount = + secondUpdater.createAccount(addressOne, 1, Wei.of(1L)).getMutable(); + secondMutableAccount.setCode(Bytes.of(0, 1, 2)); + secondMutableAccount.setStorageValue(UInt256.ONE, UInt256.ONE); + secondUpdater.commit(); + + secondWorldState.persist(null); + + assertKeyValueStorageEqual(accountStorage, secondAccountStorage); + assertKeyValueStorageEqual(codeStorage, secondCodeStorage); + assertKeyValueStorageEqual(storageStorage, secondStorageStorage); + assertKeyValueStorageEqual(trieBranchStorage, secondTrieBranchStorage); + // trie logs won't be the same, we don't delete the roll back log + assertKeyValueSubset(trieLogStorage, secondTrieLogStorage); + assertThat(secondWorldState.rootHash()).isEqualByComparingTo(worldState.rootHash()); + } + + private TrieLogLayer getTrieLogLayer(final InMemoryKeyValueStorage storage, final Bytes key) { + return storage + .get(key.toArrayUnsafe()) + .map(bytes -> TrieLogLayer.readFrom(new BytesValueRLPInput(Bytes.wrap(bytes), false))) + .get(); + } + + private static void assertKeyValueStorageEqual( + final KeyValueStorage first, final KeyValueStorage second) { + final var firstKeys = + first.getAllKeysThat(k -> true).stream().map(Bytes::wrap).collect(Collectors.toSet()); + final var secondKeys = + second.getAllKeysThat(k -> true).stream().map(Bytes::wrap).collect(Collectors.toSet()); + + assertThat(secondKeys).isEqualTo(firstKeys); + for (final Bytes key : firstKeys) { + assertThat(Bytes.wrap(second.get(key.toArrayUnsafe()).get())) + .isEqualByComparingTo(Bytes.wrap(first.get(key.toArrayUnsafe()).get())); + } + } + + private static void assertKeyValueSubset( + final KeyValueStorage largerSet, final KeyValueStorage smallerSet) { + final var largerKeys = + largerSet.getAllKeysThat(k -> true).stream().map(Bytes::wrap).collect(Collectors.toSet()); + final var smallerKeys = + smallerSet.getAllKeysThat(k -> true).stream().map(Bytes::wrap).collect(Collectors.toSet()); + + assertThat(largerKeys).containsAll(smallerKeys); + for (final Bytes key : largerKeys) { + if (smallerKeys.contains(key)) { + assertThat(Bytes.wrap(largerSet.get(key.toArrayUnsafe()).get())) + .isEqualByComparingTo(Bytes.wrap(smallerSet.get(key.toArrayUnsafe()).get())); + } + } + } +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java new file mode 100644 index 00000000000..1bbc0aee480 --- /dev/null +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/RollingImport.java @@ -0,0 +1,127 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.hyperledger.besu.ethereum.bonsai; + +import static com.google.common.base.Preconditions.checkArgument; + +import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.InMemoryStorageProvider; +import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput; +import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier; +import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage; +import org.hyperledger.besu.util.io.RollingFileReader; + +import java.io.IOException; +import java.nio.file.Path; + +import org.apache.tuweni.bytes.Bytes; + +public class RollingImport { + + public static void main(final String[] arg) throws IOException { + checkArgument(arg.length == 1, "Single argument is file prefix, like `./layer/besu-layer`"); + + final RollingFileReader reader = + new RollingFileReader((i, c) -> Path.of(String.format(arg[0] + "-%04d.rdat", i)), false); + + final InMemoryStorageProvider provider = new InMemoryStorageProvider(); + final BonsaiWorldStateArchive archive = new BonsaiWorldStateArchive(provider); + final InMemoryKeyValueStorage accountStorage = + (InMemoryKeyValueStorage) + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE); + final InMemoryKeyValueStorage codeStorage = + (InMemoryKeyValueStorage) + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE); + final InMemoryKeyValueStorage storageStorage = + (InMemoryKeyValueStorage) + provider.getStorageBySegmentIdentifier( + KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE); + final InMemoryKeyValueStorage trieBranchStorage = + (InMemoryKeyValueStorage) + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE); + final InMemoryKeyValueStorage trieLogStorage = + (InMemoryKeyValueStorage) + provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE); + final BonsaiPersistedWorldState bonsaiState = + new BonsaiPersistedWorldState( + archive, + accountStorage, + codeStorage, + storageStorage, + trieBranchStorage, + trieLogStorage); + + int count = 0; + while (!reader.isDone()) { + try { + final byte[] bytes = reader.readBytes(); + if (bytes.length < 1) { + continue; + } + final TrieLogLayer layer = + TrieLogLayer.readFrom(new BytesValueRLPInput(Bytes.wrap(bytes), false)); + final BonsaiWorldStateUpdater updater = (BonsaiWorldStateUpdater) bonsaiState.updater(); + updater.rollForward(layer); + updater.commit(); + bonsaiState.persist(Hash.wrap(layer.getBlockHash())); + if (count % 10000 == 0) { + System.out.println(". - " + count); + } else if (count % 100 == 0) { + System.out.print("."); + System.out.flush(); + } + } catch (final Exception e) { + // e.printStackTrace(System.out); + System.out.println(count); + throw e; + } + count++; + } + + System.out.printf("%nCount %d - now going backwards!%n", count); + + while (count > 0) { + try { + + count--; + reader.seek(count); + final byte[] bytes = reader.readBytes(); + final TrieLogLayer layer = + TrieLogLayer.readFrom(new BytesValueRLPInput(Bytes.wrap(bytes), false)); + final BonsaiWorldStateUpdater updater = (BonsaiWorldStateUpdater) bonsaiState.updater(); + updater.rollBack(layer); + updater.commit(); + bonsaiState.persist(Hash.wrap(layer.getBlockHash())); + if (count % 10000 == 0) { + System.out.println(". - " + count); + } else if (count % 100 == 0) { + System.out.print("."); + System.out.flush(); + } + } catch (final Exception e) { + System.out.println(count); + throw e; + } + } + System.out.printf("Back to zero!%n"); + accountStorage.dump(System.out); + codeStorage.dump(System.out); + storageStorage.dump(System.out); + trieBranchStorage.dump(System.out); + trieLogStorage.dump(System.out); + } +} diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageWorldStateStorageTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageWorldStateStorageTest.java index 88115dbb694..8dfb871b979 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageWorldStateStorageTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/storage/keyvalue/KeyValueStorageWorldStateStorageTest.java @@ -36,27 +36,29 @@ public void getCode_returnsEmpty() { @Test public void getAccountStateTrieNode_returnsEmptyNode() { final WorldStateKeyValueStorage storage = emptyStorage(); - assertThat(storage.getAccountStateTrieNode(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) + assertThat( + storage.getAccountStateTrieNode(Bytes.EMPTY, MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) .contains(MerklePatriciaTrie.EMPTY_TRIE_NODE); } @Test public void getAccountStorageTrieNode_returnsEmptyNode() { final WorldStateKeyValueStorage storage = emptyStorage(); - assertThat(storage.getAccountStorageTrieNode(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) + assertThat( + storage.getAccountStorageTrieNode(Bytes.EMPTY, MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) .contains(MerklePatriciaTrie.EMPTY_TRIE_NODE); } @Test public void getNodeData_returnsEmptyValue() { final WorldStateKeyValueStorage storage = emptyStorage(); - assertThat(storage.getNodeData(Hash.EMPTY)).contains(Bytes.EMPTY); + assertThat(storage.getNodeData(null, Hash.EMPTY)).contains(Bytes.EMPTY); } @Test public void getNodeData_returnsEmptyNode() { final WorldStateKeyValueStorage storage = emptyStorage(); - assertThat(storage.getNodeData(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) + assertThat(storage.getNodeData(Bytes.EMPTY, MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) .contains(MerklePatriciaTrie.EMPTY_TRIE_NODE); } @@ -85,22 +87,23 @@ public void getAccountStateTrieNode_saveAndGetSpecialValues() { storage .updater() .putAccountStateTrieNode( - Hash.hash(MerklePatriciaTrie.EMPTY_TRIE_NODE), MerklePatriciaTrie.EMPTY_TRIE_NODE) - .putAccountStateTrieNode(Hash.hash(Bytes.EMPTY), Bytes.EMPTY) + null, Hash.hash(MerklePatriciaTrie.EMPTY_TRIE_NODE), MerklePatriciaTrie.EMPTY_TRIE_NODE) + .putAccountStateTrieNode(null, Hash.hash(Bytes.EMPTY), Bytes.EMPTY) .commit(); - assertThat(storage.getAccountStateTrieNode(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) + assertThat( + storage.getAccountStateTrieNode(Bytes.EMPTY, MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) .contains(MerklePatriciaTrie.EMPTY_TRIE_NODE); - assertThat(storage.getAccountStateTrieNode(Hash.EMPTY)).contains(Bytes.EMPTY); + assertThat(storage.getAccountStateTrieNode(Bytes.EMPTY, Hash.EMPTY)).contains(Bytes.EMPTY); } @Test public void getAccountStateTrieNode_saveAndGetRegularValue() { final Bytes bytes = Bytes.fromHexString("0x123456"); final WorldStateKeyValueStorage storage = emptyStorage(); - storage.updater().putAccountStateTrieNode(Hash.hash(bytes), bytes).commit(); + storage.updater().putAccountStateTrieNode(null, Hash.hash(bytes), bytes).commit(); - assertThat(storage.getAccountStateTrieNode(Hash.hash(bytes))).contains(bytes); + assertThat(storage.getAccountStateTrieNode(Bytes.EMPTY, Hash.hash(bytes))).contains(bytes); } @Test @@ -109,22 +112,23 @@ public void getAccountStorageTrieNode_saveAndGetSpecialValues() { storage .updater() .putAccountStorageTrieNode( - Hash.hash(MerklePatriciaTrie.EMPTY_TRIE_NODE), MerklePatriciaTrie.EMPTY_TRIE_NODE) - .putAccountStorageTrieNode(Hash.hash(Bytes.EMPTY), Bytes.EMPTY) + null, Hash.hash(MerklePatriciaTrie.EMPTY_TRIE_NODE), MerklePatriciaTrie.EMPTY_TRIE_NODE) + .putAccountStorageTrieNode(null, Hash.hash(Bytes.EMPTY), Bytes.EMPTY) .commit(); - assertThat(storage.getAccountStorageTrieNode(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) + assertThat( + storage.getAccountStorageTrieNode(Bytes.EMPTY, MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) .contains(MerklePatriciaTrie.EMPTY_TRIE_NODE); - assertThat(storage.getAccountStorageTrieNode(Hash.EMPTY)).contains(Bytes.EMPTY); + assertThat(storage.getAccountStorageTrieNode(Bytes.EMPTY, Hash.EMPTY)).contains(Bytes.EMPTY); } @Test public void getAccountStorageTrieNode_saveAndGetRegularValue() { final Bytes bytes = Bytes.fromHexString("0x123456"); final WorldStateKeyValueStorage storage = emptyStorage(); - storage.updater().putAccountStorageTrieNode(Hash.hash(bytes), bytes).commit(); + storage.updater().putAccountStorageTrieNode(null, Hash.hash(bytes), bytes).commit(); - assertThat(storage.getAccountStateTrieNode(Hash.hash(bytes))).contains(bytes); + assertThat(storage.getAccountStateTrieNode(Bytes.EMPTY, Hash.hash(bytes))).contains(bytes); } @Test @@ -133,22 +137,22 @@ public void getNodeData_saveAndGetSpecialValues() { storage .updater() .putAccountStorageTrieNode( - Hash.hash(MerklePatriciaTrie.EMPTY_TRIE_NODE), MerklePatriciaTrie.EMPTY_TRIE_NODE) - .putAccountStorageTrieNode(Hash.hash(Bytes.EMPTY), Bytes.EMPTY) + null, Hash.hash(MerklePatriciaTrie.EMPTY_TRIE_NODE), MerklePatriciaTrie.EMPTY_TRIE_NODE) + .putAccountStorageTrieNode(null, Hash.hash(Bytes.EMPTY), Bytes.EMPTY) .commit(); - assertThat(storage.getNodeData(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) + assertThat(storage.getNodeData(Bytes.EMPTY, MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH)) .contains(MerklePatriciaTrie.EMPTY_TRIE_NODE); - assertThat(storage.getNodeData(Hash.EMPTY)).contains(Bytes.EMPTY); + assertThat(storage.getNodeData(Bytes.EMPTY, Hash.EMPTY)).contains(Bytes.EMPTY); } @Test public void getNodeData_saveAndGetRegularValue() { final Bytes bytes = Bytes.fromHexString("0x123456"); final WorldStateKeyValueStorage storage = emptyStorage(); - storage.updater().putAccountStorageTrieNode(Hash.hash(bytes), bytes).commit(); + storage.updater().putAccountStorageTrieNode(null, Hash.hash(bytes), bytes).commit(); - assertThat(storage.getNodeData(Hash.hash(bytes))).contains(bytes); + assertThat(storage.getNodeData(null, Hash.hash(bytes))).contains(bytes); } @Test diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldStateTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldStateTest.java index 4781fb85502..032d6097a49 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldStateTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/DefaultMutableWorldStateTest.java @@ -69,7 +69,7 @@ public void rootHash_Empty() { final MutableWorldState worldState = createEmpty(); assertThat(worldState.rootHash()).isEqualTo(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH); - worldState.persist(); + worldState.persist(null); assertThat(worldState.rootHash()).isEqualTo(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH); } @@ -100,7 +100,7 @@ public void removeAccount_AccountDoesNotExist() { updater.commit(); assertThat(worldState.rootHash()).isEqualTo(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH); - worldState.persist(); + worldState.persist(null); assertThat(worldState.rootHash()).isEqualTo(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH); } @@ -113,7 +113,7 @@ public void removeAccount_UpdatedAccount() { updater.commit(); assertThat(worldState.rootHash()).isEqualTo(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH); - worldState.persist(); + worldState.persist(null); assertThat(worldState.rootHash()).isEqualTo(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH); } @@ -145,7 +145,7 @@ public void removeAccount_AccountExistsAndIsPersisted() { WorldUpdater updater = worldState.updater(); updater.createAccount(ADDRESS).getMutable().setBalance(Wei.of(100000)); updater.commit(); - worldState.persist(); + worldState.persist(null); assertThat(worldState.get(ADDRESS)).isNotNull(); assertThat(worldState.rootHash()).isNotEqualTo(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH); @@ -158,7 +158,7 @@ public void removeAccount_AccountExistsAndIsPersisted() { updater.commit(); assertThat(updater.get(ADDRESS)).isNull(); // And after persisting - worldState.persist(); + worldState.persist(null); assertThat(updater.get(ADDRESS)).isNull(); assertThat(worldState.rootHash()).isEqualTo(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH); @@ -185,7 +185,7 @@ public void streamAccounts_singleAccount() { assertThat(accounts.get(0).getBalance()).isEqualTo(Wei.of(100000)); // Check again after persisting - worldState.persist(); + worldState.persist(null); accounts = worldState.streamAccounts(Bytes32.ZERO, 10).collect(Collectors.toList()); assertThat(accounts.size()).isEqualTo(1L); assertThat(accounts.get(0).getAddress()).hasValue(ADDRESS); @@ -267,7 +267,7 @@ public void commitAndPersist() { assertThat(kvWorldStateStorage.isWorldStateAvailable(worldState.rootHash())).isFalse(); // Persist and re-run assertions - worldState.persist(); + worldState.persist(null); assertThat(kvWorldStateStorage.isWorldStateAvailable(worldState.rootHash())).isTrue(); assertThat(worldState.rootHash()).isEqualTo(expectedRootHash); @@ -474,7 +474,7 @@ public void clearStorage_AfterPersisting() { account.setBalance(Wei.of(100000)); account.setStorageValue(storageKey, storageValue); updater.commit(); - worldState.persist(); + worldState.persist(null); assertThat(worldState.get(ADDRESS)).isNotNull(); assertThat(worldState.rootHash()).isNotEqualTo(MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH); @@ -548,7 +548,7 @@ public void clearStorageThenEditAfterPersisting() { account.setStorageValue(storageKey, originalStorageValue); assertThat(account.getStorageValue(storageKey)).isEqualTo(originalStorageValue); updater.commit(); - worldState.persist(); + worldState.persist(null); // Clear storage then edit account = updater.getAccount(ADDRESS).getMutable(); @@ -667,14 +667,14 @@ public void shouldCombineUnchangedAndChangedValuesWhenRetrievingStorageEntries() assertThat(updater.get(ADDRESS).storageEntriesFrom(Hash.ZERO, 10)).isEqualTo(finalEntries); assertThat(worldState.get(ADDRESS).storageEntriesFrom(Hash.ZERO, 10)).isEqualTo(initialEntries); - worldState.persist(); + worldState.persist(null); assertThat(updater.get(ADDRESS).storageEntriesFrom(Hash.ZERO, 10)).isEqualTo(finalEntries); assertThat(worldState.get(ADDRESS).storageEntriesFrom(Hash.ZERO, 10)).isEqualTo(initialEntries); updater.commit(); assertThat(worldState.get(ADDRESS).storageEntriesFrom(Hash.ZERO, 10)).isEqualTo(finalEntries); - worldState.persist(); + worldState.persist(null); assertThat(worldState.get(ADDRESS).storageEntriesFrom(Hash.ZERO, 10)).isEqualTo(finalEntries); } } diff --git a/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java b/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java index 772c663670f..fcd63a210e2 100644 --- a/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java +++ b/ethereum/eth/src/jmh/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderBenchmark.java @@ -152,8 +152,9 @@ public Optional downloadWorldState() { } peer.respondWhileOtherThreadsWork(responder, () -> !result.isDone()); result.getNow(null); - final Optional rootData = worldStateStorage.getNodeData(blockHeader.getStateRoot()); - if (!rootData.isPresent()) { + final Optional rootData = + worldStateStorage.getNodeData(Bytes.EMPTY, blockHeader.getStateRoot()); + if (rootData.isEmpty()) { throw new IllegalStateException("World state download did not complete."); } return rootData; diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/AccountTrieNodeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/AccountTrieNodeDataRequest.java index 24f2e5718ec..8d0d0ccfd35 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/AccountTrieNodeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/AccountTrieNodeDataRequest.java @@ -34,12 +34,12 @@ class AccountTrieNodeDataRequest extends TrieNodeDataRequest { @Override protected void doPersist(final Updater updater) { - updater.putAccountStateTrieNode(getHash(), getData()); + updater.putAccountStateTrieNode(null, getHash(), getData()); } @Override public Optional getExistingData(final WorldStateStorage worldStateStorage) { - return worldStateStorage.getAccountStateTrieNode(getHash()); + return worldStateStorage.getAccountStateTrieNode(null, getHash()); } @Override diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/StorageTrieNodeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/StorageTrieNodeDataRequest.java index 2ef852443ef..d18527152e9 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/StorageTrieNodeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/StorageTrieNodeDataRequest.java @@ -31,12 +31,12 @@ class StorageTrieNodeDataRequest extends TrieNodeDataRequest { @Override protected void doPersist(final Updater updater) { - updater.putAccountStorageTrieNode(getHash(), getData()); + updater.putAccountStorageTrieNode(null, getHash(), getData()); } @Override public Optional getExistingData(final WorldStateStorage worldStateStorage) { - return worldStateStorage.getAccountStorageTrieNode(getHash()); + return worldStateStorage.getAccountStorageTrieNode(null, getHash()); } @Override diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/TrieNodeDataRequest.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/TrieNodeDataRequest.java index 0f070595586..96c5339bcfc 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/TrieNodeDataRequest.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/TrieNodeDataRequest.java @@ -37,7 +37,7 @@ public Stream getChildRequests() { return Stream.empty(); } - final List> nodes = TrieNodeDecoder.decodeNodes(getData()); + final List> nodes = TrieNodeDecoder.decodeNodes(null, getData()); return nodes.stream() .flatMap( node -> { diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldDownloadState.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldDownloadState.java index 93d73f7e574..df3371c8305 100644 --- a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldDownloadState.java +++ b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldDownloadState.java @@ -189,7 +189,7 @@ public synchronized boolean checkCompletion( return false; } final Updater updater = worldStateStorage.updater(); - updater.putAccountStateTrieNode(header.getStateRoot(), rootNodeData); + updater.putAccountStateTrieNode(null, header.getStateRoot(), rootNodeData); updater.commit(); internalFuture.complete(null); // THere are no more inputs to process so make sure we wake up any threads waiting to dequeue diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/PersistDataStepTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/PersistDataStepTest.java index db9b58e3f39..9d37fe877cb 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/PersistDataStepTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/PersistDataStepTest.java @@ -110,7 +110,7 @@ private StubTask createTaskWithoutData(final Bytes data) { private void assertDataPersisted(final List> tasks) { tasks.forEach( task -> - assertThat(worldStateStorage.getNodeData(task.getData().getHash())) + assertThat(worldStateStorage.getNodeData(null, task.getData().getHash())) .isEqualTo(Optional.of(task.getData().getData()))); } } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldDownloadStateTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldDownloadStateTest.java index 4e6b29b9f51..1ed0bd329a1 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldDownloadStateTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldDownloadStateTest.java @@ -81,7 +81,7 @@ public void shouldStoreRootNodeBeforeReturnedFutureCompletes() { final CompletableFuture postFutureChecks = future.thenAccept( result -> - assertThat(worldStateStorage.getAccountStateTrieNode(ROOT_NODE_HASH)) + assertThat(worldStateStorage.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)) .contains(ROOT_NODE_DATA)); downloadState.checkCompletion(worldStateStorage, header); @@ -97,7 +97,7 @@ public void shouldNotCompleteWhenThereArePendingTasks() { downloadState.checkCompletion(worldStateStorage, header); assertThat(future).isNotDone(); - assertThat(worldStateStorage.getAccountStateTrieNode(ROOT_NODE_HASH)).isEmpty(); + assertThat(worldStateStorage.getAccountStateTrieNode(Bytes.EMPTY, ROOT_NODE_HASH)).isEmpty(); assertThat(downloadState.isDownloading()).isTrue(); } diff --git a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderTest.java b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderTest.java index 5b553815e09..fd4fd1f2941 100644 --- a/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderTest.java +++ b/ethereum/eth/src/test/java/org/hyperledger/besu/ethereum/eth/sync/worldstate/WorldStateDownloaderTest.java @@ -464,7 +464,7 @@ public void doesNotRequestKnownAccountTrieNodesFromNetwork() { allNodes.forEach( (nodeHash, node) -> { if (storeNode.get()) { - localStorageUpdater.putAccountStateTrieNode(nodeHash, node); + localStorageUpdater.putAccountStateTrieNode(null, nodeHash, node); knownNodes.add(nodeHash); } else { unknownNodes.add(nodeHash); @@ -560,7 +560,7 @@ public void doesNotRequestKnownStorageTrieNodesFromNetwork() { final Bytes32 hash = entry.getKey(); final Bytes data = entry.getValue(); if (storeNode) { - localStorageUpdater.putAccountStorageTrieNode(hash, data); + localStorageUpdater.putAccountStorageTrieNode(null, hash, data); knownNodes.add(hash); } else { unknownNodes.add(hash); @@ -774,8 +774,8 @@ private List getFirstSetOfChildNodeRequests( final WorldStateStorage storage, final Bytes32 rootHash) { final List hashesToRequest = new ArrayList<>(); - Bytes rootNodeRlp = storage.getNodeData(rootHash).get(); - TrieNodeDecoder.decodeNodes(rootNodeRlp).stream() + final Bytes rootNodeRlp = storage.getNodeData(Bytes.EMPTY, rootHash).get(); + TrieNodeDecoder.decodeNodes(Bytes.EMPTY, rootNodeRlp).stream() .filter(n -> !Objects.equals(n.getHash(), rootHash)) .filter(Node::isReferencedByHash) .forEach((n) -> hashesToRequest.add(n.getHash())); diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java index 4f603b94f25..082a6fa2117 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/BlockchainReferenceTestCaseSpec.java @@ -74,7 +74,7 @@ private static WorldStateArchive buildWorldStateArchive( } updater.commit(); - worldState.persist(); + worldState.persist(null); return worldStateArchive; } diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/GeneralStateTestCaseSpec.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/GeneralStateTestCaseSpec.java index 6b5011d3b3b..5a1523ca5dd 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/GeneralStateTestCaseSpec.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/GeneralStateTestCaseSpec.java @@ -52,7 +52,7 @@ private Map> generate( final Map> postSections, final StateTestVersionedTransaction versionedTransaction) { - initialWorldState.persist(); + initialWorldState.persist(null); final Map> res = new LinkedHashMap<>(postSections.size()); for (final Map.Entry> entry : postSections.entrySet()) { diff --git a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/VMReferenceTestCaseSpec.java b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/VMReferenceTestCaseSpec.java index e73215fdb09..712b44a2e17 100644 --- a/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/VMReferenceTestCaseSpec.java +++ b/ethereum/referencetests/src/main/java/org/hyperledger/besu/ethereum/referencetests/VMReferenceTestCaseSpec.java @@ -55,7 +55,7 @@ public VMReferenceTestCaseSpec( @JsonProperty("post") final ReferenceTestWorldState finalWorldState) { this.exec = exec; this.initialWorldState = initialWorldState; - this.initialWorldState.persist(); + this.initialWorldState.persist(null); exec.setBlockHeader(env); if (finalGas != null && out != null && finalWorldState != null) { diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/BranchNode.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/BranchNode.java index b16a3342f11..a1853e087f0 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/BranchNode.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/BranchNode.java @@ -68,6 +68,11 @@ public void accept(final NodeVisitor visitor) { visitor.visit(this); } + @Override + public void accept(final Bytes location, final LocationNodeVisitor visitor) { + visitor.visit(location, this); + } + @Override public Bytes getPath() { return Bytes.EMPTY; @@ -145,7 +150,7 @@ public Node replaceChild(final byte index, final Node updatedChild) { if (updatedChild == NULL_NODE) { if (value.isPresent() && !hasChildren()) { return nodeFactory.createLeaf(Bytes.of(index), value.get()); - } else if (!value.isPresent()) { + } else if (value.isEmpty()) { final Optional> flattened = maybeFlatten(newChildren); if (flattened.isPresent()) { return flattened.get(); diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/CommitVisitor.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/CommitVisitor.java index 26fd24081a4..3746f13e266 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/CommitVisitor.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/CommitVisitor.java @@ -16,7 +16,7 @@ import org.apache.tuweni.bytes.Bytes; -class CommitVisitor implements NodeVisitor { +public class CommitVisitor implements LocationNodeVisitor { private final NodeUpdater nodeUpdater; @@ -25,21 +25,21 @@ public CommitVisitor(final NodeUpdater nodeUpdater) { } @Override - public void visit(final ExtensionNode extensionNode) { + public void visit(final Bytes location, final ExtensionNode extensionNode) { if (!extensionNode.isDirty()) { return; } final Node child = extensionNode.getChild(); if (child.isDirty()) { - child.accept(this); + child.accept(Bytes.concatenate(location, extensionNode.getPath()), this); } - maybeStoreNode(extensionNode); + maybeStoreNode(location, extensionNode); } @Override - public void visit(final BranchNode branchNode) { + public void visit(final Bytes location, final BranchNode branchNode) { if (!branchNode.isDirty()) { return; } @@ -47,29 +47,29 @@ public void visit(final BranchNode branchNode) { for (byte i = 0; i < BranchNode.RADIX; ++i) { final Node child = branchNode.child(i); if (child.isDirty()) { - child.accept(this); + child.accept(Bytes.concatenate(location, Bytes.of(i)), this); } } - maybeStoreNode(branchNode); + maybeStoreNode(location, branchNode); } @Override - public void visit(final LeafNode leafNode) { + public void visit(final Bytes location, final LeafNode leafNode) { if (!leafNode.isDirty()) { return; } - maybeStoreNode(leafNode); + maybeStoreNode(location, leafNode); } @Override - public void visit(final NullNode nullNode) {} + public void visit(final Bytes location, final NullNode nullNode) {} - private void maybeStoreNode(final Node node) { + private void maybeStoreNode(final Bytes location, final Node node) { final Bytes nodeRLP = node.getRlp(); if (nodeRLP.size() >= 32) { - this.nodeUpdater.store(node.getHash(), nodeRLP); + this.nodeUpdater.store(location, node.getHash(), nodeRLP); } } } diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/DefaultNodeFactory.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/DefaultNodeFactory.java index 1ca7f3096e8..79d7fa02e23 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/DefaultNodeFactory.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/DefaultNodeFactory.java @@ -21,13 +21,13 @@ import org.apache.tuweni.bytes.Bytes; -class DefaultNodeFactory implements NodeFactory { +public class DefaultNodeFactory implements NodeFactory { @SuppressWarnings("rawtypes") private static final Node NULL_NODE = NullNode.instance(); private final Function valueSerializer; - DefaultNodeFactory(final Function valueSerializer) { + public DefaultNodeFactory(final Function valueSerializer) { this.valueSerializer = valueSerializer; } diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/ExtensionNode.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/ExtensionNode.java index 04ee6f0d2c6..5a398f71a4b 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/ExtensionNode.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/ExtensionNode.java @@ -55,6 +55,11 @@ public void accept(final NodeVisitor visitor) { visitor.visit(this); } + @Override + public void accept(final Bytes location, final LocationNodeVisitor visitor) { + visitor.visit(location, this); + } + @Override public Bytes getPath() { return path; @@ -131,11 +136,15 @@ public Node replacePath(final Bytes path) { @Override public String print() { final StringBuilder builder = new StringBuilder(); - builder.append("Extension:"); - builder.append("\n\tRef: ").append(getRlpRef()); - builder.append("\n\tPath: " + CompactEncoding.encode(path)); final String childRep = getChild().print().replaceAll("\n\t", "\n\t\t"); - builder.append("\n\t").append(childRep); + builder + .append("Extension:") + .append("\n\tRef: ") + .append(getRlpRef()) + .append("\n\tPath: ") + .append(CompactEncoding.encode(path)) + .append("\n\t") + .append(childRep); return builder.toString(); } diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/KeyValueMerkleStorage.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/KeyValueMerkleStorage.java index e400c8809b3..30d727bab79 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/KeyValueMerkleStorage.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/KeyValueMerkleStorage.java @@ -34,16 +34,14 @@ public KeyValueMerkleStorage(final KeyValueStorage keyValueStorage) { } @Override - public Optional get(final Bytes32 hash) { - final Optional value = - pendingUpdates.containsKey(hash) - ? Optional.of(pendingUpdates.get(hash)) - : keyValueStorage.get(hash.toArrayUnsafe()).map(Bytes::wrap); - return value; + public Optional get(final Bytes location, final Bytes32 hash) { + return pendingUpdates.containsKey(hash) + ? Optional.of(pendingUpdates.get(hash)) + : keyValueStorage.get(hash.toArrayUnsafe()).map(Bytes::wrap); } @Override - public void put(final Bytes32 hash, final Bytes value) { + public void put(final Bytes location, final Bytes32 hash, final Bytes value) { pendingUpdates.put(hash, value); } diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/LeafNode.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/LeafNode.java index f30edd85c80..a5714283c18 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/LeafNode.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/LeafNode.java @@ -59,6 +59,11 @@ public void accept(final NodeVisitor visitor) { visitor.visit(this); } + @Override + public void accept(final Bytes location, final LocationNodeVisitor visitor) { + visitor.visit(location, this); + } + @Override public Bytes getPath() { return path; diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/LocationNodeVisitor.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/LocationNodeVisitor.java new file mode 100644 index 00000000000..2e1f934ea98 --- /dev/null +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/LocationNodeVisitor.java @@ -0,0 +1,28 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.trie; + +import org.apache.tuweni.bytes.Bytes; + +interface LocationNodeVisitor { + + void visit(Bytes location, ExtensionNode extensionNode); + + void visit(Bytes location, BranchNode branchNode); + + void visit(Bytes location, LeafNode leafNode); + + void visit(Bytes location, NullNode nullNode); +} diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/MerkleStorage.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/MerkleStorage.java index 5c7f88e605d..5f08b02f8f2 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/MerkleStorage.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/MerkleStorage.java @@ -25,10 +25,11 @@ public interface MerkleStorage { /** * Returns an {@code Optional} of the content mapped to the hash if it exists; otherwise empty. * + * @param location The location in the trie. * @param hash The hash for the content. * @return an {@code Optional} of the content mapped to the hash if it exists; otherwise empty */ - Optional get(Bytes32 hash); + Optional get(Bytes location, Bytes32 hash); /** * Updates the content mapped to the specified hash, creating the mapping if one does not already @@ -37,10 +38,11 @@ public interface MerkleStorage { *

Note: if the storage implementation already contains content for the given hash, it will * replace the existing content. * + * @param location The location in the trie. * @param hash The hash for the content. * @param content The content to store. */ - void put(Bytes32 hash, Bytes content); + void put(Bytes location, Bytes32 hash, Bytes content); /** Persist accumulated changes to underlying storage. */ void commit(); diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/Node.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/Node.java index 5fd51565651..8e85e2049de 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/Node.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/Node.java @@ -26,6 +26,8 @@ public interface Node { void accept(NodeVisitor visitor); + void accept(Bytes location, LocationNodeVisitor visitor); + Bytes getPath(); Optional getValue(); diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/NodeFactory.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/NodeFactory.java index 992bdec9c6e..f8e8b9a614f 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/NodeFactory.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/NodeFactory.java @@ -19,7 +19,7 @@ import org.apache.tuweni.bytes.Bytes; -interface NodeFactory { +public interface NodeFactory { Node createExtension(Bytes path, Node child); diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/NodeLoader.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/NodeLoader.java index c00a4cbae83..bbd16f93fc6 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/NodeLoader.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/NodeLoader.java @@ -20,5 +20,5 @@ import org.apache.tuweni.bytes.Bytes32; public interface NodeLoader { - Optional getNode(Bytes32 hash); + Optional getNode(Bytes location, Bytes32 hash); } diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/NodeUpdater.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/NodeUpdater.java index 589367fa16e..1668645fd87 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/NodeUpdater.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/NodeUpdater.java @@ -18,5 +18,5 @@ import org.apache.tuweni.bytes.Bytes32; public interface NodeUpdater { - void store(Bytes32 hash, Bytes value); + void store(Bytes location, Bytes32 hash, Bytes value); } diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/NullNode.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/NullNode.java index 5dbec3804d3..0e0c07b3e9b 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/NullNode.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/NullNode.java @@ -21,14 +21,14 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -class NullNode implements Node { +public class NullNode implements Node { @SuppressWarnings("rawtypes") private static final NullNode instance = new NullNode(); private NullNode() {} @SuppressWarnings("unchecked") - static NullNode instance() { + public static NullNode instance() { return instance; } @@ -42,6 +42,11 @@ public void accept(final NodeVisitor visitor) { visitor.visit(this); } + @Override + public void accept(final Bytes location, final LocationNodeVisitor visitor) { + visitor.visit(location, this); + } + @Override public Bytes getPath() { return Bytes.EMPTY; diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/PutVisitor.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/PutVisitor.java index 125611f9b66..e8e939d2f86 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/PutVisitor.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/PutVisitor.java @@ -16,11 +16,11 @@ import org.apache.tuweni.bytes.Bytes; -class PutVisitor implements PathNodeVisitor { +public class PutVisitor implements PathNodeVisitor { private final NodeFactory nodeFactory; private final V value; - PutVisitor(final NodeFactory nodeFactory, final V value) { + public PutVisitor(final NodeFactory nodeFactory, final V value) { this.nodeFactory = nodeFactory; this.value = value; } diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/RestoreVisitor.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/RestoreVisitor.java index 45fab8edb77..3e421ab2f11 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/RestoreVisitor.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/RestoreVisitor.java @@ -169,6 +169,11 @@ public void accept(final NodeVisitor visitor) { // do nothing } + @Override + public void accept(final Bytes location, final LocationNodeVisitor visitor) { + // do nothing + } + @Override public Bytes getPath() { return path; diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/StoredMerklePatriciaTrie.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/StoredMerklePatriciaTrie.java index e24c1b2b47a..dedb95dde4b 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/StoredMerklePatriciaTrie.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/StoredMerklePatriciaTrie.java @@ -76,7 +76,7 @@ public StoredMerklePatriciaTrie( this.root = rootHash.equals(EMPTY_TRIE_NODE_HASH) ? NullNode.instance() - : new StoredNode<>(nodeFactory, rootHash); + : new StoredNode<>(nodeFactory, Bytes.EMPTY, rootHash); } @Override @@ -111,17 +111,25 @@ public void remove(final K key) { @Override public void commit(final NodeUpdater nodeUpdater) { final CommitVisitor commitVisitor = new CommitVisitor<>(nodeUpdater); - root.accept(commitVisitor); + root.accept(Bytes.EMPTY, commitVisitor); // Make sure root node was stored if (root.isDirty() && root.getRlpRef().size() < 32) { - nodeUpdater.store(root.getHash(), root.getRlpRef()); + nodeUpdater.store(Bytes.EMPTY, root.getHash(), root.getRlpRef()); } // Reset root so dirty nodes can be garbage collected final Bytes32 rootHash = root.getHash(); this.root = rootHash.equals(EMPTY_TRIE_NODE_HASH) ? NullNode.instance() - : new StoredNode<>(nodeFactory, rootHash); + : new StoredNode<>(nodeFactory, Bytes.EMPTY, rootHash); + } + + public void acceptAtRoot(final NodeVisitor visitor) { + root.accept(visitor); + } + + public void acceptAtRoot(final PathNodeVisitor visitor, final Bytes path) { + root.accept(visitor, path); } @Override diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/StoredNode.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/StoredNode.java index f452d565a9e..765def40ecf 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/StoredNode.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/StoredNode.java @@ -24,11 +24,13 @@ class StoredNode implements Node { private final StoredNodeFactory nodeFactory; + private final Bytes location; private final Bytes32 hash; private Node loaded; - StoredNode(final StoredNodeFactory nodeFactory, final Bytes32 hash) { + StoredNode(final StoredNodeFactory nodeFactory, final Bytes location, final Bytes32 hash) { this.nodeFactory = nodeFactory; + this.location = location; this.hash = hash; } @@ -57,6 +59,12 @@ public void accept(final NodeVisitor visitor) { node.accept(visitor); } + @Override + public void accept(final Bytes location, final LocationNodeVisitor visitor) { + final Node node = load(); + node.accept(location, visitor); + } + @Override public Bytes getPath() { return load().getPath(); @@ -103,7 +111,7 @@ private Node load() { if (loaded == null) { loaded = nodeFactory - .retrieve(hash) + .retrieve(location, hash) .orElseThrow( () -> new MerkleTrieException("Unable to load trie node value for hash " + hash)); } diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/StoredNodeFactory.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/StoredNodeFactory.java index 255526942a8..6518a287f01 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/StoredNodeFactory.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/StoredNodeFactory.java @@ -29,7 +29,7 @@ import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; -class StoredNodeFactory implements NodeFactory { +public class StoredNodeFactory implements NodeFactory { @SuppressWarnings("rawtypes") private static final NullNode NULL_NODE = NullNode.instance(); @@ -37,7 +37,7 @@ class StoredNodeFactory implements NodeFactory { private final Function valueSerializer; private final Function valueDeserializer; - StoredNodeFactory( + public StoredNodeFactory( final NodeLoader nodeLoader, final Function valueSerializer, final Function valueDeserializer) { @@ -90,12 +90,14 @@ private Node handleNewNode(final Node node) { return node; } - public Optional> retrieve(final Bytes32 hash) throws MerkleTrieException { + public Optional> retrieve(final Bytes location, final Bytes32 hash) + throws MerkleTrieException { return nodeLoader - .getNode(hash) + .getNode(location, hash) .map( rlp -> { - final Node node = decode(rlp, () -> format("Invalid RLP value for hash %s", hash)); + final Node node = + decode(location, rlp, () -> format("Invalid RLP value for hash %s", hash)); // recalculating the node.hash() is expensive, so we only do this as an assertion assert (hash.equals(node.getHash())) : "Node hash " + node.getHash() + " not equal to expected " + hash; @@ -103,20 +105,21 @@ public Optional> retrieve(final Bytes32 hash) throws MerkleTrieException }); } - public Node decode(final Bytes rlp) { - return decode(rlp, () -> String.format("Failed to decode value %s", rlp.toString())); + public Node decode(final Bytes location, final Bytes rlp) { + return decode(location, rlp, () -> String.format("Failed to decode value %s", rlp.toString())); } - private Node decode(final Bytes rlp, final Supplier errMessage) + private Node decode(final Bytes location, final Bytes rlp, final Supplier errMessage) throws MerkleTrieException { try { - return decode(RLP.input(rlp), errMessage); + return decode(location, RLP.input(rlp), errMessage); } catch (final RLPException ex) { throw new MerkleTrieException(errMessage.get(), ex); } } - private Node decode(final RLPInput nodeRLPs, final Supplier errMessage) { + private Node decode( + final Bytes location, final RLPInput nodeRLPs, final Supplier errMessage) { final int nodesCount = nodeRLPs.enterList(); switch (nodesCount) { case 1: @@ -139,13 +142,13 @@ private Node decode(final RLPInput nodeRLPs, final Supplier errMessag nodeRLPs.leaveList(); return leafNode; } else { - final Node extensionNode = decodeExtension(path, nodeRLPs, errMessage); + final Node extensionNode = decodeExtension(location, path, nodeRLPs, errMessage); nodeRLPs.leaveList(); return extensionNode; } case (BranchNode.RADIX + 1): - final BranchNode branchNode = decodeBranch(nodeRLPs, errMessage); + final BranchNode branchNode = decodeBranch(location, nodeRLPs, errMessage); nodeRLPs.leaveList(); return branchNode; @@ -156,31 +159,41 @@ private Node decode(final RLPInput nodeRLPs, final Supplier errMessag } private Node decodeExtension( - final Bytes path, final RLPInput valueRlp, final Supplier errMessage) { + final Bytes location, + final Bytes path, + final RLPInput valueRlp, + final Supplier errMessage) { final RLPInput childRlp = valueRlp.readAsRlp(); if (childRlp.nextIsList()) { - final Node childNode = decode(childRlp, errMessage); + final Node childNode = decode(location, childRlp, errMessage); return new ExtensionNode<>(path, childNode, this); } else { final Bytes32 childHash = childRlp.readBytes32(); - final StoredNode childNode = new StoredNode<>(this, childHash); + final StoredNode childNode = + new StoredNode<>( + this, location == null ? null : Bytes.concatenate(location, path), childHash); return new ExtensionNode<>(path, childNode, this); } } @SuppressWarnings("unchecked") - private BranchNode decodeBranch(final RLPInput nodeRLPs, final Supplier errMessage) { + private BranchNode decodeBranch( + final Bytes location, final RLPInput nodeRLPs, final Supplier errMessage) { final ArrayList> children = new ArrayList<>(BranchNode.RADIX); for (int i = 0; i < BranchNode.RADIX; ++i) { if (nodeRLPs.nextIsNull()) { nodeRLPs.skipNext(); children.add(NULL_NODE); } else if (nodeRLPs.nextIsList()) { - final Node child = decode(nodeRLPs, errMessage); + final Node child = decode(location, nodeRLPs, errMessage); children.add(child); } else { final Bytes32 childHash = nodeRLPs.readBytes32(); - children.add(new StoredNode<>(this, childHash)); + children.add( + new StoredNode<>( + this, + location == null ? null : Bytes.concatenate(location, Bytes.of((byte) i)), + childHash)); } } diff --git a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/TrieNodeDecoder.java b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/TrieNodeDecoder.java index a1635a644b5..af8215a0de4 100644 --- a/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/TrieNodeDecoder.java +++ b/ethereum/trie/src/main/java/org/hyperledger/besu/ethereum/trie/TrieNodeDecoder.java @@ -31,7 +31,7 @@ public class TrieNodeDecoder { private static final StoredNodeFactory emptyNodeFactory = - new StoredNodeFactory<>((h) -> Optional.empty(), Function.identity(), Function.identity()); + new StoredNodeFactory<>((l, h) -> Optional.empty(), Function.identity(), Function.identity()); // Hide constructor for static utility class private TrieNodeDecoder() {} @@ -39,26 +39,27 @@ private TrieNodeDecoder() {} /** * Decode an rlp-encoded trie node * + * @param location The location in the trie. * @param rlp The rlp-encoded node * @return A {@code Node} representation of the rlp data */ - public static Node decode(final Bytes rlp) { - return emptyNodeFactory.decode(rlp); + public static Node decode(final Bytes location, final Bytes rlp) { + return emptyNodeFactory.decode(location, rlp); } /** * Flattens this node and all of its inlined nodes and node references into a list. * + * @param location The location in the trie. * @param nodeRlp The bytes of the trie node to be decoded. * @return A list of nodes and node references embedded in the given rlp. */ - public static List> decodeNodes(final Bytes nodeRlp) { - Node node = decode(nodeRlp); - List> nodes = new ArrayList<>(); + public static List> decodeNodes(final Bytes location, final Bytes nodeRlp) { + final Node node = decode(location, nodeRlp); + final List> nodes = new ArrayList<>(); nodes.add(node); - final List> toProcess = new ArrayList<>(); - toProcess.addAll(node.getChildren()); + final List> toProcess = new ArrayList<>(node.getChildren()); while (!toProcess.isEmpty()) { final Node currentNode = toProcess.remove(0); if (Objects.equals(NullNode.instance(), currentNode)) { @@ -119,7 +120,10 @@ private static class BreadthFirstIterator implements Iterator> { this.nodeFactory = new StoredNodeFactory<>(nodeLoader, Function.identity(), Function.identity()); - nodeLoader.getNode(rootHash).map(TrieNodeDecoder::decode).ifPresent(currentNodes::add); + nodeLoader + .getNode(Bytes.EMPTY, rootHash) + .map(h -> TrieNodeDecoder.decode(Bytes.EMPTY, h)) + .ifPresent(currentNodes::add); } public static BreadthFirstIterator create( @@ -140,8 +144,7 @@ public Node next() { final Node nextNode = currentNodes.remove(0); - final List> children = new ArrayList<>(); - children.addAll(nextNode.getChildren()); + final List> children = new ArrayList<>(nextNode.getChildren()); while (!children.isEmpty()) { Node child = children.remove(0); if (Objects.equals(child, NullNode.instance())) { @@ -150,8 +153,8 @@ public Node next() { } if (child.isReferencedByHash()) { // Retrieve hash-referenced child - final Optional> maybeChildNode = nodeFactory.retrieve(child.getHash()); - if (!maybeChildNode.isPresent()) { + final Optional> maybeChildNode = nodeFactory.retrieve(null, child.getHash()); + if (maybeChildNode.isEmpty()) { continue; } child = maybeChildNode.get(); diff --git a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/AbstractMerklePatriciaTrieTest.java b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/AbstractMerklePatriciaTrieTest.java index 9262c513f55..6db7aff5c6f 100644 --- a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/AbstractMerklePatriciaTrieTest.java +++ b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/AbstractMerklePatriciaTrieTest.java @@ -363,8 +363,8 @@ public void getValueWithProof_forExistingValues() { assertThat(valueWithProof.getProofRelatedNodes()).hasSize(2); assertThat(valueWithProof.getValue()).contains(value1); - List> nodes = - TrieNodeDecoder.decodeNodes(valueWithProof.getProofRelatedNodes().get(1)); + final List> nodes = + TrieNodeDecoder.decodeNodes(null, valueWithProof.getProofRelatedNodes().get(1)); assertThat(new String(nodes.get(1).getValue().get().toArray(), UTF_8)).isEqualTo(value1); assertThat(new String(nodes.get(2).getValue().get().toArray(), UTF_8)).isEqualTo(value2); @@ -401,8 +401,8 @@ public void getValueWithProof_singleNodeTrie() { assertThat(valueWithProof.getValue()).contains(value1); assertThat(valueWithProof.getProofRelatedNodes()).hasSize(1); - List> nodes = - TrieNodeDecoder.decodeNodes(valueWithProof.getProofRelatedNodes().get(0)); + final List> nodes = + TrieNodeDecoder.decodeNodes(null, valueWithProof.getProofRelatedNodes().get(0)); assertThat(nodes.size()).isEqualTo(1); final String nodeValue = new String(nodes.get(0).getValue().get().toArray(), UTF_8); diff --git a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/TrieNodeDecoderTest.java b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/TrieNodeDecoderTest.java index 739f3789765..581bbfd0341 100644 --- a/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/TrieNodeDecoderTest.java +++ b/ethereum/trie/src/test/java/org/hyperledger/besu/ethereum/trie/TrieNodeDecoderTest.java @@ -38,7 +38,7 @@ public void decodeNodes() { final InMemoryKeyValueStorage storage = new InMemoryKeyValueStorage(); // Build a small trie - MerklePatriciaTrie trie = + final MerklePatriciaTrie trie = new StoredMerklePatriciaTrie<>( new BytesToByteNodeLoader(storage), Function.identity(), Function.identity()); trie.put(Bytes.fromHexString("0x100000"), Bytes.of(1)); @@ -54,19 +54,19 @@ public void decodeNodes() { // Save nodes to storage final KeyValueStorageTransaction tx = storage.startTransaction(); - trie.commit((key, value) -> tx.put(key.toArrayUnsafe(), value.toArrayUnsafe())); + trie.commit((location, key, value) -> tx.put(key.toArrayUnsafe(), value.toArrayUnsafe())); tx.commit(); // Get and flatten root node final Bytes rootNodeRlp = Bytes.wrap(storage.get(trie.getRootHash().toArrayUnsafe()).get()); - final List> nodes = TrieNodeDecoder.decodeNodes(rootNodeRlp); + final List> nodes = TrieNodeDecoder.decodeNodes(null, rootNodeRlp); // The full trie hold 10 nodes, the branch node starting with 0x3... holding 2 values will be a // hash // referenced node and so its 2 child nodes will be missing assertThat(nodes.size()).isEqualTo(8); // Collect and check values - List actualValues = + final List actualValues = nodes.stream() .filter(n -> !n.isReferencedByHash()) .map(Node::getValue) @@ -82,7 +82,7 @@ public void breadthFirstDecode_smallTrie() { final InMemoryKeyValueStorage storage = new InMemoryKeyValueStorage(); // Build a small trie - MerklePatriciaTrie trie = + final MerklePatriciaTrie trie = new StoredMerklePatriciaTrie<>( new BytesToByteNodeLoader(storage), Function.identity(), Function.identity()); trie.put(Bytes.fromHexString("0x100000"), Bytes.of(1)); @@ -95,7 +95,7 @@ public void breadthFirstDecode_smallTrie() { // Save nodes to storage final KeyValueStorageTransaction tx = storage.startTransaction(); - trie.commit((key, value) -> tx.put(key.toArrayUnsafe(), value.toArrayUnsafe())); + trie.commit((location, key, value) -> tx.put(key.toArrayUnsafe(), value.toArrayUnsafe())); tx.commit(); // First layer should just be the root node @@ -119,12 +119,12 @@ public void breadthFirstDecode_smallTrie() { // First node should be root node assertThat(depth0And1Nodes.get(0).getHash()).isEqualTo(rootNode.getHash()); // Subsequent nodes should be children of root node - List expectedNodesHashes = + final List expectedNodesHashes = rootNode.getChildren().stream() .filter(n -> !Objects.equals(n, NullNode.instance())) .map(Node::getHash) .collect(Collectors.toList()); - List actualNodeHashes = + final List actualNodeHashes = depth0And1Nodes.subList(1, expectedNodeCount).stream() .map(Node::getHash) .collect(Collectors.toList()); @@ -136,7 +136,7 @@ public void breadthFirstDecode_smallTrie() { .collect(Collectors.toList()); assertThat(allNodes.size()).isEqualTo(10); // Collect and check values - List actualValues = + final List actualValues = allNodes.stream() .map(Node::getValue) .filter(Optional::isPresent) @@ -153,23 +153,23 @@ public void breadthFirstDecode_partialTrie() { final InMemoryKeyValueStorage partialStorage = new InMemoryKeyValueStorage(); // Build a small trie - MerklePatriciaTrie trie = + final MerklePatriciaTrie trie = new StoredMerklePatriciaTrie<>( new BytesToByteNodeLoader(fullStorage), Function.identity(), Function.identity()); final Random random = new Random(1); for (int i = 0; i < 30; i++) { - byte[] key = new byte[4]; - byte[] val = new byte[4]; + final byte[] key = new byte[4]; + final byte[] val = new byte[4]; random.nextBytes(key); random.nextBytes(val); trie.put(Bytes.wrap(key), Bytes.wrap(val)); } final KeyValueStorageTransaction tx = fullStorage.startTransaction(); - trie.commit((key, value) -> tx.put(key.toArrayUnsafe(), value.toArrayUnsafe())); + trie.commit((location, key, value) -> tx.put(key.toArrayUnsafe(), value.toArrayUnsafe())); tx.commit(); // Get root node - Node rootNode = + final Node rootNode = TrieNodeDecoder.breadthFirstDecoder( new BytesToByteNodeLoader(fullStorage), trie.getRootHash()) .findFirst() @@ -189,9 +189,9 @@ public void breadthFirstDecode_partialTrie() { @Test public void breadthFirstDecode_emptyTrie() { - List> result = + final List> result = TrieNodeDecoder.breadthFirstDecoder( - (h) -> Optional.empty(), MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH) + (l, h) -> Optional.empty(), MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH) .collect(Collectors.toList()); assertThat(result.size()).isEqualTo(0); } @@ -200,31 +200,31 @@ public void breadthFirstDecode_emptyTrie() { public void breadthFirstDecode_singleNodeTrie() { final InMemoryKeyValueStorage storage = new InMemoryKeyValueStorage(); - MerklePatriciaTrie trie = + final MerklePatriciaTrie trie = new StoredMerklePatriciaTrie<>( new BytesToByteNodeLoader(storage), Function.identity(), Function.identity()); trie.put(Bytes.fromHexString("0x100000"), Bytes.of(1)); // Save nodes to storage final KeyValueStorageTransaction tx = storage.startTransaction(); - trie.commit((key, value) -> tx.put(key.toArrayUnsafe(), value.toArrayUnsafe())); + trie.commit((location, key, value) -> tx.put(key.toArrayUnsafe(), value.toArrayUnsafe())); tx.commit(); - List> result = + final List> result = TrieNodeDecoder.breadthFirstDecoder(new BytesToByteNodeLoader(storage), trie.getRootHash()) .collect(Collectors.toList()); assertThat(result.size()).isEqualTo(1); assertThat(result.get(0).getValue()).contains(Bytes.of(1)); - Bytes actualPath = CompactEncoding.pathToBytes(result.get(0).getPath()); + final Bytes actualPath = CompactEncoding.pathToBytes(result.get(0).getPath()); assertThat(actualPath).isEqualTo(Bytes.fromHexString("0x100000")); } @Test public void breadthFirstDecode_unknownTrie() { - Bytes32 randomRootHash = Bytes32.fromHexStringLenient("0x12"); - List> result = - TrieNodeDecoder.breadthFirstDecoder((h) -> Optional.empty(), randomRootHash) + final Bytes32 randomRootHash = Bytes32.fromHexStringLenient("0x12"); + final List> result = + TrieNodeDecoder.breadthFirstDecoder((l, h) -> Optional.empty(), randomRootHash) .collect(Collectors.toList()); assertThat(result.size()).isEqualTo(0); } @@ -238,7 +238,7 @@ private BytesToByteNodeLoader(final KeyValueStorage storage) { } @Override - public Optional getNode(final Bytes32 hash) { + public Optional getNode(final Bytes location, final Bytes32 hash) { final byte[] value = storage.get(hash.toArrayUnsafe()).orElse(null); return value == null ? Optional.empty() : Optional.of(Bytes.wrap(value)); } diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index c87bd2b01a8..9fba73ca219 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -64,7 +64,7 @@ Calculated : ${currentHash} tasks.register('checkAPIChanges', FileStateChecker) { description = "Checks that the API for the Plugin-API project does not change without deliberate thought" files = sourceSets.main.allJava.files - knownHash = 'Ev0Y22aT+tcysFVaGcdaJON9Mf+aYqk8+5h+aHKUzuk=' + knownHash = '/Bi8ouR+5GhYpCWaAOlobYGAeLCQvfGxHjVWz0TPSas=' } check.dependsOn('checkAPIChanges') diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BesuConfiguration.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BesuConfiguration.java index 646841c514c..05ef88dfecf 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BesuConfiguration.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/BesuConfiguration.java @@ -14,6 +14,8 @@ */ package org.hyperledger.besu.plugin.services; +import org.hyperledger.besu.plugin.Unstable; + import java.nio.file.Path; /** Generally useful configuration provided by Besu. */ @@ -32,4 +34,14 @@ public interface BesuConfiguration { * @return location of the data directory in the file system of the client. */ Path getDataPath(); + + /** + * Database version. This sets the list of segmentIdentifiers that should be initialized. + * + * @return Database version. + */ + @Unstable + default int getDatabaseVersion() { + return 1; + } } diff --git a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java index 59c784294e6..9daf97fdeb9 100644 --- a/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java +++ b/plugin-api/src/main/java/org/hyperledger/besu/plugin/services/storage/SegmentIdentifier.java @@ -36,4 +36,15 @@ public interface SegmentIdentifier { * @return unique id of the segment. */ byte[] getId(); + + /** + * Not all segments are in all DB versions. This queries the segment to see if it is in the DB + * version. + * + * @param version Version of the DB + * @return true if the segment is in that DB version + */ + default boolean includeInDatabaseVersion(final int version) { + return true; + } } diff --git a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java index 21e05c4cbee..c4e9abf69e1 100644 --- a/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java +++ b/plugins/rocksdb/src/main/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactory.java @@ -36,6 +36,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import com.google.common.base.Supplier; import org.apache.logging.log4j.LogManager; @@ -45,7 +46,7 @@ public class RocksDBKeyValueStorageFactory implements KeyValueStorageFactory { private static final Logger LOG = LogManager.getLogger(); private static final int DEFAULT_VERSION = 1; - private static final Set SUPPORTED_VERSIONS = Set.of(0, 1); + private static final Set SUPPORTED_VERSIONS = Set.of(0, 1, 2); private static final String NAME = "rocksdb"; private final RocksDBMetricsFactory rocksDBMetricsFactory; @@ -112,12 +113,17 @@ public KeyValueStorage create( return unsegmentedStorage; } case 1: + case 2: { unsegmentedStorage = null; if (segmentedStorage == null) { + final List segmentsForVersion = + segments.stream() + .filter(segmentId -> segmentId.includeInDatabaseVersion(databaseVersion)) + .collect(Collectors.toList()); segmentedStorage = new RocksDBColumnarKeyValueStorage( - rocksDBConfiguration, segments, metricsSystem, rocksDBMetricsFactory); + rocksDBConfiguration, segmentsForVersion, metricsSystem, rocksDBMetricsFactory); } return new SegmentedKeyValueStorageAdapter<>(segment, segmentedStorage); } @@ -161,7 +167,7 @@ private int readDatabaseVersion(final BesuConfiguration commonConfiguration) thr databaseVersion = DatabaseMetadata.lookUpFrom(dataDir).getVersion(); LOG.info("Existing database detected at {}. Version {}", dataDir, databaseVersion); } else { - databaseVersion = defaultVersion; + databaseVersion = commonConfiguration.getDatabaseVersion(); LOG.info("No existing database detected at {}. Using version {}", dataDir, databaseVersion); Files.createDirectories(dataDir); new DatabaseMetadata(databaseVersion).writeToDirectory(dataDir); diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java index 220594249e6..d786f9322b4 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValuePrivacyStorageFactoryTest.java @@ -86,6 +86,7 @@ public void shouldCreateCorrectMetadataFileForLatestVersion() throws Exception { final Path tempDatabaseDir = temporaryFolder.newFolder().toPath().resolve("db"); when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir); when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); + when(commonConfiguration.getDatabaseVersion()).thenReturn(DEFAULT_VERSION); final RocksDBKeyValuePrivacyStorageFactory storageFactory = new RocksDBKeyValuePrivacyStorageFactory( @@ -116,6 +117,7 @@ public void shouldUpdateCorrectMetadataFileForLatestVersion() throws Exception { final Path tempDatabaseDir = temporaryFolder.newFolder().toPath().resolve("db"); when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir); when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); + when(commonConfiguration.getDatabaseVersion()).thenReturn(DEFAULT_VERSION); final RocksDBKeyValueStorageFactory storageFactory = new RocksDBKeyValueStorageFactory( diff --git a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java index 2343c9aaf05..156ac51161a 100644 --- a/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java +++ b/plugins/rocksdb/src/test/java/org/hyperledger/besu/plugin/services/storage/rocksdb/RocksDBKeyValueStorageFactoryTest.java @@ -58,6 +58,7 @@ public void shouldCreateCorrectMetadataFileForLatestVersion() throws Exception { final Path tempDatabaseDir = temporaryFolder.newFolder().toPath().resolve("db"); when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir); when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); + when(commonConfiguration.getDatabaseVersion()).thenReturn(DEFAULT_VERSION); final RocksDBKeyValueStorageFactory storageFactory = new RocksDBKeyValueStorageFactory( @@ -95,6 +96,7 @@ public void shouldDetectCorrectVersionIfMetadataFileExists() throws Exception { Files.createDirectories(tempDataDir); when(commonConfiguration.getStoragePath()).thenReturn(tempDatabaseDir); when(commonConfiguration.getDataPath()).thenReturn(tempDataDir); + when(commonConfiguration.getDatabaseVersion()).thenReturn(DEFAULT_VERSION); final RocksDBKeyValueStorageFactory storageFactory = new RocksDBKeyValueStorageFactory( diff --git a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java index 7e2028c43c2..8610b930b3e 100644 --- a/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java +++ b/services/kvstore/src/main/java/org/hyperledger/besu/services/kvstore/InMemoryKeyValueStorage.java @@ -20,6 +20,7 @@ import org.hyperledger.besu.plugin.services.storage.KeyValueStorage; import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction; +import java.io.PrintStream; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -153,4 +154,15 @@ public void rollback() { removedKeys.clear(); } } + + public void dump(final PrintStream ps) { + final Lock lock = rwLock.readLock(); + lock.lock(); + try { + hashValueStore.forEach( + (k, v) -> ps.printf(" %s : %s%n", k.toHexString(), Bytes.wrap(v).toHexString())); + } finally { + lock.unlock(); + } + } } From c4ef564825c50b6a60f5f7e43ccb9ce01eab6c0d Mon Sep 17 00:00:00 2001 From: Abdelhamid Bakhta <45264458+abdelhamidbakhta@users.noreply.github.com> Date: Mon, 7 Dec 2020 18:29:55 +0100 Subject: [PATCH 15/16] Accept to use default port values if not in use. (#1673) * Accept to use default port values if not in use. Signed-off-by: Abdelhamid Bakhta --- .../org/hyperledger/besu/cli/BesuCommand.java | 46 ++++++++++++++----- .../hyperledger/besu/cli/BesuCommandTest.java | 30 +++++++++++- 2 files changed, 63 insertions(+), 13 deletions(-) diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 766b40d6ebf..22cb5118c7c 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -2366,21 +2366,12 @@ public EnodeDnsConfiguration getEnodeDnsConfiguration() { } private void checkPortClash() { - // List of port parameters - final List ports = - asList( - p2pPort, - graphQLHttpPort, - rpcHttpPort, - rpcWsPort, - metricsPort, - metricsPushPort, - stratumPort); - ports.stream() + getEffectivePorts().stream() .filter(Objects::nonNull) + .filter(port -> port > 0) .forEach( port -> { - if (port != 0 && !allocatedPorts.add(port)) { + if (!allocatedPorts.add(port)) { throw new ParameterException( commandLine, "Port number '" @@ -2390,6 +2381,37 @@ private void checkPortClash() { }); } + /** + * * Gets the list of effective ports (ports that are enabled). + * + * @return The list of effective ports + */ + private List getEffectivePorts() { + final List effectivePorts = new ArrayList<>(); + addPortIfEnabled(effectivePorts, p2pPort, p2pEnabled); + addPortIfEnabled(effectivePorts, graphQLHttpPort, isGraphQLHttpEnabled); + addPortIfEnabled(effectivePorts, rpcHttpPort, isRpcHttpEnabled); + addPortIfEnabled(effectivePorts, rpcWsPort, isRpcWsEnabled); + addPortIfEnabled(effectivePorts, metricsPort, isMetricsEnabled); + addPortIfEnabled(effectivePorts, metricsPushPort, isMetricsPushEnabled); + addPortIfEnabled(effectivePorts, stratumPort, iStratumMiningEnabled); + return effectivePorts; + } + + /** + * Adds port in the passed list only if enabled. + * + * @param ports The list of ports + * @param port The port value + * @param enabled true if enabled, false otherwise + */ + private void addPortIfEnabled( + final List ports, final Integer port, final boolean enabled) { + if (enabled) { + ports.add(port); + } + } + private void checkGoQuorumCompatibilityConfig(final EthNetworkConfig ethNetworkConfig) { if (isGoQuorumCompatibilityMode) { final GenesisConfigOptions genesisConfigOptions = readGenesisConfigOptions(); diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 4562b4253bc..ba02f6c9f45 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -3853,7 +3853,12 @@ public void assertThatInvalidWsTimeoutSecondsFails() { @Test public void assertThatDuplicatePortSpecifiedFails() { - parseCommand("--p2p-port=9", "--rpc-http-port=10", "--rpc-ws-port=10"); + parseCommand( + "--p2p-port=9", + "--rpc-http-enabled", + "--rpc-http-port=10", + "--rpc-ws-port=10", + "--rpc-ws-enabled"); assertThat(commandErrorOutput.toString()) .contains("Port number '10' has been specified multiple times."); } @@ -3999,4 +4004,27 @@ public void quorumInteropEnabledFailsWithMainnetChainId() throws IOException { assertThat(commandErrorOutput.toString()) .contains("GoQuorum compatibility mode (enabled) cannot be used on Mainnet"); } + + @Test + public void assertThatCheckPortClashAcceptsAsExpected() throws Exception { + // use WS port for HTTP + final int port = 8546; + parseCommand("--rpc-http-enabled", "--rpc-http-port", String.valueOf(port)); + verify(mockRunnerBuilder).jsonRpcConfiguration(jsonRpcConfigArgumentCaptor.capture()); + verify(mockRunnerBuilder).build(); + assertThat(jsonRpcConfigArgumentCaptor.getValue().getPort()).isEqualTo(port); + assertThat(commandOutput.toString()).isEmpty(); + assertThat(commandErrorOutput.toString()).isEmpty(); + } + + @Test + public void assertThatCheckPortClashRejectsAsExpected() throws Exception { + // use WS port for HTTP + final int port = 8546; + parseCommand("--rpc-http-enabled", "--rpc-http-port", String.valueOf(port), "--rpc-ws-enabled"); + assertThat(commandOutput.toString()).isEmpty(); + assertThat(commandErrorOutput.toString()) + .contains( + "Port number '8546' has been specified multiple times. Please review the supplied configuration."); + } } From 54326fdf562ca07a4f9cfa54a6b1df496cbbd649 Mon Sep 17 00:00:00 2001 From: matkt Date: Mon, 7 Dec 2020 19:22:42 +0100 Subject: [PATCH 16/16] Use gasPrice when this field is present (eth_estimateGas) (#1636) Signed-off-by: Karim TAAM karim.t2am@gmail.com --- CHANGELOG.md | 2 + .../methods/EthCallIntegrationTest.java | 42 ++--- .../EthEstimateGasIntegrationTest.java | 55 +++++-- .../pojoadapter/BlockAdapterBase.java | 9 +- .../api/jsonrpc/internal/methods/EthCall.java | 20 ++- .../internal/methods/EthEstimateGas.java | 29 +++- .../parameters/JsonCallParameter.java | 64 ++++++++ .../privacy/methods/priv/PrivCall.java | 8 +- .../internal/processor/TransactionTracer.java | 4 +- .../jsonrpc/internal/methods/EthCallTest.java | 43 ++--- .../internal/methods/EthEstimateGasTest.java | 152 +++++++++++++++--- .../privacy/methods/priv/PrivCallTest.java | 20 ++- .../mainnet/MainnetTransactionProcessor.java | 2 +- .../mainnet/MainnetTransactionValidator.java | 2 +- .../mainnet/TransactionValidationParams.java | 99 +++++------- .../ethereum/transaction/CallParameter.java | 28 ---- .../transaction/TransactionSimulator.java | 47 ++++-- .../MainnetTransactionProcessorTest.java | 4 +- .../MainnetTransactionValidatorTest.java | 4 +- .../TransactionValidationParamsTest.java | 16 +- .../transaction/TransactionSimulatorTest.java | 83 ++++++++++ 21 files changed, 520 insertions(+), 213 deletions(-) create mode 100644 ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/JsonCallParameter.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e75f7fdb08..c5b00398789 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,11 @@ ### Additions and Improvements * Added `memory` as an option to `--key-value-storage`. This ephemeral storage is intended for sync testing and debugging. [\#1617](https://github.com/hyperledger/besu/pull/1617) +* Fixed gasPrice parameter not always respected when passed to `eth_estimateGas` endpoint [#1636](https://github.com/hyperledger/besu/pull/1636) ### Bug Fixes + #### Previously identified known issues - [Fast sync when running Besu on cloud providers](KNOWN_ISSUES.md#fast-sync-when-running-besu-on-cloud-providers) diff --git a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthCallIntegrationTest.java b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthCallIntegrationTest.java index 295109d79cc..0e5a3ab2de8 100644 --- a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthCallIntegrationTest.java +++ b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthCallIntegrationTest.java @@ -23,6 +23,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.JsonRpcMethod; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonCallParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; @@ -30,7 +31,6 @@ import org.hyperledger.besu.ethereum.core.Address; import org.hyperledger.besu.ethereum.core.Gas; import org.hyperledger.besu.ethereum.core.Wei; -import org.hyperledger.besu.ethereum.transaction.CallParameter; import org.hyperledger.besu.testutil.BlockTestUtil; import java.util.Map; @@ -66,8 +66,8 @@ public void setUp() { @Test public void shouldReturnExpectedResultForCallAtLatestBlock() { - final CallParameter callParameter = - new CallParameter( + final JsonCallParameter callParameter = + new JsonCallParameter( Address.fromHexString("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"), Address.fromHexString("0x6295ee1b4f6dd65047762f924ecd367c17eabf8f"), null, @@ -75,7 +75,8 @@ public void shouldReturnExpectedResultForCallAtLatestBlock() { null, null, null, - Bytes.fromHexString("0x12a7b914")); + Bytes.fromHexString("0x12a7b914"), + null); final JsonRpcRequestContext request = requestWithParams(callParameter, "latest"); final JsonRpcResponse expectedResponse = new JsonRpcSuccessResponse( @@ -88,8 +89,8 @@ public void shouldReturnExpectedResultForCallAtLatestBlock() { @Test public void shouldReturnExpectedResultForCallAtSpecificBlock() { - final CallParameter callParameter = - new CallParameter( + final JsonCallParameter callParameter = + new JsonCallParameter( Address.fromHexString("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"), Address.fromHexString("0x6295ee1b4f6dd65047762f924ecd367c17eabf8f"), null, @@ -97,7 +98,8 @@ public void shouldReturnExpectedResultForCallAtSpecificBlock() { null, null, null, - Bytes.fromHexString("0x12a7b914")); + Bytes.fromHexString("0x12a7b914"), + null); final JsonRpcRequestContext request = requestWithParams(callParameter, "0x8"); final JsonRpcResponse expectedResponse = new JsonRpcSuccessResponse( @@ -110,8 +112,8 @@ public void shouldReturnExpectedResultForCallAtSpecificBlock() { @Test public void shouldReturnInvalidRequestWhenMissingToField() { - final CallParameter callParameter = - new CallParameter( + final JsonCallParameter callParameter = + new JsonCallParameter( Address.fromHexString("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"), null, null, @@ -119,7 +121,8 @@ public void shouldReturnInvalidRequestWhenMissingToField() { null, null, null, - Bytes.fromHexString("0x12a7b914")); + Bytes.fromHexString("0x12a7b914"), + null); final JsonRpcRequestContext request = requestWithParams(callParameter, "latest"); final Throwable thrown = catchThrowable(() -> method.response(request)); @@ -132,8 +135,8 @@ public void shouldReturnInvalidRequestWhenMissingToField() { @Test public void shouldReturnErrorWithGasLimitTooLow() { - final CallParameter callParameter = - new CallParameter( + final JsonCallParameter callParameter = + new JsonCallParameter( Address.fromHexString("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"), Address.fromHexString("0x6295ee1b4f6dd65047762f924ecd367c17eabf8f"), Gas.ZERO, @@ -141,7 +144,8 @@ public void shouldReturnErrorWithGasLimitTooLow() { null, null, null, - Bytes.fromHexString("0x12a7b914")); + Bytes.fromHexString("0x12a7b914"), + null); final JsonRpcRequestContext request = requestWithParams(callParameter, "latest"); final JsonRpcResponse expectedResponse = new JsonRpcErrorResponse(null, JsonRpcError.INTRINSIC_GAS_EXCEEDS_LIMIT); @@ -153,8 +157,8 @@ public void shouldReturnErrorWithGasLimitTooLow() { @Test public void shouldReturnErrorWithGasPriceTooHigh() { - final CallParameter callParameter = - new CallParameter( + final JsonCallParameter callParameter = + new JsonCallParameter( Address.fromHexString("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"), Address.fromHexString("0x6295ee1b4f6dd65047762f924ecd367c17eabf8f"), null, @@ -162,7 +166,8 @@ public void shouldReturnErrorWithGasPriceTooHigh() { null, null, null, - Bytes.fromHexString("0x12a7b914")); + Bytes.fromHexString("0x12a7b914"), + null); final JsonRpcRequestContext request = requestWithParams(callParameter, "latest"); final JsonRpcResponse expectedResponse = new JsonRpcErrorResponse(null, JsonRpcError.TRANSACTION_UPFRONT_COST_EXCEEDS_BALANCE); @@ -174,8 +179,8 @@ public void shouldReturnErrorWithGasPriceTooHigh() { @Test public void shouldReturnEmptyHashResultForCallWithOnlyToField() { - final CallParameter callParameter = - new CallParameter( + final JsonCallParameter callParameter = + new JsonCallParameter( null, Address.fromHexString("0x6295ee1b4f6dd65047762f924ecd367c17eabf8f"), null, @@ -183,6 +188,7 @@ public void shouldReturnEmptyHashResultForCallWithOnlyToField() { null, null, null, + null, null); final JsonRpcRequestContext request = requestWithParams(callParameter, "latest"); final JsonRpcResponse expectedResponse = new JsonRpcSuccessResponse(null, "0x"); diff --git a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthEstimateGasIntegrationTest.java b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthEstimateGasIntegrationTest.java index d0606d622d1..4bdc376fbec 100644 --- a/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthEstimateGasIntegrationTest.java +++ b/ethereum/api/src/integration-test/java/org/hyperledger/besu/ethereum/api/jsonrpc/methods/EthEstimateGasIntegrationTest.java @@ -21,12 +21,14 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.JsonRpcMethod; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonCallParameter; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.core.Address; import org.hyperledger.besu.ethereum.core.Gas; import org.hyperledger.besu.ethereum.core.Wei; -import org.hyperledger.besu.ethereum.transaction.CallParameter; import org.hyperledger.besu.testutil.BlockTestUtil; import java.util.Map; @@ -62,8 +64,8 @@ public void setUp() { @Test public void shouldReturnExpectedValueForEmptyCallParameter() { - final CallParameter callParameter = - new CallParameter(null, null, null, null, null, null, null, null); + final JsonCallParameter callParameter = + new JsonCallParameter(null, null, null, null, null, null, null, null, null); final JsonRpcRequestContext request = requestWithParams(callParameter); final JsonRpcResponse expectedResponse = new JsonRpcSuccessResponse(null, "0x5208"); @@ -74,8 +76,8 @@ public void shouldReturnExpectedValueForEmptyCallParameter() { @Test public void shouldReturnExpectedValueForTransfer() { - final CallParameter callParameter = - new CallParameter( + final JsonCallParameter callParameter = + new JsonCallParameter( Address.fromHexString("0x6295ee1b4f6dd65047762f924ecd367c17eabf8f"), Address.fromHexString("0x8888f1f195afa192cfee860698584c030f4c9db1"), null, @@ -83,6 +85,7 @@ public void shouldReturnExpectedValueForTransfer() { null, null, Wei.ONE, + null, null); final JsonRpcRequestContext request = requestWithParams(callParameter); final JsonRpcResponse expectedResponse = new JsonRpcSuccessResponse(null, "0x5208"); @@ -94,8 +97,8 @@ public void shouldReturnExpectedValueForTransfer() { @Test public void shouldReturnExpectedValueForContractDeploy() { - final CallParameter callParameter = - new CallParameter( + final JsonCallParameter callParameter = + new JsonCallParameter( Address.fromHexString("0x6295ee1b4f6dd65047762f924ecd367c17eabf8f"), null, null, @@ -104,7 +107,8 @@ public void shouldReturnExpectedValueForContractDeploy() { null, null, Bytes.fromHexString( - "0x608060405234801561001057600080fd5b50610157806100206000396000f30060806040526004361061004c576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff1680633bdab8bf146100515780639ae97baa14610068575b600080fd5b34801561005d57600080fd5b5061006661007f565b005b34801561007457600080fd5b5061007d6100b9565b005b7fa53887c1eed04528e23301f55ad49a91634ef5021aa83a97d07fd16ed71c039a60016040518082815260200191505060405180910390a1565b7fa53887c1eed04528e23301f55ad49a91634ef5021aa83a97d07fd16ed71c039a60026040518082815260200191505060405180910390a17fa53887c1eed04528e23301f55ad49a91634ef5021aa83a97d07fd16ed71c039a60036040518082815260200191505060405180910390a15600a165627a7a7230582010ddaa52e73a98c06dbcd22b234b97206c1d7ed64a7c048e10c2043a3d2309cb0029")); + "0x608060405234801561001057600080fd5b50610157806100206000396000f30060806040526004361061004c576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff1680633bdab8bf146100515780639ae97baa14610068575b600080fd5b34801561005d57600080fd5b5061006661007f565b005b34801561007457600080fd5b5061007d6100b9565b005b7fa53887c1eed04528e23301f55ad49a91634ef5021aa83a97d07fd16ed71c039a60016040518082815260200191505060405180910390a1565b7fa53887c1eed04528e23301f55ad49a91634ef5021aa83a97d07fd16ed71c039a60026040518082815260200191505060405180910390a17fa53887c1eed04528e23301f55ad49a91634ef5021aa83a97d07fd16ed71c039a60036040518082815260200191505060405180910390a15600a165627a7a7230582010ddaa52e73a98c06dbcd22b234b97206c1d7ed64a7c048e10c2043a3d2309cb0029"), + null); final JsonRpcRequestContext request = requestWithParams(callParameter); final JsonRpcResponse expectedResponse = new JsonRpcSuccessResponse(null, "0x1b551"); @@ -114,9 +118,9 @@ public void shouldReturnExpectedValueForContractDeploy() { } @Test - public void shouldIgnoreGasLimitAndGasPriceAndReturnExpectedValue() { - final CallParameter callParameter = - new CallParameter( + public void shouldIgnoreSenderBalanceAccountWhenStrictModeDisabledAndReturnExpectedValue() { + final JsonCallParameter callParameter = + new JsonCallParameter( Address.fromHexString("0x6295ee1b4f6dd65047762f924ecd367c17eabf8f"), null, Gas.of(1), @@ -125,7 +129,8 @@ public void shouldIgnoreGasLimitAndGasPriceAndReturnExpectedValue() { null, null, Bytes.fromHexString( - "0x608060405234801561001057600080fd5b50610157806100206000396000f30060806040526004361061004c576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff1680633bdab8bf146100515780639ae97baa14610068575b600080fd5b34801561005d57600080fd5b5061006661007f565b005b34801561007457600080fd5b5061007d6100b9565b005b7fa53887c1eed04528e23301f55ad49a91634ef5021aa83a97d07fd16ed71c039a60016040518082815260200191505060405180910390a1565b7fa53887c1eed04528e23301f55ad49a91634ef5021aa83a97d07fd16ed71c039a60026040518082815260200191505060405180910390a17fa53887c1eed04528e23301f55ad49a91634ef5021aa83a97d07fd16ed71c039a60036040518082815260200191505060405180910390a15600a165627a7a7230582010ddaa52e73a98c06dbcd22b234b97206c1d7ed64a7c048e10c2043a3d2309cb0029")); + "0x608060405234801561001057600080fd5b50610157806100206000396000f30060806040526004361061004c576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff1680633bdab8bf146100515780639ae97baa14610068575b600080fd5b34801561005d57600080fd5b5061006661007f565b005b34801561007457600080fd5b5061007d6100b9565b005b7fa53887c1eed04528e23301f55ad49a91634ef5021aa83a97d07fd16ed71c039a60016040518082815260200191505060405180910390a1565b7fa53887c1eed04528e23301f55ad49a91634ef5021aa83a97d07fd16ed71c039a60026040518082815260200191505060405180910390a17fa53887c1eed04528e23301f55ad49a91634ef5021aa83a97d07fd16ed71c039a60036040518082815260200191505060405180910390a15600a165627a7a7230582010ddaa52e73a98c06dbcd22b234b97206c1d7ed64a7c048e10c2043a3d2309cb0029"), + false); final JsonRpcRequestContext request = requestWithParams(callParameter); final JsonRpcResponse expectedResponse = new JsonRpcSuccessResponse(null, "0x1b551"); @@ -134,10 +139,32 @@ public void shouldIgnoreGasLimitAndGasPriceAndReturnExpectedValue() { assertThat(response).isEqualToComparingFieldByField(expectedResponse); } + @Test + public void shouldNotIgnoreSenderBalanceAccountWhenStrictModeDisabledAndThrowError() { + final JsonCallParameter callParameter = + new JsonCallParameter( + Address.fromHexString("0x6295ee1b4f6dd65047762f924ecd367c17eabf8f"), + null, + Gas.of(1), + Wei.fromHexString("0x9999999999"), + null, + null, + null, + Bytes.fromHexString( + "0x608060405234801561001057600080fd5b50610157806100206000396000f30060806040526004361061004c576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff1680633bdab8bf146100515780639ae97baa14610068575b600080fd5b34801561005d57600080fd5b5061006661007f565b005b34801561007457600080fd5b5061007d6100b9565b005b7fa53887c1eed04528e23301f55ad49a91634ef5021aa83a97d07fd16ed71c039a60016040518082815260200191505060405180910390a1565b7fa53887c1eed04528e23301f55ad49a91634ef5021aa83a97d07fd16ed71c039a60026040518082815260200191505060405180910390a17fa53887c1eed04528e23301f55ad49a91634ef5021aa83a97d07fd16ed71c039a60036040518082815260200191505060405180910390a15600a165627a7a7230582010ddaa52e73a98c06dbcd22b234b97206c1d7ed64a7c048e10c2043a3d2309cb0029"), + true); + final JsonRpcRequestContext request = requestWithParams(callParameter); + final JsonRpcResponse expectedResponse = + new JsonRpcErrorResponse(null, JsonRpcError.TRANSACTION_UPFRONT_COST_EXCEEDS_BALANCE); + + final JsonRpcResponse response = method.response(request); + assertThat(response).isEqualToComparingFieldByField(expectedResponse); + } + @Test public void shouldReturnExpectedValueForInsufficientGas() { - final CallParameter callParameter = - new CallParameter(null, null, Gas.of(1), null, null, null, null, null); + final JsonCallParameter callParameter = + new JsonCallParameter(null, null, Gas.of(1), null, null, null, null, null, null); final JsonRpcRequestContext request = requestWithParams(callParameter); final JsonRpcResponse expectedResponse = new JsonRpcSuccessResponse(null, "0x5208"); diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/BlockAdapterBase.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/BlockAdapterBase.java index 71b5d16a8bc..35cb29fad2c 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/BlockAdapterBase.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/graphql/internal/pojoadapter/BlockAdapterBase.java @@ -28,9 +28,11 @@ import org.hyperledger.besu.ethereum.core.Wei; import org.hyperledger.besu.ethereum.core.WorldState; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; +import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams; import org.hyperledger.besu.ethereum.transaction.CallParameter; import org.hyperledger.besu.ethereum.transaction.TransactionSimulator; import org.hyperledger.besu.ethereum.transaction.TransactionSimulatorResult; +import org.hyperledger.besu.ethereum.vm.OperationTracer; import java.util.ArrayList; import java.util.List; @@ -216,7 +218,12 @@ private Optional executeCall(final DataFetchingEnvironment environme final CallParameter param = new CallParameter(from, to, gasParam, gasPriceParam, valueParam, data); - final Optional opt = transactionSimulator.process(param, bn); + final Optional opt = + transactionSimulator.process( + param, + TransactionValidationParams.transactionSimulator(), + OperationTracer.NO_TRACING, + bn); if (opt.isPresent()) { final TransactionSimulatorResult result = opt.get(); long status = 0; diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java index 34df27a428c..2cd19c268ed 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCall.java @@ -19,12 +19,14 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.BlockParameter; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonCallParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; -import org.hyperledger.besu.ethereum.transaction.CallParameter; +import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams; import org.hyperledger.besu.ethereum.transaction.TransactionSimulator; +import org.hyperledger.besu.ethereum.vm.OperationTracer; public class EthCall extends AbstractBlockParameterMethod { @@ -49,10 +51,14 @@ protected BlockParameter blockParameter(final JsonRpcRequestContext request) { @Override protected Object resultByBlockNumber( final JsonRpcRequestContext request, final long blockNumber) { - final CallParameter callParams = validateAndGetCallParams(request); + final JsonCallParameter callParams = validateAndGetCallParams(request); return transactionSimulator - .process(callParams, blockNumber) + .process( + callParams, + TransactionValidationParams.transactionSimulator(), + OperationTracer.NO_TRACING, + blockNumber) .map( result -> result @@ -77,11 +83,15 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { return (JsonRpcResponse) findResultByParamType(requestContext); } - private CallParameter validateAndGetCallParams(final JsonRpcRequestContext request) { - final CallParameter callParams = request.getRequiredParameter(0, CallParameter.class); + private JsonCallParameter validateAndGetCallParams(final JsonRpcRequestContext request) { + final JsonCallParameter callParams = request.getRequiredParameter(0, JsonCallParameter.class); if (callParams.getTo() == null) { throw new InvalidJsonRpcParameters("Missing \"to\" field in call arguments"); } + if (callParams.getGasPrice() != null + && (callParams.getFeeCap().isPresent() || callParams.getGasPremium().isPresent())) { + throw new InvalidJsonRpcParameters("gasPrice cannot be used with baseFee or feeCap"); + } return callParams; } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthEstimateGas.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthEstimateGas.java index e6e63f8d093..e8b34fb5506 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthEstimateGas.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthEstimateGas.java @@ -17,6 +17,8 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.JsonRpcErrorConverter; import org.hyperledger.besu.ethereum.api.jsonrpc.RpcMethod; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonCallParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; @@ -25,6 +27,8 @@ import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.Wei; +import org.hyperledger.besu.ethereum.mainnet.ImmutableTransactionValidationParams; +import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams; import org.hyperledger.besu.ethereum.mainnet.ValidationResult; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.transaction.CallParameter; @@ -33,6 +37,7 @@ import org.hyperledger.besu.ethereum.transaction.TransactionSimulatorResult; import org.hyperledger.besu.ethereum.vm.EstimateGasOperationTracer; +import java.util.Optional; import java.util.function.Function; public class EthEstimateGas implements JsonRpcMethod { @@ -55,7 +60,7 @@ public String getName() { @Override public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { - final CallParameter callParams = requestContext.getRequiredParameter(0, CallParameter.class); + final JsonCallParameter callParams = validateAndGetCallParams(requestContext); final BlockHeader blockHeader = blockHeader(); if (blockHeader == null) { @@ -73,7 +78,14 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { final EstimateGasOperationTracer operationTracer = new EstimateGasOperationTracer(); return transactionSimulator - .process(modifiedCallParams, operationTracer, blockHeader.getNumber()) + .process( + modifiedCallParams, + ImmutableTransactionValidationParams.builder() + .from(TransactionValidationParams.transactionSimulator()) + .isAllowExceedingBalance(!callParams.isStrict()) + .build(), + operationTracer, + blockHeader.getNumber()) .map(gasEstimateResponse(requestContext, operationTracer)) .orElse(errorResponse(requestContext, JsonRpcError.INTERNAL_ERROR)); } @@ -84,12 +96,12 @@ private BlockHeader blockHeader() { } private CallParameter overrideGasLimitAndPrice( - final CallParameter callParams, final long gasLimit) { + final JsonCallParameter callParams, final long gasLimit) { return new CallParameter( callParams.getFrom(), callParams.getTo(), gasLimit, - Wei.ZERO, + Optional.ofNullable(callParams.getGasPrice()).orElse(Wei.ZERO), callParams.getGasPremium(), callParams.getFeeCap(), callParams.getValue(), @@ -150,4 +162,13 @@ private JsonRpcErrorResponse errorResponse( final JsonRpcRequestContext request, final JsonRpcError jsonRpcError) { return new JsonRpcErrorResponse(request.getRequest().getId(), jsonRpcError); } + + private JsonCallParameter validateAndGetCallParams(final JsonRpcRequestContext request) { + final JsonCallParameter callParams = request.getRequiredParameter(0, JsonCallParameter.class); + if (callParams.getGasPrice() != null + && (callParams.getFeeCap().isPresent() || callParams.getGasPremium().isPresent())) { + throw new InvalidJsonRpcParameters("gasPrice cannot be used with baseFee or feeCap"); + } + return callParams; + } } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/JsonCallParameter.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/JsonCallParameter.java new file mode 100644 index 00000000000..c9ff32a3444 --- /dev/null +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/parameters/JsonCallParameter.java @@ -0,0 +1,64 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters; + +import static java.lang.Boolean.FALSE; + +import org.hyperledger.besu.ethereum.core.Address; +import org.hyperledger.besu.ethereum.core.Gas; +import org.hyperledger.besu.ethereum.core.Wei; +import org.hyperledger.besu.ethereum.core.deserializer.GasDeserializer; +import org.hyperledger.besu.ethereum.core.deserializer.HexStringDeserializer; +import org.hyperledger.besu.ethereum.transaction.CallParameter; + +import java.util.Optional; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.apache.tuweni.bytes.Bytes; + +public class JsonCallParameter extends CallParameter { + + private final boolean strict; + + @JsonCreator + public JsonCallParameter( + @JsonProperty("from") final Address from, + @JsonProperty("to") final Address to, + @JsonDeserialize(using = GasDeserializer.class) @JsonProperty("gas") final Gas gasLimit, + @JsonProperty("gasPrice") final Wei gasPrice, + @JsonProperty("gasPremium") final Wei gasPremium, + @JsonProperty("feeCap") final Wei feeCap, + @JsonProperty("value") final Wei value, + @JsonDeserialize(using = HexStringDeserializer.class) @JsonProperty("data") + final Bytes payload, + @JsonProperty("strict") final Boolean strict) { + super( + from, + to, + gasLimit != null ? gasLimit.toLong() : -1, + gasPrice, + Optional.ofNullable(gasPremium), + Optional.ofNullable(feeCap), + value, + payload); + this.strict = Optional.ofNullable(strict).orElse(FALSE); + } + + public boolean isStrict() { + return strict; + } +} diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivCall.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivCall.java index 3f2ba205292..5e78ebd191b 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivCall.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivCall.java @@ -20,13 +20,13 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.methods.AbstractBlockParameterMethod; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.BlockParameter; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonCallParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.privacy.methods.EnclavePublicKeyProvider; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.query.BlockchainQueries; import org.hyperledger.besu.ethereum.privacy.PrivacyController; -import org.hyperledger.besu.ethereum.transaction.CallParameter; public class PrivCall extends AbstractBlockParameterMethod { @@ -55,7 +55,7 @@ protected BlockParameter blockParameter(final JsonRpcRequestContext request) { @Override protected Object resultByBlockNumber( final JsonRpcRequestContext request, final long blockNumber) { - final CallParameter callParams = validateAndGetCallParams(request); + final JsonCallParameter callParams = validateAndGetCallParams(request); final String privacyGroupId = request.getRequiredParameter(0, String.class); final String enclavePublicKey = enclavePublicKeyProvider.getEnclaveKey(request.getUser()); @@ -89,8 +89,8 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) { return (JsonRpcResponse) findResultByParamType(requestContext); } - private CallParameter validateAndGetCallParams(final JsonRpcRequestContext request) { - final CallParameter callParams = request.getRequiredParameter(1, CallParameter.class); + private JsonCallParameter validateAndGetCallParams(final JsonRpcRequestContext request) { + final JsonCallParameter callParams = request.getRequiredParameter(1, JsonCallParameter.class); if (callParams.getTo() == null) { throw new InvalidJsonRpcParameters("Missing \"to\" field in call arguments"); } diff --git a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/TransactionTracer.java b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/TransactionTracer.java index dc3e1e6e09a..60c7bbc2f82 100644 --- a/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/TransactionTracer.java +++ b/ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/processor/TransactionTracer.java @@ -23,8 +23,8 @@ import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.WorldUpdater; import org.hyperledger.besu.ethereum.debug.TraceOptions; +import org.hyperledger.besu.ethereum.mainnet.ImmutableTransactionValidationParams; import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; -import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.vm.BlockHashLookup; import org.hyperledger.besu.ethereum.vm.DebugOperationTracer; @@ -169,6 +169,6 @@ private TransactionProcessingResult processTransaction( tracer, new BlockHashLookup(header, blockchain), false, - new TransactionValidationParams.Builder().allowFutureNonce(true).build()); + ImmutableTransactionValidationParams.builder().isAllowFutureNonce(true).build()); } } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCallTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCallTest.java index 900c9e97c76..51460b98888 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCallTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthCallTest.java @@ -26,6 +26,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonCallParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.results.Quantity; @@ -67,8 +68,8 @@ public void shouldReturnCorrectMethodName() { @Test public void shouldThrowInvalidJsonRpcParametersExceptionWhenMissingToField() { - final CallParameter callParameter = - new CallParameter( + final JsonCallParameter callParameter = + new JsonCallParameter( Address.fromHexString("0x0"), null, Gas.ZERO, @@ -76,7 +77,8 @@ public void shouldThrowInvalidJsonRpcParametersExceptionWhenMissingToField() { null, null, Wei.ZERO, - Bytes.EMPTY); + Bytes.EMPTY, + null); final JsonRpcRequestContext request = ethCallRequest(callParameter, "latest"); final Throwable thrown = catchThrowable(() -> method.response(request)); @@ -92,18 +94,19 @@ public void shouldReturnNullWhenProcessorReturnsEmpty() { final JsonRpcRequestContext request = ethCallRequest(callParameter(), "latest"); final JsonRpcResponse expectedResponse = new JsonRpcSuccessResponse(null, null); - when(transactionSimulator.process(any(), anyLong())).thenReturn(Optional.empty()); + when(transactionSimulator.process(any(), any(), any(), anyLong())).thenReturn(Optional.empty()); final JsonRpcResponse response = method.response(request); assertThat(response).isEqualToComparingFieldByField(expectedResponse); - verify(transactionSimulator).process(any(), anyLong()); + verify(transactionSimulator).process(any(), any(), any(), anyLong()); } @Test public void shouldAcceptRequestWhenMissingOptionalFields() { - final CallParameter callParameter = - new CallParameter(null, Address.fromHexString("0x0"), null, null, null, null, null, null); + final JsonCallParameter callParameter = + new JsonCallParameter( + null, Address.fromHexString("0x0"), null, null, null, null, null, null, null); final JsonRpcRequestContext request = ethCallRequest(callParameter, "latest"); final JsonRpcResponse expectedResponse = new JsonRpcSuccessResponse(null, Bytes.of().toString()); @@ -113,7 +116,7 @@ public void shouldAcceptRequestWhenMissingOptionalFields() { final JsonRpcResponse response = method.response(request); assertThat(response).isEqualToComparingFieldByFieldRecursively(expectedResponse); - verify(transactionSimulator).process(eq(callParameter), anyLong()); + verify(transactionSimulator).process(eq(callParameter), any(), any(), anyLong()); } @Test @@ -126,41 +129,41 @@ public void shouldReturnExecutionResultWhenExecutionIsSuccessful() { final JsonRpcResponse response = method.response(request); assertThat(response).isEqualToComparingFieldByFieldRecursively(expectedResponse); - verify(transactionSimulator).process(eq(callParameter()), anyLong()); + verify(transactionSimulator).process(eq(callParameter()), any(), any(), anyLong()); } @Test public void shouldUseCorrectBlockNumberWhenLatest() { final JsonRpcRequestContext request = ethCallRequest(callParameter(), "latest"); when(blockchainQueries.headBlockNumber()).thenReturn(11L); - when(transactionSimulator.process(any(), anyLong())).thenReturn(Optional.empty()); + when(transactionSimulator.process(any(), any(), any(), anyLong())).thenReturn(Optional.empty()); method.response(request); - verify(transactionSimulator).process(any(), eq(11L)); + verify(transactionSimulator).process(any(), any(), any(), eq(11L)); } @Test public void shouldUseCorrectBlockNumberWhenEarliest() { final JsonRpcRequestContext request = ethCallRequest(callParameter(), "earliest"); - when(transactionSimulator.process(any(), anyLong())).thenReturn(Optional.empty()); + when(transactionSimulator.process(any(), any(), any(), anyLong())).thenReturn(Optional.empty()); method.response(request); - verify(transactionSimulator).process(any(), eq(0L)); + verify(transactionSimulator).process(any(), any(), any(), eq(0L)); } @Test public void shouldUseCorrectBlockNumberWhenSpecified() { final JsonRpcRequestContext request = ethCallRequest(callParameter(), Quantity.create(13L)); - when(transactionSimulator.process(any(), anyLong())).thenReturn(Optional.empty()); + when(transactionSimulator.process(any(), any(), any(), anyLong())).thenReturn(Optional.empty()); method.response(request); - verify(transactionSimulator).process(any(), eq(13L)); + verify(transactionSimulator).process(any(), any(), any(), eq(13L)); } - private CallParameter callParameter() { - return new CallParameter( + private JsonCallParameter callParameter() { + return new JsonCallParameter( Address.fromHexString("0x0"), Address.fromHexString("0x0"), Gas.ZERO, @@ -168,7 +171,8 @@ private CallParameter callParameter() { null, null, Wei.ZERO, - Bytes.EMPTY); + Bytes.EMPTY, + null); } private JsonRpcRequestContext ethCallRequest( @@ -182,6 +186,7 @@ private void mockTransactionProcessorSuccessResult(final Bytes output) { when(result.getValidationResult()).thenReturn(ValidationResult.valid()); when(result.getOutput()).thenReturn(output); - when(transactionSimulator.process(any(), anyLong())).thenReturn(Optional.of(result)); + when(transactionSimulator.process(any(), any(), any(), anyLong())) + .thenReturn(Optional.of(result)); } } diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthEstimateGasTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthEstimateGasTest.java index 91cc809ae8c..a05541bbe6a 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthEstimateGasTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/EthEstimateGasTest.java @@ -18,10 +18,13 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonCallParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcError; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcErrorResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; @@ -33,6 +36,8 @@ import org.hyperledger.besu.ethereum.core.BlockHeader; import org.hyperledger.besu.ethereum.core.Gas; import org.hyperledger.besu.ethereum.core.Wei; +import org.hyperledger.besu.ethereum.mainnet.ImmutableTransactionValidationParams; +import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams; import org.hyperledger.besu.ethereum.mainnet.ValidationResult; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.transaction.CallParameter; @@ -83,9 +88,13 @@ public void shouldReturnCorrectMethodName() { @Test public void shouldReturnErrorWhenTransientLegacyTransactionProcessorReturnsEmpty() { - final JsonRpcRequestContext request = ethEstimateGasRequest(legacyTransactionCallParameter()); + final JsonRpcRequestContext request = + ethEstimateGasRequest(defaultLegacyTransactionCallParameter(Wei.ZERO)); when(transactionSimulator.process( - eq(modifiedLegacyTransactionCallParameter()), any(OperationTracer.class), eq(1L))) + eq(modifiedLegacyTransactionCallParameter(Wei.ZERO)), + any(TransactionValidationParams.class), + any(OperationTracer.class), + eq(1L))) .thenReturn(Optional.empty()); final JsonRpcResponse expectedResponse = @@ -99,7 +108,10 @@ public void shouldReturnErrorWhenTransientLegacyTransactionProcessorReturnsEmpty public void shouldReturnErrorWhenTransientEip1559TransactionProcessorReturnsEmpty() { final JsonRpcRequestContext request = ethEstimateGasRequest(eip1559TransactionCallParameter()); when(transactionSimulator.process( - eq(modifiedEip1559TransactionCallParameter()), any(OperationTracer.class), eq(1L))) + eq(modifiedEip1559TransactionCallParameter()), + any(TransactionValidationParams.class), + any(OperationTracer.class), + eq(1L))) .thenReturn(Optional.empty()); final JsonRpcResponse expectedResponse = @@ -111,7 +123,8 @@ public void shouldReturnErrorWhenTransientEip1559TransactionProcessorReturnsEmpt @Test public void shouldReturnGasEstimateWhenTransientLegacyTransactionProcessorReturnsResultSuccess() { - final JsonRpcRequestContext request = ethEstimateGasRequest(legacyTransactionCallParameter()); + final JsonRpcRequestContext request = + ethEstimateGasRequest(defaultLegacyTransactionCallParameter(Wei.ZERO)); mockTransientProcessorResultGasEstimate(1L, true, false); final JsonRpcResponse expectedResponse = new JsonRpcSuccessResponse(null, Quantity.create(1L)); @@ -120,6 +133,29 @@ public void shouldReturnGasEstimateWhenTransientLegacyTransactionProcessorReturn .isEqualToComparingFieldByField(expectedResponse); } + @Test + public void shouldUseGasPriceParameterWhenIsPresent() { + final Wei gasPrice = Wei.of(1000); + final JsonRpcRequestContext request = + ethEstimateGasRequest(defaultLegacyTransactionCallParameter(gasPrice)); + mockTransientProcessorResultGasEstimate(1L, true, false, gasPrice); + + final JsonRpcResponse expectedResponse = new JsonRpcSuccessResponse(null, Quantity.create(1L)); + + Assertions.assertThat(method.response(request)) + .isEqualToComparingFieldByField(expectedResponse); + } + + @Test + public void shouldReturnGasEstimateErrorWhenGasPricePresentForEip1559Transaction() { + final JsonRpcRequestContext request = + ethEstimateGasRequest(eip1559TransactionCallParameter(Optional.of(Wei.of(10)))); + mockTransientProcessorResultGasEstimate(1L, false, false); + Assertions.assertThatThrownBy(() -> method.response(request)) + .isInstanceOf(InvalidJsonRpcParameters.class) + .hasMessageContaining("gasPrice cannot be used with baseFee or feeCap"); + } + @Test public void shouldReturnGasEstimateWhenTransientEip1559TransactionProcessorReturnsResultSuccess() { @@ -134,7 +170,8 @@ public void shouldReturnGasEstimateWhenTransientLegacyTransactionProcessorReturn @Test public void shouldReturnGasEstimateErrorWhenTransientLegacyTransactionProcessorReturnsResultFailure() { - final JsonRpcRequestContext request = ethEstimateGasRequest(legacyTransactionCallParameter()); + final JsonRpcRequestContext request = + ethEstimateGasRequest(defaultLegacyTransactionCallParameter(Wei.ZERO)); mockTransientProcessorResultGasEstimate(1L, false, false); final JsonRpcResponse expectedResponse = @@ -159,7 +196,8 @@ public void shouldReturnGasEstimateWhenTransientLegacyTransactionProcessorReturn @Test public void shouldReturnErrorWhenLegacyTransactionProcessorReturnsTxInvalidReason() { - final JsonRpcRequestContext request = ethEstimateGasRequest(legacyTransactionCallParameter()); + final JsonRpcRequestContext request = + ethEstimateGasRequest(defaultLegacyTransactionCallParameter(Wei.ZERO)); mockTransientProcessorResultTxInvalidReason( TransactionInvalidReason.UPFRONT_COST_EXCEEDS_BALANCE); @@ -186,7 +224,8 @@ public void shouldReturnErrorWhenEip1559TransactionProcessorReturnsTxInvalidReas @Test public void shouldReturnErrorWhenWorldStateIsNotAvailable() { when(worldStateArchive.isWorldStateAvailable(any())).thenReturn(false); - final JsonRpcRequestContext request = ethEstimateGasRequest(legacyTransactionCallParameter()); + final JsonRpcRequestContext request = + ethEstimateGasRequest(defaultLegacyTransactionCallParameter(Wei.ZERO)); mockTransientProcessorResultGasEstimate(1L, false, false); final JsonRpcResponse expectedResponse = @@ -198,7 +237,8 @@ public void shouldReturnErrorWhenWorldStateIsNotAvailable() { @Test public void shouldReturnErrorWhenTransactionReverted() { - final JsonRpcRequestContext request = ethEstimateGasRequest(legacyTransactionCallParameter()); + final JsonRpcRequestContext request = + ethEstimateGasRequest(defaultLegacyTransactionCallParameter(Wei.ZERO)); mockTransientProcessorResultGasEstimate(1L, false, true); final JsonRpcResponse expectedResponse = @@ -208,25 +248,82 @@ public void shouldReturnErrorWhenTransactionReverted() { .isEqualToComparingFieldByField(expectedResponse); } + @Test + public void shouldIgnoreSenderBalanceAccountWhenStrictModeDisabled() { + final JsonRpcRequestContext request = + ethEstimateGasRequest(defaultLegacyTransactionCallParameter(Wei.ZERO)); + mockTransientProcessorResultGasEstimate(1L, false, true); + + method.response(request); + + verify(transactionSimulator) + .process( + eq(modifiedLegacyTransactionCallParameter(Wei.ZERO)), + eq( + ImmutableTransactionValidationParams.builder() + .from(TransactionValidationParams.transactionSimulator()) + .isAllowExceedingBalance(true) + .build()), + any(OperationTracer.class), + eq(1L)); + } + + @Test + public void shouldNotIgnoreSenderBalanceAccountWhenStrictModeDisabled() { + final JsonRpcRequestContext request = + ethEstimateGasRequest(legacyTransactionCallParameter(Wei.ZERO, true)); + mockTransientProcessorResultGasEstimate(1L, false, true); + + method.response(request); + + verify(transactionSimulator) + .process( + eq(modifiedLegacyTransactionCallParameter(Wei.ZERO)), + eq( + ImmutableTransactionValidationParams.builder() + .from(TransactionValidationParams.transactionSimulator()) + .isAllowExceedingBalance(false) + .build()), + any(OperationTracer.class), + eq(1L)); + } + private void mockTransientProcessorResultTxInvalidReason(final TransactionInvalidReason reason) { final TransactionSimulatorResult mockTxSimResult = - getMockTransactionSimulatorResult(false, false, 0); + getMockTransactionSimulatorResult(false, false, 0, Wei.ZERO); when(mockTxSimResult.getValidationResult()).thenReturn(ValidationResult.invalid(reason)); } private void mockTransientProcessorResultGasEstimate( final long estimateGas, final boolean isSuccessful, final boolean isReverted) { - getMockTransactionSimulatorResult(isSuccessful, isReverted, estimateGas); + mockTransientProcessorResultGasEstimate(estimateGas, isSuccessful, isReverted, Wei.ZERO); + } + + private void mockTransientProcessorResultGasEstimate( + final long estimateGas, + final boolean isSuccessful, + final boolean isReverted, + final Wei gasPrice) { + getMockTransactionSimulatorResult(isSuccessful, isReverted, estimateGas, gasPrice); } private TransactionSimulatorResult getMockTransactionSimulatorResult( - final boolean isSuccessful, final boolean isReverted, final long estimateGas) { + final boolean isSuccessful, + final boolean isReverted, + final long estimateGas, + final Wei gasPrice) { final TransactionSimulatorResult mockTxSimResult = mock(TransactionSimulatorResult.class); when(transactionSimulator.process( - eq(modifiedLegacyTransactionCallParameter()), any(OperationTracer.class), eq(1L))) + eq(modifiedLegacyTransactionCallParameter(gasPrice)), + any(TransactionValidationParams.class), + any(OperationTracer.class), + eq(1L))) .thenReturn(Optional.of(mockTxSimResult)); when(transactionSimulator.process( - eq(modifiedEip1559TransactionCallParameter()), any(OperationTracer.class), eq(1L))) + eq(modifiedEip1559TransactionCallParameter()), + any(TransactionValidationParams.class), + any(OperationTracer.class), + eq(1L))) .thenReturn(Optional.of(mockTxSimResult)); final TransactionProcessingResult mockResult = mock(TransactionProcessingResult.class); when(mockResult.getEstimateGasUsedByTransaction()).thenReturn(estimateGas); @@ -237,24 +334,30 @@ private TransactionSimulatorResult getMockTransactionSimulatorResult( return mockTxSimResult; } - private CallParameter legacyTransactionCallParameter() { - return new CallParameter( + private JsonCallParameter defaultLegacyTransactionCallParameter(final Wei gasPrice) { + return legacyTransactionCallParameter(gasPrice, false); + } + + private JsonCallParameter legacyTransactionCallParameter( + final Wei gasPrice, final boolean isStrict) { + return new JsonCallParameter( Address.fromHexString("0x0"), Address.fromHexString("0x0"), Gas.ZERO, - Wei.ZERO, + gasPrice, null, null, Wei.ZERO, - Bytes.EMPTY); + Bytes.EMPTY, + isStrict); } - private CallParameter modifiedLegacyTransactionCallParameter() { + private CallParameter modifiedLegacyTransactionCallParameter(final Wei gasPrice) { return new CallParameter( Address.fromHexString("0x0"), Address.fromHexString("0x0"), Long.MAX_VALUE, - Wei.ZERO, + gasPrice, Optional.empty(), Optional.empty(), Wei.ZERO, @@ -262,15 +365,20 @@ private CallParameter modifiedLegacyTransactionCallParameter() { } private CallParameter eip1559TransactionCallParameter() { - return new CallParameter( + return eip1559TransactionCallParameter(Optional.empty()); + } + + private JsonCallParameter eip1559TransactionCallParameter(final Optional gasPrice) { + return new JsonCallParameter( Address.fromHexString("0x0"), Address.fromHexString("0x0"), null, - Wei.ZERO, + gasPrice.orElse(null), Wei.fromHexString("0x10"), Wei.fromHexString("0x10"), Wei.ZERO, - Bytes.EMPTY); + Bytes.EMPTY, + false); } private CallParameter modifiedEip1559TransactionCallParameter() { diff --git a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivCallTest.java b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivCallTest.java index 1aaca3a57de..27063676fb2 100644 --- a/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivCallTest.java +++ b/ethereum/api/src/test/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/privacy/methods/priv/PrivCallTest.java @@ -28,6 +28,7 @@ import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequest; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.JsonRpcRequestContext; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.exception.InvalidJsonRpcParameters; +import org.hyperledger.besu.ethereum.api.jsonrpc.internal.parameters.JsonCallParameter; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.privacy.methods.EnclavePublicKeyProvider; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcResponse; import org.hyperledger.besu.ethereum.api.jsonrpc.internal.response.JsonRpcSuccessResponse; @@ -76,8 +77,8 @@ public void shouldReturnCorrectMethodName() { @Test public void shouldThrowInvalidJsonRpcParametersExceptionWhenMissingToField() { - final CallParameter callParameter = - new CallParameter( + final JsonCallParameter callParameter = + new JsonCallParameter( Address.fromHexString("0x0"), null, Gas.ZERO, @@ -85,7 +86,8 @@ public void shouldThrowInvalidJsonRpcParametersExceptionWhenMissingToField() { null, null, Wei.ZERO, - Bytes.EMPTY); + Bytes.EMPTY, + null); final JsonRpcRequestContext request = ethCallRequest(privacyGroupId, callParameter, "latest"); final Throwable thrown = catchThrowable(() -> method.response(request)); @@ -109,8 +111,9 @@ public void shouldReturnNullWhenProcessorReturnsEmpty() { @Test public void shouldAcceptRequestWhenMissingOptionalFields() { - final CallParameter callParameter = - new CallParameter(null, Address.fromHexString("0x0"), null, null, null, null, null, null); + final JsonCallParameter callParameter = + new JsonCallParameter( + null, Address.fromHexString("0x0"), null, null, null, null, null, null, null); final JsonRpcRequestContext request = ethCallRequest(privacyGroupId, callParameter, "latest"); final JsonRpcResponse expectedResponse = new JsonRpcSuccessResponse(null, Bytes.of().toString()); @@ -192,8 +195,8 @@ public void multiTenancyCheckFailure() { .isInstanceOf(MultiTenancyValidationException.class); } - private CallParameter callParameter() { - return new CallParameter( + private JsonCallParameter callParameter() { + return new JsonCallParameter( Address.fromHexString("0x0"), Address.fromHexString("0x0"), Gas.ZERO, @@ -201,7 +204,8 @@ private CallParameter callParameter() { null, null, Wei.ZERO, - Bytes.EMPTY); + Bytes.EMPTY, + null); } private JsonRpcRequestContext ethCallRequest( diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java index 372808f9b93..db39f9c5d8b 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessor.java @@ -172,7 +172,7 @@ public TransactionProcessingResult processTransaction( operationTracer, blockHashLookup, isPersistingPrivateState, - new TransactionValidationParams.Builder().build()); + ImmutableTransactionValidationParams.builder().build()); } /** diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidator.java index eed28ec2257..6202051eca3 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidator.java @@ -266,7 +266,7 @@ public void setTransactionFilter(final TransactionFilter transactionFilter) { public ValidationResult validateForSender( final Transaction transaction, final Account sender, final boolean allowFutureNonce) { final TransactionValidationParams validationParams = - new TransactionValidationParams.Builder().allowFutureNonce(allowFutureNonce).build(); + ImmutableTransactionValidationParams.builder().isAllowFutureNonce(allowFutureNonce).build(); return validateForSender(transaction, sender, validationParams); } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/TransactionValidationParams.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/TransactionValidationParams.java index 43940c27d65..fb282a09599 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/TransactionValidationParams.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/TransactionValidationParams.java @@ -14,87 +14,64 @@ */ package org.hyperledger.besu.ethereum.mainnet; -public class TransactionValidationParams { - - private static final TransactionValidationParams processingBlockParams = - new TransactionValidationParams(false, true, false); - private static final TransactionValidationParams transactionPoolParams = - new TransactionValidationParams(true, false, true); - private static final TransactionValidationParams miningParams = - new TransactionValidationParams(false, true, true); - private static final TransactionValidationParams blockReplayParams = - new TransactionValidationParams(false, false, false); - private static final TransactionValidationParams transactionSimulatorParams = - new TransactionValidationParams(false, false, false); - private final boolean allowFutureNonce; - private final boolean checkOnchainPermissions; - private final boolean checkLocalPermissions; - - private TransactionValidationParams( - final boolean allowFutureNonce, - final boolean checkOnchainPermissions, - final boolean checkLocalPermissions) { - this.allowFutureNonce = allowFutureNonce; - this.checkOnchainPermissions = checkOnchainPermissions; - this.checkLocalPermissions = checkLocalPermissions; +import org.immutables.value.Value; + +@Value.Immutable +@Value.Style(allParameters = true) +public interface TransactionValidationParams { + + TransactionValidationParams processingBlockParams = + ImmutableTransactionValidationParams.of(false, false, true, false); + + TransactionValidationParams transactionPoolParams = + ImmutableTransactionValidationParams.of(true, false, false, true); + + TransactionValidationParams miningParams = + ImmutableTransactionValidationParams.of(false, false, true, true); + + TransactionValidationParams blockReplayParams = + ImmutableTransactionValidationParams.of(false, false, false, false); + + TransactionValidationParams transactionSimulatorParams = + ImmutableTransactionValidationParams.of(false, false, false, false); + + @Value.Default + default boolean isAllowFutureNonce() { + return false; } - public boolean isAllowFutureNonce() { - return allowFutureNonce; + @Value.Default + default boolean isAllowExceedingBalance() { + return false; } - public boolean checkOnchainPermissions() { - return checkOnchainPermissions; + @Value.Default + default boolean checkOnchainPermissions() { + return false; } - public boolean checkLocalPermissions() { - return checkLocalPermissions; + @Value.Default + default boolean checkLocalPermissions() { + return true; } - public static TransactionValidationParams transactionSimulator() { + static TransactionValidationParams transactionSimulator() { return transactionSimulatorParams; } - public static TransactionValidationParams processingBlock() { + static TransactionValidationParams processingBlock() { return processingBlockParams; } - public static TransactionValidationParams transactionPool() { + static TransactionValidationParams transactionPool() { return transactionPoolParams; } - public static TransactionValidationParams mining() { + static TransactionValidationParams mining() { return miningParams; } - public static TransactionValidationParams blockReplay() { + static TransactionValidationParams blockReplay() { return blockReplayParams; } - - public static class Builder { - - private boolean allowFutureNonce = false; - private boolean checkOnchainPermissions = false; - private boolean checkLocalPermissions = true; - - public Builder allowFutureNonce(final boolean allowFutureNonce) { - this.allowFutureNonce = allowFutureNonce; - return this; - } - - public Builder checkOnchainPermissions(final boolean checkOnchainPermissions) { - this.checkOnchainPermissions = checkOnchainPermissions; - return this; - } - - public Builder checkLocalPermissions(final boolean checkLocalPermissions) { - this.checkLocalPermissions = checkLocalPermissions; - return this; - } - - public TransactionValidationParams build() { - return new TransactionValidationParams( - allowFutureNonce, checkOnchainPermissions, checkLocalPermissions); - } - } } diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/CallParameter.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/CallParameter.java index 55389563477..516bd3e8be2 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/CallParameter.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/CallParameter.java @@ -15,17 +15,11 @@ package org.hyperledger.besu.ethereum.transaction; import org.hyperledger.besu.ethereum.core.Address; -import org.hyperledger.besu.ethereum.core.Gas; import org.hyperledger.besu.ethereum.core.Wei; -import org.hyperledger.besu.ethereum.core.deserializer.GasDeserializer; -import org.hyperledger.besu.ethereum.core.deserializer.HexStringDeserializer; import java.util.Objects; import java.util.Optional; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import org.apache.tuweni.bytes.Bytes; // Represents parameters for a eth_call or eth_estimateGas JSON-RPC methods. @@ -47,28 +41,6 @@ public class CallParameter { private final Bytes payload; - @JsonCreator - public CallParameter( - @JsonProperty("from") final Address from, - @JsonProperty("to") final Address to, - @JsonDeserialize(using = GasDeserializer.class) @JsonProperty("gas") final Gas gasLimit, - @JsonProperty("gasPrice") final Wei gasPrice, - @JsonProperty("gasPremium") final Wei gasPremium, - @JsonProperty("feeCap") final Wei feeCap, - @JsonProperty("value") final Wei value, - @JsonDeserialize(using = HexStringDeserializer.class) @JsonProperty("data") - final Bytes payload) { - this( - from, - to, - gasLimit != null ? gasLimit.toLong() : -1, - gasPrice, - Optional.ofNullable(gasPremium), - Optional.ofNullable(feeCap), - value, - payload); - } - public CallParameter( final Address from, final Address to, diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java index 31cb54973dc..0695ac89e77 100644 --- a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java +++ b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulator.java @@ -23,6 +23,7 @@ import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.Wei; +import org.hyperledger.besu.ethereum.core.WorldUpdater; import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; @@ -67,30 +68,44 @@ public TransactionSimulator( } public Optional process( - final CallParameter callParams, final Hash blockHeaderHash) { - final BlockHeader header = blockchain.getBlockHeader(blockHeaderHash).orElse(null); - return process(callParams, OperationTracer.NO_TRACING, header); + final CallParameter callParams, + final TransactionValidationParams transactionValidationParams, + final OperationTracer operationTracer, + final long blockNumber) { + final BlockHeader header = blockchain.getBlockHeader(blockNumber).orElse(null); + return process(callParams, transactionValidationParams, operationTracer, header); } public Optional process( - final CallParameter callParams, final long blockNumber) { - return process(callParams, OperationTracer.NO_TRACING, blockNumber); + final CallParameter callParams, final Hash blockHeaderHash) { + final BlockHeader header = blockchain.getBlockHeader(blockHeaderHash).orElse(null); + return process( + callParams, + TransactionValidationParams.transactionSimulator(), + OperationTracer.NO_TRACING, + header); } public Optional process( - final CallParameter callParams, - final OperationTracer operationTracer, - final long blockNumber) { - final BlockHeader header = blockchain.getBlockHeader(blockNumber).orElse(null); - return process(callParams, operationTracer, header); + final CallParameter callParams, final long blockNumber) { + return process( + callParams, + TransactionValidationParams.transactionSimulator(), + OperationTracer.NO_TRACING, + blockNumber); } public Optional processAtHead(final CallParameter callParams) { - return process(callParams, OperationTracer.NO_TRACING, blockchain.getChainHeadHeader()); + return process( + callParams, + TransactionValidationParams.transactionSimulator(), + OperationTracer.NO_TRACING, + blockchain.getChainHeadHeader()); } private Optional process( final CallParameter callParams, + final TransactionValidationParams transactionValidationParams, final OperationTracer operationTracer, final BlockHeader header) { if (header == null) { @@ -112,6 +127,12 @@ private Optional process( final Wei value = callParams.getValue() != null ? callParams.getValue() : Wei.ZERO; final Bytes payload = callParams.getPayload() != null ? callParams.getPayload() : Bytes.EMPTY; + final WorldUpdater updater = worldState.updater(); + + if (transactionValidationParams.isAllowExceedingBalance()) { + updater.getOrCreate(senderAddress).getMutable().incrementBalance(Wei.of(Long.MAX_VALUE)); + } + final Transaction.Builder transactionBuilder = Transaction.builder() .nonce(nonce) @@ -134,13 +155,13 @@ private Optional process( final TransactionProcessingResult result = transactionProcessor.processTransaction( blockchain, - worldState.updater(), + updater, header, transaction, protocolSpec.getMiningBeneficiaryCalculator().calculateBeneficiary(header), new BlockHashLookup(header, blockchain), false, - TransactionValidationParams.transactionSimulator(), + transactionValidationParams, operationTracer); return Optional.of(new TransactionSimulatorResult(transaction, result)); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessorTest.java index 462dccc213f..772981c54eb 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionProcessorTest.java @@ -76,7 +76,7 @@ public void shouldCallTransactionValidatorWithExpectedTransactionValidationParam transactionValidationParamCaptor(); final TransactionValidationParams expectedValidationParams = - new TransactionValidationParams.Builder().build(); + ImmutableTransactionValidationParams.builder().build(); transactionProcessor.processTransaction( blockchain, @@ -86,7 +86,7 @@ public void shouldCallTransactionValidatorWithExpectedTransactionValidationParam Address.fromHexString("1"), blockHashLookup, false, - new TransactionValidationParams.Builder().build()); + ImmutableTransactionValidationParams.builder().build()); assertThat(txValidationParamCaptor.getValue()) .isEqualToComparingFieldByField(expectedValidationParams); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidatorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidatorTest.java index dc25d0e931c..c037859698f 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidatorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidatorTest.java @@ -222,7 +222,7 @@ public void shouldPropagateCorrectStateChangeParamToTransactionFilter() { validator.setTransactionFilter(transactionFilter); final TransactionValidationParams validationParams = - new TransactionValidationParams.Builder().checkOnchainPermissions(true).build(); + ImmutableTransactionValidationParams.builder().checkOnchainPermissions(true).build(); validator.validateForSender(basicTransaction, accountWithNonce(0), validationParams); @@ -240,7 +240,7 @@ public void shouldNotCheckAccountPermissionIfBothValidationParamsCheckPermission validator.setTransactionFilter(transactionFilter); final TransactionValidationParams validationParams = - new TransactionValidationParams.Builder() + ImmutableTransactionValidationParams.builder() .checkOnchainPermissions(false) .checkLocalPermissions(false) .build(); diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/TransactionValidationParamsTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/TransactionValidationParamsTest.java index e5d3d90358d..d00f105b4df 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/TransactionValidationParamsTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/mainnet/TransactionValidationParamsTest.java @@ -23,14 +23,14 @@ public class TransactionValidationParamsTest { @Test public void isAllowFutureNonce() { assertThat( - new TransactionValidationParams.Builder() - .allowFutureNonce(true) + ImmutableTransactionValidationParams.builder() + .isAllowFutureNonce(true) .build() .isAllowFutureNonce()) .isTrue(); assertThat( - new TransactionValidationParams.Builder() - .allowFutureNonce(false) + ImmutableTransactionValidationParams.builder() + .isAllowFutureNonce(false) .build() .isAllowFutureNonce()) .isFalse(); @@ -39,13 +39,13 @@ public void isAllowFutureNonce() { @Test public void checkOnchainPermissions() { assertThat( - new TransactionValidationParams.Builder() + ImmutableTransactionValidationParams.builder() .checkOnchainPermissions(true) .build() .checkOnchainPermissions()) .isTrue(); assertThat( - new TransactionValidationParams.Builder() + ImmutableTransactionValidationParams.builder() .checkOnchainPermissions(false) .build() .checkOnchainPermissions()) @@ -55,13 +55,13 @@ public void checkOnchainPermissions() { @Test public void checkLocalPermissions() { assertThat( - new TransactionValidationParams.Builder() + ImmutableTransactionValidationParams.builder() .checkLocalPermissions(true) .build() .checkLocalPermissions()) .isTrue(); assertThat( - new TransactionValidationParams.Builder() + ImmutableTransactionValidationParams.builder() .checkLocalPermissions(false) .build() .checkLocalPermissions()) diff --git a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java index b6cc7639e4d..9ebd9fc3f97 100644 --- a/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java +++ b/ethereum/core/src/test/java/org/hyperledger/besu/ethereum/transaction/TransactionSimulatorTest.java @@ -20,6 +20,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import org.hyperledger.besu.crypto.SECP256K1; @@ -27,15 +28,20 @@ import org.hyperledger.besu.ethereum.core.Account; import org.hyperledger.besu.ethereum.core.Address; import org.hyperledger.besu.ethereum.core.BlockHeader; +import org.hyperledger.besu.ethereum.core.EvmAccount; import org.hyperledger.besu.ethereum.core.Hash; +import org.hyperledger.besu.ethereum.core.MutableAccount; import org.hyperledger.besu.ethereum.core.MutableWorldState; import org.hyperledger.besu.ethereum.core.Transaction; import org.hyperledger.besu.ethereum.core.Wei; +import org.hyperledger.besu.ethereum.core.WorldUpdater; +import org.hyperledger.besu.ethereum.mainnet.ImmutableTransactionValidationParams; import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor; import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule; import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult; import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult.Status; +import org.hyperledger.besu.ethereum.vm.OperationTracer; import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive; import java.util.Optional; @@ -65,6 +71,7 @@ public class TransactionSimulatorTest { @Mock private Blockchain blockchain; @Mock private WorldStateArchive worldStateArchive; @Mock private MutableWorldState worldState; + @Mock private WorldUpdater worldUpdater; @Mock private ProtocolSchedule protocolSchedule; @Mock private ProtocolSpec protocolSpec; @Mock private MainnetTransactionProcessor transactionProcessor; @@ -112,6 +119,72 @@ public void shouldReturnSuccessfulResultWhenProcessingIsSuccessful() { verifyTransactionWasProcessed(expectedTransaction); } + @Test + public void shouldIncreaseBalanceAccountWhenExceedingBalanceAllowed() { + final CallParameter callParameter = legacyTransactionCallParameter(); + + mockBlockchainForBlockHeader(Hash.ZERO, 1L); + mockWorldStateForAccount(Hash.ZERO, callParameter.getFrom(), 1L); + + final Transaction expectedTransaction = + Transaction.builder() + .nonce(1L) + .gasPrice(callParameter.getGasPrice()) + .gasLimit(callParameter.getGasLimit()) + .to(callParameter.getTo()) + .sender(callParameter.getFrom()) + .value(callParameter.getValue()) + .payload(callParameter.getPayload()) + .signature(FAKE_SIGNATURE) + .build(); + + mockProcessorStatusForTransaction(1L, expectedTransaction, Status.SUCCESSFUL); + + final MutableAccount mutableAccount = + mockWorldUpdaterForAccount(Hash.ZERO, callParameter.getFrom()); + + transactionSimulator.process( + callParameter, + ImmutableTransactionValidationParams.builder().isAllowExceedingBalance(true).build(), + OperationTracer.NO_TRACING, + 1L); + + verify(mutableAccount).incrementBalance(Wei.of(Long.MAX_VALUE)); + } + + @Test + public void shouldNotIncreaseBalanceAccountWhenExceedingBalanceIsNotAllowed() { + final CallParameter callParameter = legacyTransactionCallParameter(); + + mockBlockchainForBlockHeader(Hash.ZERO, 1L); + mockWorldStateForAccount(Hash.ZERO, callParameter.getFrom(), 1L); + + final Transaction expectedTransaction = + Transaction.builder() + .nonce(1L) + .gasPrice(callParameter.getGasPrice()) + .gasLimit(callParameter.getGasLimit()) + .to(callParameter.getTo()) + .sender(callParameter.getFrom()) + .value(callParameter.getValue()) + .payload(callParameter.getPayload()) + .signature(FAKE_SIGNATURE) + .build(); + + mockProcessorStatusForTransaction(1L, expectedTransaction, Status.SUCCESSFUL); + + final MutableAccount mutableAccount = + mockWorldUpdaterForAccount(Hash.ZERO, callParameter.getFrom()); + + transactionSimulator.process( + callParameter, + ImmutableTransactionValidationParams.builder().isAllowExceedingBalance(false).build(), + OperationTracer.NO_TRACING, + 1L); + + verifyNoInteractions(mutableAccount); + } + @Test public void shouldUseDefaultValuesWhenMissingOptionalFields() { final CallParameter callParameter = legacyTransactionCallParameter(); @@ -345,6 +418,16 @@ private void mockWorldStateForAbsentAccount(final Hash stateRoot) { when(worldState.get(any())).thenReturn(null); } + private MutableAccount mockWorldUpdaterForAccount(final Hash stateRoot, final Address address) { + final EvmAccount account = mock(EvmAccount.class); + final MutableAccount mutableAccount = mock(MutableAccount.class); + when(worldStateArchive.getMutable(eq(stateRoot))).thenReturn(Optional.of(worldState)); + when(worldState.updater()).thenReturn(worldUpdater); + when(worldUpdater.getOrCreate(eq(address))).thenReturn(account); + when(account.getMutable()).thenReturn(mutableAccount); + return mutableAccount; + } + private void mockBlockchainForBlockHeader(final Hash stateRoot, final long blockNumber) { mockBlockchainForBlockHeader(stateRoot, blockNumber, Hash.ZERO); }