Skip to content

Commit

Permalink
4844: read blobs as raw bytes not as unit256 and fix blob versioned h…
Browse files Browse the repository at this point in the history 
…ashes validation (hyperledger#5071)

* Fix: read blobs as raw bytes not as unit256

Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net>

* Do not load a trusted setup more than once

Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net>

* Update evm/src/main/java/org/hyperledger/besu/evm/precompile/KZGPointEvalPrecompiledContract.java

Co-authored-by: Justin Florentine <justin+github@florentine.us>
Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net>

* Ignore tests that fails due to mixing c-kzg presets

Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net>

* Revert failing unit test

Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net>

---------

Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net>
Co-authored-by: Justin Florentine <justin+github@florentine.us>
  • Loading branch information
@fab-10 @jflo
fab-10 and jflo authored Feb 10, 2023
1 parent 031db82 commit cd67140
Show file tree
Hide file tree
Showing 6 changed files with 4,250 additions and 94 deletions.
8 changes: 1 addition & 7 deletions ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/Transaction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1369,13 +1369,7 @@ public BlobsWithCommitments(

public List<Bytes> getBlobs() {
return blobs.getElements().stream()
.map(
blob -> {
return blob.getElements().stream()
.map(sszuInt256Wrapper -> (Bytes) sszuInt256Wrapper.getData().toBytes())
.reduce(Bytes::concatenate)
.orElseThrow();
})
.map(TransactionNetworkPayload.Blob::getBytes)
.collect(Collectors.toList());
}

Expand Down
36 changes: 17 additions & 19 deletions .../main/java/org/hyperledger/besu/ethereum/core/encoding/ssz/TransactionNetworkPayload.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.bytes.Bytes32;
import org.apache.tuweni.ssz.SSZFixedSizeTypeList;
import org.apache.tuweni.ssz.SSZFixedSizeVector;
import org.apache.tuweni.ssz.SSZReadable;
import org.apache.tuweni.ssz.SSZReader;
import org.apache.tuweni.ssz.SSZVariableSizeTypeList;
Expand Down Expand Up @@ -335,8 +334,8 @@ public void setData(final Bytes data) {

public static class AccessTuple implements SSZReadable, SSZWritable {
Bytes address;
SSZFixedSizeTypeList<SSZUInt256Wrapper> storageKeys =
new SSZFixedSizeTypeList<>(ELEMENT_SIZE, SSZUInt256Wrapper::new);
SSZFixedSizeTypeList<SSZByte32Wrapper> storageKeys =
new SSZFixedSizeTypeList<>(ELEMENT_SIZE, SSZByte32Wrapper::new);

@Override
public boolean isFixed() {
Expand All @@ -359,7 +358,7 @@ public Address getAddress() {

public List<Bytes32> getStorageKeys() {
return storageKeys.getElements().stream()
.map(sszuInt256Wrapper -> sszuInt256Wrapper.getData().toBytes())
.map(sszByte32Wrapper -> sszByte32Wrapper.getData())
.collect(Collectors.toList());
}

Expand All @@ -375,9 +374,9 @@ public void setStorageKeys(final List<Bytes32> storageKeys) {
storageKeys.stream()
.map(
bytes32 -> {
SSZUInt256Wrapper sszuInt256Wrapper = new SSZUInt256Wrapper();
sszuInt256Wrapper.setData(UInt256.fromBytes(bytes32));
return sszuInt256Wrapper;
SSZByte32Wrapper sszByte32Wrapper = new SSZByte32Wrapper();
sszByte32Wrapper.setData(UInt256.fromBytes(bytes32));
return sszByte32Wrapper;
})
.collect(Collectors.toList()));
}
Expand Down Expand Up @@ -480,42 +479,41 @@ public Bytes getData() {
}

public static class Blob implements SSZReadable, SSZWritable {
SSZFixedSizeVector<SSZUInt256Wrapper> vector =
new SSZFixedSizeVector<>(FIELD_ELEMENTS_PER_BLOB, ELEMENT_SIZE, SSZUInt256Wrapper::new);
Bytes bytes;

@Override
public void populateFromReader(final SSZReader reader) {
vector.populateFromReader(reader);
bytes = reader.readFixedBytes(FIELD_ELEMENTS_PER_BLOB * ELEMENT_SIZE);
}

@Override
public void writeTo(final SSZWriter writer) {
vector.writeTo(writer);
writer.writeFixedBytes(bytes);
}

public List<SSZUInt256Wrapper> getElements() {
return vector.getElements();
public Bytes getBytes() {
return bytes;
}
}

public static class SSZUInt256Wrapper implements SSZReadable, SSZWritable {
UInt256 data;
public static class SSZByte32Wrapper implements SSZReadable, SSZWritable {
Bytes32 data;

@Override
public void populateFromReader(final SSZReader reader) {
data = reader.readUInt256();
data = Bytes32.wrap(reader.readFixedBytes(32));
}

@Override
public void writeTo(final SSZWriter writer) {
writer.writeUInt256(data);
writer.writeBytes(data);
}

public UInt256 getData() {
public Bytes32 getData() {
return data;
}

public void setData(final UInt256 data) {
public void setData(final Bytes32 data) {
this.data = data;
}
}
Expand Down
54 changes: 22 additions & 32 deletions ...core/src/main/java/org/hyperledger/besu/ethereum/mainnet/MainnetTransactionValidator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,8 +135,7 @@ public ValidationResult<TransactionInvalidReason> validate(
return signatureResult;
}

if (transaction.getType().equals(TransactionType.BLOB)
&& transaction.getBlobsWithCommitments().isPresent()) {
if (transaction.getType().supportsBlob() && transaction.getBlobsWithCommitments().isPresent()) {
final ValidationResult<TransactionInvalidReason> blobsResult =
validateTransactionsBlobs(transaction);
if (!blobsResult.isValid()) {
Expand Down Expand Up @@ -334,13 +333,7 @@ public ValidationResult<TransactionInvalidReason> validateTransactionSignature(
public ValidationResult<TransactionInvalidReason> validateTransactionsBlobs(
final Transaction transaction) {

if (transaction.getBlobsWithCommitments().isEmpty()) {
return ValidationResult.invalid(
TransactionInvalidReason.INVALID_BLOBS,
"transaction blobs are empty, cannot verify without blobs");
}

Transaction.BlobsWithCommitments blobsWithCommitments =
final Transaction.BlobsWithCommitments blobsWithCommitments =
transaction.getBlobsWithCommitments().get();

final long blobsLimit = gasLimitCalculator.currentDataGasLimit() / gasCalculator.dataGasCost(1);
Expand All @@ -360,57 +353,54 @@ public ValidationResult<TransactionInvalidReason> validateTransactionsBlobs(
"transaction blobs and commitments are not the same size");
}

List<TransactionNetworkPayload.KZGCommitment> commitments =
blobsWithCommitments.kzgCommitments.getElements();
for (TransactionNetworkPayload.KZGCommitment commitment : commitments) {
if (commitment.getData().get(0) != BLOB_COMMITMENT_VERSION_KZG) {
return ValidationResult.invalid(
TransactionInvalidReason.INVALID_BLOBS,
"transaction blobs commitment version is not supported");
}
}
if (transaction.getVersionedHashes().isEmpty()) {
return ValidationResult.invalid(
TransactionInvalidReason.INVALID_BLOBS,
"transaction versioned hashes are empty, cannot verify without versioned hashes");
}
List<Hash> versionedHashes = transaction.getVersionedHashes().get();
final List<Hash> versionedHashes = transaction.getVersionedHashes().get();

for (int i = 0; i < versionedHashes.size(); i++) {
TransactionNetworkPayload.KZGCommitment commitment =
final TransactionNetworkPayload.KZGCommitment commitment =
blobsWithCommitments.kzgCommitments.getElements().get(i);
Hash versionedHash = versionedHashes.get(i);
Hash calculatedVersionedHash = hashCommitment(commitment);
final Hash versionedHash = versionedHashes.get(i);

if (versionedHash.get(0) != BLOB_COMMITMENT_VERSION_KZG) {
return ValidationResult.invalid(
TransactionInvalidReason.INVALID_BLOBS,
"transaction blobs commitment version is not supported. Expected "
+ BLOB_COMMITMENT_VERSION_KZG
+ ", found "
+ versionedHash.get(0));
}

final Hash calculatedVersionedHash = hashCommitment(commitment);
if (!calculatedVersionedHash.equals(versionedHash)) {
return ValidationResult.invalid(
TransactionInvalidReason.INVALID_BLOBS,
"transaction blobs commitment hash does not match commitment");
}
}

Bytes blobs =
final Bytes blobs =
blobsWithCommitments.blobs.getElements().stream()
.map(
blob ->
blob.getElements().stream()
.map(sszuInt256Wrapper -> (Bytes) sszuInt256Wrapper.getData().toBytes())
.reduce(Bytes::concatenate)
.orElseThrow())
.map(TransactionNetworkPayload.Blob::getBytes)
.reduce(Bytes::concatenate)
.orElseThrow();

Bytes kzgCommitments =
final Bytes kzgCommitments =
blobsWithCommitments.kzgCommitments.getElements().stream()
.map(commitment -> commitment.getData())
.reduce(Bytes::concatenate)
.orElseThrow();

boolean kzgVerification =
final boolean kzgVerification =
CKZG4844JNI.verifyAggregateKzgProof(
blobs.toArrayUnsafe(),
kzgCommitments.toArrayUnsafe(),
blobsWithCommitments.blobs.getElements().size(),
blobsWithCommitments.kzgProof.getBytes().toArrayUnsafe());

if (!kzgVerification) {
return ValidationResult.invalid(
TransactionInvalidReason.INVALID_BLOBS,
Expand All @@ -421,7 +411,7 @@ public ValidationResult<TransactionInvalidReason> validateTransactionsBlobs(
}

private Hash hashCommitment(final TransactionNetworkPayload.KZGCommitment commitment) {
SHA256Digest digest = new SHA256Digest();
final SHA256Digest digest = new SHA256Digest();
digest.update(commitment.getData().toArrayUnsafe(), 0, commitment.getData().size());

final byte[] dig = new byte[digest.getDigestSize()];
Expand Down
77 changes: 41 additions & 36 deletions evm/src/main/java/org/hyperledger/besu/evm/precompile/KZGPointEvalPrecompiledContract.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;

import com.google.common.annotations.VisibleForTesting;
import ethereum.ckzg4844.CKZG4844JNI;
Expand All @@ -38,6 +39,8 @@
/** The KZGPointEval precompile contract. */
public class KZGPointEvalPrecompiledContract implements PrecompiledContract {

private static final AtomicBoolean trustedSetupLoaded = new AtomicBoolean(false);

/** Instantiates a new KZGPointEval precompile contract. */
public KZGPointEvalPrecompiledContract() {
this(Optional.empty());
Expand All @@ -49,53 +52,55 @@ public KZGPointEvalPrecompiledContract() {
* @param pathToTrustedSetup the trusted setup path
*/
public KZGPointEvalPrecompiledContract(final Optional<Path> pathToTrustedSetup) {

String absolutePathToSetup;
CKZG4844JNI.Preset bitLength;
if (pathToTrustedSetup.isPresent()) {
Path pathToSetup = pathToTrustedSetup.get();
absolutePathToSetup = pathToSetup.toAbsolutePath().toString();
} else {
InputStream is =
KZGPointEvalPrecompiledContract.class.getResourceAsStream(
"mainnet_kzg_trusted_setup_4096.txt");
try {
File jniWillLoadFrom = File.createTempFile("kzgTrustedSetup", "txt");
jniWillLoadFrom.deleteOnExit();
Files.copy(is, jniWillLoadFrom.toPath(), REPLACE_EXISTING);
is.close();
absolutePathToSetup = jniWillLoadFrom.getAbsolutePath();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
try (BufferedReader setupFile =
Files.newBufferedReader(Paths.get(absolutePathToSetup), Charset.defaultCharset())) {
String firstLine = setupFile.readLine();
if ("4".equals(firstLine)) {
bitLength = CKZG4844JNI.Preset.MINIMAL;
} else if ("4096".equals(firstLine)) {
bitLength = CKZG4844JNI.Preset.MAINNET;
if (trustedSetupLoaded.compareAndSet(false, true)) {
String absolutePathToSetup;
CKZG4844JNI.Preset bitLength;
if (pathToTrustedSetup.isPresent()) {
Path pathToSetup = pathToTrustedSetup.get();
absolutePathToSetup = pathToSetup.toAbsolutePath().toString();
} else {
throw new IllegalArgumentException("provided file not a setup for either 4 or 4096 bits");
InputStream is =
KZGPointEvalPrecompiledContract.class.getResourceAsStream(
"mainnet_kzg_trusted_setup_4096.txt");
try {
File jniWillLoadFrom = File.createTempFile("kzgTrustedSetup", ".txt");
jniWillLoadFrom.deleteOnExit();
Files.copy(is, jniWillLoadFrom.toPath(), REPLACE_EXISTING);
is.close();
absolutePathToSetup = jniWillLoadFrom.getAbsolutePath();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
CKZG4844JNI.loadNativeLibrary(bitLength);
try {
CKZG4844JNI.loadTrustedSetup(absolutePathToSetup);
} catch (RuntimeException mightBeAlreadyLoaded) {
if (!mightBeAlreadyLoaded.getMessage().contains("Trusted Setup is already loaded")) {
throw mightBeAlreadyLoaded;
try (BufferedReader setupFile =
Files.newBufferedReader(Paths.get(absolutePathToSetup), Charset.defaultCharset())) {
String firstLine = setupFile.readLine();
if ("4".equals(firstLine)) {
bitLength = CKZG4844JNI.Preset.MINIMAL;
} else if ("4096".equals(firstLine)) {
bitLength = CKZG4844JNI.Preset.MAINNET;
} else {
throw new IllegalArgumentException("provided file not a setup for either 4 or 4096 bits");
}
CKZG4844JNI.loadNativeLibrary(bitLength);
try {
CKZG4844JNI.loadTrustedSetup(absolutePathToSetup);
} catch (RuntimeException mightBeAlreadyLoaded) {
if (!mightBeAlreadyLoaded.getMessage().contains("Trusted Setup is already loaded")) {
throw mightBeAlreadyLoaded;
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}

/** free up resources. */
@VisibleForTesting
public void tearDown() {
CKZG4844JNI.freeTrustedSetup();
trustedSetupLoaded.set(false);
}

@Override
Expand Down
Loading

0 comments on commit cd67140

Please sign in to comment.