Rust API for EIP4844 - Ethereum KZG commitment (#320)

* KZG parallel: swap context and threadpool argument order for more natural method call syntax

* reduce stdlib surface being compiled in

* KZG go API now with parallelism

* generate rust bindings

* KZG parallel: fix parallel golang tests messing up with the threadpool thread-local storage

* ethereum KZG: expose Rust API

* kzg-rust: properly deserialize test vectors

* kzg-rust: detailed output for cargo test -- --nocapture

* kzg-rust: passing more tests

* kzg-rust: passing all the serial tests

* KZG: add inputs lengths error even if not used in C (will be used in Rust and Nim

* kzg-rust: qualified enum for windows

* kzg-rust: add tests for parallel API

.github/workflows/ci.yml

@@ -327,14 +327,14 @@ jobs:
         # We need to deactivate the assembly (used by default for benches)
         run: |
           cd constantine
-          cargo test --no-default-features halo2curves --features halo2curves/multicore
+          cargo test -- --nocapture
       - name: Run Constantine as Rust library tests (NO Assembly)
         if: matrix.target.BACKEND == 'NO_ASM' && matrix.target.cpu != 'i386'
         shell: bash
         # We need to deactivate the assembly (used by default for benches)
         run: |
           cd constantine
-          CTT_ASM=0 cargo test --no-default-features halo2curves --features halo2curves/multicore
+          CTT_ASM=0 cargo test -- --nocapture
 
       - name: Run Constantine in-depth tests (Unix - with GMP, with Assembly)
         if: runner.os != 'Windows' && matrix.target.BACKEND == 'ASM'

Cargo.toml

@@ -2,7 +2,8 @@
 resolver = "2"
 members = [
     "constantine-rust/constantine-sys",
-    "constantine-rust/constantine-zal-halo2kzg",
+    "constantine-rust/constantine-halo2-zal",
+    "constantine-rust/constantine-ethereum-kzg",
 ]
 
 # If Nim static library is compiled with Clang ThinLTO, enable it on Rust side

benchmarks/bench_ethereum_eip4844_kzg.nim

@@ -53,7 +53,6 @@ proc randomize(rng: var RngState, blob: var Blob) =
 proc new(T: type BenchSet, ctx: ptr EthereumKZGContext): T =
   new(result)
   for i in 0 ..< result.N:
-    let t {.noInit.} = rng.random_unsafe(Fr[BLS12_381])
     rng.randomize(result.blobs[i])
     discard ctx.blob_to_kzg_commitment(result.commitments[i], result.blobs[i].addr)
     discard ctx.compute_blob_kzg_proof(result.proofs[i], result.blobs[i].addr, result.commitments[i])
@@ -80,9 +79,11 @@ proc benchBlobToKzgCommitment(b: BenchSet, ctx: ptr EthereumKZGContext, iters: i
   block:
     bench("blob_to_kzg_commitment", $tp.numThreads & " threads", iters):
       var commitment {.noInit.}: array[48, byte]
-      doAssert cttEthKzg_Success == tp.blob_to_kzg_commitment_parallel(ctx, commitment, b.blobs[0].addr)
+      doAssert cttEthKzg_Success == ctx.blob_to_kzg_commitment_parallel(tp, commitment, b.blobs[0].addr)
   let stopParallel = getMonotime()
 
+  tp.shutdown()
+
   let perfSerial = inNanoseconds((stopSerial-startSerial) div iters)
   let perfParallel = inNanoseconds((stopParallel-startParallel) div iters)
 
@@ -107,9 +108,11 @@ proc benchComputeKzgProof(b: BenchSet, ctx: ptr EthereumKZGContext, iters: int)
     bench("compute_kzg_proof", $tp.numThreads & " threads", iters):
       var proof {.noInit.}: array[48, byte]
       var eval_at_challenge {.noInit.}: array[32, byte]
-      doAssert cttEthKzg_Success == tp.compute_kzg_proof_parallel(ctx, proof, eval_at_challenge, b.blobs[0].addr, b.challenge)
+      doAssert cttEthKzg_Success == ctx.compute_kzg_proof_parallel(tp, proof, eval_at_challenge, b.blobs[0].addr, b.challenge)
   let stopParallel = getMonotime()
 
+  tp.shutdown()
+
   let perfSerial = inNanoseconds((stopSerial-startSerial) div iters)
   let perfParallel = inNanoseconds((stopParallel-startParallel) div iters)
 
@@ -132,9 +135,11 @@ proc benchComputeBlobKzgProof(b: BenchSet, ctx: ptr EthereumKZGContext, iters: i
   block:
     bench("compute_blob_kzg_proof", $tp.numThreads & " threads", iters):
       var proof {.noInit.}: array[48, byte]
-      doAssert cttEthKzg_Success == tp.compute_blob_kzg_proof_parallel(ctx, proof, b.blobs[0].addr, b.commitments[0])
+      doAssert cttEthKzg_Success == ctx.compute_blob_kzg_proof_parallel(tp, proof, b.blobs[0].addr, b.commitments[0])
   let stopParallel = getMonotime()
 
+  tp.shutdown()
+
   let perfSerial = inNanoseconds((stopSerial-startSerial) div iters)
   let perfParallel = inNanoseconds((stopParallel-startParallel) div iters)
 
@@ -162,9 +167,11 @@ proc benchVerifyBlobKzgProof(b: BenchSet, ctx: ptr EthereumKZGContext, iters: in
   let startParallel = getMonotime()
   block:
     bench("verify_blob_kzg_proof", $tp.numThreads & " threads", iters):
-      discard tp.verify_blob_kzg_proof_parallel(ctx, b.blobs[0].addr, b.commitments[0], b.proofs[0])
+      discard ctx.verify_blob_kzg_proof_parallel(tp, b.blobs[0].addr, b.commitments[0], b.proofs[0])
   let stopParallel = getMonotime()
 
+  tp.shutdown()
+
   let perfSerial = inNanoseconds((stopSerial-startSerial) div iters)
   let perfParallel = inNanoseconds((stopParallel-startParallel) div iters)
 
@@ -198,15 +205,17 @@ proc benchVerifyBlobKzgProofBatch(b: BenchSet, ctx: ptr EthereumKZGContext, iter
     let startParallel = getMonotime()
     block:
       bench("verify_blob_kzg_proof (batch " & $i & ')', $tp.numThreads & " threads", iters):
-        discard tp.verify_blob_kzg_proof_batch_parallel(
-                  ctx,
+        discard ctx.verify_blob_kzg_proof_batch_parallel(
+                  tp,
                   b.blobs.asUnchecked(),
                   b.commitments.asUnchecked(),
                   b.proofs.asUnchecked(),
                   i,
                   secureRandomBytes)
     let stopParallel = getMonotime()
 
+    tp.shutdown()
+
     let perfSerial = inNanoseconds((stopSerial-startSerial) div iters)
     let perfParallel = inNanoseconds((stopParallel-startParallel) div iters)
 

bindings/lib_constantine.nim

@@ -23,6 +23,7 @@ import
   ../constantine/ethereum_bls_signatures,
   ../constantine/trusted_setups/ethereum_kzg_srs,
   ../constantine/ethereum_eip4844_kzg,
+  ../constantine/ethereum_eip4844_kzg_parallel,
 
   # Ensure globals like proc from kernel32.dll are populated at library load time
   ./lib_autoload

constantine-go/constantine.go

@@ -178,3 +178,91 @@ func (ctx EthKzgContext) VerifyBlobKzgProofBatch(blobs []EthBlob, commitments []
 	}
 	return true, nil
 }
+
+// Ethereum EIP-4844 KZG API - Parallel
+// -----------------------------------------------------
+
+
+func (ctx EthKzgContext) BlobToKZGCommitmentParallel(tp Threadpool, blob EthBlob) (commitment EthKzgCommitment, err error) {
+	status := C.ctt_eth_kzg_blob_to_kzg_commitment_parallel(
+		ctx.cCtx, tp.ctx,
+		(*C.ctt_eth_kzg_commitment)(unsafe.Pointer(&commitment)),
+		(*C.ctt_eth_kzg_blob)(unsafe.Pointer(&blob)),
+	)
+	if status != C.cttEthKzg_Success {
+		err = errors.New(
+			C.GoString(C.ctt_eth_kzg_status_to_string(status)),
+		)
+	}
+	return commitment, err
+}
+
+func (ctx EthKzgContext) ComputeKzgProofParallel(tp Threadpool, blob EthBlob, z EthKzgChallenge) (proof EthKzgProof, y EthKzgEvalAtChallenge, err error) {
+	status := C.ctt_eth_kzg_compute_kzg_proof_parallel(
+		ctx.cCtx, tp.ctx,
+		(*C.ctt_eth_kzg_proof)(unsafe.Pointer(&proof)),
+		(*C.ctt_eth_kzg_eval_at_challenge)(unsafe.Pointer(&y)),
+		(*C.ctt_eth_kzg_blob)(unsafe.Pointer(&blob)),
+		(*C.ctt_eth_kzg_challenge)(unsafe.Pointer(&z)),
+	)
+	if status != C.cttEthKzg_Success {
+		err = errors.New(
+			C.GoString(C.ctt_eth_kzg_status_to_string(status)),
+		)
+	}
+	return proof, y, err
+}
+
+func (ctx EthKzgContext) ComputeBlobKzgProofParallel(tp Threadpool, blob EthBlob, commitment EthKzgCommitment) (proof EthKzgProof, err error) {
+	status := C.ctt_eth_kzg_compute_blob_kzg_proof_parallel(
+		ctx.cCtx, tp.ctx,
+		(*C.ctt_eth_kzg_proof)(unsafe.Pointer(&proof)),
+		(*C.ctt_eth_kzg_blob)(unsafe.Pointer(&blob)),
+		(*C.ctt_eth_kzg_commitment)(unsafe.Pointer(&commitment)),
+	)
+	if status != C.cttEthKzg_Success {
+		err = errors.New(
+			C.GoString(C.ctt_eth_kzg_status_to_string(status)),
+		)
+	}
+	return proof, err
+}
+
+func (ctx EthKzgContext) VerifyBlobKzgProofParallel(tp Threadpool, blob EthBlob, commitment EthKzgCommitment, proof EthKzgProof) (bool, error) {
+	status := C.ctt_eth_kzg_verify_blob_kzg_proof_parallel(
+		ctx.cCtx, tp.ctx,
+		(*C.ctt_eth_kzg_blob)(unsafe.Pointer(&blob)),
+		(*C.ctt_eth_kzg_commitment)(unsafe.Pointer(&commitment)),
+		(*C.ctt_eth_kzg_proof)(unsafe.Pointer(&proof)),
+	)
+	if status != C.cttEthKzg_Success {
+		err := errors.New(
+			C.GoString(C.ctt_eth_kzg_status_to_string(status)),
+		)
+		return false, err
+	}
+	return true, nil
+}
+
+func (ctx EthKzgContext) VerifyBlobKzgProofBatchParallel(tp Threadpool, blobs []EthBlob, commitments []EthKzgCommitment, proofs []EthKzgProof, secureRandomBytes [32]byte) (bool, error) {
+
+	if len(blobs) != len(commitments) || len(blobs) != len(proofs) {
+		return false, errors.New("VerifyBlobKzgProofBatch: Lengths of inputs do not match.")
+	}
+
+	status := C.ctt_eth_kzg_verify_blob_kzg_proof_batch_parallel(
+		ctx.cCtx, tp.ctx,
+		*(**C.ctt_eth_kzg_blob)(unsafe.Pointer(&blobs)),
+		*(**C.ctt_eth_kzg_commitment)(unsafe.Pointer(&commitments)),
+		*(**C.ctt_eth_kzg_proof)(unsafe.Pointer(&proofs)),
+		(C.size_t)(len(blobs)),
+		(*C.uint8_t)(unsafe.Pointer(&secureRandomBytes)),
+	)
+	if status != C.cttEthKzg_Success {
+		err := errors.New(
+			C.GoString(C.ctt_eth_kzg_status_to_string(status)),
+		)
+		return false, err
+	}
+	return true, nil
+}