From 276ea453d5baff357040139dc08b75a34bfba6c5 Mon Sep 17 00:00:00 2001 From: Koby Date: Thu, 13 Jul 2023 19:16:07 +0200 Subject: [PATCH 01/10] feat: move to bincode and GzEncoding for artifacts --- Cargo.toml | 1 + acir/Cargo.toml | 3 +- acir/src/circuit/mod.rs | 30 +++++++++++++++++++- acir/src/native_types/witness_map.rs | 41 +++++++++++++++++++++++++--- 4 files changed, 69 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f45b9f362..569552d70 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ stdlib = { package = "acvm_stdlib", version = "0.18.1", path = "stdlib", default brillig = { version = "0.18.1", path = "brillig", default-features = false } blackbox_solver = { package = "acvm_blackbox_solver", version = "0.18.1", path = "blackbox_solver", default-features = false } +bincode = { version = "2.0.0-rc.3", features = ["serde"] } rmp-serde = "1.1.0" num-bigint = "0.4" diff --git a/acir/Cargo.toml b/acir/Cargo.toml index 7a66a449c..36beeef73 100644 --- a/acir/Cargo.toml +++ b/acir/Cargo.toml @@ -15,8 +15,9 @@ acir_field.workspace = true brillig.workspace = true serde.workspace = true thiserror.workspace = true -rmp-serde.workspace = true +rmp-serde = { workspace = true, optional = true } flate2 = "1.0.24" +bincode.workspace = true [dev-dependencies] serde_json = "1.0" diff --git a/acir/src/circuit/mod.rs b/acir/src/circuit/mod.rs index 22f140999..c4a5f84cb 100644 --- a/acir/src/circuit/mod.rs +++ b/acir/src/circuit/mod.rs @@ -6,7 +6,16 @@ pub mod opcodes; use crate::native_types::Witness; pub use opcodes::Opcode; -use flate2::{read::DeflateDecoder, write::DeflateEncoder, Compression}; + +#[cfg(feature="serialize-messagepack")] +use flate2::{read::DeflateDecoder, write::DeflateEncoder}; +use std::io::prelude::*; + +#[cfg(not(feature="serialize-messagepack"))] +use flate2::write::GzEncoder; +use flate2::Compression; + +// use flate2::{read::DeflateDecoder, write::DeflateEncoder, Compression}; use serde::{Deserialize, Serialize}; use std::collections::BTreeSet; use std::io::prelude::*; @@ -49,6 +58,7 @@ impl Circuit { PublicInputs(public_inputs) } + #[cfg(feature="serialize-messagepack")] pub fn write(&self, writer: W) -> std::io::Result<()> { let buf = rmp_serde::to_vec(&self).unwrap(); let mut deflater = DeflateEncoder::new(writer, Compression::best()); @@ -56,6 +66,7 @@ impl Circuit { Ok(()) } + #[cfg(feature="serialize-messagepack")] pub fn read(reader: R) -> std::io::Result { let mut deflater = DeflateDecoder::new(reader); let mut buf_d = Vec::new(); @@ -64,6 +75,23 @@ impl Circuit { Ok(circuit) } + #[cfg(not(feature="serialize-messagepack"))] + pub fn write(&self, writer: W) -> std::io::Result<()> { + let buf = bincode::serde::encode_to_vec(&self, bincode::config::standard()).unwrap(); + let mut encoder = GzEncoder::new(writer, Compression::default()); + encoder.write_all(&buf).unwrap(); + Ok(()) + } + + #[cfg(not(feature="serialize-messagepack"))] + pub fn read(reader: R) -> std::io::Result { + let mut gz_decoder = flate2::read::GzDecoder::new(reader); + let mut buf_d = Vec::new(); + gz_decoder.read_to_end(&mut buf_d).unwrap(); + let (circuit, _len): (Circuit, usize) = bincode::serde::decode_from_slice(buf_d.as_slice(), bincode::config::standard()).unwrap(); + Ok(circuit) + } + /// Initial list of labels attached to opcodes. pub fn initial_opcode_labels(&self) -> Vec { (0..self.opcodes.len()).map(|label| OpcodeLabel::Resolved(label as u64)).collect() diff --git a/acir/src/native_types/witness_map.rs b/acir/src/native_types/witness_map.rs index 4df598dbd..b8f1e6026 100644 --- a/acir/src/native_types/witness_map.rs +++ b/acir/src/native_types/witness_map.rs @@ -5,15 +5,15 @@ use std::{ }; use acir_field::FieldElement; -use flate2::{ - bufread::{DeflateDecoder, DeflateEncoder}, - Compression, -}; +use flate2::bufread::GzDecoder; +#[cfg(feature="serialize-messagepack")] +use flate2::{bufread::DeflateDecoder, write::DeflateEncoder, Compression}; use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::native_types::Witness; +#[cfg(feature="serialize-messagepack")] #[derive(Debug, Error)] enum SerializationError { #[error(transparent)] @@ -26,6 +26,13 @@ enum SerializationError { Deflate(#[from] std::io::Error), } +#[cfg(not(feature="serialize-messagepack"))] +#[derive(Debug, Error)] +enum SerializationError { + #[error(transparent)] + Deflate(#[from] std::io::Error), +} + #[derive(Debug, Error)] #[error(transparent)] pub struct WitnessMapError(#[from] SerializationError); @@ -85,6 +92,7 @@ impl From> for WitnessMap { } } +#[cfg(feature="serialize-messagepack")] impl TryFrom for Vec { type Error = WitnessMapError; @@ -97,6 +105,17 @@ impl TryFrom for Vec { } } +#[cfg(not(feature="serialize-messagepack"))] +impl TryFrom for Vec { + type Error = WitnessMapError; + + fn try_from(val: WitnessMap) -> Result { + let buf = bincode::serde::encode_to_vec(&val, bincode::config::standard()).unwrap(); + Ok(buf) + } +} + +#[cfg(feature="serialize-messagepack")] impl TryFrom<&[u8]> for WitnessMap { type Error = WitnessMapError; @@ -109,3 +128,17 @@ impl TryFrom<&[u8]> for WitnessMap { Ok(Self(witness_map)) } } + +#[cfg(not(feature="serialize-messagepack"))] +impl TryFrom<&[u8]> for WitnessMap { + type Error = WitnessMapError; + + fn try_from(bytes: &[u8]) -> Result { + let mut deflater = GzDecoder::new(bytes); + let mut buf_d = Vec::new(); + deflater.read_to_end(&mut buf_d).map_err(|err| WitnessMapError(err.into()))?; + let (witness_map, _len) = + bincode::serde::decode_from_slice(buf_d.as_slice(), bincode::config::standard()).unwrap(); + Ok(Self(witness_map)) + } +} From 864635873244a64a5f830d8f38bdf3c18a400c45 Mon Sep 17 00:00:00 2001 From: Koby Date: Fri, 14 Jul 2023 00:26:11 +0200 Subject: [PATCH 02/10] fix: ensure output done --- acir/src/circuit/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/acir/src/circuit/mod.rs b/acir/src/circuit/mod.rs index c4a5f84cb..8a9014e05 100644 --- a/acir/src/circuit/mod.rs +++ b/acir/src/circuit/mod.rs @@ -80,6 +80,7 @@ impl Circuit { let buf = bincode::serde::encode_to_vec(&self, bincode::config::standard()).unwrap(); let mut encoder = GzEncoder::new(writer, Compression::default()); encoder.write_all(&buf).unwrap(); + encoder.finish().unwrap(); Ok(()) } From 5a008bcf516337d84e7ae585cc6d1b3b4d8f2112 Mon Sep 17 00:00:00 2001 From: Koby Date: Fri, 14 Jul 2023 13:21:52 +0200 Subject: [PATCH 03/10] fix: witness GzEncoding --- acir/src/native_types/witness_map.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/acir/src/native_types/witness_map.rs b/acir/src/native_types/witness_map.rs index b8f1e6026..d35d045b2 100644 --- a/acir/src/native_types/witness_map.rs +++ b/acir/src/native_types/witness_map.rs @@ -111,7 +111,10 @@ impl TryFrom for Vec { fn try_from(val: WitnessMap) -> Result { let buf = bincode::serde::encode_to_vec(&val, bincode::config::standard()).unwrap(); - Ok(buf) + let mut deflater = GzEncoder::new(buf.as_slice(), Compression::best()); + let mut buf_c = Vec::new(); + deflater.read_to_end(&mut buf_c).map_err(|err| WitnessMapError(err.into()))?; + Ok(buf_c) } } From efc7e686bf67a6874341664aa59f76bf531f1e94 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 14 Jul 2023 11:41:48 +0000 Subject: [PATCH 04/10] Fix bincode encoding. --- Cargo.toml | 2 +- acir/src/circuit/mod.rs | 4 ++-- acir/src/native_types/witness_map.rs | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 569552d70..932ddb50b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ stdlib = { package = "acvm_stdlib", version = "0.18.1", path = "stdlib", default brillig = { version = "0.18.1", path = "brillig", default-features = false } blackbox_solver = { package = "acvm_blackbox_solver", version = "0.18.1", path = "blackbox_solver", default-features = false } -bincode = { version = "2.0.0-rc.3", features = ["serde"] } +bincode = "*" rmp-serde = "1.1.0" num-bigint = "0.4" diff --git a/acir/src/circuit/mod.rs b/acir/src/circuit/mod.rs index 8a9014e05..ca215b1a2 100644 --- a/acir/src/circuit/mod.rs +++ b/acir/src/circuit/mod.rs @@ -77,7 +77,7 @@ impl Circuit { #[cfg(not(feature="serialize-messagepack"))] pub fn write(&self, writer: W) -> std::io::Result<()> { - let buf = bincode::serde::encode_to_vec(&self, bincode::config::standard()).unwrap(); + let buf = bincode::serialize(&self).unwrap(); let mut encoder = GzEncoder::new(writer, Compression::default()); encoder.write_all(&buf).unwrap(); encoder.finish().unwrap(); @@ -89,7 +89,7 @@ impl Circuit { let mut gz_decoder = flate2::read::GzDecoder::new(reader); let mut buf_d = Vec::new(); gz_decoder.read_to_end(&mut buf_d).unwrap(); - let (circuit, _len): (Circuit, usize) = bincode::serde::decode_from_slice(buf_d.as_slice(), bincode::config::standard()).unwrap(); + let (circuit, _len): (Circuit, usize) = bincode::deserialize(buf_d.as_slice()).unwrap(); Ok(circuit) } diff --git a/acir/src/native_types/witness_map.rs b/acir/src/native_types/witness_map.rs index b8f1e6026..0c186592b 100644 --- a/acir/src/native_types/witness_map.rs +++ b/acir/src/native_types/witness_map.rs @@ -110,7 +110,7 @@ impl TryFrom for Vec { type Error = WitnessMapError; fn try_from(val: WitnessMap) -> Result { - let buf = bincode::serde::encode_to_vec(&val, bincode::config::standard()).unwrap(); + let buf = bincode::serialize(&val).unwrap(); Ok(buf) } } @@ -137,8 +137,8 @@ impl TryFrom<&[u8]> for WitnessMap { let mut deflater = GzDecoder::new(bytes); let mut buf_d = Vec::new(); deflater.read_to_end(&mut buf_d).map_err(|err| WitnessMapError(err.into()))?; - let (witness_map, _len) = - bincode::serde::decode_from_slice(buf_d.as_slice(), bincode::config::standard()).unwrap(); + let witness_map = + bincode::deserialize(buf_d.as_slice()).unwrap(); Ok(Self(witness_map)) } } From f4e2d1b4d98319f67ac1bfd8eb32044359884ab6 Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Fri, 14 Jul 2023 12:06:16 +0000 Subject: [PATCH 05/10] fix. --- acir/src/native_types/witness_map.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/acir/src/native_types/witness_map.rs b/acir/src/native_types/witness_map.rs index a9fe15fbf..080cac731 100644 --- a/acir/src/native_types/witness_map.rs +++ b/acir/src/native_types/witness_map.rs @@ -6,6 +6,8 @@ use std::{ use acir_field::FieldElement; use flate2::bufread::GzDecoder; +use flate2::bufread::GzEncoder; +use flate2::Compression; #[cfg(feature="serialize-messagepack")] use flate2::{bufread::DeflateDecoder, write::DeflateEncoder, Compression}; use serde::{Deserialize, Serialize}; From f6306fc5aa3ec53026cec0766593a382d6a50d79 Mon Sep 17 00:00:00 2001 From: Koby Date: Fri, 14 Jul 2023 17:39:01 +0200 Subject: [PATCH 06/10] chore: simpler use alignment --- acir/src/circuit/mod.rs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/acir/src/circuit/mod.rs b/acir/src/circuit/mod.rs index ca215b1a2..f5dacebac 100644 --- a/acir/src/circuit/mod.rs +++ b/acir/src/circuit/mod.rs @@ -6,19 +6,13 @@ pub mod opcodes; use crate::native_types::Witness; pub use opcodes::Opcode; - -#[cfg(feature="serialize-messagepack")] -use flate2::{read::DeflateDecoder, write::DeflateEncoder}; use std::io::prelude::*; -#[cfg(not(feature="serialize-messagepack"))] -use flate2::write::GzEncoder; use flate2::Compression; -// use flate2::{read::DeflateDecoder, write::DeflateEncoder, Compression}; use serde::{Deserialize, Serialize}; use std::collections::BTreeSet; -use std::io::prelude::*; + #[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Default)] pub struct Circuit { @@ -61,14 +55,14 @@ impl Circuit { #[cfg(feature="serialize-messagepack")] pub fn write(&self, writer: W) -> std::io::Result<()> { let buf = rmp_serde::to_vec(&self).unwrap(); - let mut deflater = DeflateEncoder::new(writer, Compression::best()); + let mut deflater = flate2::write::DeflateEncoder::new(writer, Compression::best()); deflater.write_all(&buf).unwrap(); Ok(()) } #[cfg(feature="serialize-messagepack")] pub fn read(reader: R) -> std::io::Result { - let mut deflater = DeflateDecoder::new(reader); + let mut deflater = flate2::read::DeflateDecoder::new(reader); let mut buf_d = Vec::new(); deflater.read_to_end(&mut buf_d).unwrap(); let circuit = rmp_serde::from_slice(buf_d.as_slice()).unwrap(); @@ -78,7 +72,7 @@ impl Circuit { #[cfg(not(feature="serialize-messagepack"))] pub fn write(&self, writer: W) -> std::io::Result<()> { let buf = bincode::serialize(&self).unwrap(); - let mut encoder = GzEncoder::new(writer, Compression::default()); + let mut encoder = flate2::write::GzEncoder::new(writer, Compression::default()); encoder.write_all(&buf).unwrap(); encoder.finish().unwrap(); Ok(()) From 1b50f3255c1eeb65098fb71e819956bfdef29560 Mon Sep 17 00:00:00 2001 From: Koby Date: Fri, 14 Jul 2023 17:43:19 +0200 Subject: [PATCH 07/10] chore: use specific bincode version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 932ddb50b..cc3efb165 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ stdlib = { package = "acvm_stdlib", version = "0.18.1", path = "stdlib", default brillig = { version = "0.18.1", path = "brillig", default-features = false } blackbox_solver = { package = "acvm_blackbox_solver", version = "0.18.1", path = "blackbox_solver", default-features = false } -bincode = "*" +bincode = "1.3.3" rmp-serde = "1.1.0" num-bigint = "0.4" From cac3bc5d541d0b3df7fcb3446eb6c425d7974449 Mon Sep 17 00:00:00 2001 From: Koby Date: Fri, 14 Jul 2023 17:57:17 +0200 Subject: [PATCH 08/10] chore: hide unused use --- acir/src/native_types/witness_map.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/acir/src/native_types/witness_map.rs b/acir/src/native_types/witness_map.rs index 080cac731..9e30fabc7 100644 --- a/acir/src/native_types/witness_map.rs +++ b/acir/src/native_types/witness_map.rs @@ -8,8 +8,6 @@ use acir_field::FieldElement; use flate2::bufread::GzDecoder; use flate2::bufread::GzEncoder; use flate2::Compression; -#[cfg(feature="serialize-messagepack")] -use flate2::{bufread::DeflateDecoder, write::DeflateEncoder, Compression}; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -97,10 +95,10 @@ impl From> for WitnessMap { #[cfg(feature="serialize-messagepack")] impl TryFrom for Vec { type Error = WitnessMapError; - + fn try_from(val: WitnessMap) -> Result { let buf = rmp_serde::to_vec(&val).map_err(|err| WitnessMapError(err.into()))?; - let mut deflater = DeflateEncoder::new(buf.as_slice(), Compression::best()); + let mut deflater = flate2::write::DeflateEncoder::new(buf.as_slice(), Compression::best()); let mut buf_c = Vec::new(); deflater.read_to_end(&mut buf_c).map_err(|err| WitnessMapError(err.into()))?; Ok(buf_c) @@ -123,9 +121,9 @@ impl TryFrom for Vec { #[cfg(feature="serialize-messagepack")] impl TryFrom<&[u8]> for WitnessMap { type Error = WitnessMapError; - + fn try_from(bytes: &[u8]) -> Result { - let mut deflater = DeflateDecoder::new(bytes); + let mut deflater = flate2::bufread::DeflateDecoder::new(bytes); let mut buf_d = Vec::new(); deflater.read_to_end(&mut buf_d).map_err(|err| WitnessMapError(err.into()))?; let witness_map = From 22245921b170000c4db328736a38817e03fedb86 Mon Sep 17 00:00:00 2001 From: Koby Date: Fri, 14 Jul 2023 20:08:52 +0200 Subject: [PATCH 09/10] fix: serialisation roudtrip --- acir/src/circuit/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acir/src/circuit/mod.rs b/acir/src/circuit/mod.rs index f5dacebac..10cf6da85 100644 --- a/acir/src/circuit/mod.rs +++ b/acir/src/circuit/mod.rs @@ -83,7 +83,7 @@ impl Circuit { let mut gz_decoder = flate2::read::GzDecoder::new(reader); let mut buf_d = Vec::new(); gz_decoder.read_to_end(&mut buf_d).unwrap(); - let (circuit, _len): (Circuit, usize) = bincode::deserialize(buf_d.as_slice()).unwrap(); + let circuit = bincode::deserialize(&buf_d).unwrap(); Ok(circuit) } From e40fb722ed5b0f613bca964d796e16de51677e37 Mon Sep 17 00:00:00 2001 From: Koby Date: Fri, 14 Jul 2023 21:46:10 +0200 Subject: [PATCH 10/10] fix: fmt --- acir/src/circuit/mod.rs | 9 ++++----- acir/src/native_types/witness_map.rs | 19 +++++++++---------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/acir/src/circuit/mod.rs b/acir/src/circuit/mod.rs index 10cf6da85..a9cd30b8f 100644 --- a/acir/src/circuit/mod.rs +++ b/acir/src/circuit/mod.rs @@ -13,7 +13,6 @@ use flate2::Compression; use serde::{Deserialize, Serialize}; use std::collections::BTreeSet; - #[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Default)] pub struct Circuit { // current_witness_index is the highest witness index in the circuit. The next witness to be added to this circuit @@ -52,7 +51,7 @@ impl Circuit { PublicInputs(public_inputs) } - #[cfg(feature="serialize-messagepack")] + #[cfg(feature = "serialize-messagepack")] pub fn write(&self, writer: W) -> std::io::Result<()> { let buf = rmp_serde::to_vec(&self).unwrap(); let mut deflater = flate2::write::DeflateEncoder::new(writer, Compression::best()); @@ -60,7 +59,7 @@ impl Circuit { Ok(()) } - #[cfg(feature="serialize-messagepack")] + #[cfg(feature = "serialize-messagepack")] pub fn read(reader: R) -> std::io::Result { let mut deflater = flate2::read::DeflateDecoder::new(reader); let mut buf_d = Vec::new(); @@ -69,7 +68,7 @@ impl Circuit { Ok(circuit) } - #[cfg(not(feature="serialize-messagepack"))] + #[cfg(not(feature = "serialize-messagepack"))] pub fn write(&self, writer: W) -> std::io::Result<()> { let buf = bincode::serialize(&self).unwrap(); let mut encoder = flate2::write::GzEncoder::new(writer, Compression::default()); @@ -78,7 +77,7 @@ impl Circuit { Ok(()) } - #[cfg(not(feature="serialize-messagepack"))] + #[cfg(not(feature = "serialize-messagepack"))] pub fn read(reader: R) -> std::io::Result { let mut gz_decoder = flate2::read::GzDecoder::new(reader); let mut buf_d = Vec::new(); diff --git a/acir/src/native_types/witness_map.rs b/acir/src/native_types/witness_map.rs index 9e30fabc7..400e0a8ca 100644 --- a/acir/src/native_types/witness_map.rs +++ b/acir/src/native_types/witness_map.rs @@ -13,7 +13,7 @@ use thiserror::Error; use crate::native_types::Witness; -#[cfg(feature="serialize-messagepack")] +#[cfg(feature = "serialize-messagepack")] #[derive(Debug, Error)] enum SerializationError { #[error(transparent)] @@ -26,7 +26,7 @@ enum SerializationError { Deflate(#[from] std::io::Error), } -#[cfg(not(feature="serialize-messagepack"))] +#[cfg(not(feature = "serialize-messagepack"))] #[derive(Debug, Error)] enum SerializationError { #[error(transparent)] @@ -92,10 +92,10 @@ impl From> for WitnessMap { } } -#[cfg(feature="serialize-messagepack")] +#[cfg(feature = "serialize-messagepack")] impl TryFrom for Vec { type Error = WitnessMapError; - + fn try_from(val: WitnessMap) -> Result { let buf = rmp_serde::to_vec(&val).map_err(|err| WitnessMapError(err.into()))?; let mut deflater = flate2::write::DeflateEncoder::new(buf.as_slice(), Compression::best()); @@ -105,7 +105,7 @@ impl TryFrom for Vec { } } -#[cfg(not(feature="serialize-messagepack"))] +#[cfg(not(feature = "serialize-messagepack"))] impl TryFrom for Vec { type Error = WitnessMapError; @@ -118,10 +118,10 @@ impl TryFrom for Vec { } } -#[cfg(feature="serialize-messagepack")] +#[cfg(feature = "serialize-messagepack")] impl TryFrom<&[u8]> for WitnessMap { type Error = WitnessMapError; - + fn try_from(bytes: &[u8]) -> Result { let mut deflater = flate2::bufread::DeflateDecoder::new(bytes); let mut buf_d = Vec::new(); @@ -132,7 +132,7 @@ impl TryFrom<&[u8]> for WitnessMap { } } -#[cfg(not(feature="serialize-messagepack"))] +#[cfg(not(feature = "serialize-messagepack"))] impl TryFrom<&[u8]> for WitnessMap { type Error = WitnessMapError; @@ -140,8 +140,7 @@ impl TryFrom<&[u8]> for WitnessMap { let mut deflater = GzDecoder::new(bytes); let mut buf_d = Vec::new(); deflater.read_to_end(&mut buf_d).map_err(|err| WitnessMapError(err.into()))?; - let witness_map = - bincode::deserialize(buf_d.as_slice()).unwrap(); + let witness_map = bincode::deserialize(buf_d.as_slice()).unwrap(); Ok(Self(witness_map)) } }