feat!: Update to acvm 0.12.0 (#165)

* feat!: update to target acvm-84b5d18d * chore: update to use new black box solver interface * use patch syntax * update to latest changes * feat!: update to acvm with non-homogeneous bb calls (#169) * feat: update acvm * feat!: updated acvm to latest master * chore: update cargo toml * Update src/barretenberg_structures.rs Co-authored-by: Tom French <15848336+TomAFrench@users.noreply.github.com> --------- Co-authored-by: Tom French <15848336+TomAFrench@users.noreply.github.com> * fix bad rebase * feat!: Update backend to rely on WitnessMap (#152) Co-authored-by: Tom French <tom@tomfren.ch> * update acvm * feat!: Implement CommonReferenceString trait (#139) * chore!: Remove filesystem access and utilize async fetch with range requests Co-authored-by: Tom French <tom@tomfren.ch> * chore: Switch to tokio test macro for async function (#191) * chore: Remove `sled` & `tempfile` dev-dependencies (#190) * chore: Remove sled & tempfile dev-dependencies * chore: space out functions --------- Co-authored-by: Tom French <tom@tomfren.ch> * chore: bump to crates.io release * Update src/crs.rs * chore: Remove streaming CRS download & indicator (#194) * Update src/acvm_interop/common_reference_string.rs * code review --------- Co-authored-by: Blaine Bublitz <blaine.bublitz@gmail.com> Co-authored-by: Álvaro Rodríguez <sirasistant@gmail.com>
noir-lang · May 19, 2023 · d613c79 · d613c79
1 parent 146a842
commit d613c79
Show file tree

Hide file tree

Showing 15 changed files with 917 additions and 842 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -10,19 +10,13 @@ license = "MIT OR Apache-2.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-acvm = { version = "0.11.0", features = ["bn254"] }
+acvm = { version = "0.12.0", features = ["bn254"] }
+bincode = "1.3.3"
+bytesize = "1.2"
+reqwest = { version = "0.11.16", default-features = false, features = ["rustls-tls"] }
+serde = { version = "1.0.136", features = ["derive"] }
 thiserror = "1.0.21"
 
-blake2 = "0.9.1"
-dirs = { version = "3.0", optional = true }
-reqwest = { version = "0.11.16", optional = true, default-features = false, features = [
-    "stream",
-    "rustls-tls",
-] }
-tokio = { version = "1.0", optional = true }
-futures-util = { version = "0.3.14", optional = true }
-indicatif = { version = "0.17.3", optional = true }
-
 # Native
 barretenberg-sys = { version = "0.1.2", optional = true }
 
@@ -38,15 +32,14 @@ getrandom = { version = "0.2", optional = true }
 [build-dependencies]
 pkg-config = "0.3"
 
+[dev-dependencies]
+blake2 = "0.10.6"
+tokio = { version = "1.0", features = [ "macros" ] }
+
 [features]
 default = ["native"]
 native = [
-    "dep:barretenberg-sys",
-    "dep:reqwest",
-    "dep:tokio",
-    "dep:futures-util",
-    "dep:dirs",
-    "dep:indicatif",
+    "dep:barretenberg-sys"
 ]
 wasm = [
     "wasmer",
@@ -56,11 +49,11 @@ wasm = [
     "wasmer/cranelift",
     "wasmer/default-compiler",
     "wasmer/default-cranelift",
-    "wasmer/default-universal",
-    "dep:reqwest",
-    "dep:tokio",
-    "dep:futures-util",
-    "dep:dirs",
-    "dep:indicatif",
+    "wasmer/default-universal"
+]
+js = [
+    "wasmer",
+    "dep:rust-embed",
+    "dep:getrandom",
+    "wasmer/js-default"
 ]
-js = ["wasmer", "dep:rust-embed", "dep:getrandom", "wasmer/js-default"]
diff --git a/cspell.json b/cspell.json
@@ -55,6 +55,7 @@
         "envrc",
         "subshell",
         "thiserror",
+        "bincode",
         // In Solidity
         //
         "addmod",
@@ -72,7 +73,7 @@
         "acir",
         "acvm",
         "barretenberg",
-        "indicatif",
+        "bytesize",
         "wasmer",
         "getrandom"
     ]

diff --git a/flake.nix b/flake.nix
@@ -75,10 +75,6 @@
         # Note: Setting this allows for consistent behavior across build and shells, but is mostly
         # hidden from the developer - i.e. when they see the command being run via `nix flake check`
         RUST_TEST_THREADS = "1";
-
-        # We provide `barretenberg-transcript00` from the overlay to the build.
-        # This is necessary because the Nix sandbox is read-only and downloading during tests would fail
-        BARRETENBERG_TRANSCRIPT = pkgs.barretenberg-transcript00;
       };
 
       nativeEnvironment = sharedEnvironment // {
@@ -153,6 +149,27 @@
         buildInputs = [ ] ++ extraBuildInputs;
       };
 
+      # The `port` is parameterized to support parallel test runs without colliding static servers
+      networkTestArgs = port: {
+        # We provide `barretenberg-transcript00` from the overlay to the tests as a URL hosted via a static server
+        # This is necessary because the Nix sandbox has no network access and downloading during tests would fail
+        TRANSCRIPT_URL = "http://0.0.0.0:${toString port}/${builtins.baseNameOf pkgs.barretenberg-transcript00}";
+
+        # This copies the `barretenberg-transcript00` from the Nix store into this sandbox
+        # which avoids exposing the entire Nix store to the static server it starts
+        # The static server is moved to the background and killed after checks are completed
+        preCheck = ''
+          cp ${pkgs.barretenberg-transcript00} .
+          echo "Starting simple static server"
+          ${pkgs.simple-http-server}/bin/simple-http-server --port ${toString port} --silent &
+          HTTP_SERVER_PID=$!
+        '';
+
+        postCheck = ''
+          kill $HTTP_SERVER_PID
+        '';
+      };
+
       # Build *just* the cargo dependencies, so we can reuse all of that work between runs
       native-cargo-artifacts = craneLib.buildDepsOnly nativeArgs;
       wasm-cargo-artifacts = craneLib.buildDepsOnly wasmArgs;
@@ -167,25 +184,37 @@
     rec {
       checks = {
         cargo-clippy-native = craneLib.cargoClippy (nativeArgs // {
+          # Crane appends "clippy"
+          pname = "native";
+
           cargoArtifacts = native-cargo-artifacts;
 
           cargoClippyExtraArgs = "--all-targets -- -D warnings";
         });
 
-        cargo-test-native = craneLib.cargoTest (nativeArgs // {
+        cargo-test-native = craneLib.cargoTest (nativeArgs // (networkTestArgs 8000) // {
+          # Crane appends "test"
+          pname = "native";
+
           cargoArtifacts = native-cargo-artifacts;
 
           # It's unclear why doCheck needs to be enabled for tests to run but not clippy
           doCheck = true;
         });
 
         cargo-clippy-wasm = craneLib.cargoClippy (wasmArgs // {
+          # Crane appends "clippy"
+          pname = "wasm";
+
           cargoArtifacts = wasm-cargo-artifacts;
 
           cargoClippyExtraArgs = "--all-targets -- -D warnings";
         });
 
-        cargo-test-wasm = craneLib.cargoTest (wasmArgs // {
+        cargo-test-wasm = craneLib.cargoTest (wasmArgs // (networkTestArgs 8001) // {
+          # Crane appends "test"
+          pname = "wasm";
+
           cargoArtifacts = wasm-cargo-artifacts;
 
           # It's unclear why doCheck needs to be enabled for tests to run but not clippy

diff --git a/src/acvm_interop/common_reference_string.rs b/src/acvm_interop/common_reference_string.rs
@@ -0,0 +1,34 @@
+use acvm::{acir::circuit::Circuit, async_trait, CommonReferenceString};
+
+use crate::{composer::Composer, BackendError, Barretenberg};
+
+// TODO(#185): Ensure CRS download works in JS
+#[async_trait]
+impl CommonReferenceString for Barretenberg {
+    type Error = BackendError;
+
+    async fn generate_common_reference_string(
+        &self,
+        circuit: &Circuit,
+    ) -> Result<Vec<u8>, Self::Error> {
+        let constraint_system = &circuit.try_into()?;
+        let common_reference_string = self.get_crs(constraint_system).await?.try_into()?;
+        // Separated to have nicer coercion on error types
+        Ok(common_reference_string)
+    }
+
+    async fn update_common_reference_string(
+        &self,
+        common_reference_string: Vec<u8>,
+        circuit: &Circuit,
+    ) -> Result<Vec<u8>, Self::Error> {
+        let mut crs = common_reference_string.try_into()?;
+        let constraint_system = &circuit.try_into()?;
+        let common_reference_string = self
+            .update_crs(&mut crs, constraint_system)
+            .await?
+            .try_into()?;
+        // Separated to have nicer coercion on error types
+        Ok(common_reference_string)
+    }
+}
diff --git a/src/acvm_interop/mod.rs b/src/acvm_interop/mod.rs
@@ -1,5 +1,6 @@
 use crate::Barretenberg;
 
+mod common_reference_string;
 mod proof_system;
 mod pwg;
 mod smart_contract;

diff --git a/src/acvm_interop/proof_system.rs b/src/acvm_interop/proof_system.rs
@@ -1,11 +1,9 @@
-use acvm::acir::{circuit::Circuit, native_types::Witness, BlackBoxFunc};
+use acvm::acir::circuit::Opcode;
+use acvm::acir::{circuit::Circuit, native_types::WitnessMap, BlackBoxFunc};
 use acvm::FieldElement;
 use acvm::{Language, ProofSystemCompiler};
-use std::collections::BTreeMap;
 
-use crate::barretenberg_structures::Assignments;
-use crate::composer::Composer;
-use crate::{BackendError, Barretenberg};
+use crate::{barretenberg_structures::Assignments, composer::Composer, BackendError, Barretenberg};
 
 impl ProofSystemCompiler for Barretenberg {
     type Error = BackendError;
@@ -21,58 +19,77 @@ impl ProofSystemCompiler for Barretenberg {
         )?)
     }
 
-    fn black_box_function_supported(&self, opcode: &BlackBoxFunc) -> bool {
+    fn supports_opcode(&self, opcode: &Opcode) -> bool {
         match opcode {
-            BlackBoxFunc::AND
-            | BlackBoxFunc::XOR
-            | BlackBoxFunc::RANGE
-            | BlackBoxFunc::SHA256
-            | BlackBoxFunc::Blake2s
-            | BlackBoxFunc::Keccak256
-            | BlackBoxFunc::ComputeMerkleRoot
-            | BlackBoxFunc::SchnorrVerify
-            | BlackBoxFunc::Pedersen
-            | BlackBoxFunc::HashToField128Security
-            | BlackBoxFunc::EcdsaSecp256k1
-            | BlackBoxFunc::FixedBaseScalarMul => true,
-
-            BlackBoxFunc::AES => false,
+            Opcode::Arithmetic(_) => true,
+            Opcode::Directive(_) => true,
+            Opcode::Block(_) => false,
+            Opcode::ROM(_) => true,
+            Opcode::RAM(_) => true,
+            Opcode::Oracle(_) => true,
+            Opcode::BlackBoxFuncCall(func) => match func.get_black_box_func() {
+                BlackBoxFunc::AND
+                | BlackBoxFunc::XOR
+                | BlackBoxFunc::RANGE
+                | BlackBoxFunc::SHA256
+                | BlackBoxFunc::Blake2s
+                | BlackBoxFunc::Keccak256
+                | BlackBoxFunc::ComputeMerkleRoot
+                | BlackBoxFunc::SchnorrVerify
+                | BlackBoxFunc::Pedersen
+                | BlackBoxFunc::HashToField128Security
+                | BlackBoxFunc::EcdsaSecp256k1
+                | BlackBoxFunc::FixedBaseScalarMul => true,
+
+                BlackBoxFunc::AES => false,
+            },
         }
     }
 
-    fn preprocess(&self, circuit: &Circuit) -> Result<(Vec<u8>, Vec<u8>), Self::Error> {
+    fn preprocess(
+        &self,
+        common_reference_string: &[u8],
+        circuit: &Circuit,
+    ) -> Result<(Vec<u8>, Vec<u8>), Self::Error> {
+        let crs = common_reference_string.try_into()?;
         let constraint_system = &circuit.try_into()?;
 
         let proving_key = self.compute_proving_key(constraint_system)?;
-        let verification_key = self.compute_verification_key(constraint_system, &proving_key)?;
+        let verification_key = self.compute_verification_key(&crs, &proving_key)?;
 
         Ok((proving_key, verification_key))
     }
 
     fn prove_with_pk(
         &self,
+        common_reference_string: &[u8],
         circuit: &Circuit,
-        witness_values: BTreeMap<Witness, FieldElement>,
+        witness_values: WitnessMap,
         proving_key: &[u8],
     ) -> Result<Vec<u8>, Self::Error> {
+        let crs = common_reference_string.try_into()?;
         let assignments = flatten_witness_map(circuit, witness_values);
 
-        Ok(self.create_proof_with_pk(&circuit.try_into()?, assignments, proving_key)?)
+        Ok(self.create_proof_with_pk(&crs, &circuit.try_into()?, assignments, proving_key)?)
     }
 
     fn verify_with_vk(
         &self,
+        common_reference_string: &[u8],
         proof: &[u8],
-        public_inputs: BTreeMap<Witness, FieldElement>,
+        public_inputs: WitnessMap,
         circuit: &Circuit,
         verification_key: &[u8],
     ) -> Result<bool, Self::Error> {
+        let crs = common_reference_string.try_into()?;
         // Unlike when proving, we omit any unassigned witnesses.
         // Witness values should be ordered by their index but we skip over any indices without an assignment.
-        let flattened_public_inputs: Vec<FieldElement> = public_inputs.into_values().collect();
+        let flattened_public_inputs: Vec<FieldElement> =
+            public_inputs.into_iter().map(|(_, el)| el).collect();
 
         Ok(Composer::verify_with_vk(
             self,
+            &crs,
             &circuit.try_into()?,
             proof,
             flattened_public_inputs.into(),
@@ -82,10 +99,7 @@ impl ProofSystemCompiler for Barretenberg {
 }
 
 /// Flatten a witness map into a vector of witness assignments.
-fn flatten_witness_map(
-    circuit: &Circuit,
-    witness_values: BTreeMap<Witness, FieldElement>,
-) -> Assignments {
+fn flatten_witness_map(circuit: &Circuit, witness_values: WitnessMap) -> Assignments {
     let num_witnesses = circuit.num_vars();
 
     // Note: The witnesses are sorted via their witness index
@@ -94,10 +108,10 @@ fn flatten_witness_map(
         .map(|witness_index| {
             // Get the value if it exists. If i does not, then we fill it with the zero value
             witness_values
-                .get(&Witness(witness_index))
+                .get_index(witness_index)
                 .map_or(FieldElement::zero(), |field| *field)
         })
         .collect();
 
-    Assignments::from(witness_assignments)
+    witness_assignments.into()
 }