From fe892a45c49674e046c7f24270c83d4565907b47 Mon Sep 17 00:00:00 2001 From: AztecBot Date: Wed, 6 Mar 2024 22:37:10 +0000 Subject: [PATCH] feat: Integrated native ACVM (https://github.com/AztecProtocol/aztec-packages/pull/4903) This PR creates a cli entrypoint for the ACVM simulator, builds it and makes it available for use in e2e tests both locally and on CI. This native simulator is used to execute sequencer-side protocol circuits, in parallel where possible. --- .aztec-sync-commit | 2 +- Cargo.lock | 20 +++++ Cargo.toml | 4 +- aztec_macros/src/lib.rs | 8 +- bootstrap.sh | 22 ++++++ bootstrap_cache.sh | 13 +++ .../noirc_evaluator/src/brillig/brillig_ir.rs | 3 +- docs/scripts/codegen_nargo_reference.sh | 2 +- .../Nargo.toml | 5 ++ tooling/acvm_cli/Cargo.toml | 38 +++++++++ tooling/acvm_cli/src/cli/execute_cmd.rs | 79 +++++++++++++++++++ tooling/acvm_cli/src/cli/fs/inputs.rs | 54 +++++++++++++ tooling/acvm_cli/src/cli/fs/mod.rs | 2 + tooling/acvm_cli/src/cli/fs/witness.rs | 36 +++++++++ tooling/acvm_cli/src/cli/mod.rs | 41 ++++++++++ tooling/acvm_cli/src/errors.rs | 52 ++++++++++++ tooling/acvm_cli/src/main.rs | 36 +++++++++ 17 files changed, 412 insertions(+), 5 deletions(-) create mode 100755 bootstrap.sh create mode 100755 bootstrap_cache.sh create mode 100644 test_programs/noir_test_failure/should_fail_suite_with_one_failure/Nargo.toml create mode 100644 tooling/acvm_cli/Cargo.toml create mode 100644 tooling/acvm_cli/src/cli/execute_cmd.rs create mode 100644 tooling/acvm_cli/src/cli/fs/inputs.rs create mode 100644 tooling/acvm_cli/src/cli/fs/mod.rs create mode 100644 tooling/acvm_cli/src/cli/fs/witness.rs create mode 100644 tooling/acvm_cli/src/cli/mod.rs create mode 100644 tooling/acvm_cli/src/errors.rs create mode 100644 tooling/acvm_cli/src/main.rs diff --git a/.aztec-sync-commit b/.aztec-sync-commit index 6841c89b691..d9741c19d2c 100644 --- a/.aztec-sync-commit +++ b/.aztec-sync-commit @@ -1 +1 @@ -7ff9b71d8d87fc93ae7dbd8ba63f5176b0cd17be +3fd7025ab43e705cab4aa67ca057e54316a1715b diff --git a/Cargo.lock b/Cargo.lock index b2b6f8037bb..83ac3744274 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -66,6 +66,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "acvm_cli" +version = "0.40.0" +dependencies = [ + "acir", + "acvm", + "bn254_blackbox_solver", + "clap", + "color-eyre", + "const_format", + "nargo", + "paste", + "proptest", + "rand 0.8.5", + "thiserror", + "toml 0.7.6", + "tracing-appender", + "tracing-subscriber", +] + [[package]] name = "acvm_js" version = "0.40.0" diff --git a/Cargo.toml b/Cargo.toml index 7d5da7b00d0..572042f1a6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ members = [ "tooling/nargo_toml", "tooling/noirc_abi", "tooling/noirc_abi_wasm", + "tooling/acvm_cli", # ACVM "acvm-repo/acir_field", "acvm-repo/acir", @@ -36,7 +37,7 @@ members = [ "acvm-repo/blackbox_solver", "acvm-repo/bn254_blackbox_solver", ] -default-members = ["tooling/nargo_cli"] +default-members = ["tooling/nargo_cli", "tooling/acvm_cli"] resolver = "2" [workspace.package] @@ -78,6 +79,7 @@ noir_lsp = { path = "tooling/lsp" } noir_debugger = { path = "tooling/debugger" } noirc_abi = { path = "tooling/noirc_abi" } bb_abstraction_leaks = { path = "tooling/bb_abstraction_leaks" } +acvm_cli = { path = "tooling/acvm_cli" } # LSP async-lsp = { version = "0.1.0", default-features = false } diff --git a/aztec_macros/src/lib.rs b/aztec_macros/src/lib.rs index f9df3f10129..012995d6ed4 100644 --- a/aztec_macros/src/lib.rs +++ b/aztec_macros/src/lib.rs @@ -727,8 +727,14 @@ fn transform_function( /// Transform a function to work with AVM bytecode fn transform_vm_function( func: &mut NoirFunction, - _storage_defined: bool, + storage_defined: bool, ) -> Result<(), AztecMacroError> { + // Create access to storage + if storage_defined { + let storage = abstract_storage("public_vm", true); + func.def.body.0.insert(0, storage); + } + // Push Avm context creation to the beginning of the function let create_context = create_avm_context()?; func.def.body.0.insert(0, create_context); diff --git a/bootstrap.sh b/bootstrap.sh new file mode 100755 index 00000000000..54129c3d61a --- /dev/null +++ b/bootstrap.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -eu + +cd $(dirname "$0") + +CMD=${1:-} + +if [ -n "$CMD" ]; then + if [ "$CMD" = "clean" ]; then + git clean -fdx + exit 0 + else + echo "Unknown command: $CMD" + exit 1 + fi +fi + +# Attempt to just pull artefacts from CI and exit on success. +[ -n "${USE_CACHE:-}" ] && ./bootstrap_cache.sh && exit + +./scripts/bootstrap_native.sh +./scripts/bootstrap_packages.sh diff --git a/bootstrap_cache.sh b/bootstrap_cache.sh new file mode 100755 index 00000000000..1cec6c81d8e --- /dev/null +++ b/bootstrap_cache.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -eu + +cd "$(dirname "$0")" +source ../build-system/scripts/setup_env '' '' mainframe_$USER > /dev/null + +echo -e "\033[1mRetrieving noir packages from remote cache...\033[0m" +extract_repo noir-packages /usr/src/noir/packages ./ +echo -e "\033[1mRetrieving nargo from remote cache...\033[0m" +extract_repo noir /usr/src/noir/target/release ./target/ + +remove_old_images noir-packages +remove_old_images noir diff --git a/compiler/noirc_evaluator/src/brillig/brillig_ir.rs b/compiler/noirc_evaluator/src/brillig/brillig_ir.rs index f1a8f24ed03..662dc074d98 100644 --- a/compiler/noirc_evaluator/src/brillig/brillig_ir.rs +++ b/compiler/noirc_evaluator/src/brillig/brillig_ir.rs @@ -32,7 +32,7 @@ use num_bigint::BigUint; /// The Brillig VM does not apply a limit to the memory address space, /// As a convention, we take use 64 bits. This means that we assume that /// memory has 2^64 memory slots. -pub(crate) const BRILLIG_MEMORY_ADDRESSING_BIT_SIZE: u32 = 32; +pub(crate) const BRILLIG_MEMORY_ADDRESSING_BIT_SIZE: u32 = 64; // Registers reserved in runtime for special purposes. pub(crate) enum ReservedRegisters { @@ -562,6 +562,7 @@ impl BrilligContext { bit_size: u32, ) { self.debug_show.const_instruction(result, constant); + self.push_opcode(BrilligOpcode::Const { destination: result, value: constant, bit_size }); } diff --git a/docs/scripts/codegen_nargo_reference.sh b/docs/scripts/codegen_nargo_reference.sh index 4ff7d43d142..6a9fda9420b 100755 --- a/docs/scripts/codegen_nargo_reference.sh +++ b/docs/scripts/codegen_nargo_reference.sh @@ -30,4 +30,4 @@ sidebar_position: 0 --- " > $NARGO_REFERENCE -cargo run -F codegen-docs -- info >> $NARGO_REFERENCE +cargo run --bin nargo -F codegen-docs -- info >> $NARGO_REFERENCE diff --git a/test_programs/noir_test_failure/should_fail_suite_with_one_failure/Nargo.toml b/test_programs/noir_test_failure/should_fail_suite_with_one_failure/Nargo.toml new file mode 100644 index 00000000000..3d2cf2c6096 --- /dev/null +++ b/test_programs/noir_test_failure/should_fail_suite_with_one_failure/Nargo.toml @@ -0,0 +1,5 @@ +[package] +name = "should_fail_with_mismatch" +type = "bin" +authors = [""] +[dependencies] diff --git a/tooling/acvm_cli/Cargo.toml b/tooling/acvm_cli/Cargo.toml new file mode 100644 index 00000000000..72424405d36 --- /dev/null +++ b/tooling/acvm_cli/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "acvm_cli" +description = "The entrypoint for executing the ACVM" +# x-release-please-start-version +version = "0.40.0" +# x-release-please-end +authors.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +# Rename binary from `acvm_cli` to `acvm` +[[bin]] +name = "acvm" +path = "src/main.rs" + +[dependencies] +thiserror.workspace = true +toml.workspace = true +color-eyre = "0.6.2" +clap.workspace = true +acvm.workspace = true +nargo.workspace = true +const_format.workspace = true +bn254_blackbox_solver.workspace = true +acir.workspace = true + +# Logs +tracing-subscriber.workspace = true +tracing-appender = "0.2.3" + +[dev-dependencies] +rand = "0.8.5" +proptest = "1.2.0" +paste = "1.0.14" diff --git a/tooling/acvm_cli/src/cli/execute_cmd.rs b/tooling/acvm_cli/src/cli/execute_cmd.rs new file mode 100644 index 00000000000..f6337c2eb35 --- /dev/null +++ b/tooling/acvm_cli/src/cli/execute_cmd.rs @@ -0,0 +1,79 @@ +use std::io::{self, Write}; + +use acir::circuit::Circuit; +use acir::native_types::WitnessMap; +use bn254_blackbox_solver::Bn254BlackBoxSolver; +use clap::Args; + +use crate::cli::fs::inputs::{read_bytecode_from_file, read_inputs_from_file}; +use crate::cli::fs::witness::save_witness_to_dir; +use crate::errors::CliError; +use nargo::ops::{execute_circuit, DefaultForeignCallExecutor}; + +use super::fs::witness::create_output_witness_string; + +/// Executes a circuit to calculate its return value +#[derive(Debug, Clone, Args)] +pub(crate) struct ExecuteCommand { + /// Write the execution witness to named file + #[clap(long, short)] + output_witness: Option, + + /// The name of the toml file which contains the input witness map + #[clap(long, short)] + input_witness: String, + + /// The name of the binary file containing circuit bytecode + #[clap(long, short)] + bytecode: String, + + /// The working directory + #[clap(long, short)] + working_directory: String, + + /// Set to print output witness to stdout + #[clap(long, short, action)] + print: bool, +} + +fn run_command(args: ExecuteCommand) -> Result { + let bytecode = read_bytecode_from_file(&args.working_directory, &args.bytecode)?; + let circuit_inputs = read_inputs_from_file(&args.working_directory, &args.input_witness)?; + let output_witness = execute_program_from_witness(&circuit_inputs, &bytecode, None)?; + let output_witness_string = create_output_witness_string(&output_witness)?; + if args.output_witness.is_some() { + save_witness_to_dir( + &output_witness_string, + &args.working_directory, + &args.output_witness.unwrap(), + )?; + } + Ok(output_witness_string) +} + +pub(crate) fn run(args: ExecuteCommand) -> Result { + let print = args.print; + let output_witness_string = run_command(args)?; + if print { + io::stdout().write_all(output_witness_string.as_bytes()).unwrap(); + } + Ok(output_witness_string) +} + +pub(crate) fn execute_program_from_witness( + inputs_map: &WitnessMap, + bytecode: &Vec, + foreign_call_resolver_url: Option<&str>, +) -> Result { + let blackbox_solver = Bn254BlackBoxSolver::new(); + let circuit: Circuit = Circuit::deserialize_circuit(&bytecode) + .map_err(|_| CliError::CircuitDeserializationError())?; + let result = execute_circuit( + &circuit, + inputs_map.clone(), + &blackbox_solver, + &mut DefaultForeignCallExecutor::new(true, foreign_call_resolver_url), + ) + .map_err(|e| CliError::CircuitExecutionError(e)); + result +} diff --git a/tooling/acvm_cli/src/cli/fs/inputs.rs b/tooling/acvm_cli/src/cli/fs/inputs.rs new file mode 100644 index 00000000000..2a46cfba884 --- /dev/null +++ b/tooling/acvm_cli/src/cli/fs/inputs.rs @@ -0,0 +1,54 @@ +use acir::{ + native_types::{Witness, WitnessMap}, + FieldElement, +}; +use toml::Table; + +use crate::errors::{CliError, FilesystemError}; +use std::{fs::read, path::Path}; + +/// Returns the circuit's parameters parsed from a toml file at the given location +pub(crate) fn read_inputs_from_file>( + working_directory: P, + file_name: &String, +) -> Result { + let file_path = working_directory.as_ref().join(file_name); + if !file_path.exists() { + return Err(CliError::FilesystemError(FilesystemError::MissingTomlFile( + file_name.to_owned(), + file_path, + ))); + } + + let input_string = std::fs::read_to_string(file_path) + .map_err(|_| FilesystemError::InvalidTomlFile(file_name.clone()))?; + let input_map = input_string + .parse::() + .map_err(|_| FilesystemError::InvalidTomlFile(file_name.clone()))?; + let mut witnesses: WitnessMap = WitnessMap::new(); + for (key, value) in input_map.into_iter() { + let index = + Witness(key.trim().parse().map_err(|_| CliError::WitnessIndexError(key.clone()))?); + if !value.is_str() { + return Err(CliError::WitnessValueError(key.clone())); + } + let field = FieldElement::from_hex(value.as_str().unwrap()).unwrap(); + witnesses.insert(index, field); + } + + Ok(witnesses) +} + +/// Returns the circuit's bytecode read from the file at the given location +pub(crate) fn read_bytecode_from_file>( + working_directory: P, + file_name: &String, +) -> Result, FilesystemError> { + let file_path = working_directory.as_ref().join(file_name); + if !file_path.exists() { + return Err(FilesystemError::MissingBytecodeFile(file_name.to_owned(), file_path)); + } + let bytecode: Vec = + read(file_path).map_err(|_| FilesystemError::InvalidBytecodeFile(file_name.clone()))?; + Ok(bytecode) +} diff --git a/tooling/acvm_cli/src/cli/fs/mod.rs b/tooling/acvm_cli/src/cli/fs/mod.rs new file mode 100644 index 00000000000..f23ba06fd8b --- /dev/null +++ b/tooling/acvm_cli/src/cli/fs/mod.rs @@ -0,0 +1,2 @@ +pub(super) mod inputs; +pub(super) mod witness; diff --git a/tooling/acvm_cli/src/cli/fs/witness.rs b/tooling/acvm_cli/src/cli/fs/witness.rs new file mode 100644 index 00000000000..2daaa5a3a58 --- /dev/null +++ b/tooling/acvm_cli/src/cli/fs/witness.rs @@ -0,0 +1,36 @@ +use std::{ + collections::BTreeMap, + fs::File, + io::Write, + path::{Path, PathBuf}, +}; + +use acvm::acir::native_types::WitnessMap; + +use crate::errors::{CliError, FilesystemError}; + +/// Saves the provided output witnesses to a toml file created at the given location +pub(crate) fn save_witness_to_dir>( + output_witness: &String, + witness_dir: P, + file_name: &String, +) -> Result { + let witness_path = witness_dir.as_ref().join(file_name); + + let mut file = File::create(&witness_path) + .map_err(|_| FilesystemError::OutputWitnessCreationFailed(file_name.clone()))?; + write!(file, "{}", output_witness) + .map_err(|_| FilesystemError::OutputWitnessWriteFailed(file_name.clone()))?; + + Ok(witness_path) +} + +/// Creates a toml representation of the provided witness map +pub(crate) fn create_output_witness_string(witnesses: &WitnessMap) -> Result { + let mut witness_map: BTreeMap = BTreeMap::new(); + for (key, value) in witnesses.clone().into_iter() { + witness_map.insert(key.0.to_string(), format!("0x{}", value.to_hex())); + } + + toml::to_string(&witness_map).map_err(|_| CliError::OutputWitnessSerializationFailed()) +} diff --git a/tooling/acvm_cli/src/cli/mod.rs b/tooling/acvm_cli/src/cli/mod.rs new file mode 100644 index 00000000000..a610b08ab77 --- /dev/null +++ b/tooling/acvm_cli/src/cli/mod.rs @@ -0,0 +1,41 @@ +use clap::{Parser, Subcommand}; +use color_eyre::eyre; +use const_format::formatcp; + +mod execute_cmd; +mod fs; + +const ACVM_VERSION: &str = env!("CARGO_PKG_VERSION"); + +static VERSION_STRING: &str = formatcp!("version = {}\n", ACVM_VERSION,); + +#[derive(Parser, Debug)] +#[command(name="acvm", author, version=VERSION_STRING, about, long_about = None)] +struct ACVMCli { + #[command(subcommand)] + command: ACVMCommand, +} + +#[non_exhaustive] +#[derive(Subcommand, Clone, Debug)] +enum ACVMCommand { + Execute(execute_cmd::ExecuteCommand), +} + +#[cfg(not(feature = "codegen-docs"))] +pub(crate) fn start_cli() -> eyre::Result<()> { + let ACVMCli { command } = ACVMCli::parse(); + + match command { + ACVMCommand::Execute(args) => execute_cmd::run(args), + }?; + + Ok(()) +} + +#[cfg(feature = "codegen-docs")] +pub(crate) fn start_cli() -> eyre::Result<()> { + let markdown: String = clap_markdown::help_markdown::(); + println!("{markdown}"); + Ok(()) +} diff --git a/tooling/acvm_cli/src/errors.rs b/tooling/acvm_cli/src/errors.rs new file mode 100644 index 00000000000..035388d05f7 --- /dev/null +++ b/tooling/acvm_cli/src/errors.rs @@ -0,0 +1,52 @@ +use nargo::NargoError; +use std::path::PathBuf; +use thiserror::Error; + +#[derive(Debug, Error)] +pub(crate) enum FilesystemError { + #[error( + " Error: cannot find {0} in expected location {1:?}.\n Please generate this file at the expected location." + )] + MissingTomlFile(String, PathBuf), + #[error(" Error: failed to parse toml file {0}.")] + InvalidTomlFile(String), + #[error( + " Error: cannot find {0} in expected location {1:?}.\n Please generate this file at the expected location." + )] + MissingBytecodeFile(String, PathBuf), + + #[error(" Error: failed to read bytecode file {0}.")] + InvalidBytecodeFile(String), + + #[error(" Error: failed to create output witness file {0}.")] + OutputWitnessCreationFailed(String), + + #[error(" Error: failed to write output witness file {0}.")] + OutputWitnessWriteFailed(String), +} + +#[derive(Debug, Error)] +pub(crate) enum CliError { + /// Filesystem errors + #[error(transparent)] + FilesystemError(#[from] FilesystemError), + + /// Error related to circuit deserialization + #[error("Error: failed to deserialize circuit")] + CircuitDeserializationError(), + + /// Error related to circuit execution + #[error(transparent)] + CircuitExecutionError(#[from] NargoError), + + /// Input Witness Value Error + #[error("Error: failed to parse witness value {0}")] + WitnessValueError(String), + + /// Input Witness Index Error + #[error("Error: failed to parse witness index {0}")] + WitnessIndexError(String), + + #[error(" Error: failed to serialize output witness.")] + OutputWitnessSerializationFailed(), +} diff --git a/tooling/acvm_cli/src/main.rs b/tooling/acvm_cli/src/main.rs new file mode 100644 index 00000000000..33cadc73a7c --- /dev/null +++ b/tooling/acvm_cli/src/main.rs @@ -0,0 +1,36 @@ +#![forbid(unsafe_code)] +#![warn(unreachable_pub)] +#![warn(clippy::semicolon_if_nothing_returned)] +#![cfg_attr(not(test), warn(unused_crate_dependencies, unused_extern_crates))] + +mod cli; +mod errors; + +use std::env; + +use tracing_appender::rolling; +use tracing_subscriber::{fmt::format::FmtSpan, EnvFilter}; + +fn main() { + // Setup tracing + if let Ok(log_dir) = env::var("ACVM_LOG_DIR") { + let debug_file = rolling::daily(log_dir, "acvm-log"); + tracing_subscriber::fmt() + .with_span_events(FmtSpan::ACTIVE) + .with_writer(debug_file) + .with_ansi(false) + .with_env_filter(EnvFilter::from_default_env()) + .init(); + } else { + tracing_subscriber::fmt() + .with_span_events(FmtSpan::ACTIVE) + .with_ansi(true) + .with_env_filter(EnvFilter::from_env("NOIR_LOG")) + .init(); + } + + if let Err(report) = cli::start_cli() { + eprintln!("{report}"); + std::process::exit(1); + } +}