From 37f28858006119c059ec1957725c616c597504ba Mon Sep 17 00:00:00 2001 From: lonerapier Date: Thu, 12 Sep 2024 01:16:20 +0530 Subject: [PATCH 01/11] feat(cli): add extractor witness gen --- src/http.rs | 58 +++++++++++++++++++++++++----- src/main.rs | 4 +-- src/witness.rs | 98 +++++++++++++++++++++++++++++++++----------------- 3 files changed, 116 insertions(+), 44 deletions(-) diff --git a/src/http.rs b/src/http.rs index f0cbb65..36c033e 100644 --- a/src/http.rs +++ b/src/http.rs @@ -11,7 +11,7 @@ pub enum HttpData { Response(Response), } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Deserialize)] pub struct Request { method: String, target: String, @@ -21,7 +21,7 @@ pub struct Request { headers: HashMap, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Deserialize)] pub struct Response { version: String, status: String, @@ -49,6 +49,46 @@ impl std::fmt::Debug for HttpData { } } +impl Serialize for Request { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + use serde::ser::SerializeMap; + let mut map = serializer.serialize_map(Some(3 + self.headers.len() * 2))?; + + map.serialize_entry("method", self.method.as_bytes())?; + map.serialize_entry("target", self.target.as_bytes())?; + map.serialize_entry("version", self.version.as_bytes())?; + + for (i, (key, value)) in self.headers.iter().enumerate() { + map.serialize_entry(&format!("key{}", i + 1), key.as_bytes())?; + map.serialize_entry(&format!("value{}", i + 1), value.as_bytes())?; + } + map.end() + } +} + +impl Serialize for Response { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + use serde::ser::SerializeMap; + let mut map = serializer.serialize_map(Some(3 + self.headers.len() * 2))?; + + map.serialize_entry("version", self.version.as_bytes())?; + map.serialize_entry("status", self.status.as_bytes())?; + map.serialize_entry("message", self.message.as_bytes())?; + + for (i, (key, value)) in self.headers.iter().enumerate() { + map.serialize_entry(&format!("key{}", i + 1), key)?; + map.serialize_entry(&format!("value{}", i + 1), value)?; + } + map.end() + } +} + fn deserialize_headers<'de, D>(deserializer: D) -> Result, D::Error> where D: serde::Deserializer<'de>, @@ -267,7 +307,7 @@ fn build_http_circuit( methodIsEqual[data_idx] === 1; } - // Get the target bytes + // Get the target bytes startLineMask[data_idx] <== inStartLine()(State[data_idx].parsing_start); targetMask[data_idx] <== inStartMiddle()(State[data_idx].parsing_start); versionMask[data_idx] <== inStartEnd()(State[data_idx].parsing_start); @@ -286,7 +326,7 @@ fn build_http_circuit( versionIsEqual[data_idx] === 1; } - // Get the status bytes + // Get the status bytes startLineMask[data_idx] <== inStartLine()(State[data_idx].parsing_start); statusMask[data_idx] <== inStartMiddle()(State[data_idx].parsing_start); messageMask[data_idx] <== inStartEnd()(State[data_idx].parsing_start); @@ -346,7 +386,7 @@ fn build_http_circuit( // Get the output body bytes { if let HttpData::Response(_) = data { - circuit_buffer += r#" + circuit_buffer += r#" signal bodyStartingIndex[DATA_BYTES]; signal isZeroMask[DATA_BYTES]; signal isPrevStartingIndex[DATA_BYTES]; @@ -384,13 +424,13 @@ fn build_http_circuit( signal targetMatch <== SubstringMatchWithIndex(DATA_BYTES, targetLen)(data, target, 100, target_start_counter); targetMatch === 1; targetLen === target_end_counter - target_start_counter - 1; - + // Check version is correct by substring match and length check // TODO: change r signal versionMatch <== SubstringMatchWithIndex(DATA_BYTES, versionLen)(data, version, 100, target_end_counter); versionMatch === 1; // -2 here for the CRLF - versionLen === version_end_counter - target_end_counter - 2; + versionLen === version_end_counter - target_end_counter - 2; "#; } HttpData::Response(_) => { @@ -403,13 +443,13 @@ fn build_http_circuit( signal statusMatch <== SubstringMatchWithIndex(DATA_BYTES, statusLen)(data, status, 100, status_start_counter); statusMatch === 1; statusLen === status_end_counter - status_start_counter - 1; - + // Check message is correct by substring match and length check // TODO: change r signal messageMatch <== SubstringMatchWithIndex(DATA_BYTES, messageLen)(data, message, 100, status_end_counter); messageMatch === 1; // -2 here for the CRLF - messageLen === message_end_counter - status_end_counter - 2; + messageLen === message_end_counter - status_end_counter - 2; "#; } } diff --git a/src/main.rs b/src/main.rs index 5a1757d..3837a05 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,7 +35,7 @@ pub struct ParserWitnessArgs { output_dir: PathBuf, /// Output filename (will be created if it doesn't exist) - #[arg(global = true, long, default_value = "output.json")] + #[arg(global = true, long, default_value = "input.json")] output_filename: String, } @@ -57,7 +57,7 @@ pub struct ExtractorWitnessArgs { output_dir: PathBuf, /// Output filename (will be created if it doesn't exist) - #[arg(global = true, long, default_value = "output.json")] + #[arg(global = true, long, default_value = "input.json")] output_filename: String, } diff --git a/src/witness.rs b/src/witness.rs index 522e6c8..5a972ae 100644 --- a/src/witness.rs +++ b/src/witness.rs @@ -1,19 +1,30 @@ use json::JsonLockfile; +use super::http::HttpData; use super::*; use std::{collections::HashMap, io::Write}; #[derive(serde::Serialize)] -pub struct Witness { +pub struct ParserWitness { data: Vec, } #[derive(serde::Serialize)] -pub struct ExtractorWitness { +pub struct JsonExtractorWitness { data: Vec, + + #[serde(flatten)] keys: HashMap>, } +#[derive(serde::Serialize)] +pub struct HttpExtractorWitness { + data: Vec, + + #[serde(flatten)] + http_data: HttpData, +} + fn print_boxed_output(lines: Vec) { // Determine the maximum length of the lines let max_length = lines.iter().map(|line| line.len()).max().unwrap_or(0); @@ -30,11 +41,14 @@ fn print_boxed_output(lines: Vec) { println!("{}", bottom_border); } -pub fn parser_witness(args: ParserWitnessArgs) -> Result<(), Box> { - let data = match &args.subcommand { - WitnessSubcommand::Json => std::fs::read(args.input_file)?, +fn read_input_file_as_bytes( + file_type: WitnessSubcommand, + file_path: PathBuf, +) -> Result, Box> { + match file_type { + WitnessSubcommand::Json => Ok(std::fs::read(file_path)?), WitnessSubcommand::Http => { - let mut data = std::fs::read(args.input_file)?; + let mut data = std::fs::read(file_path)?; let mut i = 0; // convert LF to CRLF while i < data.len() { @@ -45,11 +59,15 @@ pub fn parser_witness(args: ParserWitnessArgs) -> Result<(), Box Result<(), Box> { + let data = read_input_file_as_bytes(args.subcommand, args.input_file)?; + + let witness = ParserWitness { data: data.clone() }; if !args.output_dir.exists() { std::fs::create_dir_all(&args.output_dir)?; @@ -69,35 +87,13 @@ pub fn parser_witness(args: ParserWitnessArgs) -> Result<(), Box Result, Box> { - match file_type { - WitnessSubcommand::Json => Ok(std::fs::read(file_path)?), - WitnessSubcommand::Http => { - let mut data = std::fs::read(file_path)?; - let mut i = 0; - // convert LF to CRLF - while i < data.len() { - if data[i] == 10 && (i == 0 || data[i - 1] != 13) { - data.insert(i, 13); - i += 2; - } else { - i += 1; - } - } - Ok(data) - } - } -} -pub fn extractor_witness(args: ExtractorWitnessArgs) -> Result<(), Box> { +fn json_extractor_witness(args: ExtractorWitnessArgs) -> Result<(), Box> { let input_data = read_input_file_as_bytes(args.subcommand, args.input_file)?; let lockfile_data = std::fs::read(&args.lockfile)?; let lockfile: JsonLockfile = serde_json::from_slice(&lockfile_data)?; - let witness = ExtractorWitness { + let witness = JsonExtractorWitness { data: input_data.clone(), keys: lockfile.as_bytes(), }; @@ -119,3 +115,39 @@ pub fn extractor_witness(args: ExtractorWitnessArgs) -> Result<(), Box Result<(), Box> { + let input_data = read_input_file_as_bytes(args.subcommand, args.input_file)?; + + let lockfile_data = std::fs::read(&args.lockfile)?; + let http_data: HttpData = serde_json::from_slice(&lockfile_data)?; + + let witness = HttpExtractorWitness { + data: input_data.clone(), + http_data, + }; + + if !args.output_dir.exists() { + std::fs::create_dir_all(&args.output_dir)?; + } + + let output_file = args.output_dir.join(args.output_filename); + let mut file = std::fs::File::create(output_file)?; + file.write_all(serde_json::to_string_pretty(&witness)?.as_bytes())?; + + // Prepare lines to print + let mut lines = Vec::new(); + lines.push(format!("Data length: {}", input_data.len())); + + // Print the output inside a nicely formatted box + print_boxed_output(lines); + + Ok(()) +} + +pub fn extractor_witness(args: ExtractorWitnessArgs) -> Result<(), Box> { + match args.subcommand { + WitnessSubcommand::Json => json_extractor_witness(args), + WitnessSubcommand::Http => http_extractor_witness(args), + } +} From df300d9dcfd7cf7db84ca2b7f7daf331ecb28b29 Mon Sep 17 00:00:00 2001 From: lonerapier Date: Thu, 12 Sep 2024 18:54:26 +0530 Subject: [PATCH 02/11] fix: select subarray param bug --- circuits/http/extractor.circom | 2 +- circuits/test/http/extractor.test.ts | 11 +---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/circuits/http/extractor.circom b/circuits/http/extractor.circom index 03ad0c2..c2b0925 100644 --- a/circuits/http/extractor.circom +++ b/circuits/http/extractor.circom @@ -78,7 +78,7 @@ template ExtractResponse(DATA_BYTES, maxContentLength) { valueStartingIndex[i] <== valueStartingIndex[i-1] + i * (1-isZeroMask[i]) * isPrevStartingIndex[i]; } - response <== SelectSubArray(DATA_BYTES, maxContentLength)(dataMask, valueStartingIndex[DATA_BYTES-1]+1, DATA_BYTES - valueStartingIndex[DATA_BYTES-1]); + response <== SelectSubArray(DATA_BYTES, maxContentLength)(dataMask, valueStartingIndex[DATA_BYTES-1]+1, maxContentLength); } template ExtractHeaderValue(DATA_BYTES, headerNameLength, maxValueLength) { diff --git a/circuits/test/http/extractor.test.ts b/circuits/test/http/extractor.test.ts index c69c7a7..6dbc0a4 100644 --- a/circuits/test/http/extractor.test.ts +++ b/circuits/test/http/extractor.test.ts @@ -31,7 +31,7 @@ describe("HTTP :: body Extractor", async () => { let output3 = parsedHttp.bodyBytes.slice(0); output3.pop(); - // output3.pop(); // TODO: fails due to shift subarray bug + output3.pop(); generatePassCase(parsedHttp.input, output3, "output length less than actual length"); }); @@ -75,15 +75,6 @@ describe("HTTP :: header Extractor", async () => { let parsedHttp = readHTTPInputFile("get_response.http"); generatePassCase(parsedHttp.input, toByte("Content-Length"), toByte(parsedHttp.headers["Content-Length"]), ""); - - // let output2 = parsedHttp.bodyBytes.slice(0); - // output2.push(0, 0, 0, 0); - // generatePassCase(parsedHttp.input, output2, "output length more than actual length"); - - // let output3 = parsedHttp.bodyBytes.slice(0); - // output3.pop(); - // // output3.pop(); // TODO: fails due to shift subarray bug - // generatePassCase(parsedHttp.input, output3, "output length less than actual length"); }); }); From e8749327cc3eea7219ff41e6874083d23d72e2f5 Mon Sep 17 00:00:00 2001 From: lonerapier Date: Thu, 12 Sep 2024 18:55:12 +0530 Subject: [PATCH 03/11] feat: add circuit config to `circuits.json`. Pending: http extraction --- examples/json/lockfile/value_string.json | 2 +- src/codegen.rs | 43 +++++++++ src/http.rs | 2 +- src/json.rs | 114 ++++++++++++++++++++--- src/main.rs | 68 ++++++-------- src/witness.rs | 43 ++++++--- 6 files changed, 200 insertions(+), 72 deletions(-) create mode 100644 src/codegen.rs diff --git a/examples/json/lockfile/value_string.json b/examples/json/lockfile/value_string.json index b25e4b5..c94b050 100644 --- a/examples/json/lockfile/value_string.json +++ b/examples/json/lockfile/value_string.json @@ -1,6 +1,6 @@ { "keys": [ - "a" + "k" ], "value_type": "string" } \ No newline at end of file diff --git a/src/codegen.rs b/src/codegen.rs new file mode 100644 index 0000000..584ca0e --- /dev/null +++ b/src/codegen.rs @@ -0,0 +1,43 @@ +use std::{collections::HashMap, env}; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CircomkitCircuitsInput { + pub file: String, + pub template: String, + pub params: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CircomkitCircuits(HashMap); + +pub fn write_circuit_config( + name: String, + circomkit_input: &CircomkitCircuitsInput, +) -> Result<(), Box> { + let mut circomkit_circuits_config = env::current_dir()?; + circomkit_circuits_config.push("circuits.json"); + + let _ = std::fs::File::create_new(circomkit_circuits_config.clone()); + + let mut circomkit_circuits: CircomkitCircuits = + serde_json::from_slice(&std::fs::read(circomkit_circuits_config.clone())?)?; + + if let Some(circuits_inputs) = circomkit_circuits.0.get_mut(&name) { + *circuits_inputs = circomkit_input.clone(); + } else { + let _ = circomkit_circuits + .0 + .insert(name.clone(), circomkit_input.clone()); + } + + std::fs::write( + circomkit_circuits_config, + serde_json::to_string_pretty(&circomkit_circuits)?, + )?; + + println!("circomkit circuits config updated for {}", name); + + Ok(()) +} diff --git a/src/http.rs b/src/http.rs index 36c033e..a3be5bc 100644 --- a/src/http.rs +++ b/src/http.rs @@ -482,7 +482,7 @@ fn build_http_circuit( } // TODO: This needs to codegen a circuit now. -pub fn http_circuit(args: HttpArgs) -> Result<(), Box> { +pub fn http_circuit(args: ExtractorArgs) -> Result<(), Box> { let data = std::fs::read(&args.lockfile)?; let http_data: HttpData = serde_json::from_slice(&data)?; diff --git a/src/json.rs b/src/json.rs index 7a21e31..3ab9bae 100644 --- a/src/json.rs +++ b/src/json.rs @@ -1,13 +1,13 @@ +use super::*; use std::{ + cmp::max_by, + collections::HashMap, fs::{self, create_dir_all}, str::FromStr, }; -use std::collections::HashMap; - -use super::*; - -const PRAGMA: &str = "pragma circom 2.1.9;\n\n"; +use codegen::{write_circuit_config, CircomkitCircuitsInput}; +use serde_json::Value; #[derive(Debug, Deserialize)] pub enum ValueType { @@ -21,7 +21,7 @@ pub enum ValueType { #[serde(untagged)] pub enum Key { String(String), - Num(i64), + Num(usize), } #[derive(Debug, Deserialize)] @@ -43,7 +43,7 @@ impl JsonLockfile { } } -fn extract_string(data: JsonLockfile, circuit_buffer: &mut String, debug: bool) { +fn extract_string(data: &JsonLockfile, circuit_buffer: &mut String, debug: bool) { *circuit_buffer += "template ExtractStringValue(DATA_BYTES, MAX_STACK_HEIGHT, "; for (i, key) in data.keys.iter().enumerate() { match key { @@ -108,7 +108,7 @@ fn extract_string(data: JsonLockfile, circuit_buffer: &mut String, debug: bool) "#; } -fn extract_number(data: JsonLockfile, circuit_buffer: &mut String, debug: bool) { +fn extract_number(data: &JsonLockfile, circuit_buffer: &mut String, debug: bool) { *circuit_buffer += "template ExtractNumValue(DATA_BYTES, MAX_STACK_HEIGHT, "; for (i, key) in data.keys.iter().enumerate() { match key { @@ -184,8 +184,8 @@ fn extract_number(data: JsonLockfile, circuit_buffer: &mut String, debug: bool) } fn build_json_circuit( - data: JsonLockfile, - output_filename: String, + data: &JsonLockfile, + output_filename: &String, debug: bool, ) -> Result<(), Box> { let mut circuit_buffer = String::new(); @@ -195,7 +195,7 @@ fn build_json_circuit( circuit_buffer += &format!("{:#?}", data); circuit_buffer += "\n*/\n"; - circuit_buffer += PRAGMA; + circuit_buffer += "pragma circom 2.1.9;\n\n"; circuit_buffer += "include \"../json/interpreter.circom\";\n\n"; // template ExtractValue(DATA_BYTES, MAX_STACK_HEIGHT, keyLen1, depth1, index2, depth2, keyLen3, depth3, index4, depth4, maxValueLen) { @@ -536,11 +536,95 @@ fn build_json_circuit( Ok(()) } -pub fn json_circuit(args: JsonArgs) -> Result<(), Box> { - let data = std::fs::read(&args.template)?; - let json_data: JsonLockfile = serde_json::from_slice(&data)?; +fn build_circuit_config( + args: &ExtractorArgs, + lockfile: &JsonLockfile, +) -> Result> { + let input = fs::read(args.input_file.clone())?; + + let circuit_template_name = match lockfile.value_type { + ValueType::String => String::from("ExtractStringValue"), + ValueType::Number => String::from("ExtractNumValue"), + }; + + let mut max_stack_height = 1; + let mut curr_stack_height = 1; + let mut inside_string: bool = false; + + for (i, char) in input[1..].iter().enumerate() { + match char { + b'"' if input[i - 1] != b'\\' => inside_string = !inside_string, + b'{' | b'[' if !inside_string => { + curr_stack_height += 1; + max_stack_height = max_by(max_stack_height, curr_stack_height, |x, y| x.cmp(y)); + } + b'}' | b']' if !inside_string => curr_stack_height -= 1, + _ => {} + } + } + + let mut params = vec![input.len(), max_stack_height]; + + let mut current_value: Value = serde_json::from_slice(&input)?; + for (i, key) in lockfile.keys.iter().enumerate() { + match key { + Key::String(key) => { + if let Some(value) = current_value.get_mut(key) { + // update circuit params + params.push(key.len()); + + // update current object value inside key + current_value = value.to_owned(); + } else { + return Err(String::from("provided key not present in input JSON").into()); + } + } + Key::Num(index) => { + if let Some(value) = current_value.get_mut(index) { + params.push(index.to_string().as_bytes().len()); + current_value = value.to_owned(); + } else { + return Err(String::from("provided index not present in input JSON").into()); + } + } + } + params.push(i); + } + + let value_bytes = match lockfile.value_type { + ValueType::Number => { + if !current_value.is_u64() { + return Err(String::from("value type doesn't match").into()); + } + + current_value.as_u64().unwrap().to_string() + } + ValueType::String => { + if !current_value.is_string() { + return Err(String::from("value type doesn't match").into()); + } + + current_value.as_str().unwrap().to_string() + } + }; + + params.push(value_bytes.as_bytes().len()); + + Ok(CircomkitCircuitsInput { + file: format!("main/{}", args.output_filename), + template: circuit_template_name, + params, + }) +} + +pub fn json_circuit(args: ExtractorArgs) -> Result<(), Box> { + let lockfile: JsonLockfile = serde_json::from_slice(&std::fs::read(&args.lockfile)?)?; + + build_json_circuit(&lockfile, &args.output_filename, args.debug)?; + + let circomkit_circuit_input = build_circuit_config(&args, &lockfile)?; - build_json_circuit(json_data, args.output_filename, args.debug)?; + write_circuit_config(args.circuit_name, &circomkit_circuit_input)?; Ok(()) } diff --git a/src/main.rs b/src/main.rs index 3837a05..4d38157 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,7 @@ use clap::{Parser, Subcommand}; use serde::{Deserialize, Serialize}; use std::{error::Error, path::PathBuf}; +pub mod codegen; pub mod http; pub mod json; pub mod witness; @@ -17,78 +18,63 @@ pub struct Args { pub enum Command { ParserWitness(ParserWitnessArgs), ExtractorWitness(ExtractorWitnessArgs), - Json(JsonArgs), - Http(HttpArgs), + Json(ExtractorArgs), + Http(ExtractorArgs), } #[derive(Parser, Debug)] pub struct ParserWitnessArgs { - #[arg(global = true, value_enum)] - subcommand: WitnessSubcommand, + #[arg(value_enum)] + subcommand: WitnessType, /// Path to the JSON file - #[arg(global = true, long)] + #[arg(long)] input_file: PathBuf, - /// Output directory (will be created if it doesn't exist) - #[arg(global = true, long, default_value = ".")] - output_dir: PathBuf, - - /// Output filename (will be created if it doesn't exist) - #[arg(global = true, long, default_value = "input.json")] - output_filename: String, + /// Name of the circuit (to be used in circomkit config) + #[arg(long)] + circuit_name: String, } #[derive(Parser, Debug)] pub struct ExtractorWitnessArgs { - #[arg(global = true, value_enum)] - subcommand: WitnessSubcommand, + #[arg(value_enum)] + subcommand: WitnessType, + + /// Name of the circuit (to be used in circomkit config) + #[arg(long)] + circuit_name: String, /// Path to the JSON file - #[arg(global = true, long)] + #[arg(long)] input_file: PathBuf, /// Path to the lockfile - #[arg(global = true, long)] + #[arg(long)] lockfile: PathBuf, - - /// Output directory (will be created if it doesn't exist) - #[arg(global = true, long, default_value = ".")] - output_dir: PathBuf, - - /// Output filename (will be created if it doesn't exist) - #[arg(global = true, long, default_value = "input.json")] - output_filename: String, } #[derive(clap::ValueEnum, Clone, Debug)] -pub enum WitnessSubcommand { +pub enum WitnessType { Json, Http, } #[derive(Parser, Debug)] -pub struct JsonArgs { - /// Path to the JSON file selective-disclosure template - #[arg(long, short)] - template: PathBuf, - - /// Output circuit file name - #[arg(long, short, default_value = "extractor")] - output_filename: String, - - /// Optional circuit debug logs - #[arg(long, short, action = clap::ArgAction::SetTrue)] - debug: bool, -} +pub struct ExtractorArgs { + /// Name of the circuit (to be used in circomkit config) + #[arg(long)] + circuit_name: String, -#[derive(Parser, Debug)] -pub struct HttpArgs { /// Path to the JSON file #[arg(long)] + input_file: PathBuf, + + /// Path to the lockfile + #[arg(long)] lockfile: PathBuf, - /// Output circuit file name + /// Output circuit file name (located in circuits/main/) #[arg(long, short, default_value = "extractor")] output_filename: String, diff --git a/src/witness.rs b/src/witness.rs index 5a972ae..c3378eb 100644 --- a/src/witness.rs +++ b/src/witness.rs @@ -42,12 +42,12 @@ fn print_boxed_output(lines: Vec) { } fn read_input_file_as_bytes( - file_type: WitnessSubcommand, + file_type: WitnessType, file_path: PathBuf, ) -> Result, Box> { match file_type { - WitnessSubcommand::Json => Ok(std::fs::read(file_path)?), - WitnessSubcommand::Http => { + WitnessType::Json => Ok(std::fs::read(file_path)?), + WitnessType::Http => { let mut data = std::fs::read(file_path)?; let mut i = 0; // convert LF to CRLF @@ -69,12 +69,17 @@ pub fn parser_witness(args: ParserWitnessArgs) -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { match args.subcommand { - WitnessSubcommand::Json => json_extractor_witness(args), - WitnessSubcommand::Http => http_extractor_witness(args), + WitnessType::Json => json_extractor_witness(args), + WitnessType::Http => http_extractor_witness(args), } } From cf0581f40559585691f04eac0e823f4dc4b8bb28 Mon Sep 17 00:00:00 2001 From: lonerapier Date: Thu, 12 Sep 2024 23:26:08 +0530 Subject: [PATCH 04/11] feat: add http extractor support --- Cargo.lock | 39 ++++++++++++++++++ Cargo.toml | 1 + src/http.rs | 107 +++++++++++++++++++++++++++++++++++++++++++++++-- src/main.rs | 2 +- src/witness.rs | 2 +- 5 files changed, 145 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 06d63c3..4433a59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "anstream" version = "0.6.15" @@ -126,6 +135,7 @@ name = "pabuild" version = "0.0.0" dependencies = [ "clap", + "regex", "serde", "serde_json", ] @@ -148,6 +158,35 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "regex" +version = "1.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" + [[package]] name = "ryu" version = "1.0.18" diff --git a/Cargo.toml b/Cargo.toml index d252b64..3d0f952 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,3 +6,4 @@ edition = "2021" serde = { version = "1.0.209", features = ["derive"] } serde_json = "1.0.127" clap = { version = "4.5.16", features = ["derive"] } +regex = "1.10.6" diff --git a/src/http.rs b/src/http.rs index a3be5bc..efed67f 100644 --- a/src/http.rs +++ b/src/http.rs @@ -1,9 +1,15 @@ +use codegen::CircomkitCircuitsInput; +use regex::Regex; +use witness::read_input_file_as_bytes; + use super::*; use std::{ collections::HashMap, fs::{self, create_dir_all}, }; +use super::codegen::write_circuit_config; + #[derive(Serialize, Deserialize)] #[serde(untagged)] pub enum HttpData { @@ -111,8 +117,8 @@ where const PRAGMA: &str = "pragma circom 2.1.9;\n\n"; fn build_http_circuit( - data: HttpData, - output_filename: String, + data: &HttpData, + output_filename: &String, debug: bool, ) -> Result<(), Box> { let mut circuit_buffer = String::new(); @@ -481,13 +487,106 @@ fn build_http_circuit( Ok(()) } -// TODO: This needs to codegen a circuit now. +fn parse_http_file( + locfile: &HttpData, + input: Vec, +) -> Result<(HttpData, Vec), Box> { + let input_string = String::from_utf8(input)?; + + let parts: Vec<&str> = input_string.split("\r\n\r\n").collect(); + assert!(parts.len() <= 2); + + let mut body = vec![]; + if parts.len() == 2 { + body = parts[1].as_bytes().to_vec(); + } + + let headers: Vec<&str> = parts[0].split("\r\n").collect(); + let start_line: Vec<&str> = headers[0].split(" ").collect(); + assert_eq!(start_line.len(), 3); + + let (_, headers) = headers.split_at(1); + let mut headers_map = HashMap::::new(); + let re = Regex::new(r":\s(.+)").unwrap(); + for &header in headers { + let key_value: Vec<&str> = re.split(header).collect(); + assert_eq!(key_value.len(), 2); + headers_map.insert(key_value[0].to_string(), key_value[1].to_string()); + } + + let http_data = match locfile { + HttpData::Request(_) => HttpData::Request(Request { + method: start_line[0].to_string(), + target: start_line[1].to_string(), + version: start_line[2].to_string(), + headers: headers_map, + }), + HttpData::Response(_) => HttpData::Response(Response { + version: start_line[0].to_string(), + status: start_line[1].to_string(), + message: start_line[2].to_string(), + headers: headers_map, + }), + }; + + Ok((http_data, body)) +} + +fn build_circuit_config( + args: &ExtractorArgs, + lockfile: &HttpData, +) -> Result> { + let input = read_input_file_as_bytes(WitnessType::Http, args.input_file.clone())?; + + let (_, http_body) = parse_http_file(lockfile, input.clone())?; + + let circuit_template_name = match lockfile { + HttpData::Request(_) => String::from("LockHTTPRequest"), + HttpData::Response(_) => String::from("LockHTTPResponse"), + }; + + let mut params = vec![input.len()]; + + match lockfile { + HttpData::Request(request) => { + params.push(request.method.len()); + params.push(request.target.len()); + params.push(request.version.len()); + for (key, value) in request.headers.iter() { + params.push(key.len()); + params.push(value.len()); + } + } + HttpData::Response(response) => { + params.push(http_body.len()); + params.push(response.version.len()); + params.push(response.status.len()); + params.push(response.message.len()); + for (key, value) in response.headers.iter() { + println!("{}, {}", key, value); + params.push(key.len()); + params.push(value.len()); + } + } + } + + Ok(CircomkitCircuitsInput { + file: format!("main/{}", args.output_filename), + template: circuit_template_name, + params, + }) +} + pub fn http_circuit(args: ExtractorArgs) -> Result<(), Box> { let data = std::fs::read(&args.lockfile)?; let http_data: HttpData = serde_json::from_slice(&data)?; - build_http_circuit(http_data, args.output_filename, args.debug)?; + build_http_circuit(&http_data, &args.output_filename, args.debug)?; + + let circomkit_circuit_input = build_circuit_config(&args, &http_data)?; + + write_circuit_config(args.circuit_name, &circomkit_circuit_input)?; Ok(()) } diff --git a/src/main.rs b/src/main.rs index 4d38157..ca8afce 100644 --- a/src/main.rs +++ b/src/main.rs @@ -66,7 +66,7 @@ pub struct ExtractorArgs { #[arg(long)] circuit_name: String, - /// Path to the JSON file + /// Path to the JSON/HTTP file #[arg(long)] input_file: PathBuf, diff --git a/src/witness.rs b/src/witness.rs index c3378eb..3f18a52 100644 --- a/src/witness.rs +++ b/src/witness.rs @@ -41,7 +41,7 @@ fn print_boxed_output(lines: Vec) { println!("{}", bottom_border); } -fn read_input_file_as_bytes( +pub fn read_input_file_as_bytes( file_type: WitnessType, file_path: PathBuf, ) -> Result, Box> { From dc9bbc879f0d97fc0c50008dc7b065c7d47eb9b0 Mon Sep 17 00:00:00 2001 From: lonerapier Date: Fri, 13 Sep 2024 00:01:22 +0530 Subject: [PATCH 05/11] fix: tests --- circuits/test/http/codegen.test.ts | 15 +++++----- .../test/json/extractor/extractor.test.ts | 30 +++++++++++-------- src/codegen.rs | 4 +-- src/http.rs | 1 - src/json.rs | 4 +-- 5 files changed, 29 insertions(+), 25 deletions(-) diff --git a/circuits/test/http/codegen.test.ts b/circuits/test/http/codegen.test.ts index d46ec48..0355107 100644 --- a/circuits/test/http/codegen.test.ts +++ b/circuits/test/http/codegen.test.ts @@ -41,11 +41,12 @@ interface Response { } -function executeCodegen(inputFilename: string, outputFilename: string) { +function executeCodegen(circuitName: string, inputFileName: string, lockfileName: string, outputFilename: string) { return new Promise((resolve, reject) => { - const inputPath = join(__dirname, "..", "..", "..", "examples", "http", "lockfile", inputFilename); + const inputFilePath = join(__dirname, "..", "..", "..", "examples", "http", inputFileName); + const lockfilePath = join(__dirname, "..", "..", "..", "examples", "http", "lockfile", lockfileName); - const codegen = spawn("cargo", ["run", "http", "--lockfile", inputPath, "--output-filename", outputFilename]); + const codegen = spawn("cargo", ["run", "http", "--circuit-name", circuitName, "--input-file", inputFilePath, "--lockfile", lockfilePath, "--output-filename", outputFilename]); codegen.stdout.on('data', (data) => { console.log(`stdout: ${data}`); @@ -73,7 +74,7 @@ describe("HTTP :: Codegen :: Request", async () => { let inputfile = "get_request.http"; // generate extractor circuit using codegen - await executeCodegen(`${lockfile}.json`, lockfile); + await executeCodegen("get_request_test", inputfile, `${lockfile}.json`, lockfile); const lockData = readLockFile(`${lockfile}.json`); console.log("lockData: ", JSON.stringify(lockData)); @@ -115,7 +116,7 @@ describe("HTTP :: Codegen :: Request", async () => { let inputfile = "get_request.http"; // generate extractor circuit using codegen - await executeCodegen(`${lockfile}.json`, lockfile); + await executeCodegen("get_request_test", inputfile, `${lockfile}.json`, lockfile); const lockData = readLockFile(`${lockfile}.json`); @@ -161,7 +162,7 @@ describe("HTTP :: Codegen :: Response", async () => { let inputfile = "get_response.http"; // generate extractor circuit using codegen - await executeCodegen(`${lockfile}.json`, lockfile); + await executeCodegen("get_response_test", inputfile, `${lockfile}.json`, lockfile); const lockData = readLockFile(`${lockfile}.json`); console.log("lockData: ", JSON.stringify(lockData)); @@ -207,7 +208,7 @@ describe("HTTP :: Codegen :: Response", async () => { let inputfile = "get_response.http"; // generate extractor circuit using codegen - await executeCodegen(`${lockfile}.json`, lockfile); + await executeCodegen("get_response_test", inputfile, `${lockfile}.json`, lockfile); const lockData = readLockFile(`${lockfile}.json`); diff --git a/circuits/test/json/extractor/extractor.test.ts b/circuits/test/json/extractor/extractor.test.ts index 9b9b6b3..eb27317 100644 --- a/circuits/test/json/extractor/extractor.test.ts +++ b/circuits/test/json/extractor/extractor.test.ts @@ -3,11 +3,12 @@ import { join } from "path"; import { spawn } from "child_process"; -function executeCodegen(inputFilename: string, outputFilename: string) { +function executeCodegen(circuitName: string, inputFileName: string, lockfileName: string, outputFilename: string) { return new Promise((resolve, reject) => { - const inputPath = join(__dirname, "..", "..", "..", "..", "examples", "json", "lockfile", inputFilename); + const inputFilePath = join(__dirname, "..", "..", "..", "..", "examples", "json", "test", inputFileName); + const lockfilePath = join(__dirname, "..", "..", "..", "..", "examples", "json", "lockfile", lockfileName); - const codegen = spawn("cargo", ["run", "json", "--template", inputPath, "--output-filename", outputFilename]); + const codegen = spawn("cargo", ["run", "json", "--circuit-name", circuitName, "--input-file", inputFilePath, "--lockfile", lockfilePath, "--output-filename", outputFilename]); codegen.stdout.on('data', (data) => { console.log(`stdout: ${data}`); @@ -34,7 +35,7 @@ describe("ExtractValue", async () => { let filename = "value_string"; // generate extractor circuit using codegen - await executeCodegen(`${filename}.json`, filename); + await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`, filename); // read JSON input file into bytes let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["k"]); @@ -56,7 +57,7 @@ describe("ExtractValue", async () => { it("two_keys: {\"key1\": \"abc\", \"key2\": \"def\" }", async () => { let filename = "two_keys" - await executeCodegen(`${filename}.json`, filename); + await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`, filename); let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["key2"]); circuit = await circomkit.WitnessTester(`Extract`, { @@ -71,7 +72,7 @@ describe("ExtractValue", async () => { it("value_number: {\"k\": 69 }", async () => { let filename = "value_number"; - await executeCodegen(`${filename}.json`, filename); + await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`, filename); let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["k"]); circuit = await circomkit.WitnessTester(`Extract`, { @@ -88,10 +89,11 @@ describe("ExtractValue", async () => { it("value_array_string: { \"k\" : [ 420 , 69 , 4200 , 600 ], \"b\": [ \"ab\" , \"ba\", \"ccc\", \"d\" ] }", async () => { let filename = "value_array_string"; - await executeCodegen(`${filename}.json`, filename); + let inputFileName = "value_array.json"; + await executeCodegen(`${filename}_test`, `${inputFileName}`, `${filename}.json`, filename); for (let i = 0; i < 4; i++) { - let [input, keyUnicode, output] = readJSONInputFile("value_array.json", ["b", i]); + let [input, keyUnicode, output] = readJSONInputFile(`${inputFileName}`, ["b", i]); circuit = await circomkit.WitnessTester(`Extract`, { file: `main/${filename}`, @@ -106,10 +108,12 @@ describe("ExtractValue", async () => { it("value_array_number: { \"k\" : [ 420 , 69 , 4200 , 600 ], \"b\": [ \"ab\" , \"ba\", \"ccc\", \"d\" ] }", async () => { let filename = "value_array_number"; - await executeCodegen(`${filename}.json`, filename); + let inputFileName = "value_array.json"; + + await executeCodegen(`${filename}_test`, `${inputFileName}`, `${filename}.json`, filename); for (let i = 0; i < 4; i++) { - let [input, keyUnicode, output] = readJSONInputFile("value_array.json", ["k", i]); + let [input, keyUnicode, output] = readJSONInputFile(`${inputFileName}`, ["k", i]); circuit = await circomkit.WitnessTester(`Extract`, { file: `main/${filename}`, @@ -125,7 +129,7 @@ describe("ExtractValue", async () => { it("value_array_nested: { \"a\": [[1,0],[0,1,3]] }", async () => { let filename = "value_array_nested"; - await executeCodegen(`${filename}.json`, filename); + await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`, filename); let index_0 = 1; let index_1 = 0; let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["a", index_0, index_1]); @@ -150,7 +154,7 @@ describe("ExtractValueMultiDepth", () => { it("value_object: { \"a\": { \"d\" : \"e\", \"e\": \"c\" }, \"e\": { \"f\": \"a\", \"e\": \"2\" } }", async () => { let filename = "value_object"; - await executeCodegen(`${filename}.json`, filename); + await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`, filename); let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["e", "e"]); @@ -176,7 +180,7 @@ describe("ExtractValueArrayObject", () => { it("value_array_object: {\"a\":[{\"b\":[1,4]},{\"c\":\"b\"}]}", async () => { let filename = "value_array_object"; - await executeCodegen(`${filename}.json`, filename); + await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`, filename); let index_0 = 0; let index_1 = 0; diff --git a/src/codegen.rs b/src/codegen.rs index 584ca0e..86a2b09 100644 --- a/src/codegen.rs +++ b/src/codegen.rs @@ -33,11 +33,11 @@ pub fn write_circuit_config( } std::fs::write( - circomkit_circuits_config, + circomkit_circuits_config.clone(), serde_json::to_string_pretty(&circomkit_circuits)?, )?; - println!("circomkit circuits config updated for {}", name); + println!("config updated: {}", circomkit_circuits_config.display()); Ok(()) } diff --git a/src/http.rs b/src/http.rs index efed67f..a82ee5d 100644 --- a/src/http.rs +++ b/src/http.rs @@ -563,7 +563,6 @@ fn build_circuit_config( params.push(response.status.len()); params.push(response.message.len()); for (key, value) in response.headers.iter() { - println!("{}, {}", key, value); params.push(key.len()); params.push(value.len()); } diff --git a/src/json.rs b/src/json.rs index 3ab9bae..5c95c38 100644 --- a/src/json.rs +++ b/src/json.rs @@ -551,9 +551,9 @@ fn build_circuit_config( let mut curr_stack_height = 1; let mut inside_string: bool = false; - for (i, char) in input[1..].iter().enumerate() { + for (i, char) in input.iter().skip(1).enumerate() { match char { - b'"' if input[i - 1] != b'\\' => inside_string = !inside_string, + b'"' if input[i] != b'\\' => inside_string = !inside_string, b'{' | b'[' if !inside_string => { curr_stack_height += 1; max_stack_height = max_by(max_stack_height, curr_stack_height, |x, y| x.cmp(y)); From 335a756e0a70e796e80266017c678974f4de08be Mon Sep 17 00:00:00 2001 From: lonerapier Date: Fri, 13 Sep 2024 00:22:35 +0530 Subject: [PATCH 06/11] refactor files a bit --- circuits/test/http/codegen.test.ts | 12 ++++----- .../test/json/extractor/extractor.test.ts | 24 +++++++++--------- src/{ => codegen}/http.rs | 23 ++++++++++------- src/{ => codegen}/json.rs | 18 ++++++++----- src/{codegen.rs => codegen/mod.rs} | 3 +++ src/main.rs | 25 ++++++++----------- src/witness.rs | 14 +++++------ 7 files changed, 64 insertions(+), 55 deletions(-) rename src/{ => codegen}/http.rs (97%) rename src/{ => codegen}/json.rs (98%) rename src/{codegen.rs => codegen/mod.rs} (97%) diff --git a/circuits/test/http/codegen.test.ts b/circuits/test/http/codegen.test.ts index 0355107..1a3c2ea 100644 --- a/circuits/test/http/codegen.test.ts +++ b/circuits/test/http/codegen.test.ts @@ -41,12 +41,12 @@ interface Response { } -function executeCodegen(circuitName: string, inputFileName: string, lockfileName: string, outputFilename: string) { +function executeCodegen(circuitName: string, inputFileName: string, lockfileName: string) { return new Promise((resolve, reject) => { const inputFilePath = join(__dirname, "..", "..", "..", "examples", "http", inputFileName); const lockfilePath = join(__dirname, "..", "..", "..", "examples", "http", "lockfile", lockfileName); - const codegen = spawn("cargo", ["run", "http", "--circuit-name", circuitName, "--input-file", inputFilePath, "--lockfile", lockfilePath, "--output-filename", outputFilename]); + const codegen = spawn("cargo", ["run", "codegen", "http", "--circuit-name", circuitName, "--input-file", inputFilePath, "--lockfile", lockfilePath]); codegen.stdout.on('data', (data) => { console.log(`stdout: ${data}`); @@ -74,7 +74,7 @@ describe("HTTP :: Codegen :: Request", async () => { let inputfile = "get_request.http"; // generate extractor circuit using codegen - await executeCodegen("get_request_test", inputfile, `${lockfile}.json`, lockfile); + await executeCodegen("get_request_test", inputfile, `${lockfile}.json`); const lockData = readLockFile(`${lockfile}.json`); console.log("lockData: ", JSON.stringify(lockData)); @@ -116,7 +116,7 @@ describe("HTTP :: Codegen :: Request", async () => { let inputfile = "get_request.http"; // generate extractor circuit using codegen - await executeCodegen("get_request_test", inputfile, `${lockfile}.json`, lockfile); + await executeCodegen("get_request_test", inputfile, `${lockfile}.json`); const lockData = readLockFile(`${lockfile}.json`); @@ -162,7 +162,7 @@ describe("HTTP :: Codegen :: Response", async () => { let inputfile = "get_response.http"; // generate extractor circuit using codegen - await executeCodegen("get_response_test", inputfile, `${lockfile}.json`, lockfile); + await executeCodegen("get_response_test", inputfile, `${lockfile}.json`); const lockData = readLockFile(`${lockfile}.json`); console.log("lockData: ", JSON.stringify(lockData)); @@ -208,7 +208,7 @@ describe("HTTP :: Codegen :: Response", async () => { let inputfile = "get_response.http"; // generate extractor circuit using codegen - await executeCodegen("get_response_test", inputfile, `${lockfile}.json`, lockfile); + await executeCodegen("get_response_test", inputfile, `${lockfile}.json`); const lockData = readLockFile(`${lockfile}.json`); diff --git a/circuits/test/json/extractor/extractor.test.ts b/circuits/test/json/extractor/extractor.test.ts index eb27317..ea7fd9a 100644 --- a/circuits/test/json/extractor/extractor.test.ts +++ b/circuits/test/json/extractor/extractor.test.ts @@ -3,12 +3,12 @@ import { join } from "path"; import { spawn } from "child_process"; -function executeCodegen(circuitName: string, inputFileName: string, lockfileName: string, outputFilename: string) { +function executeCodegen(circuitName: string, inputFileName: string, lockfileName: string) { return new Promise((resolve, reject) => { const inputFilePath = join(__dirname, "..", "..", "..", "..", "examples", "json", "test", inputFileName); const lockfilePath = join(__dirname, "..", "..", "..", "..", "examples", "json", "lockfile", lockfileName); - const codegen = spawn("cargo", ["run", "json", "--circuit-name", circuitName, "--input-file", inputFilePath, "--lockfile", lockfilePath, "--output-filename", outputFilename]); + const codegen = spawn("cargo", ["run", "codegen", "json", "--circuit-name", circuitName, "--input-file", inputFilePath, "--lockfile", lockfilePath]); codegen.stdout.on('data', (data) => { console.log(`stdout: ${data}`); @@ -35,7 +35,7 @@ describe("ExtractValue", async () => { let filename = "value_string"; // generate extractor circuit using codegen - await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`, filename); + await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`); // read JSON input file into bytes let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["k"]); @@ -57,7 +57,7 @@ describe("ExtractValue", async () => { it("two_keys: {\"key1\": \"abc\", \"key2\": \"def\" }", async () => { let filename = "two_keys" - await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`, filename); + await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`); let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["key2"]); circuit = await circomkit.WitnessTester(`Extract`, { @@ -72,7 +72,7 @@ describe("ExtractValue", async () => { it("value_number: {\"k\": 69 }", async () => { let filename = "value_number"; - await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`, filename); + await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`); let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["k"]); circuit = await circomkit.WitnessTester(`Extract`, { @@ -90,10 +90,10 @@ describe("ExtractValue", async () => { it("value_array_string: { \"k\" : [ 420 , 69 , 4200 , 600 ], \"b\": [ \"ab\" , \"ba\", \"ccc\", \"d\" ] }", async () => { let filename = "value_array_string"; let inputFileName = "value_array.json"; - await executeCodegen(`${filename}_test`, `${inputFileName}`, `${filename}.json`, filename); + await executeCodegen(`${filename}_test`, inputFileName, `${filename}.json`); for (let i = 0; i < 4; i++) { - let [input, keyUnicode, output] = readJSONInputFile(`${inputFileName}`, ["b", i]); + let [input, keyUnicode, output] = readJSONInputFile(inputFileName, ["b", i]); circuit = await circomkit.WitnessTester(`Extract`, { file: `main/${filename}`, @@ -110,10 +110,10 @@ describe("ExtractValue", async () => { let filename = "value_array_number"; let inputFileName = "value_array.json"; - await executeCodegen(`${filename}_test`, `${inputFileName}`, `${filename}.json`, filename); + await executeCodegen(`${filename}_test`, inputFileName, `${filename}.json`); for (let i = 0; i < 4; i++) { - let [input, keyUnicode, output] = readJSONInputFile(`${inputFileName}`, ["k", i]); + let [input, keyUnicode, output] = readJSONInputFile(inputFileName, ["k", i]); circuit = await circomkit.WitnessTester(`Extract`, { file: `main/${filename}`, @@ -129,7 +129,7 @@ describe("ExtractValue", async () => { it("value_array_nested: { \"a\": [[1,0],[0,1,3]] }", async () => { let filename = "value_array_nested"; - await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`, filename); + await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`); let index_0 = 1; let index_1 = 0; let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["a", index_0, index_1]); @@ -154,7 +154,7 @@ describe("ExtractValueMultiDepth", () => { it("value_object: { \"a\": { \"d\" : \"e\", \"e\": \"c\" }, \"e\": { \"f\": \"a\", \"e\": \"2\" } }", async () => { let filename = "value_object"; - await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`, filename); + await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`); let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["e", "e"]); @@ -180,7 +180,7 @@ describe("ExtractValueArrayObject", () => { it("value_array_object: {\"a\":[{\"b\":[1,4]},{\"c\":\"b\"}]}", async () => { let filename = "value_array_object"; - await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`, filename); + await executeCodegen(`${filename}_test`, `${filename}.json`, `${filename}.json`); let index_0 = 0; let index_1 = 0; diff --git a/src/http.rs b/src/codegen/http.rs similarity index 97% rename from src/http.rs rename to src/codegen/http.rs index a82ee5d..753c604 100644 --- a/src/http.rs +++ b/src/codegen/http.rs @@ -1,15 +1,17 @@ -use codegen::CircomkitCircuitsInput; +use crate::{ + codegen::{write_circuit_config, CircomkitCircuitsInput}, + witness::read_input_file_as_bytes, + ExtractorArgs, FileType, +}; use regex::Regex; -use witness::read_input_file_as_bytes; +use serde::{Deserialize, Serialize}; -use super::*; use std::{ collections::HashMap, + error::Error, fs::{self, create_dir_all}, }; -use super::codegen::write_circuit_config; - #[derive(Serialize, Deserialize)] #[serde(untagged)] pub enum HttpData { @@ -535,8 +537,9 @@ fn parse_http_file( fn build_circuit_config( args: &ExtractorArgs, lockfile: &HttpData, + codegen_filename: String, ) -> Result> { - let input = read_input_file_as_bytes(WitnessType::Http, args.input_file.clone())?; + let input = read_input_file_as_bytes(FileType::Http, args.input_file.clone())?; let (_, http_body) = parse_http_file(lockfile, input.clone())?; @@ -570,7 +573,7 @@ fn build_circuit_config( } Ok(CircomkitCircuitsInput { - file: format!("main/{}", args.output_filename), + file: codegen_filename, template: circuit_template_name, params, }) @@ -581,9 +584,11 @@ pub fn http_circuit(args: ExtractorArgs) -> Result<(), Box> { let http_data: HttpData = serde_json::from_slice(&data)?; - build_http_circuit(&http_data, &args.output_filename, args.debug)?; + let codegen_filename = format!("http_{}", args.circuit_name); + + build_http_circuit(&http_data, &codegen_filename, args.debug)?; - let circomkit_circuit_input = build_circuit_config(&args, &http_data)?; + let circomkit_circuit_input = build_circuit_config(&args, &http_data, codegen_filename)?; write_circuit_config(args.circuit_name, &circomkit_circuit_input)?; diff --git a/src/json.rs b/src/codegen/json.rs similarity index 98% rename from src/json.rs rename to src/codegen/json.rs index 5c95c38..e584e5c 100644 --- a/src/json.rs +++ b/src/codegen/json.rs @@ -1,4 +1,5 @@ -use super::*; +use serde::{Deserialize, Serialize}; +use serde_json::Value; use std::{ cmp::max_by, collections::HashMap, @@ -6,8 +7,10 @@ use std::{ str::FromStr, }; -use codegen::{write_circuit_config, CircomkitCircuitsInput}; -use serde_json::Value; +use crate::{ + codegen::{write_circuit_config, CircomkitCircuitsInput}, + ExtractorArgs, +}; #[derive(Debug, Deserialize)] pub enum ValueType { @@ -539,6 +542,7 @@ fn build_json_circuit( fn build_circuit_config( args: &ExtractorArgs, lockfile: &JsonLockfile, + codegen_filename: String, ) -> Result> { let input = fs::read(args.input_file.clone())?; @@ -611,7 +615,7 @@ fn build_circuit_config( params.push(value_bytes.as_bytes().len()); Ok(CircomkitCircuitsInput { - file: format!("main/{}", args.output_filename), + file: format!("main/{}", codegen_filename), template: circuit_template_name, params, }) @@ -620,9 +624,11 @@ fn build_circuit_config( pub fn json_circuit(args: ExtractorArgs) -> Result<(), Box> { let lockfile: JsonLockfile = serde_json::from_slice(&std::fs::read(&args.lockfile)?)?; - build_json_circuit(&lockfile, &args.output_filename, args.debug)?; + let codegen_filename = format!("json_{}", args.circuit_name); + + build_json_circuit(&lockfile, &codegen_filename, args.debug)?; - let circomkit_circuit_input = build_circuit_config(&args, &lockfile)?; + let circomkit_circuit_input = build_circuit_config(&args, &lockfile, codegen_filename)?; write_circuit_config(args.circuit_name, &circomkit_circuit_input)?; diff --git a/src/codegen.rs b/src/codegen/mod.rs similarity index 97% rename from src/codegen.rs rename to src/codegen/mod.rs index 86a2b09..97729c9 100644 --- a/src/codegen.rs +++ b/src/codegen/mod.rs @@ -2,6 +2,9 @@ use std::{collections::HashMap, env}; use serde::{Deserialize, Serialize}; +pub mod http; +pub mod json; + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CircomkitCircuitsInput { pub file: String, diff --git a/src/main.rs b/src/main.rs index ca8afce..a295dc9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,7 @@ use clap::{Parser, Subcommand}; -use serde::{Deserialize, Serialize}; use std::{error::Error, path::PathBuf}; pub mod codegen; -pub mod http; -pub mod json; pub mod witness; #[derive(Parser, Debug)] @@ -18,14 +15,13 @@ pub struct Args { pub enum Command { ParserWitness(ParserWitnessArgs), ExtractorWitness(ExtractorWitnessArgs), - Json(ExtractorArgs), - Http(ExtractorArgs), + Codegen(ExtractorArgs), } #[derive(Parser, Debug)] pub struct ParserWitnessArgs { #[arg(value_enum)] - subcommand: WitnessType, + subcommand: FileType, /// Path to the JSON file #[arg(long)] @@ -39,7 +35,7 @@ pub struct ParserWitnessArgs { #[derive(Parser, Debug)] pub struct ExtractorWitnessArgs { #[arg(value_enum)] - subcommand: WitnessType, + subcommand: FileType, /// Name of the circuit (to be used in circomkit config) #[arg(long)] @@ -55,13 +51,16 @@ pub struct ExtractorWitnessArgs { } #[derive(clap::ValueEnum, Clone, Debug)] -pub enum WitnessType { +pub enum FileType { Json, Http, } #[derive(Parser, Debug)] pub struct ExtractorArgs { + #[arg(value_enum)] + subcommand: FileType, + /// Name of the circuit (to be used in circomkit config) #[arg(long)] circuit_name: String, @@ -74,10 +73,6 @@ pub struct ExtractorArgs { #[arg(long)] lockfile: PathBuf, - /// Output circuit file name (located in circuits/main/) - #[arg(long, short, default_value = "extractor")] - output_filename: String, - /// Optional circuit debug logs #[arg(long, short, action = clap::ArgAction::SetTrue)] debug: bool, @@ -86,8 +81,10 @@ pub struct ExtractorArgs { pub fn main() -> Result<(), Box> { match Args::parse().command { Command::ParserWitness(args) => witness::parser_witness(args), - Command::Json(args) => json::json_circuit(args), - Command::Http(args) => http::http_circuit(args), + Command::Codegen(args) => match args.subcommand { + FileType::Http => codegen::http::http_circuit(args), + FileType::Json => codegen::json::json_circuit(args), + }, Command::ExtractorWitness(args) => witness::extractor_witness(args), } } diff --git a/src/witness.rs b/src/witness.rs index 3f18a52..849e6f6 100644 --- a/src/witness.rs +++ b/src/witness.rs @@ -1,7 +1,5 @@ -use json::JsonLockfile; - -use super::http::HttpData; use super::*; +use crate::codegen::{http::HttpData, json::JsonLockfile}; use std::{collections::HashMap, io::Write}; #[derive(serde::Serialize)] @@ -42,12 +40,12 @@ fn print_boxed_output(lines: Vec) { } pub fn read_input_file_as_bytes( - file_type: WitnessType, + file_type: FileType, file_path: PathBuf, ) -> Result, Box> { match file_type { - WitnessType::Json => Ok(std::fs::read(file_path)?), - WitnessType::Http => { + FileType::Json => Ok(std::fs::read(file_path)?), + FileType::Http => { let mut data = std::fs::read(file_path)?; let mut i = 0; // convert LF to CRLF @@ -162,7 +160,7 @@ fn http_extractor_witness(args: ExtractorWitnessArgs) -> Result<(), Box Result<(), Box> { match args.subcommand { - WitnessType::Json => json_extractor_witness(args), - WitnessType::Http => http_extractor_witness(args), + FileType::Json => json_extractor_witness(args), + FileType::Http => http_extractor_witness(args), } } From e073ed8d6c7c66f67443a2245c35956adbafef57 Mon Sep 17 00:00:00 2001 From: lonerapier Date: Fri, 13 Sep 2024 00:46:01 +0530 Subject: [PATCH 07/11] fix tests again, arghhhhhhhgstgst --- circuits/test/http/codegen.test.ts | 20 +++++++++++-------- .../test/json/extractor/extractor.test.ts | 16 +++++++-------- src/main.rs | 12 +++++------ 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/circuits/test/http/codegen.test.ts b/circuits/test/http/codegen.test.ts index 1a3c2ea..aa209c6 100644 --- a/circuits/test/http/codegen.test.ts +++ b/circuits/test/http/codegen.test.ts @@ -72,9 +72,10 @@ describe("HTTP :: Codegen :: Request", async () => { it("(valid) GET:", async () => { let lockfile = "request.lock"; let inputfile = "get_request.http"; + let circuitName = "get_request_test"; // generate extractor circuit using codegen - await executeCodegen("get_request_test", inputfile, `${lockfile}.json`); + await executeCodegen(circuitName, inputfile, `${lockfile}.json`); const lockData = readLockFile(`${lockfile}.json`); console.log("lockData: ", JSON.stringify(lockData)); @@ -90,7 +91,7 @@ describe("HTTP :: Codegen :: Request", async () => { circuit = await circomkit.WitnessTester(`Extract`, { - file: `main/${lockfile}`, + file: `main/http_${circuitName}`, template: "LockHTTPRequest", params: params, }); @@ -114,9 +115,10 @@ describe("HTTP :: Codegen :: Request", async () => { it("(invalid) GET:", async () => { let lockfile = "request.lock"; let inputfile = "get_request.http"; + let circuitName = "get_request_test"; // generate extractor circuit using codegen - await executeCodegen("get_request_test", inputfile, `${lockfile}.json`); + await executeCodegen(circuitName, inputfile, `${lockfile}.json`); const lockData = readLockFile(`${lockfile}.json`); @@ -131,7 +133,7 @@ describe("HTTP :: Codegen :: Request", async () => { circuit = await circomkit.WitnessTester(`Extract`, { - file: `main/${lockfile}`, + file: `main/http_${circuitName}`, template: "LockHTTPRequest", params: params, }); @@ -160,9 +162,10 @@ describe("HTTP :: Codegen :: Response", async () => { it("(valid) GET:", async () => { let lockfile = "response.lock"; let inputfile = "get_response.http"; + let circuitName = "get_response_test"; // generate extractor circuit using codegen - await executeCodegen("get_response_test", inputfile, `${lockfile}.json`); + await executeCodegen(circuitName, inputfile, `${lockfile}.json`); const lockData = readLockFile(`${lockfile}.json`); console.log("lockData: ", JSON.stringify(lockData)); @@ -180,7 +183,7 @@ describe("HTTP :: Codegen :: Response", async () => { circuit = await circomkit.WitnessTester(`Extract`, { - file: `main/${lockfile}`, + file: `main/http_${circuitName}`, template: "LockHTTPResponse", params: params, }); @@ -206,9 +209,10 @@ describe("HTTP :: Codegen :: Response", async () => { it("(invalid) GET:", async () => { let lockfile = "response.lock"; let inputfile = "get_response.http"; + let circuitName = "get_response_test"; // generate extractor circuit using codegen - await executeCodegen("get_response_test", inputfile, `${lockfile}.json`); + await executeCodegen(circuitName, inputfile, `${lockfile}.json`); const lockData = readLockFile(`${lockfile}.json`); @@ -225,7 +229,7 @@ describe("HTTP :: Codegen :: Response", async () => { circuit = await circomkit.WitnessTester(`Extract`, { - file: `main/${lockfile}`, + file: `main/http_${circuitName}`, template: "LockHTTPResponse", params: params, }); diff --git a/circuits/test/json/extractor/extractor.test.ts b/circuits/test/json/extractor/extractor.test.ts index ea7fd9a..c67a70b 100644 --- a/circuits/test/json/extractor/extractor.test.ts +++ b/circuits/test/json/extractor/extractor.test.ts @@ -41,7 +41,7 @@ describe("ExtractValue", async () => { let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["k"]); circuit = await circomkit.WitnessTester(`Extract`, { - file: `main/${filename}`, + file: `main/json_${filename}_test`, template: "ExtractStringValue", params: [input.length, 1, 1, 0, 1], }); @@ -61,7 +61,7 @@ describe("ExtractValue", async () => { let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["key2"]); circuit = await circomkit.WitnessTester(`Extract`, { - file: `main/${filename}`, + file: `main/json_${filename}_test`, template: "ExtractStringValue", params: [input.length, 1, 4, 0, 3], }); @@ -76,7 +76,7 @@ describe("ExtractValue", async () => { let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["k"]); circuit = await circomkit.WitnessTester(`Extract`, { - file: `main/${filename}`, + file: `main/json_${filename}_test`, template: "ExtractNumValue", params: [input.length, 1, 1, 0, 2], }); @@ -96,7 +96,7 @@ describe("ExtractValue", async () => { let [input, keyUnicode, output] = readJSONInputFile(inputFileName, ["b", i]); circuit = await circomkit.WitnessTester(`Extract`, { - file: `main/${filename}`, + file: `main/json_${filename}_test`, template: "ExtractStringValue", params: [input.length, 2, 1, 0, i, 1, output.length], }); @@ -116,7 +116,7 @@ describe("ExtractValue", async () => { let [input, keyUnicode, output] = readJSONInputFile(inputFileName, ["k", i]); circuit = await circomkit.WitnessTester(`Extract`, { - file: `main/${filename}`, + file: `main/json_${filename}_test`, template: "ExtractNumValue", params: [input.length, 2, 1, 0, i, 1, output.length], }); @@ -135,7 +135,7 @@ describe("ExtractValue", async () => { let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["a", index_0, index_1]); circuit = await circomkit.WitnessTester(`Extract`, { - file: `main/${filename}`, + file: `main/json_${filename}_test`, template: "ExtractNumValue", params: [input.length, 3, 1, 0, index_0, 1, index_1, 2, 1], }); @@ -159,7 +159,7 @@ describe("ExtractValueMultiDepth", () => { let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["e", "e"]); circuit = await circomkit.WitnessTester(`Extract`, { - file: `main/${filename}`, + file: `main/json_${filename}_test`, template: "ExtractStringValue", params: [input.length, 3, 1, 0, 1, 1, 1], }); @@ -187,7 +187,7 @@ describe("ExtractValueArrayObject", () => { let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["a", index_0, "b", index_1]); circuit = await circomkit.WitnessTester(`Extract`, { - file: `main/${filename}`, + file: `main/json_${filename}_test`, template: "ExtractNumValue", params: [input.length, 4, 1, 0, index_0, 1, 1, 2, index_1, 3, 1], }); diff --git a/src/main.rs b/src/main.rs index a295dc9..1acb47b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,6 +18,12 @@ pub enum Command { Codegen(ExtractorArgs), } +#[derive(clap::ValueEnum, Clone, Debug)] +pub enum FileType { + Json, + Http, +} + #[derive(Parser, Debug)] pub struct ParserWitnessArgs { #[arg(value_enum)] @@ -50,12 +56,6 @@ pub struct ExtractorWitnessArgs { lockfile: PathBuf, } -#[derive(clap::ValueEnum, Clone, Debug)] -pub enum FileType { - Json, - Http, -} - #[derive(Parser, Debug)] pub struct ExtractorArgs { #[arg(value_enum)] From 8a84f4fd27540754714a4cb007b5452c12eddab9 Mon Sep 17 00:00:00 2001 From: lonerapier Date: Fri, 13 Sep 2024 14:00:59 +0530 Subject: [PATCH 08/11] simplify cli a bit more --- src/main.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main.rs b/src/main.rs index 1acb47b..033fbd5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,8 +13,8 @@ pub struct Args { #[derive(Subcommand, Debug)] pub enum Command { - ParserWitness(ParserWitnessArgs), - ExtractorWitness(ExtractorWitnessArgs), + #[command(subcommand)] + Witness(WitnessType), Codegen(ExtractorArgs), } @@ -24,6 +24,12 @@ pub enum FileType { Http, } +#[derive(Debug, Parser)] +pub enum WitnessType { + Parser(ParserWitnessArgs), + Extractor(ExtractorWitnessArgs), +} + #[derive(Parser, Debug)] pub struct ParserWitnessArgs { #[arg(value_enum)] @@ -80,11 +86,13 @@ pub struct ExtractorArgs { pub fn main() -> Result<(), Box> { match Args::parse().command { - Command::ParserWitness(args) => witness::parser_witness(args), + Command::Witness(witness_type) => match witness_type { + WitnessType::Parser(args) => witness::parser_witness(args), + WitnessType::Extractor(args) => witness::extractor_witness(args), + }, Command::Codegen(args) => match args.subcommand { FileType::Http => codegen::http::http_circuit(args), FileType::Json => codegen::json::json_circuit(args), }, - Command::ExtractorWitness(args) => witness::extractor_witness(args), } } From 97f53ff44f7f266ecb634a8ddf3565420347d3b1 Mon Sep 17 00:00:00 2001 From: lonerapier Date: Fri, 13 Sep 2024 17:15:17 +0530 Subject: [PATCH 09/11] more cli cleanup --- src/circuit_config.rs | 50 +++++++++++++++++++++++++++++++++++++ src/codegen/http.rs | 10 ++++---- src/codegen/json.rs | 58 ++++++++++++++++++++++++------------------- src/codegen/mod.rs | 46 +--------------------------------- src/main.rs | 3 ++- src/witness.rs | 47 ++++++++++++++++++++++++----------- 6 files changed, 123 insertions(+), 91 deletions(-) create mode 100644 src/circuit_config.rs diff --git a/src/circuit_config.rs b/src/circuit_config.rs new file mode 100644 index 0000000..7d43497 --- /dev/null +++ b/src/circuit_config.rs @@ -0,0 +1,50 @@ +use serde::{Deserialize, Serialize}; +use std::{collections::HashMap, env}; + +/// circuit config used for circomkit support +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CircomkitCircuitConfig { + /// file name containing the circuit template + pub file: String, + /// circuit template name + pub template: String, + /// circuit parameters + pub params: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CircomkitConfig(HashMap); + +/// Writes config to `circuits.json` for circomkit support +/// # Inputs +/// - `name`: circuit name +/// - `circuit_config`: [`CircomkitCircuitConfig`] +pub fn write_config( + name: String, + circuit_config: &CircomkitCircuitConfig, +) -> Result<(), Box> { + let mut circomkit_config = env::current_dir()?; + circomkit_config.push("circuits.json"); + + let _ = std::fs::File::create_new(&circomkit_config); + + let mut circomkit_circuits: CircomkitConfig = + serde_json::from_slice(&std::fs::read(&circomkit_config)?)?; + + if let Some(circuits_inputs) = circomkit_circuits.0.get_mut(&name) { + *circuits_inputs = circuit_config.clone(); + } else { + let _ = circomkit_circuits + .0 + .insert(name.clone(), circuit_config.clone()); + } + + std::fs::write( + circomkit_config.clone(), + serde_json::to_string_pretty(&circomkit_circuits)?, + )?; + + println!("Config updated: {}", circomkit_config.display()); + + Ok(()) +} diff --git a/src/codegen/http.rs b/src/codegen/http.rs index 753c604..d1b24c4 100644 --- a/src/codegen/http.rs +++ b/src/codegen/http.rs @@ -1,5 +1,5 @@ use crate::{ - codegen::{write_circuit_config, CircomkitCircuitsInput}, + circuit_config::{write_config, CircomkitCircuitConfig}, witness::read_input_file_as_bytes, ExtractorArgs, FileType, }; @@ -538,8 +538,8 @@ fn build_circuit_config( args: &ExtractorArgs, lockfile: &HttpData, codegen_filename: String, -) -> Result> { - let input = read_input_file_as_bytes(FileType::Http, args.input_file.clone())?; +) -> Result> { + let input = read_input_file_as_bytes(&FileType::Http, args.input_file.clone())?; let (_, http_body) = parse_http_file(lockfile, input.clone())?; @@ -572,7 +572,7 @@ fn build_circuit_config( } } - Ok(CircomkitCircuitsInput { + Ok(CircomkitCircuitConfig { file: codegen_filename, template: circuit_template_name, params, @@ -590,7 +590,7 @@ pub fn http_circuit(args: ExtractorArgs) -> Result<(), Box> { let circomkit_circuit_input = build_circuit_config(&args, &http_data, codegen_filename)?; - write_circuit_config(args.circuit_name, &circomkit_circuit_input)?; + write_config(args.circuit_name, &circomkit_circuit_input)?; Ok(()) } diff --git a/src/codegen/json.rs b/src/codegen/json.rs index e584e5c..491921b 100644 --- a/src/codegen/json.rs +++ b/src/codegen/json.rs @@ -8,7 +8,7 @@ use std::{ }; use crate::{ - codegen::{write_circuit_config, CircomkitCircuitsInput}, + circuit_config::{write_config, CircomkitCircuitConfig}, ExtractorArgs, }; @@ -28,12 +28,12 @@ pub enum Key { } #[derive(Debug, Deserialize)] -pub struct JsonLockfile { +pub struct Lockfile { keys: Vec, value_type: ValueType, } -impl JsonLockfile { +impl Lockfile { pub fn as_bytes(&self) -> HashMap> { let mut keys = HashMap::>::new(); for (i, key) in self.keys.iter().enumerate() { @@ -46,7 +46,7 @@ impl JsonLockfile { } } -fn extract_string(data: &JsonLockfile, circuit_buffer: &mut String, debug: bool) { +fn extract_string(data: &Lockfile, circuit_buffer: &mut String, debug: bool) { *circuit_buffer += "template ExtractStringValue(DATA_BYTES, MAX_STACK_HEIGHT, "; for (i, key) in data.keys.iter().enumerate() { match key { @@ -111,7 +111,7 @@ fn extract_string(data: &JsonLockfile, circuit_buffer: &mut String, debug: bool) "#; } -fn extract_number(data: &JsonLockfile, circuit_buffer: &mut String, debug: bool) { +fn extract_number(data: &Lockfile, circuit_buffer: &mut String, debug: bool) { *circuit_buffer += "template ExtractNumValue(DATA_BYTES, MAX_STACK_HEIGHT, "; for (i, key) in data.keys.iter().enumerate() { match key { @@ -187,7 +187,7 @@ fn extract_number(data: &JsonLockfile, circuit_buffer: &mut String, debug: bool) } fn build_json_circuit( - data: &JsonLockfile, + data: &Lockfile, output_filename: &String, debug: bool, ) -> Result<(), Box> { @@ -539,18 +539,7 @@ fn build_json_circuit( Ok(()) } -fn build_circuit_config( - args: &ExtractorArgs, - lockfile: &JsonLockfile, - codegen_filename: String, -) -> Result> { - let input = fs::read(args.input_file.clone())?; - - let circuit_template_name = match lockfile.value_type { - ValueType::String => String::from("ExtractStringValue"), - ValueType::Number => String::from("ExtractNumValue"), - }; - +pub fn json_max_stack_height(input: &[u8]) -> usize { let mut max_stack_height = 1; let mut curr_stack_height = 1; let mut inside_string: bool = false; @@ -567,7 +556,25 @@ fn build_circuit_config( } } - let mut params = vec![input.len(), max_stack_height]; + max_stack_height +} + +/// Builds circuit config for circomkit support. +pub fn build_circuit_config( + args: &ExtractorArgs, + lockfile: &Lockfile, + codegen_filename: String, +) -> Result> { + let input = fs::read(args.input_file.clone())?; + + let circuit_template_name = match lockfile.value_type { + ValueType::String => String::from("ExtractStringValue"), + ValueType::Number => String::from("ExtractNumValue"), + }; + + // build circuit arguments + // [DATA_BYTES, MAX_STACK_HEIGHT, keyLen1, depth1, ..., maxValueLen] + let mut params = vec![input.len(), json_max_stack_height(&input)]; let mut current_value: Value = serde_json::from_slice(&input)?; for (i, key) in lockfile.keys.iter().enumerate() { @@ -595,26 +602,25 @@ fn build_circuit_config( params.push(i); } + // get value of specified key + // Currently only supports number, string let value_bytes = match lockfile.value_type { ValueType::Number => { if !current_value.is_u64() { return Err(String::from("value type doesn't match").into()); } - current_value.as_u64().unwrap().to_string() } ValueType::String => { if !current_value.is_string() { return Err(String::from("value type doesn't match").into()); } - current_value.as_str().unwrap().to_string() } }; - params.push(value_bytes.as_bytes().len()); - Ok(CircomkitCircuitsInput { + Ok(CircomkitCircuitConfig { file: format!("main/{}", codegen_filename), template: circuit_template_name, params, @@ -622,15 +628,15 @@ fn build_circuit_config( } pub fn json_circuit(args: ExtractorArgs) -> Result<(), Box> { - let lockfile: JsonLockfile = serde_json::from_slice(&std::fs::read(&args.lockfile)?)?; + let lockfile: Lockfile = serde_json::from_slice(&std::fs::read(&args.lockfile)?)?; let codegen_filename = format!("json_{}", args.circuit_name); build_json_circuit(&lockfile, &codegen_filename, args.debug)?; - let circomkit_circuit_input = build_circuit_config(&args, &lockfile, codegen_filename)?; + let config = build_circuit_config(&args, &lockfile, codegen_filename)?; - write_circuit_config(args.circuit_name, &circomkit_circuit_input)?; + write_config(args.circuit_name, &config)?; Ok(()) } diff --git a/src/codegen/mod.rs b/src/codegen/mod.rs index 97729c9..9550283 100644 --- a/src/codegen/mod.rs +++ b/src/codegen/mod.rs @@ -1,46 +1,2 @@ -use std::{collections::HashMap, env}; - -use serde::{Deserialize, Serialize}; - pub mod http; -pub mod json; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CircomkitCircuitsInput { - pub file: String, - pub template: String, - pub params: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CircomkitCircuits(HashMap); - -pub fn write_circuit_config( - name: String, - circomkit_input: &CircomkitCircuitsInput, -) -> Result<(), Box> { - let mut circomkit_circuits_config = env::current_dir()?; - circomkit_circuits_config.push("circuits.json"); - - let _ = std::fs::File::create_new(circomkit_circuits_config.clone()); - - let mut circomkit_circuits: CircomkitCircuits = - serde_json::from_slice(&std::fs::read(circomkit_circuits_config.clone())?)?; - - if let Some(circuits_inputs) = circomkit_circuits.0.get_mut(&name) { - *circuits_inputs = circomkit_input.clone(); - } else { - let _ = circomkit_circuits - .0 - .insert(name.clone(), circomkit_input.clone()); - } - - std::fs::write( - circomkit_circuits_config.clone(), - serde_json::to_string_pretty(&circomkit_circuits)?, - )?; - - println!("config updated: {}", circomkit_circuits_config.display()); - - Ok(()) -} +pub mod json; \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 033fbd5..3678922 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ use clap::{Parser, Subcommand}; use std::{error::Error, path::PathBuf}; +pub mod circuit_config; pub mod codegen; pub mod witness; @@ -18,7 +19,7 @@ pub enum Command { Codegen(ExtractorArgs), } -#[derive(clap::ValueEnum, Clone, Debug)] +#[derive(clap::ValueEnum, Clone, Debug, PartialEq)] pub enum FileType { Json, Http, diff --git a/src/witness.rs b/src/witness.rs index 849e6f6..ec09bd9 100644 --- a/src/witness.rs +++ b/src/witness.rs @@ -1,6 +1,11 @@ -use super::*; -use crate::codegen::{http::HttpData, json::JsonLockfile}; -use std::{collections::HashMap, io::Write}; +use crate::{ + codegen::{ + http::HttpData, + json::{json_max_stack_height, Lockfile}, + }, + ExtractorWitnessArgs, FileType, ParserWitnessArgs, +}; +use std::{collections::HashMap, io::Write, path::PathBuf}; #[derive(serde::Serialize)] pub struct ParserWitness { @@ -40,7 +45,7 @@ fn print_boxed_output(lines: Vec) { } pub fn read_input_file_as_bytes( - file_type: FileType, + file_type: &FileType, file_path: PathBuf, ) -> Result, Box> { match file_type { @@ -63,7 +68,7 @@ pub fn read_input_file_as_bytes( } pub fn parser_witness(args: ParserWitnessArgs) -> Result<(), Box> { - let data = read_input_file_as_bytes(args.subcommand, args.input_file)?; + let data = read_input_file_as_bytes(&args.subcommand, args.input_file)?; let witness = ParserWitness { data: data.clone() }; @@ -84,6 +89,12 @@ pub fn parser_witness(args: ParserWitnessArgs) -> Result<(), Box Result<(), Box Result<(), Box> { - let input_data = read_input_file_as_bytes(args.subcommand, args.input_file)?; + // read input and lockfile + let input_data = read_input_file_as_bytes(&args.subcommand, args.input_file)?; let lockfile_data = std::fs::read(&args.lockfile)?; - let lockfile: JsonLockfile = serde_json::from_slice(&lockfile_data)?; + let lockfile: Lockfile = serde_json::from_slice(&lockfile_data)?; + // create extractor witness data let witness = JsonExtractorWitness { data: input_data.clone(), keys: lockfile.as_bytes(), }; + // create output dir if not exists let mut output_dir = std::env::current_dir()?; output_dir.push("inputs"); - output_dir.push(args.circuit_name); - + output_dir.push(&args.circuit_name); if !output_dir.exists() { std::fs::create_dir_all(&output_dir)?; } + // write input file let output_file = output_dir.join("inputs.json"); let mut file = std::fs::File::create(output_file)?; - file.write_all(serde_json::to_string_pretty(&witness)?.as_bytes())?; // Prepare lines to print let mut lines = Vec::new(); lines.push(format!("Data length: {}", input_data.len())); + lines.push(format!( + "Max stack height: {}", + json_max_stack_height(&input_data) + )); // Print the output inside a nicely formatted box print_boxed_output(lines); @@ -125,27 +142,29 @@ fn json_extractor_witness(args: ExtractorWitnessArgs) -> Result<(), Box Result<(), Box> { - let input_data = read_input_file_as_bytes(args.subcommand, args.input_file)?; + // read input and lockfile + let input_data = read_input_file_as_bytes(&args.subcommand, args.input_file)?; let lockfile_data = std::fs::read(&args.lockfile)?; let http_data: HttpData = serde_json::from_slice(&lockfile_data)?; + // create witness data let witness = HttpExtractorWitness { data: input_data.clone(), http_data, }; + // create witness dir let mut output_dir = std::env::current_dir()?; output_dir.push("inputs"); - output_dir.push(args.circuit_name); - + output_dir.push(&args.circuit_name); if !output_dir.exists() { std::fs::create_dir_all(&output_dir)?; } + // write witness to file let output_file = output_dir.join("inputs.json"); let mut file = std::fs::File::create(output_file)?; - file.write_all(serde_json::to_string_pretty(&witness)?.as_bytes())?; // Prepare lines to print From 0d5f010f114e42bb198197300e1c4125caf5c7dd Mon Sep 17 00:00:00 2001 From: lonerapier Date: Fri, 13 Sep 2024 17:30:14 +0530 Subject: [PATCH 10/11] add json proof docs --- README.md | 10 +++--- docs/pabuild.md | 89 ++++++++++++++++++++++++++++++++----------------- 2 files changed, 64 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index f9265aa..3314937 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,13 @@ @@ -120,10 +120,10 @@ From the root of this repository, run: ```sh cargo install --path . ``` -to install the `wpbuild` binary. +to install the `pabuild` binary. You can see a help menu with the subcommands by: ```sh -wpbuild --help +pabuild --help ``` This is our local Rust command line application. Please see the [documentation](docs/pabuild.md) for how to use this alongside the other tools. diff --git a/docs/pabuild.md b/docs/pabuild.md index 97191c2..19bee24 100644 --- a/docs/pabuild.md +++ b/docs/pabuild.md @@ -17,14 +17,17 @@ To get the basic idea, run ```sh pabuild witness --help ``` -It can process and generate JSON files to be used for these circuits. +It can process and generate input JSON files to be used for parser/extractor circuits. + +> [!NOTE] +> `circuit-name` need to be **same** for witness generator and codegen. ### Examples **JSON Parsing:** If we have a given JSON file we want to parse such as [`examples/json/test/example.json`](../examples/json/test/example.json) for the `json-parser` circuit (see [`circuits.json`](../circuits.json)), then we can: ```sh -pabuild witness json --input-file examples/json/test/example.json --output-dir inputs/json-parser --output-filename input.json json +pabuild witness parser json --input-file examples/json/test/example.json --circuit-name json-parser ``` Afterwards, you can run `npx circomkit compile json-parser` then `circomkit witness json-parser input`. @@ -33,11 +36,25 @@ Afterwards, you can run `npx circomkit compile json-parser` then `circomkit witn If we have a given HTTP request/response (as a file) we want to parse such as [`examples/http/get_request.http`](../examples/http/get_request.http) for the `http-parser` circuit (see `circuits.json`), then we can: ```sh -pabuild witness http --input-file examples/json/get_request.http --output-dir inputs/http-parser --output-filename input.json http +pabuild witness parser http --input-file examples/http/get_request.http --circuit-name http-parser ``` Afterwards, you can run `npx circomkit compile http-parser` then `circomkit witness http-parser input`. +**JSON Extractor:** +To extract a value out of a JSON, we need a lockfile that contains keys and value type. + +```sh +pabuild witness extractor json --input-file examples/json/test/value_string.json --lockfile examples/json/lockfile/value_string.json --circuit-name value_string +``` + +**HTTP Extractor:** +To extract reponse from HTTP, a lockfile need to be given with start line (method, status, version) and headers to be matched. Example can be found in [examples/http/lockfile](../examples/http/lockfile/). + +```sh +pabuild witness extractor http --input-file examples/http/get_response.http --lockfile examples/http/lockfile/response.lock.json --circuit-name http-get-response +``` + ## Codegen ### JSON Extraction @@ -45,67 +62,79 @@ JSON extractor circuit is generated using rust to handle arbitrary keys and arra Run: ```sh -pabuild json --help +pabuild codegen --help ``` to get options: ``` -Usage: pabuild json [OPTIONS] --template