diff --git a/Cargo.lock b/Cargo.lock index f7d2691..06d63c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,6 +121,15 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "pabuild" +version = "0.0.0" +dependencies = [ + "clap", + "serde", + "serde_json", +] + [[package]] name = "proc-macro2" version = "1.0.86" @@ -278,12 +287,3 @@ name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "wpbuild" -version = "0.0.0" -dependencies = [ - "clap", - "serde", - "serde_json", -] diff --git a/Cargo.toml b/Cargo.toml index dbfe087..d252b64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "wpbuild" +name = "pabuild" edition = "2021" [dependencies] diff --git a/README.md b/README.md index 6b88400..fc344aa 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- SPARK + Parser Attestor

@@ -16,67 +16,118 @@ ## Overview -SPARK (Succinct Parser Attestation for Reconciliation of Knowledge) is a project focused on implementing parsers and extractors for various data formats using zero-knowledge proofs. +`parser-attestor` is a project focused on implementing parsers and extractors/selective-disclosure for various data formats inside of zero-knowledge circuits. ## Repository Structure -- `src/bin`: Binaries - - `witness`: Used for witness generation - - `codegen`: Used for generating extractor circuits based on input - `circuits/`: Current implementation of circuits - `http`: HTTP parser and extractor - `json`: JSON parser and extractor + - `json` has its own documentation [here](docs/json.md) - `utils`: Utility circuits - `test`: Circuit tests +- `src/`: Rust `pabuild` binary + - `pabuild` has its own documentation [here](docs/pabuild.md) - `examples/`: Reference examples for JSON and HTTP parsers +Documentation, in general, can be found in the `docs` directory. +We will add to this over time to make working with `parser-attestor` easier. + ## Getting Started ### Prerequisites -To use this repo, you need to install the following: +To use this repo, you will need to install the following dependencies. +These instructions should work on Linux/GNU and MacOS, but aren't guaranteed to work on Windows. -1. `circom` and `snarkjs`: - ```sh - git clone https://github.com/iden3/circom.git - cd circom - cargo build --release - cargo install --path circom - npm install -g snarkjs +#### Install Rust +To install Rust, you need to run: +```sh +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +exec $SHELL +``` +Check this is installed by running: +```sh +rustc --version && cargo --version +``` +to see the path to your Rust compiler and Cargo package manager. -### Circomkit -You will need `yarn` on your system (brew, or apt-get or something). -Then run: `npm install` to get everything else. +#### Install Circom +Succinctly, `cd` to a directory of your choosing and run: +```sh +git clone https://github.com/iden3/circom.git +cd circom +cargo build --release +cargo install --path circom +``` +in order to install `circom` globally. -#### Commands -To see what you can do, I suggest running: +#### Install Node +First, install `nvm` by running: +```sh +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash +exec $SHELL ``` -npx circomkit help +Now with `nvm` installed, run: +```sh +nvm install --lts +nvm use --lts +node --version && npm --version ``` -from the repository root. -#### Compiling and Witnessgen -For example, to compile the extractor, you can: +#### Node packages +From the root of the repository, you can now run: +```sh +npm install ``` -npx circomkit compile extract +which will install all the necessary packages for working with Circom. +This includes executables `circomkit`, `snarkjs`, and `mocha` which are accessible with Node: `npx`. + +##### Circomkit +This repository uses `circomkit` to manage Circom circuits. +To see what you can do with `circomkit`, we suggest running: ``` -Then you can do +npx circomkit help ``` -npx circomkit witness extract witness +`circomkit` can essentially do everything you would want to do with these Circuits, though we can't guarantee all commands work properly. + +**Example:** +For example, to compile the `json-parser`, you can run the following from the repository root: ``` -And even: +npx circomkit compile json-parser ``` -npx circomkit prove extract witness +which implicitly checks the `circuits.json` for an object that points to the circuit's code itself. + +If you are having trouble with `circomkit`, consider: + +##### SNARKJS +Likewise, `snarkjs` is used to handle proofs and verification under the hood. +There is [documentation](https://docs.circom.io/getting-started/compiling-circuits/) on Circom's usage to work with this. +We suggest starting at that link and carrying through to "Proving circuits with ZK". + +##### Mocha +`mocha` will also be installed from before. +Running +```sh +npx mocha ``` +will run every circuit test. +To filter tests, you can use the `-g` flag (very helpful!). -To clean up, just run: + +### Install `pabuild` +From the root of this repository, run: +```sh +cargo install --path . ``` -npx circomkit clean extract +to install the `wpbuild` binary. +You can see a help menu with the subcommands by: +```sh +wpbuild --help ``` +This is our local Rust command line application. +Please see the [documentation](docs/pabuild.md) for how to use this alongside the other tools. -All of the above should be ran from repository root. - -## Binaries ### Rust Example Witness JSON Creation To generate example input JSON files for the Circom circuits, run: @@ -101,111 +152,6 @@ witness http --input-file examples/http/get_request.http --output-dir inputs/get Afterwards, you can run `circomkit compile get_request` then `circomkit witness get_request input`. -### Codegen - -JSON extractor circuit is generated using rust to handle arbitrary keys and array indices. - -Run: -```bash -cargo run --bin codegen -- --help -``` -to get options: -``` -Usage: codegen [OPTIONS] --json-file - -Options: - -j, --json-file Path to the JSON file - -o, --output-filename Output circuit file name [default: extractor] -``` -Takes input 2 arguments: -- `json-file`: input json file. Examples are located in [codegen](./examples/json/test/codegen/) -- `output-filename`: circuit filename to save. Located in [circuits/main](./circuits/main/). If not given, defaults to `extractor.circom`. - -To test an end-to-end JSON extraction proof: -- Run codegen to generate circuits. Replace `value_string` with input filename. - ```bash - cargo run --bin codegen -- --json-file ./examples/json/test/codegen/value_string.json --output-filename value_string - ``` - -- Compile circom circuit using - ``` - circom ./circuits/main/value_string.circom --r1cs --wasm - ``` - -- To use circomkit: add circuit config to [circuits.json](./circuits.json). and input file to [inputs](./inputs/) - -- Generate witness: - ```bash - node build/json_extract_value_string/json_extract_value_string_js/generate_witness inputs/json_extract_value_string/value_string.json build/json_extract_value_string/witness/ - ``` - or generate using circomkit: - ```bash - npx circomkit witness json_extract_value_string value_string - ``` - -- create trusted setup: - ```bash - npx circomkit setup json_extract_value_string - # OR - snarkjs groth16 setup build/json_extract_value_string/json_extract_value_string.r1cs ptau/powersOfTau28_hez_final_14.ptau build/json_extract_value_string/groth16_pkey.zkey - ``` - -- create proof: - ```bash - npx circomkit prove json_extract_value_string value_string - # OR - snarkjs groth16 prove build/json_extract_value_string/groth16_pkey.zkey build/json_extract_value_string/value_string/witness.wtns build/json_extract_value_string/value_string/groth16_proof.json inputs/json_extract_value_string/value_string.json - ``` - -- verify proof: - ```bash - npx circomkit verify json_extract_value_string value_string - # OR - snarkjs groth16 verify build/json_extract_value_string/groth16_vkey.json inputs/json_extract_value_string/value_string.json build/json_extract_value_string/value_string/groth16_proof.json - ``` - -## Testing -To test, you can just run -``` -npx mocha -``` -from the repository root. - -To run specific tests, use the `-g` flag for `mocha`, e.g., to run any proof described with "State" we can pass: -``` -npx mocha -g State -``` - -> [!NOTE] -> Currently [search](./circuits/search.circom) circuit isn't working with circomkit, so you might have to compile using circom: `circom circuits/main/search.circom --r1cs --wasm -l node_modules/ -o build/search/` - -## (MOSTLY DEPRECATED DUE TO CIRCOMKIT) Running an example -``` -circom extract.circom --r1cs --wasm - -# in rust? circom witness rs -node extract_js/generate_witness.js extract_js/extract.wasm input.json witness.wtns - -## -# IF YOU NEED A NEW pot (works for all circuits) -snarkjs powersoftau new bn128 14 pot14_0000.ptau -v -snarkjs powersoftau contribute pot14_0000.ptau pot14_0001.ptau --name="First contribution" -v -snarkjs powersoftau prepare phase2 pot14_0001.ptau pot14_final.ptau -v -## - -snarkjs groth16 setup extract.r1cs pot14_final.ptau extract_0000.zkey - -snarkjs zkey contribute extract_0000.zkey extract_0001.zkey --name="1st Contributor Name" -v - -snarkjs zkey export verificationkey extractor_0001.zkey verification_key.json - -# in rust -snarkjs groth16 prove extractor_0001.zkey witness.wtns proof.json public.json - -# in rust -snarkjs groth16 verify verification_key.json public.json proof.json -``` - ## Contributing diff --git a/circuits.json b/circuits.json index 9139100..09f0948 100644 --- a/circuits.json +++ b/circuits.json @@ -1,5 +1,5 @@ { - "extract": { + "json-parser": { "file": "json/parser/parser", "template": "Parser", "params": [ @@ -7,93 +7,11 @@ 13 ] }, - "value_string": { - "file": "json/parser/parser", - "template": "Parser", - "params": [ - 12, - 1 - ] - }, - "value_number": { - "file": "json/parser/parser", - "template": "Parser", - "params": [ - 12, - 2 - ] - }, - "value_array": { - "file": "json/parser/parser", - "template": "Parser", - "params": [ - 18, - 2 - ] - }, - "value_array_nested": { - "file": "json/parser/parser", - "template": "Parser", - "params": [ - 24, - 4 - ] - }, - "value_array_object": { - "file": "json/parser/parser", - "template": "Parser", - "params": [ - 25, - 4 - ] - }, - "value_array_object_array": { - "file": "json/parser/parser", - "template": "Parser", - "params": [ - 31, - 5 - ] - }, - "value_object": { - "file": "json/parser/parser", - "template": "Parser", - "params": [ - 21, - 3 - ] - }, - "search": { - "file": "search", - "template": "SubstringMatch", - "params": [ - 787, - 10 - ] - }, - "get_request": { + "http-parser": { "file": "http/parser/parser", "template": "Parser", "params": [ 60 ] - }, - "get_response": { - "file": "http/parser/parser", - "template": "Parser", - "params": [ - 89 - ] - }, - "json_extract_value_string": { - "file": "main/value_string", - "template": "ExtractStringValue", - "params": [ - 12, - 1, - 1, - 0, - 1 - ] } } \ No newline at end of file diff --git a/circuits/http/extractor.circom b/circuits/http/extractor.circom index d6b3dd6..03ad0c2 100644 --- a/circuits/http/extractor.circom +++ b/circuits/http/extractor.circom @@ -72,7 +72,7 @@ template ExtractResponse(DATA_BYTES, maxContentLength) { signal isPrevStartingIndex[DATA_BYTES]; valueStartingIndex[0] <== 0; isZeroMask[0] <== IsZero()(dataMask[0]); - for (var i=1 ; i(filename: string): T { + const filePath = join(__dirname, "..", "..", "..", "examples", "http", "lockfile", filename); const jsonString = readFileSync(filePath, 'utf-8'); const jsonData = JSON.parse(jsonString); return jsonData; } -interface HttpData { - request: Request; - response: Response; +function getHeaders(data: Request | Response): [string, string][] { + const headers: [string, string][] = []; + let i = 1; + while (true) { + const nameKey = `headerName${i}`; + const valueKey = `headerValue${i}`; + if (nameKey in data && valueKey in data) { + headers.push([data[nameKey], data[valueKey]]); + i++; + } else { + break; + } + } + return headers; } interface Request { method: string, target: string, version: string, - headers: [string, string][], + [key: string]: string, } interface Response { version: string, status: string, message: string, - headers: [string, string][], + [key: string]: string, } function executeCodegen(inputFilename: string, outputFilename: string) { return new Promise((resolve, reject) => { - const inputPath = join(__dirname, "..", "..", "..", "examples", "lockfile", inputFilename); + const inputPath = join(__dirname, "..", "..", "..", "examples", "http", "lockfile", inputFilename); - const codegen = spawn("cargo", ["run", "http-lock", "--lockfile", inputPath, "--output-filename", outputFilename]); + const codegen = spawn("cargo", ["run", "http", "--lockfile", inputPath, "--output-filename", outputFilename]); codegen.stdout.on('data', (data) => { console.log(`stdout: ${data}`); @@ -54,86 +65,184 @@ function executeCodegen(inputFilename: string, outputFilename: string) { }); } -describe("HTTP :: Codegen", async () => { - let circuit: WitnessTester<["data", "beginning", "middle", "final", "header1", "value1", "header2", "value2"], []>; +describe("HTTP :: Codegen :: Request", async () => { + let circuit: WitnessTester<["data", "method", "target", "version", "header1", "value1", "header2", "value2"], []>; - it("(valid) get_request:", async () => { - let lockfile = "test.lock"; + it("(valid) GET:", async () => { + let lockfile = "request.lock"; let inputfile = "get_request.http"; // generate extractor circuit using codegen await executeCodegen(`${lockfile}.json`, lockfile); - const lockData = await readLockFile(`${lockfile}.json`); + const lockData = readLockFile(`${lockfile}.json`); console.log("lockData: ", JSON.stringify(lockData)); - const input = await readHTTPInputFile(`${inputfile}`).input + const input = readHTTPInputFile(`${inputfile}`).input; - const params = [input.length, lockData.request.method.length, lockData.request.target.length, lockData.request.version.length]; - lockData.request.headers.forEach(header => { - params.push(header[0].length); // Header name length - params.push(header[1].length); // Header value length - console.log("header: ", header[0]); - console.log("value: ", header[1]); + const headers = getHeaders(lockData); + const params = [input.length, lockData.method.length, lockData.target.length, lockData.version.length]; + headers.forEach(header => { + params.push(header[0].length); + params.push(header[1].length); }); circuit = await circomkit.WitnessTester(`Extract`, { - file: `circuits/main/${lockfile}`, - template: "LockHTTP", + file: `main/${lockfile}`, + template: "LockHTTPRequest", params: params, }); console.log("#constraints:", await circuit.getConstraintCount()); // match circuit output to original JSON value - await circuit.expectPass({ + const circuitInput: any = { data: input, - beginning: toByte(lockData.request.method), - middle: toByte(lockData.request.target), - final: toByte(lockData.request.version), - header1: toByte(lockData.request.headers[0][0]), - value1: toByte(lockData.request.headers[0][1]), - header2: toByte(lockData.request.headers[1][0]), - value2: toByte(lockData.request.headers[1][1]) - }, - {} - ); + method: toByte(lockData.method), + target: toByte(lockData.target), + version: toByte(lockData.version), + }; + + headers.forEach((header, index) => { + circuitInput[`header${index + 1}`] = toByte(header[0]); + circuitInput[`value${index + 1}`] = toByte(header[1]); + }); + await circuit.expectPass(circuitInput, {}); }); - it("(invalid) get_request:", async () => { - let lockfile = "test.lock"; + it("(invalid) GET:", async () => { + let lockfile = "request.lock"; let inputfile = "get_request.http"; // generate extractor circuit using codegen await executeCodegen(`${lockfile}.json`, lockfile); - const lockData = await readLockFile(`${lockfile}.json`); + const lockData = readLockFile(`${lockfile}.json`); - const input = await readHTTPInputFile(`${inputfile}`).input + const input = readHTTPInputFile(`${inputfile}`).input - const params = [input.length, lockData.request.method.length, lockData.request.target.length, lockData.request.version.length]; - lockData.request.headers.forEach(header => { - params.push(header[0].length); // Header name length - params.push(header[1].length); // Header value length + const headers = getHeaders(lockData); + const params = [input.length, lockData.method.length, lockData.target.length, lockData.version.length]; + headers.forEach(header => { + params.push(header[0].length); + params.push(header[1].length); }); circuit = await circomkit.WitnessTester(`Extract`, { - file: `circuits/main/${lockfile}`, - template: "LockHTTP", + file: `main/${lockfile}`, + template: "LockHTTPRequest", params: params, }); console.log("#constraints:", await circuit.getConstraintCount()); - await circuit.expectFail({ - data: input.slice(0), - beginning: toByte(lockData.request.method), - middle: toByte(lockData.request.target), - final: toByte(lockData.request.version), - header1: toByte(lockData.request.headers[0][0]), - value1: toByte("/aip"), - header2: toByte(lockData.request.headers[1][0]), - value2: toByte(lockData.request.headers[1][1]) + const circuitInput: any = { + data: input, + method: toByte(lockData.method), + target: toByte(lockData.target), + version: toByte(lockData.version), + }; + + headers.forEach((header, index) => { + circuitInput[`header${index + 1}`] = toByte(header[0]); + circuitInput[`value${index + 1}`] = toByte(header[1]); }); + + circuitInput.value1 = toByte("/aip"); + await circuit.expectFail(circuitInput); }); }); + +describe("HTTP :: Codegen :: Response", async () => { + let circuit: WitnessTester<["data", "version", "status", "message", "header1", "value1", "header2", "value2"], ["body"]>; + + it("(valid) GET:", async () => { + let lockfile = "response.lock"; + let inputfile = "get_response.http"; + + // generate extractor circuit using codegen + await executeCodegen(`${lockfile}.json`, lockfile); + + const lockData = readLockFile(`${lockfile}.json`); + console.log("lockData: ", JSON.stringify(lockData)); + + const http = readHTTPInputFile(`${inputfile}`); + const input = http.input; + + const headers = getHeaders(lockData); + + const params = [input.length, parseInt(http.headers["Content-Length"]), lockData.version.length, lockData.status.length, lockData.message.length]; + headers.forEach(header => { + params.push(header[0].length); + params.push(header[1].length); + }); + + + circuit = await circomkit.WitnessTester(`Extract`, { + file: `main/${lockfile}`, + template: "LockHTTPResponse", + params: params, + }); + console.log("#constraints:", await circuit.getConstraintCount()); + + // match circuit output to original JSON value + const circuitInput: any = { + data: input, + version: toByte(lockData.version), + status: toByte(lockData.status), + message: toByte(lockData.message), + }; + + headers.forEach((header, index) => { + circuitInput[`header${index + 1}`] = toByte(header[0]); + circuitInput[`value${index + 1}`] = toByte(header[1]); + }); + + + await circuit.expectPass(circuitInput, { body: http.bodyBytes }); + }); + + it("(invalid) GET:", async () => { + let lockfile = "response.lock"; + let inputfile = "get_response.http"; + + // generate extractor circuit using codegen + await executeCodegen(`${lockfile}.json`, lockfile); + + const lockData = readLockFile(`${lockfile}.json`); + + const http = readHTTPInputFile(`${inputfile}`); + const input = http.input; + + const headers = getHeaders(lockData); + + const params = [input.length, parseInt(http.headers["Content-Length"]), lockData.version.length, lockData.status.length, lockData.message.length]; + headers.forEach(header => { + params.push(header[0].length); + params.push(header[1].length); + }); + + + circuit = await circomkit.WitnessTester(`Extract`, { + file: `main/${lockfile}`, + template: "LockHTTPResponse", + params: params, + }); + console.log("#constraints:", await circuit.getConstraintCount()); + + const circuitInput: any = { + data: input, + version: toByte(lockData.version), + status: toByte(lockData.status), + message: toByte(lockData.message), + }; + + headers.forEach((header, index) => { + circuitInput[`header${index + 1}`] = toByte(header[0]); + circuitInput[`value${index + 1}`] = toByte(header[1]); + }); + + circuitInput.value1 = toByte("/aip"); + await circuit.expectFail(circuitInput); + }); +}); \ No newline at end of file diff --git a/circuits/test/http/extractor.test.ts b/circuits/test/http/extractor.test.ts index 6619de4..c69c7a7 100644 --- a/circuits/test/http/extractor.test.ts +++ b/circuits/test/http/extractor.test.ts @@ -9,7 +9,7 @@ describe("HTTP :: body Extractor", async () => { it(`(valid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`ExtractResponseData`, { - file: "circuits/http/extractor", + file: "http/extractor", template: "ExtractResponse", params: [input.length, expected.length], }); @@ -60,7 +60,7 @@ describe("HTTP :: header Extractor", async () => { it(`(valid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`ExtractHeaderValue`, { - file: "circuits/http/extractor", + file: "http/extractor", template: "ExtractHeaderValue", params: [input.length, headerName.length, headerValue.length], }); diff --git a/circuits/test/http/interpreter.test.ts b/circuits/test/http/interpreter.test.ts index 5c46d95..ac1a330 100644 --- a/circuits/test/http/interpreter.test.ts +++ b/circuits/test/http/interpreter.test.ts @@ -9,7 +9,7 @@ describe("HTTP :: Interpreter", async () => { it(`(valid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`LockRequestLineData`, { - file: "circuits/http/interpreter", + file: "http/interpreter", template: "MethodMatch", params: [input.length, method.length], }); @@ -24,7 +24,7 @@ describe("HTTP :: Interpreter", async () => { it(`(invalid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`LockRequestLineData`, { - file: "circuits/http/interpreter", + file: "http/interpreter", template: "MethodMatch", params: [input.length, method.length], }); diff --git a/circuits/test/http/locker.test.ts b/circuits/test/http/locker.test.ts index 1a67fc1..4969b39 100644 --- a/circuits/test/http/locker.test.ts +++ b/circuits/test/http/locker.test.ts @@ -8,7 +8,7 @@ describe("HTTP :: Locker :: Request Line", async () => { it(`(valid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`LockStartLine`, { - file: "circuits/http/locker", + file: "http/locker", template: "LockStartLine", params: [input.length, beginning.length, middle.length, final.length], }); @@ -23,7 +23,7 @@ describe("HTTP :: Locker :: Request Line", async () => { it(`(invalid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`LockStartLine`, { - file: "circuits/http/locker", + file: "http/locker", template: "LockStartLine", params: [input.length, beginning.length, middle.length, final.length], }); @@ -58,7 +58,7 @@ describe("HTTP :: Locker :: Status Line", async () => { it(`(valid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`LockStartLine`, { - file: "circuits/http/locker", + file: "http/locker", template: "LockStartLine", params: [input.length, beginning.length, middle.length, final.length], }); @@ -73,7 +73,7 @@ describe("HTTP :: Locker :: Status Line", async () => { it(`(invalid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`LockStartLine`, { - file: "circuits/http/locker", + file: "http/locker", template: "LockStartLine", params: [input.length, beginning.length, middle.length, final.length], }); @@ -100,7 +100,7 @@ describe("HTTP :: Locker :: Header", async () => { it(`(valid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`LockHeader`, { - file: "circuits/http/locker", + file: "http/locker", template: "LockHeader", params: [input.length, header.length, value.length], }); @@ -115,7 +115,7 @@ describe("HTTP :: Locker :: Header", async () => { it(`(invalid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`LockHeader`, { - file: "circuits/http/locker", + file: "http/locker", template: "LockHeader", params: [input.length, header.length, value.length], }); diff --git a/circuits/test/json/extractor/extractor.test.ts b/circuits/test/json/extractor/extractor.test.ts index 4d47ea7..9b9b6b3 100644 --- a/circuits/test/json/extractor/extractor.test.ts +++ b/circuits/test/json/extractor/extractor.test.ts @@ -5,9 +5,9 @@ import { spawn } from "child_process"; function executeCodegen(inputFilename: string, outputFilename: string) { return new Promise((resolve, reject) => { - const inputPath = join(__dirname, "..", "..", "..", "..", "examples", "extractor", inputFilename); + const inputPath = join(__dirname, "..", "..", "..", "..", "examples", "json", "lockfile", inputFilename); - const codegen = spawn("cargo", ["run", "extractor", "--template", inputPath, "--output-filename", outputFilename]); + const codegen = spawn("cargo", ["run", "json", "--template", inputPath, "--output-filename", outputFilename]); codegen.stdout.on('data', (data) => { console.log(`stdout: ${data}`); @@ -40,7 +40,7 @@ describe("ExtractValue", async () => { let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["k"]); circuit = await circomkit.WitnessTester(`Extract`, { - file: `circuits/main/${filename}`, + file: `main/${filename}`, template: "ExtractStringValue", params: [input.length, 1, 1, 0, 1], }); @@ -60,7 +60,7 @@ describe("ExtractValue", async () => { let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["key2"]); circuit = await circomkit.WitnessTester(`Extract`, { - file: `circuits/main/${filename}`, + file: `main/${filename}`, template: "ExtractStringValue", params: [input.length, 1, 4, 0, 3], }); @@ -75,7 +75,7 @@ describe("ExtractValue", async () => { let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["k"]); circuit = await circomkit.WitnessTester(`Extract`, { - file: `circuits/main/${filename}`, + file: `main/${filename}`, template: "ExtractNumValue", params: [input.length, 1, 1, 0, 2], }); @@ -94,7 +94,7 @@ describe("ExtractValue", async () => { let [input, keyUnicode, output] = readJSONInputFile("value_array.json", ["b", i]); circuit = await circomkit.WitnessTester(`Extract`, { - file: `circuits/main/${filename}`, + file: `main/${filename}`, template: "ExtractStringValue", params: [input.length, 2, 1, 0, i, 1, output.length], }); @@ -112,7 +112,7 @@ describe("ExtractValue", async () => { let [input, keyUnicode, output] = readJSONInputFile("value_array.json", ["k", i]); circuit = await circomkit.WitnessTester(`Extract`, { - file: `circuits/main/${filename}`, + file: `main/${filename}`, template: "ExtractNumValue", params: [input.length, 2, 1, 0, i, 1, output.length], }); @@ -131,7 +131,7 @@ describe("ExtractValue", async () => { let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["a", index_0, index_1]); circuit = await circomkit.WitnessTester(`Extract`, { - file: `circuits/main/${filename}`, + file: `main/${filename}`, template: "ExtractNumValue", params: [input.length, 3, 1, 0, index_0, 1, index_1, 2, 1], }); @@ -155,7 +155,7 @@ describe("ExtractValueMultiDepth", () => { let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["e", "e"]); circuit = await circomkit.WitnessTester(`Extract`, { - file: `circuits/main/${filename}`, + file: `main/${filename}`, template: "ExtractStringValue", params: [input.length, 3, 1, 0, 1, 1, 1], }); @@ -183,7 +183,7 @@ describe("ExtractValueArrayObject", () => { let [input, keyUnicode, output] = readJSONInputFile(`${filename}.json`, ["a", index_0, "b", index_1]); circuit = await circomkit.WitnessTester(`Extract`, { - file: `circuits/main/${filename}`, + file: `main/${filename}`, template: "ExtractNumValue", params: [input.length, 4, 1, 0, index_0, 1, 1, 2, index_1, 3, 1], }); diff --git a/circuits/test/json/extractor/interpreter.test.ts b/circuits/test/json/extractor/interpreter.test.ts index cb2bfd6..26643b1 100644 --- a/circuits/test/json/extractor/interpreter.test.ts +++ b/circuits/test/json/extractor/interpreter.test.ts @@ -7,7 +7,7 @@ describe("Interpreter", async () => { before(async () => { circuit = await circomkit.WitnessTester(`InsideKey`, { - file: "circuits/json/interpreter", + file: "json/interpreter", template: "InsideKey", params: [4], }); @@ -46,7 +46,7 @@ describe("Interpreter", async () => { before(async () => { circuit = await circomkit.WitnessTester(`InsideValue`, { - file: "circuits/json/interpreter", + file: "json/interpreter", template: "InsideValue", params: [4], }); @@ -88,7 +88,7 @@ describe("Interpreter", async () => { it(`(valid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`InsideValueAtDepth`, { - file: "circuits/json/interpreter", + file: "json/interpreter", template: "InsideValueAtDepth", params: [4, depth], }); @@ -125,7 +125,7 @@ describe("Interpreter", async () => { it(`(valid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`InsideArrayIndex`, { - file: "circuits/json/interpreter", + file: "json/interpreter", template: "InsideArrayIndex", params: [4, index], }); @@ -165,7 +165,7 @@ describe("Interpreter", async () => { it(`(valid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`InsideArrayIndexAtDepth`, { - file: "circuits/json/interpreter", + file: "json/interpreter", template: "InsideArrayIndexAtDepth", params: [4, index, depth], }); @@ -199,7 +199,7 @@ describe("Interpreter", async () => { before(async () => { circuit = await circomkit.WitnessTester(`NextKVPair`, { - file: "circuits/json/interpreter", + file: "json/interpreter", template: "NextKVPair", params: [4], }); @@ -239,7 +239,7 @@ describe("Interpreter", async () => { it(`(valid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`NextKVPairAtDepth`, { - file: "circuits/json/interpreter", + file: "json/interpreter", template: "NextKVPairAtDepth", params: [4, depth], }); @@ -273,7 +273,7 @@ describe("Interpreter", async () => { it(`(valid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`KeyMatch`, { - file: "circuits/json/interpreter", + file: "json/interpreter", template: "KeyMatch", params: [input.data.length, input.key.length], }); @@ -314,7 +314,7 @@ describe("Interpreter", async () => { it(`(valid) witness: ${description} ${desc}`, async () => { circuit = await circomkit.WitnessTester(`KeyMatchAtDepth`, { - file: "circuits/json/interpreter", + file: "json/interpreter", template: "KeyMatchAtDepth", params: [input.data.length, 4, input.key.length, depth], }); diff --git a/circuits/test/json/parser/parsing_types.test.ts b/circuits/test/json/parser/parsing_types.test.ts index a5783dc..88d892e 100644 --- a/circuits/test/json/parser/parsing_types.test.ts +++ b/circuits/test/json/parser/parsing_types.test.ts @@ -19,7 +19,7 @@ describe("StateUpdate", () => { before(async () => { circuit = await circomkit.WitnessTester(`StateUpdate`, { - file: "circuits/json/parser/machine", + file: "json/parser/machine", template: "StateUpdate", params: [4], }); diff --git a/circuits/test/json/parser/stack.test.ts b/circuits/test/json/parser/stack.test.ts index f719582..860737e 100644 --- a/circuits/test/json/parser/stack.test.ts +++ b/circuits/test/json/parser/stack.test.ts @@ -5,7 +5,7 @@ describe("GetTopOfStack", () => { let circuit: WitnessTester<["stack"], ["value", "pointer"]>; before(async () => { circuit = await circomkit.WitnessTester(`GetTopOfStack`, { - file: "circuits/json/parser/machine", + file: "json/parser/machine", template: "GetTopOfStack", params: [4], }); @@ -34,7 +34,7 @@ describe("StateUpdate :: RewriteStack", () => { >; before(async () => { circuit = await circomkit.WitnessTester(`GetTopOfStack`, { - file: "circuits/json/parser/machine", + file: "json/parser/machine", template: "StateUpdate", params: [4], }); diff --git a/circuits/test/json/parser/values.test.ts b/circuits/test/json/parser/values.test.ts index 2bca379..069525b 100644 --- a/circuits/test/json/parser/values.test.ts +++ b/circuits/test/json/parser/values.test.ts @@ -8,7 +8,7 @@ describe("StateUpdate :: Values", () => { >; before(async () => { circuit = await circomkit.WitnessTester(`GetTopOfStack`, { - file: "circuits/json/parser/machine", + file: "json/parser/machine", template: "StateUpdate", params: [4], }); diff --git a/circuits/test/utils/array.test.ts b/circuits/test/utils/array.test.ts index ae4a42e..c951452 100644 --- a/circuits/test/utils/array.test.ts +++ b/circuits/test/utils/array.test.ts @@ -5,7 +5,7 @@ describe("array", () => { let circuit: WitnessTester<["in"], ["out"]>; before(async () => { circuit = await circomkit.WitnessTester(`Slice`, { - file: "circuits/utils/array", + file: "utils/array", template: "Slice", params: [10, 2, 4], }); @@ -33,7 +33,7 @@ describe("IsEqualArray", () => { let circuit: WitnessTester<["in"], ["out"]>; before(async () => { circuit = await circomkit.WitnessTester(`IsEqualArray`, { - file: "circuits/utils/array", + file: "utils/array", template: "IsEqualArray", params: [3], }); @@ -87,7 +87,7 @@ describe("Contains", () => { let circuit: WitnessTester<["in", "array"], ["out"]>; before(async () => { circuit = await circomkit.WitnessTester(`Contains`, { - file: "circuits/utils/array", + file: "utils/array", template: "Contains", params: [3], }); @@ -128,7 +128,7 @@ describe("ArrayAdd", () => { let circuit: WitnessTester<["lhs", "rhs"], ["out"]>; before(async () => { circuit = await circomkit.WitnessTester(`ArrayAdd`, { - file: "circuits/utils/array", + file: "utils/array", template: "ArrayAdd", params: [3], }); @@ -148,7 +148,7 @@ describe("ArrayMul", () => { let circuit: WitnessTester<["lhs", "rhs"], ["out"]>; before(async () => { circuit = await circomkit.WitnessTester(`ArrayMul`, { - file: "circuits/utils/array", + file: "utils/array", template: "ArrayMul", params: [3], }); @@ -168,7 +168,7 @@ describe("GenericArrayAdd", () => { let circuit: WitnessTester<["arrays"], ["out"]>; before(async () => { circuit = await circomkit.WitnessTester(`ArrayAdd`, { - file: "circuits/utils/array", + file: "utils/array", template: "GenericArrayAdd", params: [3, 2], }); diff --git a/circuits/test/utils/bytes.test.ts b/circuits/test/utils/bytes.test.ts index 5ee55ef..d0690a3 100644 --- a/circuits/test/utils/bytes.test.ts +++ b/circuits/test/utils/bytes.test.ts @@ -4,7 +4,7 @@ describe("ASCII", () => { let circuit: WitnessTester<["in"], ["out"]>; before(async () => { circuit = await circomkit.WitnessTester(`ASCII`, { - file: "circuits/utils/bytes", + file: "utils/bytes", template: "ASCII", params: [13], }); diff --git a/circuits/test/utils/hash.test.ts b/circuits/test/utils/hash.test.ts index e834cc7..1661367 100644 --- a/circuits/test/utils/hash.test.ts +++ b/circuits/test/utils/hash.test.ts @@ -7,7 +7,7 @@ describe("hash", () => { before(async () => { circuit = await circomkit.WitnessTester(`PoseidonModular`, { - file: "circuits/utils/hash", + file: "utils/hash", template: "PoseidonModular", params: [16], }); @@ -30,7 +30,7 @@ describe("hash", () => { before(async () => { circuit = await circomkit.WitnessTester(`PoseidonModular`, { - file: "circuits/utils/hash", + file: "utils/hash", template: "PoseidonModular", params: [379], }); diff --git a/circuits/test/utils/operators.test.ts b/circuits/test/utils/operators.test.ts index 7c82c61..67c0069 100644 --- a/circuits/test/utils/operators.test.ts +++ b/circuits/test/utils/operators.test.ts @@ -4,7 +4,7 @@ describe("SwitchArray", () => { let circuit: WitnessTester<["case", "branches", "vals"], ["match", "out"]>; before(async () => { circuit = await circomkit.WitnessTester(`SwitchArray`, { - file: "circuits/utils/operators", + file: "utils/operators", template: "SwitchArray", params: [3, 2], }); @@ -59,7 +59,7 @@ describe("Switch", () => { let circuit: WitnessTester<["case", "branches", "vals"], ["match", "out"]>; before(async () => { circuit = await circomkit.WitnessTester(`Switch`, { - file: "circuits/utils/operators", + file: "utils/operators", template: "Switch", params: [3], }); @@ -101,7 +101,7 @@ describe("InRange", () => { let circuit: WitnessTester<["in", "range"], ["out"]>; before(async () => { circuit = await circomkit.WitnessTester(`InRange`, { - file: "circuits/utils/operators", + file: "utils/operators", template: "InRange", params: [8], }); diff --git a/circuits/test/utils/search.test.ts b/circuits/test/utils/search.test.ts index f0938db..e6cbbe1 100644 --- a/circuits/test/utils/search.test.ts +++ b/circuits/test/utils/search.test.ts @@ -14,7 +14,7 @@ describe("search", () => { const hashResult = PoseidonModular(concatenatedInput); circuit = await circomkit.WitnessTester(`SubstringSearch`, { - file: "circuits/utils/search", + file: "utils/search", template: "SubstringSearch", params: [data.length, key.length], }); @@ -32,7 +32,7 @@ describe("search", () => { const hashResult = PoseidonModular(concatenatedInput); circuit = await circomkit.WitnessTester(`SubstringSearch`, { - file: "circuits/utils/search", + file: "utils/search", template: "SubstringSearch", params: [data.length, key.length], }); @@ -51,7 +51,7 @@ describe("search", () => { const key = [1, 0]; circuit = await circomkit.WitnessTester(`SubstringSearch`, { - file: "circuits/utils/search", + file: "utils/search", template: "SubstringSearch", params: [data.length, key.length], }); @@ -67,7 +67,7 @@ describe("search", () => { const hashResult = PoseidonModular(concatenatedInput); circuit = await circomkit.WitnessTester(`SubstringSearch`, { - file: "circuits/utils/search", + file: "utils/search", template: "SubstringSearch", params: [witness["data"].length, witness["key"].length], }); @@ -85,7 +85,7 @@ describe("search", () => { before(async () => { circuit = await circomkit.WitnessTester(`SubstringSearch`, { - file: "circuits/utils/search", + file: "utils/search", template: "SubstringMatchWithIndex", params: [787, 10], }); @@ -122,7 +122,7 @@ describe("search", () => { before(async () => { circuit = await circomkit.WitnessTester(`SubstringSearch`, { - file: "circuits/utils/search", + file: "utils/search", template: "SubstringMatch", params: [787, 10], }); diff --git a/docs/http.md b/docs/http.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/pabuild.md b/docs/pabuild.md new file mode 100644 index 0000000..97191c2 --- /dev/null +++ b/docs/pabuild.md @@ -0,0 +1,113 @@ +# `pabuild` CLI Tool +This repository contains a small Rust CLI tool called `pabuild`. + +## Install `pabuild` +From the root of this repository, run: +```sh +cargo install --path . +``` +to install the `pabuild` binary. +You can see a help menu with the subcommands by: +```sh +pabuild --help +``` + +## Witnessgen +To get the basic idea, run +```sh +pabuild witness --help +``` +It can process and generate JSON files to be used for these circuits. + +### Examples +**JSON Parsing:** +If we have a given JSON file we want to parse such as [`examples/json/test/example.json`](../examples/json/test/example.json) for the `json-parser` circuit (see [`circuits.json`](../circuits.json)), then we can: + +```sh +pabuild witness json --input-file examples/json/test/example.json --output-dir inputs/json-parser --output-filename input.json json +``` + +Afterwards, you can run `npx circomkit compile json-parser` then `circomkit witness json-parser input`. + +**HTTP Parsing:** +If we have a given HTTP request/response (as a file) we want to parse such as [`examples/http/get_request.http`](../examples/http/get_request.http) for the `http-parser` circuit (see `circuits.json`), then we can: + +```sh +pabuild witness http --input-file examples/json/get_request.http --output-dir inputs/http-parser --output-filename input.json http +``` + +Afterwards, you can run `npx circomkit compile http-parser` then `circomkit witness http-parser input`. + +## Codegen + +### JSON Extraction +JSON extractor circuit is generated using rust to handle arbitrary keys and array indices. + +Run: +```sh +pabuild json --help +``` +to get options: +``` +Usage: pabuild json [OPTIONS] --template