Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: general value extractor #95

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
+890 −48
Open

feat: general value extractor #95

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d524cce
init example
lonerapier Oct 7, 2024
9dbc27a
feat: add single value extractor
lonerapier Oct 8, 2024
cbf5595
test: add example
lonerapier Oct 8, 2024
6984bc9
feat: remove keyLen and add maxKeyLen
lonerapier Oct 8, 2024
b9feeeb
add optimisations from web-prover
lonerapier Oct 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
229 changes: 214 additions & 15 deletions circuits.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,216 @@
{
"json-parser": {
"file": "json/parser/parser",
"template": "Parser",
"params": [
157,
13
]
},
"http-parser": {
"file": "http/parser/parser",
"template": "Parser",
"params": [
60
]
}
"two_keys_test": {
"file": "main/json_two_keys_test",
"template": "ExtractStringValue",
"params": [
40,
1,
4,
0,
3
]
},
"value_array_number_test": {
"file": "main/json_value_array_number_test",
"template": "ExtractNumValue",
"params": [
73,
2,
1,
0,
2,
1,
4
]
},
"value_number_test": {
"file": "main/json_value_number_test",
"template": "ExtractNumValue",
"params": [
12,
1,
1,
0,
2
]
},
"http-parser": {
"file": "http/parser/parser",
"template": "Parser",
"params": [
60
]
},
"get_request_test": {
"file": "main/http_get_request_test",
"template": "LockHTTPRequest",
"params": [
60,
3,
4,
8,
6,
16,
4,
9
]
},
"value_array_string_test": {
"file": "main/json_value_array_string_test",
"template": "ExtractStringValue",
"params": [
73,
2,
1,
0,
1,
1,
2
]
},
"value_array_object_test": {
"file": "main/json_value_array_object_test",
"template": "ExtractNumValue",
"params": [
29,
4,
1,
0,
0,
1,
1,
2,
0,
3,
1
]
},
"spotify_top_artists_test": {
"file": "main/http_spotify_top_artists_test",
"template": "LockHTTPResponse",
"params": [
203,
85,
8,
3,
2,
12,
31
]
},
"spotify_test": {
"file": "main/json_spotify_test",
"template": "ExtractStringValue",
"params": [
85,
5,
4,
0,
5,
1,
0,
2,
7,
3,
4,
4,
12
]
},
"value_string": {
"file": "main/json_value_string",
"template": "ExtractStringValue",
"params": [
12,
1,
1,
0,
1
]
},
"get_response_test": {
"file": "main/http_get_response_test",
"template": "LockHTTPResponse",
"params": [
89,
18,
8,
3,
2,
12,
16
]
},
"value_array_nested_test": {
"file": "main/json_value_array_nested_test",
"template": "ExtractNumValue",
"params": [
24,
3,
1,
0,
0,
1,
0,
2,
1
]
},
"value_string_test": {
"file": "main/json_value_string_test",
"template": "ExtractStringValue",
"params": [
12,
1,
1,
0,
1
]
},
"spotify_top_artists": {
"file": "main/extended_spotify_top_artists",
"template": "HttpJson",
"params": [
203,
85,
8,
3,
2,
12,
31,
5,
4,
0,
5,
1,
0,
2,
7,
3,
4,
4,
12
]
},
"json-parser": {
"file": "json/parser/parser",
"template": "Parser",
"params": [
157,
13
]
},
"value_object_test": {
"file": "main/json_value_object_test",
"template": "ExtractStringValue",
"params": [
88,
3,
1,
0,
1,
1,
1
]
}
}
164 changes: 164 additions & 0 deletions circuits/json/extractor.circom
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
pragma circom 2.1.9;

include "interpreter.circom";

template ObjectExtractor(DATA_BYTES, MAX_STACK_HEIGHT, maxKeyLen, maxValueLen) {
assert(MAX_STACK_HEIGHT >= 2);

// Declaration of signals.
signal input data[DATA_BYTES];
signal input key[maxKeyLen];
signal input keyLen;

signal output value[maxValueLen];

// Constraints.
signal value_starting_index[DATA_BYTES - maxKeyLen];
// flag determining whether this byte is matched value
signal is_value_match[DATA_BYTES - maxKeyLen];
// final mask
signal mask[DATA_BYTES - maxKeyLen];

component State[DATA_BYTES - maxKeyLen];
State[0] = StateUpdate(MAX_STACK_HEIGHT);
State[0].byte <== data[0];
for(var i = 0; i < MAX_STACK_HEIGHT; i++) {
State[0].stack[i] <== [0,0];
}
State[0].parsing_string <== 0;
State[0].parsing_number <== 0;

signal parsing_key[DATA_BYTES - maxKeyLen];
signal parsing_value[DATA_BYTES - maxKeyLen];
signal parsing_object_value[DATA_BYTES - maxKeyLen];
signal is_key_match[DATA_BYTES - maxKeyLen];
signal is_key_match_for_value[DATA_BYTES+1 - maxKeyLen];
is_key_match_for_value[0] <== 0;
signal is_next_pair_at_depth[DATA_BYTES - maxKeyLen];
signal or[DATA_BYTES - maxKeyLen];

// initialise first iteration

// check inside key or value
parsing_key[0] <== InsideKey()(State[0].next_stack[0], State[0].next_parsing_string, State[0].next_parsing_number);
parsing_value[0] <== InsideValueObject()(State[0].next_stack[0], State[0].next_stack[1], State[0].next_parsing_string, State[0].next_parsing_number);

is_key_match[0] <== 0;
is_next_pair_at_depth[0] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(State[0].next_stack, data[0]);
is_key_match_for_value[1] <== Mux1()([is_key_match_for_value[0] * (1-is_next_pair_at_depth[0]), is_key_match[0] * (1-is_next_pair_at_depth[0])], is_key_match[0]);
is_value_match[0] <== parsing_value[0] * is_key_match_for_value[1];

mask[0] <== data[0] * is_value_match[0];

for(var data_idx = 1; data_idx < DATA_BYTES - maxKeyLen; data_idx++) {
State[data_idx] = StateUpdate(MAX_STACK_HEIGHT);
State[data_idx].byte <== data[data_idx];
State[data_idx].stack <== State[data_idx - 1].next_stack;
State[data_idx].parsing_string <== State[data_idx - 1].next_parsing_string;
State[data_idx].parsing_number <== State[data_idx - 1].next_parsing_number;

// - parsing key
// - parsing value (different for string/numbers and array)
// - key match (key 1, key 2)
// - is next pair
// - is key match for value
// - value_mask
// - mask

// check if inside key or not
parsing_key[data_idx] <== InsideKey()(State[data_idx].next_stack[0], State[data_idx].next_parsing_string, State[data_idx].next_parsing_number);
// check if inside value
parsing_value[data_idx] <== InsideValueObject()(State[data_idx].next_stack[0], State[data_idx].next_stack[1], State[data_idx].next_parsing_string, State[data_idx].next_parsing_number);

// to get correct value, check:
// - key matches at current index and depth of key is as specified
// - whether next KV pair starts
// - whether key matched for a value (propogate key match until new KV pair of lower depth starts)
is_key_match[data_idx] <== KeyMatchAtIndex(DATA_BYTES, maxKeyLen, data_idx)(data, key, keyLen, parsing_key[data_idx]);
is_next_pair_at_depth[data_idx] <== NextKVPairAtDepth(MAX_STACK_HEIGHT, 0)(State[data_idx].next_stack, data[data_idx]);
is_key_match_for_value[data_idx+1] <== Mux1()([is_key_match_for_value[data_idx] * (1-is_next_pair_at_depth[data_idx]), is_key_match[data_idx] * (1-is_next_pair_at_depth[data_idx])], is_key_match[data_idx]);
is_value_match[data_idx] <== is_key_match_for_value[data_idx+1] * parsing_value[data_idx];

or[data_idx] <== OR()(is_value_match[data_idx], is_value_match[data_idx - 1]);

// mask = currently parsing value and all subsequent keys matched
mask[data_idx] <== data[data_idx] * or[data_idx];
}

// find starting index of value in data by matching mask
signal is_zero_mask[DATA_BYTES];
signal is_prev_starting_index[DATA_BYTES];
value_starting_index[0] <== 0;
is_prev_starting_index[0] <== 0;
is_zero_mask[0] <== IsZero()(mask[0]);
for (var i=1 ; i<DATA_BYTES - maxKeyLen ; i++) {
is_zero_mask[i] <== IsZero()(mask[i]);
is_prev_starting_index[i] <== IsZero()(value_starting_index[i-1]);
value_starting_index[i] <== value_starting_index[i-1] + i * (1-is_zero_mask[i]) * is_prev_starting_index[i];
}

log("value starting index", value_starting_index[DATA_BYTES-1 - maxKeyLen]);
value <== SelectSubArray(DATA_BYTES - maxKeyLen, maxValueLen)(mask, value_starting_index[DATA_BYTES-1 - maxKeyLen], maxValueLen);
for (var i = 0 ; i < maxValueLen ; i++) {
log(i, value[i]);
}
}

template ArrayIndexExtractor(DATA_BYTES, MAX_STACK_HEIGHT, maxValueLen) {
assert(MAX_STACK_HEIGHT >= 2);

signal input data[DATA_BYTES];
signal input index;

signal output value[maxValueLen];

// value starting index in `data`
signal value_starting_index[DATA_BYTES];
// final mask
signal mask[DATA_BYTES];

component State[DATA_BYTES];
State[0] = StateUpdate(MAX_STACK_HEIGHT);
State[0].byte <== data[0];
for(var i = 0; i < MAX_STACK_HEIGHT; i++) {
State[0].stack[i] <== [0,0];
}
State[0].parsing_string <== 0;
State[0].parsing_number <== 0;

signal parsing_array[DATA_BYTES];
signal or[DATA_BYTES];

parsing_array[0] <== InsideArrayIndexObject()(State[0].next_stack[0], State[0].next_stack[1], State[0].next_parsing_string, State[0].next_parsing_number, index);
mask[0] <== data[0] * parsing_array[0];

for(var data_idx = 1; data_idx < DATA_BYTES; data_idx++) {
State[data_idx] = StateUpdate(MAX_STACK_HEIGHT);
State[data_idx].byte <== data[data_idx];
State[data_idx].stack <== State[data_idx - 1].next_stack;
State[data_idx].parsing_string <== State[data_idx - 1].next_parsing_string;
State[data_idx].parsing_number <== State[data_idx - 1].next_parsing_number;

parsing_array[data_idx] <== InsideArrayIndexObject()(State[data_idx].next_stack[0], State[data_idx].next_stack[1], State[data_idx].next_parsing_string, State[data_idx].next_parsing_number, index);

or[data_idx] <== OR()(parsing_array[data_idx], parsing_array[data_idx - 1]);
mask[data_idx] <== data[data_idx] * or[data_idx];
}

signal is_zero_mask[DATA_BYTES];
signal is_prev_starting_index[DATA_BYTES];
value_starting_index[0] <== 0;
is_prev_starting_index[0] <== 0;
is_zero_mask[0] <== IsZero()(mask[0]);
for (var i=1 ; i<DATA_BYTES ; i++) {
is_zero_mask[i] <== IsZero()(mask[i]);
is_prev_starting_index[i] <== IsZero()(value_starting_index[i-1]);
value_starting_index[i] <== value_starting_index[i-1] + i * (1-is_zero_mask[i]) * is_prev_starting_index[i];
}

log("value starting index", value_starting_index[DATA_BYTES-1]);
value <== SelectSubArray(DATA_BYTES, maxValueLen)(mask, value_starting_index[DATA_BYTES-1], maxValueLen);
for (var i = 0 ; i < maxValueLen ; i++) {
log(i, value[i]);
}
}
Loading