From 8e7bd77ab595dac82c5adbe4bad01ecdf6784c25 Mon Sep 17 00:00:00 2001 From: pkoppstein Date: Fri, 14 Jul 2023 04:16:56 -0400 Subject: [PATCH 1/5] redefine last/0; add to jq.test Modify last/0 to allow e.g. pick(last) Include some tests for input, inputs, debug, debug(), pick() Also some tidying of spacing in builtin.jq --- src/builtin.jq | 94 +++++++++++++++++++++++++------------------------- tests/jq.test | 32 +++++++++++++++++ 2 files changed, 79 insertions(+), 47 deletions(-) diff --git a/src/builtin.jq b/src/builtin.jq index a13d7845bf..73c0aec0d3 100644 --- a/src/builtin.jq +++ b/src/builtin.jq @@ -62,9 +62,9 @@ def nulls: select(. == null); def values: select(. != null); def scalars: select(type|. != "array" and . != "object"); def join($x): reduce .[] as $i (null; - (if .==null then "" else .+$x end) + - ($i | if type=="boolean" or type=="number" then tostring else .//"" end) - ) // ""; + (if .==null then "" else .+$x end) + + ($i | if type=="boolean" or type=="number" then tostring else .//"" end) + ) // ""; def _flatten($x): reduce .[] as $i ([]; if $i | type == "array" and $x != 0 then . + ($i | _flatten($x-1)) else . + [$i] end); def flatten($x): if $x < 0 then error("flatten depth must not be negative") else _flatten($x) end; def flatten: _flatten(-1); @@ -75,25 +75,25 @@ def fromdate: fromdateiso8601; def todate: todateiso8601; def match(re; mode): _match_impl(re; mode; false)|.[]; def match($val): ($val|type) as $vt | if $vt == "string" then match($val; null) - elif $vt == "array" and ($val | length) > 1 then match($val[0]; $val[1]) - elif $vt == "array" and ($val | length) > 0 then match($val[0]; null) - else error( $vt + " not a string or array") end; + elif $vt == "array" and ($val | length) > 1 then match($val[0]; $val[1]) + elif $vt == "array" and ($val | length) > 0 then match($val[0]; null) + else error( $vt + " not a string or array") end; def test(re; mode): _match_impl(re; mode; true); def test($val): ($val|type) as $vt | if $vt == "string" then test($val; null) - elif $vt == "array" and ($val | length) > 1 then test($val[0]; $val[1]) - elif $vt == "array" and ($val | length) > 0 then test($val[0]; null) - else error( $vt + " not a string or array") end; + elif $vt == "array" and ($val | length) > 1 then test($val[0]; $val[1]) + elif $vt == "array" and ($val | length) > 0 then test($val[0]; null) + else error( $vt + " not a string or array") end; def capture(re; mods): match(re; mods) | reduce ( .captures | .[] | select(.name != null) | { (.name) : .string } ) as $pair ({}; . + $pair); def capture($val): ($val|type) as $vt | if $vt == "string" then capture($val; null) - elif $vt == "array" and ($val | length) > 1 then capture($val[0]; $val[1]) - elif $vt == "array" and ($val | length) > 0 then capture($val[0]; null) - else error( $vt + " not a string or array") end; + elif $vt == "array" and ($val | length) > 1 then capture($val[0]; $val[1]) + elif $vt == "array" and ($val | length) > 0 then capture($val[0]; null) + else error( $vt + " not a string or array") end; def scan($re; $flags): match($re; "g" + $flags) - | if (.captures|length > 0) - then [ .captures | .[] | .string ] - else .string - end; + | if (.captures|length > 0) + then [ .captures | .[] | .string ] + else .string + end; def scan($re): scan($re; null); # # If input is an array, then emit a stream of successive subarrays of length n (or less), @@ -118,17 +118,17 @@ def split($re; flags): [ splits($re; flags) ]; # If s contains capture variables, then create a capture object and pipe it to s, bearing # in mind that s could be a stream def sub($re; s; $flags): - . as $in - | (reduce match($re; $flags) as $edit - ({result: [], previous: 0}; - $in[ .previous: ($edit | .offset) ] as $gap - # create the "capture" objects (one per item in s) - | [reduce ( $edit | .captures | .[] | select(.name != null) | { (.name) : .string } ) as $pair + . as $in + | (reduce match($re; $flags) as $edit + ({result: [], previous: 0}; + $in[ .previous: ($edit | .offset) ] as $gap + # create the "capture" objects (one per item in s) + | [reduce ( $edit | .captures | .[] | select(.name != null) | { (.name) : .string } ) as $pair ({}; . + $pair) | s ] as $inserts - | reduce range(0; $inserts|length) as $ix (.; .result[$ix] += $gap + $inserts[$ix]) - | .previous = ($edit | .offset + .length ) ) - | .result[] + $in[.previous:] ) - // $in; + | reduce range(0; $inserts|length) as $ix (.; .result[$ix] += $gap + $inserts[$ix]) + | .previous = ($edit | .offset + .length ) ) + | .result[] + $in[.previous:] ) + // $in; # def sub($re; s): sub($re; s; ""); # @@ -138,17 +138,17 @@ def gsub($re; s): sub($re; s; "g"); ######################################################################## # generic iterator/generator def while(cond; update): - def _while: - if cond then ., (update | _while) else empty end; - _while; + def _while: + if cond then ., (update | _while) else empty end; + _while; def until(cond; next): - def _until: - if cond then . else (next|_until) end; - _until; + def _until: + if cond then . else (next|_until) end; + _until; def limit($n; exp): - if $n > 0 then label $out | foreach exp as $item ($n; .-1; $item, if . <= 0 then break $out else empty end) - elif $n == 0 then empty - else exp end; + if $n > 0 then label $out | foreach exp as $item ($n; .-1; $item, if . <= 0 then break $out else empty end) + elif $n == 0 then empty + else exp end; # range/3, with a `by` expression argument def range($init; $upto; $by): if $by > 0 then $init|while(. < $upto; . + $by) @@ -167,18 +167,18 @@ def nth($n; g): if $n < 0 then error("nth doesn't support negative indices") else label $out | foreach g as $item ($n + 1; . - 1; if . <= 0 then $item, break $out else empty end) end; def first: .[0]; -def last: .[-1]; +def last: .[length-1]; # support pick(last) def nth($n): .[$n]; def combinations: - if length == 0 then [] else - .[0][] as $x - | (.[1:] | combinations) as $y - | [$x] + $y - end; + if length == 0 then [] + else .[0][] as $x + | (.[1:] | combinations) as $y + | [$x] + $y + end; def combinations(n): - . as $dot - | [range(n) | $dot] - | combinations; + . as $dot + | [range(n) | $dot] + | combinations; # transpose a possibly jagged matrix, quickly; # rows are padded with nulls so the result is always rectangular. def transpose: @@ -192,9 +192,9 @@ def transpose: def in(xs): . as $x | xs | has($x); def inside(xs): . as $x | xs | contains($x); def repeat(exp): - def _repeat: - exp, _repeat; - _repeat; + def _repeat: + exp, _repeat; + _repeat; def inputs: try repeat(input) catch if .=="break" then empty else error end; # like ruby's downcase - only characters A to Z are affected def ascii_downcase: diff --git a/tests/jq.test b/tests/jq.test index 4a8bfeee0c..a91dd0812b 100644 --- a/tests/jq.test +++ b/tests/jq.test @@ -1814,3 +1814,35 @@ true tojson | fromjson {"a":nan} {"a":null} + +pick(.a.b.c) +null +{"a":{"b":{"c":null}}} + +pick(first) +[1,2] +[1] + +pick(first|first) +[[10,20],30] +[[10]] + +pick(last) +[[10,20],30] +[null,30] + +[inputs] +null +[] + +[input?] +null +[] + +debug +1 +1 + +debug(.,.) +1 +1 From 35874f5a0bcffae0c9b485d749b29e53dbbfe27f Mon Sep 17 00:00:00 2001 From: pkoppstein Date: Sat, 15 Jul 2023 21:50:19 -0400 Subject: [PATCH 2/5] execute.c: $array[-1]; revert change to def last: Incorporate execute.c update by @nicowilliams to resolve pick(last) issue --- src/builtin.jq | 2 +- src/execute.c | 8 ++++++++ tests/jq.test | 43 +++++++++++++++++++++++++------------------ 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/src/builtin.jq b/src/builtin.jq index 73c0aec0d3..6c31c26d98 100644 --- a/src/builtin.jq +++ b/src/builtin.jq @@ -167,7 +167,7 @@ def nth($n; g): if $n < 0 then error("nth doesn't support negative indices") else label $out | foreach g as $item ($n + 1; . - 1; if . <= 0 then $item, break $out else empty end) end; def first: .[0]; -def last: .[length-1]; # support pick(last) +def last: .[-1]; def nth($n): .[$n]; def combinations: if length == 0 then [] diff --git a/src/execute.c b/src/execute.c index e1894aa5e8..a93402f513 100644 --- a/src/execute.c +++ b/src/execute.c @@ -684,6 +684,14 @@ jv jq_next(jq_state *jq) { set_error(jq, jv_invalid_with_msg(msg)); goto do_backtrack; } + // $array | .[-1] + if (jv_get_kind(k) == JV_KIND_NUMBER && jv_get_kind(t) == JV_KIND_ARRAY) { + int idx = jv_number_value(k); + if (idx < 0) { + jv_free(k); + k = jv_number(jv_array_length(jv_copy(t)) + idx); + } + } jv v = jv_get(t, jv_copy(k)); if (jv_is_valid(v)) { path_append(jq, k, jv_copy(v)); diff --git a/tests/jq.test b/tests/jq.test index 649c6847fc..b3dfb2c5a8 100644 --- a/tests/jq.test +++ b/tests/jq.test @@ -1002,6 +1002,27 @@ del(.[1], .[-6], .[2], .[-3:9]) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] [0, 3, 5, 6, 9] +# negative index +setpath([-1]; 1) +[0] +[1] + +pick(.a.b.c) +null +{"a":{"b":{"c":null}}} + +pick(first) +[1,2] +[1] + +pick(first|first) +[[10,20],30] +[[10]] + +pick(last) +[[10,20],30] +[null,30] + # # Assignment # @@ -1819,30 +1840,16 @@ tojson | fromjson {"a":nan} {"a":null} -pick(.a.b.c) -null -{"a":{"b":{"c":null}}} - -pick(first) -[1,2] -[1] - -pick(first|first) -[[10,20],30] -[[10]] - -pick(last) -[[10,20],30] -[null,30] - -[inputs] +# input and inputs +[input?] null [] -[input?] +[inputs] null [] +# debug should emit its input debug 1 1 From 53ff5716b58be2fc3cc670c3bdbee4b35596ca52 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Sat, 15 Jul 2023 20:00:00 -0500 Subject: [PATCH 3/5] Allow .[-1] in path expressions --- src/execute.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/execute.c b/src/execute.c index adf3773799..0486223bbe 100644 --- a/src/execute.c +++ b/src/execute.c @@ -684,6 +684,14 @@ jv jq_next(jq_state *jq) { set_error(jq, jv_invalid_with_msg(msg)); goto do_backtrack; } + // $array | .[-1] + if (jv_get_kind(k) == JV_KIND_NUMBER && jv_get_kind(t) == JV_KIND_ARRAY) { + int idx = jv_number_value(k); + if (idx < 0) { + jv_free(k); + k = jv_number(jv_array_length(jv_copy(t)) + idx); + } + } jv v = jv_get(t, jv_copy(k)); if (jv_is_valid(v)) { path_append(jq, k, jv_copy(v)); From 93b0ad550bc54683726923197ecd5a99fc8e4a58 Mon Sep 17 00:00:00 2001 From: pkoppstein Date: Sat, 15 Jul 2023 21:50:19 -0400 Subject: [PATCH 4/5] Test negative indices in path expressions --- tests/jq.test | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/tests/jq.test b/tests/jq.test index f43f93842f..3b82562c52 100644 --- a/tests/jq.test +++ b/tests/jq.test @@ -1002,6 +1002,28 @@ del(.[1], .[-6], .[2], .[-3:9]) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] [0, 3, 5, 6, 9] +# negative index +setpath([-1]; 1) +[0] +[1] + +pick(.a.b.c) +null +{"a":{"b":{"c":null}}} + +pick(first) +[1,2] +[1] + +pick(first|first) +[[10,20],30] +[[10]] + +# negative indices in path expressions (since last/1 is .[-1]) +pick(last) +[[10,20],30] +[null,30] + # # Assignment # @@ -1819,13 +1841,28 @@ tojson | fromjson {"a":nan} {"a":null} - # calling input/0, or debug/0 in a test doesn't crash jq try input catch . null "break" +pick(.a.b.c) +null +{"a":{"b":{"c":null}}} + +pick(first) +[1,2] +[1] + +pick(first|first) +[[10,20],30] +[[10]] + +pick(last) +[[10,20],30] +[null,30] + debug 1 1 From 8f5d3aee13128c169ace637e06ce3fd4405d1960 Mon Sep 17 00:00:00 2001 From: pkoppstein Date: Sat, 15 Jul 2023 21:50:19 -0400 Subject: [PATCH 5/5] Add trivial tests of input/inputs --- tests/jq.test | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/jq.test b/tests/jq.test index 3b82562c52..c78252154b 100644 --- a/tests/jq.test +++ b/tests/jq.test @@ -1863,6 +1863,16 @@ pick(last) [[10,20],30] [null,30] +# input and inputs +[input?] +null +[] + +[inputs] +null +[] + +# debug should emit its input debug 1 1