From 9977893bf0b99dc6a6cad4b6d80713ea9bfe7f4b Mon Sep 17 00:00:00 2001 From: itchyny Date: Sat, 22 Jul 2023 11:00:20 +0900 Subject: [PATCH] Improve manual in various ways (inputs, sort_by, foreach sections, etc.) - Add error/0 and mentions null input behavior (close #2231) - Explain value iterator suffix syntax .foo[] (close #1047) - Mention array slicing is also zero-based (close #2094) - Add examples of input and inputs filters (close #2216, close #2470) - Improve sort_by about multiple values (close #2103, close #2467, close #2474) - Improve foreach section and simplify examples (close #1148, close #2169) - Fix recurse/1 document on how it is identical using recurse/2 (close #2036, close #2412) - Add non-string examples of index/1, rindex/1 (close #1422) - Simplify the example of truncate_stream/1 (close #1736) --- docs/content/download/default.yml | 2 +- docs/content/manual/manual.yml | 206 +++++++++++++++++-------- docs/content/manual/v1.3/manual.yml | 41 +++-- docs/content/manual/v1.4/manual.yml | 39 +++-- docs/content/manual/v1.5/manual.yml | 188 +++++++++++++++-------- docs/content/manual/v1.6/manual.yml | 196 ++++++++++++++++-------- jq.1.prebuilt | 227 ++++++++++++++++++++-------- tests/jq.test | 8 + tests/man.test | 93 ++++++++++-- 9 files changed, 698 insertions(+), 302 deletions(-) diff --git a/docs/content/download/default.yml b/docs/content/download/default.yml index 5a08568c85..d46455e529 100644 --- a/docs/content/download/default.yml +++ b/docs/content/download/default.yml @@ -155,7 +155,7 @@ body: You can build it using the usual `./configure && make && sudo make install` rigmarole. - If you're interested in using the lastest development version, try: + If you're interested in using the latest development version, try: git clone --recursive https://github.com/jqlang/jq.git cd jq diff --git a/docs/content/manual/manual.yml b/docs/content/manual/manual.yml index 157bdc4270..d629900f6b 100644 --- a/docs/content/manual/manual.yml +++ b/docs/content/manual/manual.yml @@ -79,8 +79,8 @@ sections: jq filters run on a stream of JSON data. The input to jq is parsed as a sequence of whitespace-separated JSON values which are passed through the provided filter one at a time. The - output(s) of the filter are written to standard out, again as a - sequence of whitespace-separated JSON data. + output(s) of the filter are written to standard output, as a + sequence of newline-separated JSON data. Note: it is important to mind the shell's quoting rules. As a general rule it's best to always quote (with single-quote @@ -132,7 +132,7 @@ sections: * `--stream-errors`: - Like `--stream`, but invalid JSON inputs yield array calues + Like `--stream`, but invalid JSON inputs yield array values where the first element is the error and the second is a path. For example, `["a",n]` produces ["Invalid literal at line 1, column 9",[1]]`. @@ -334,7 +334,7 @@ sections: input, jq processing can sometimes make it appear as though it does. For example, using the current implementation of jq, we would see that the expression: - + 1E1234567890 | . produces `1.7976931348623157e+308` on at least one platform. @@ -372,7 +372,7 @@ sections: output: ['"Hello, world!"'] - program: '.' - input: '0.12345678901234567890123456789' + input: '0.12345678901234567890123456789' output: ['0.12345678901234567890123456789'] - program: '[., tojson]' @@ -398,7 +398,7 @@ sections: JSON object (aka dictionary or hash) as input, `.foo` produces the value at the key "foo" if the key is present, or null otherwise. - A filter of the form `.foo.bar` is equivalent to `.foo|.bar`. + A filter of the form `.foo.bar` is equivalent to `.foo | .bar`. The `.foo` syntax only works for simple, identifier-like keys, that is, keys that are all made of alphanumeric characters and @@ -484,6 +484,7 @@ sections: Either index may be negative (in which case it counts backwards from the end of the array), or omitted (in which case it refers to the start or end of the array). + Indices are zero-based. examples: - program: '.[2:4]' @@ -509,7 +510,8 @@ sections: entirely, it will return *all* of the elements of an array. Running `.[]` with the input `[1,2,3]` will produce the numbers as three separate results, rather than as a single - array. + array. A filter of the form `.foo[]` is equivalent to + `.foo | .[]`. You can also use this on an object, and it will return all the values of the object. @@ -525,6 +527,10 @@ sections: input: '[]' output: [] + - program: '.foo[]' + input: '{"foo":[1,2,3]}' + output: ['1','2','3'] + - program: '.[]' input: '{"a": 1, "b": 1}' output: ['1', '1'] @@ -533,7 +539,8 @@ sections: body: | Like `.[]`, but no errors will be output if . is not an array - or object. + or object. A filter of the form `.foo[]?` is equivalent to + `.foo | .[]?`. - title: "Comma: `,`" body: | @@ -728,15 +735,15 @@ sections: Recursively descends `.`, producing every value. This is the same as the zero-argument `recurse` builtin (see below). This is intended to resemble the XPath `//` operator. Note that - `..a` does not work; use `..|.a` instead. In the example - below we use `..|.a?` to find all the values of object keys + `..a` does not work; use `.. | .a` instead. In the example + below we use `.. | .a?` to find all the values of object keys "a" in any object found "below" `.`. This is particularly useful in conjunction with `path(EXP)` (also see below) and the `?` operator. examples: - - program: '..|.a?' + - program: '.. | .a?' input: '[[{"a":1}]]' output: ['1'] @@ -1190,12 +1197,25 @@ sections: input: 'null' output: ['[1,2,3]'] - - title: "`error(message)`" + - title: "`error`, `error(message)`" body: | - Produces an error, just like `.a` applied to values other than - null and objects would, but with the given message as the - error's value. Errors can be caught with try/catch; see below. + Produces an error with the input value, or with the message + given as the argument. Errors can be caught with try/catch; + see below. + + When the error value is `null`, it produces nothing and works + just like `empty`. So `[null | error]` and `[error(null)]` both + emit `[]`. + + examples: + - program: 'try error catch .' + input: '"error message"' + output: ['"error message"'] + + - program: 'try error("invalid value: \(.)") catch .' + input: '42' + output: ['"invalid value: 42"'] - title: "`halt`" body: | @@ -1481,19 +1501,25 @@ sections: sorted order), and if their keys are equal then the values are compared key by key. - `sort` may be used to sort by a particular field of an - object, or by applying any jq filter. - - `sort_by(f)` compares two elements by comparing the result of - `f` on each element. + `sort_by` may be used to sort by a particular field of an + object, or by applying any jq filter. `sort_by(f)` compares + two elements by comparing the result of `f` on each element. + When `f` produces multiple values, it firstly compares the + first values, and the second values if the first values are + equal, and so on. examples: - program: 'sort' input: '[8,3,null,6]' output: ['[null,3,6,8]'] + - program: 'sort_by(.foo)' - input: '[{"foo":4, "bar":10}, {"foo":3, "bar":100}, {"foo":2, "bar":1}]' - output: ['[{"foo":2, "bar":1}, {"foo":3, "bar":100}, {"foo":4, "bar":10}]'] + input: '[{"foo":4, "bar":10}, {"foo":3, "bar":10}, {"foo":2, "bar":1}]' + output: ['[{"foo":2, "bar":1}, {"foo":3, "bar":10}, {"foo":4, "bar":10}]'] + + - program: 'sort_by(.foo, .bar)' + input: '[{"foo":4, "bar":10}, {"foo":3, "bar":20}, {"foo":2, "bar":1}, {"foo":3, "bar":10}]' + output: ['[{"foo":2, "bar":1}, {"foo":3, "bar":10}, {"foo":3, "bar":20}, {"foo":4, "bar":10}]'] - title: "`group_by(path_expression)`" body: | @@ -1620,9 +1646,21 @@ sections: - program: 'index(", ")' input: '"a,b, cd, efg, hijk"' output: ['3'] + - program: 'index(1)' + input: '[0,1,2,1,3,1,4]' + output: ['1'] + - program: 'index([1,2])' + input: '[0,1,2,3,1,4,2,5,1,2,6,7]' + output: ['1'] - program: 'rindex(", ")' input: '"a,b, cd, efg, hijk"' output: ['12'] + - program: 'rindex(1)' + input: '[0,1,2,1,3,1,4]' + output: ['5'] + - program: 'rindex([1,2])' + input: '[0,1,2,3,1,4,2,5,1,2,6,7]' + output: ['8'] - title: "`inside`" body: | @@ -1846,7 +1884,7 @@ sections: When called without an argument, `recurse` is equivalent to `recurse(.[]?)`. - `recurse(f)` is identical to `recurse(f; . != null)` and can be + `recurse(f)` is identical to `recurse(f; true)` and can be used without concerns about recursion depth. `recurse(f; condition)` is a generator which begins by @@ -2861,29 +2899,6 @@ sections: $times_three | [. + $times_three]) | ...`: here the binding `$times_three` is _not_ visible past the closing parenthesis. - - title: Reduce - body: | - - The `reduce` syntax in jq allows you to combine all of the - results of an expression by accumulating them into a single - answer. As an example, we'll pass `[3,2,1]` to this expression: - - reduce .[] as $item (0; . + $item) - - For each result that `.[]` produces, `. + $item` is run to - accumulate a running total, starting from 0. In this - example, `.[]` produces the results 3, 2, and 1, so the - effect is similar to running something like this: - - 0 | (3 as $item | . + $item) | - (2 as $item | . + $item) | - (1 as $item | . + $item) - - examples: - - program: 'reduce .[] as $item (0; . + $item)' - input: '[10,2,5,3]' - output: ['20'] - - title: "`isempty(exp)`" body: | @@ -2894,6 +2909,14 @@ sections: input: 'null' output: ['true'] + - program: 'isempty(.[])' + input: '[]' + output: ['true'] + + - program: 'isempty(.[])' + input: '[1,2,3]' + output: ['false'] + - title: "`limit(n; exp)`" body: | @@ -2931,32 +2954,79 @@ sections: input: '10' output: ['[0,9,5]'] + - title: "`reduce`" + body: | + + The `reduce` syntax allows you to combine all of the results of + an expression by accumulating them into a single answer. + The form is `reduce EXP as $var (INIT; UPDATE)`. + As an example, we'll pass `[1,2,3]` to this expression: + + reduce .[] as $item (0; . + $item) + + For each result that `.[]` produces, `. + $item` is run to + accumulate a running total, starting from 0 as the input value. + In this example, `.[]` produces the results `1`, `2`, and `3`, + so the effect is similar to running something like this: + + 0 | 1 as $item | . + $item | + 2 as $item | . + $item | + 3 as $item | . + $item + + examples: + - program: 'reduce .[] as $item (0; . + $item)' + input: '[1,2,3,4,5]' + output: ['15'] + + - program: 'reduce .[] as [$i,$j] (0; . + $i * $j)' + input: '[[1,2],[3,4],[5,6]]' + output: ['44'] + + - program: 'reduce .[] as {$x,$y} (null; .x += $x | .y += [$y])' + input: '[{"x":"a","y":1},{"x":"b","y":2},{"x":"c","y":3}]' + output: ['{"x":"abc","y":[1,2,3]}'] + - title: "`foreach`" body: | The `foreach` syntax is similar to `reduce`, but intended to allow the construction of `limit` and reducers that produce - intermediate results (see example). + intermediate results. The form is `foreach EXP as $var (INIT; UPDATE; EXTRACT)`. - Like `reduce`, `INIT` is evaluated once to produce a state - value, then each output of `EXP` is bound to `$var`, `UPDATE` - is evaluated for each output of `EXP` with the current state - and with `$var` visible. Each value output by `UPDATE` - replaces the previous state. Finally, `EXTRACT` is evaluated - for each new state to extract an output of `foreach`. + As an example, we'll pass `[1,2,3]` to this expression: + + foreach .[] as $item (0; . + $item; [$item, . * 2]) + + Like the `reduce` syntax, `. + $item` is run for each result + that `.[]` produces, but `[$item, . * 2]` is run for each + intermediate values. In this example, since the intermediate + values are `1`, `3`, and `6`, the `foreach` expression produces + `[1,2]`, `[2,6]`, and `[3,12]`. So the effect is similar + to running something like this: - This is mostly useful only for constructing `reduce`- and - `limit`-like functions. But it is much more general, as it - allows for partial reductions (see the example below). + 0 | 1 as $item | . + $item | [$item, . * 2], + 2 as $item | . + $item | [$item, . * 2], + 3 as $item | . + $item | [$item, . * 2] + + When `EXTRACT` is omitted, the identity filter is used. + That is, it outputs the intermediate values as they are. examples: - - program: '[foreach .[] as $item - ([[],[]]; - if $item == null then [[],.[0]] else [(.[0] + [$item]),[]] end; - if $item == null then .[1] else empty end)]' - input: '[1,2,3,4,null,"a","b",null]' - output: ['[[1,2,3,4],["a","b"]]'] + - program: 'foreach .[] as $item (0; . + $item)' + input: '[1,2,3,4,5]' + output: ['1','3','6','10','15'] + + - program: 'foreach .[] as $item (0; . + $item; [$item, . * 2])' + input: '[1,2,3,4,5]' + output: ['[1,2]','[2,6]','[3,12]','[4,20]','[5,30]'] + + - program: 'foreach .[] as $item (0; . + 1; {index: ., $item})' + input: '["foo", "bar", "baz"]' + output: + - '{"index":1,"item":"foo"}' + - '{"index":2,"item":"bar"}' + - '{"index":3,"item":"baz"}' - title: Recursion body: | @@ -3098,9 +3168,11 @@ sections: Outputs one new input. Note that when using `input` it is generally be necessary to - invoke jq with the -n command-line option, otherwise + invoke jq with the `-n` command-line option, otherwise the first entity will be lost. + echo 1 2 3 4 | jq '[., input]' # [1,2] [3,4] + - title: "`inputs`" body: | @@ -3108,9 +3180,11 @@ sections: This is primarily useful for reductions over a program's inputs. Note that when using `inputs` it is generally necessary - to invoke jq with the -n command-line option, otherwise + to invoke jq with the `-n` command-line option, otherwise the first entity will be lost. + echo 1 2 3 | jq -n 'reduce inputs as $i (0; . + $i)' # 6 + - title: "`debug`, `debug(msgs)`" body: | @@ -3188,9 +3262,9 @@ sections: given streaming expression. examples: - - program: '[1|truncate_stream([[0],1],[[1,0],2],[[1,0]],[[1]])]' + - program: 'truncate_stream([[0],1],[[1,0],2],[[1,0]],[[1]])' input: '1' - output: ['[[[0],2],[[0]]]'] + output: ['[[0],2]', '[[0]]'] - title: "`fromstream(stream_expression)`" body: | diff --git a/docs/content/manual/v1.3/manual.yml b/docs/content/manual/v1.3/manual.yml index 5679a2997b..375799be7a 100644 --- a/docs/content/manual/v1.3/manual.yml +++ b/docs/content/manual/v1.3/manual.yml @@ -76,8 +76,8 @@ sections: jq filters run on a stream of JSON data. The input to jq is parsed as a sequence of whitespace-separated JSON values which are passed through the provided filter one at a time. The - output(s) of the filter are written to standard out, again as a - sequence of whitespace-separated JSON data. + output(s) of the filter are written to standard output, as a + sequence of newline-separated JSON data. You can affect how jq reads and writes its input and output using some command-line options: @@ -172,9 +172,8 @@ sections: You can also look up fields of an object using syntax like `.["foo"]` (`.foo` above is a shorthand version of this). This - one works for arrays as well, if the key is an - integer. Arrays are zero-based (like javascript), so `.[2]` - returns the third element of the array. + one works for arrays as well, if the key is an integer. Arrays + are zero-based, so `.[2]` returns the third element of the array. The `.[10:15]` syntax can be used to return a subarray of an array. The array returned by `.[10:15]` will be of length 5, @@ -279,7 +278,7 @@ sections: hashes with only string keys), and "null". Booleans, null, strings and numbers are written the same way as - in javascript. Just like everything else in jq, these simple + in JSON. Just like everything else in jq, these simple values take an input and produce an output - `42` is a valid jq expression that takes an input, ignores it, and returns 42 instead. @@ -907,7 +906,7 @@ sections: means that you'll sometimes have to be more explicit about the condition you want: you can't test whether, e.g. a string is empty using `if .name then A else B end`, you'll - need something more like `if (.name | count) > 0 then A else + need something more like `if (.name | length) > 0 then A else B end` instead. If the condition `A` produces multiple results, then `B` is evaluated @@ -1040,7 +1039,7 @@ sections: produces its sum, and the `length` expression is given the array and produces its length. - So, there's generally a cleaner way to solve most problems in jq that + So, there's generally a cleaner way to solve most problems in jq than defining variables. Still, sometimes they do make things easier, so jq lets you define variables using `expression as $variable`. All variable names start with `$`. Here's a slightly uglier version of the @@ -1128,29 +1127,29 @@ sections: input: '[[1,2],[10,20]]' output: ['[[1,2,1,2], [10,20,1,2]]'] - - title: Reduce + - title: "`reduce`" body: | - The `reduce` syntax in jq allows you to combine all of the - results of an expression by accumulating them into a single - answer. As an example, we'll pass `[3,2,1]` to this expression: + The `reduce` syntax allows you to combine all of the results of + an expression by accumulating them into a single answer. + The form is `reduce EXP as $var (INIT; UPDATE)`. + As an example, we'll pass `[1,2,3]` to this expression: reduce .[] as $item (0; . + $item) For each result that `.[]` produces, `. + $item` is run to - accumulate a running total, starting from 0. In this - example, `.[]` produces the results 3, 2, and 1, so the - effect is similar to running something like this: + accumulate a running total, starting from 0 as the input value. + In this example, `.[]` produces the results `1`, `2`, and `3`, + so the effect is similar to running something like this: - 0 | (3 as $item | . + $item) | - (2 as $item | . + $item) | - (1 as $item | . + $item) + 0 | 1 as $item | . + $item | + 2 as $item | . + $item | + 3 as $item | . + $item examples: - program: 'reduce .[] as $item (0; . + $item)' - input: '[10,2,5,3]' - output: ['20'] - + input: '[1,2,3,4,5]' + output: ['15'] - title: Assignment body: | diff --git a/docs/content/manual/v1.4/manual.yml b/docs/content/manual/v1.4/manual.yml index ee8c6a9f71..0af395bf24 100644 --- a/docs/content/manual/v1.4/manual.yml +++ b/docs/content/manual/v1.4/manual.yml @@ -76,8 +76,8 @@ sections: jq filters run on a stream of JSON data. The input to jq is parsed as a sequence of whitespace-separated JSON values which are passed through the provided filter one at a time. The - output(s) of the filter are written to standard out, again as a - sequence of whitespace-separated JSON data. + output(s) of the filter are written to standard output, as a + sequence of newline-separated JSON data. Note: it is important to mind the shell's quoting rules. As a general rule it's best to always quote (with single-quote @@ -253,9 +253,8 @@ sections: You can also look up fields of an object using syntax like `.["foo"]` (`.foo` above is a shorthand version of this). This - one works for arrays as well, if the key is an - integer. Arrays are zero-based (like javascript), so `.[2]` - returns the third element of the array. + one works for arrays as well, if the key is an integer. Arrays + are zero-based, so `.[2]` returns the third element of the array. The `.[10:15]` syntax can be used to return a subarray of an array or substring of a string. The array returned by @@ -263,7 +262,7 @@ sections: index 10 (inclusive) to index 15 (exclusive). Either index may be negative (in which case it counts backwards from the end of the array), or omitted (in which case it refers to the start - or end of the array). + or end of the array). Indices are zero-based. The `?` "operator" can also be used with the slice operator, as in `.[10:15]?`, which outputs values where the inputs are @@ -375,7 +374,7 @@ sections: hashes with only string keys), and "null". Booleans, null, strings and numbers are written the same way as - in javascript. Just like everything else in jq, these simple + in JSON. Just like everything else in jq, these simple values take an input and produce an output - `42` is a valid jq expression that takes an input, ignores it, and returns 42 instead. @@ -1530,29 +1529,29 @@ sections: input: '[[1,2],[10,20]]' output: ['[[1,2,1,2], [10,20,1,2]]'] - - title: Reduce + - title: "`reduce`" body: | - The `reduce` syntax in jq allows you to combine all of the - results of an expression by accumulating them into a single - answer. As an example, we'll pass `[3,2,1]` to this expression: + The `reduce` syntax allows you to combine all of the results of + an expression by accumulating them into a single answer. + The form is `reduce EXP as $var (INIT; UPDATE)`. + As an example, we'll pass `[1,2,3]` to this expression: reduce .[] as $item (0; . + $item) For each result that `.[]` produces, `. + $item` is run to - accumulate a running total, starting from 0. In this - example, `.[]` produces the results 3, 2, and 1, so the - effect is similar to running something like this: + accumulate a running total, starting from 0 as the input value. + In this example, `.[]` produces the results `1`, `2`, and `3`, + so the effect is similar to running something like this: - 0 | (3 as $item | . + $item) | - (2 as $item | . + $item) | - (1 as $item | . + $item) + 0 | 1 as $item | . + $item | + 2 as $item | . + $item | + 3 as $item | . + $item examples: - program: 'reduce .[] as $item (0; . + $item)' - input: '[10,2,5,3]' - output: ['20'] - + input: '[1,2,3,4,5]' + output: ['15'] - title: Assignment body: | diff --git a/docs/content/manual/v1.5/manual.yml b/docs/content/manual/v1.5/manual.yml index cb93058f10..71ce31bb11 100644 --- a/docs/content/manual/v1.5/manual.yml +++ b/docs/content/manual/v1.5/manual.yml @@ -78,8 +78,8 @@ sections: jq filters run on a stream of JSON data. The input to jq is parsed as a sequence of whitespace-separated JSON values which are passed through the provided filter one at a time. The - output(s) of the filter are written to standard out, again as a - sequence of whitespace-separated JSON data. + output(s) of the filter are written to standard output, as a + sequence of newline-separated JSON data. Note: it is important to mind the shell's quoting rules. As a general rule it's best to always quote (with single-quote @@ -319,9 +319,8 @@ sections: You can also look up fields of an object using syntax like `.["foo"]` (`.foo` above is a shorthand version of this). This - one works for arrays as well, if the key is an - integer. Arrays are zero-based (like javascript), so `.[2]` - returns the third element of the array. + one works for arrays as well, if the key is an integer. Arrays + are zero-based, so `.[2]` returns the third element of the array. The `.[10:15]` syntax can be used to return a subarray of an array or substring of a string. The array returned by @@ -329,7 +328,7 @@ sections: index 10 (inclusive) to index 15 (exclusive). Either index may be negative (in which case it counts backwards from the end of the array), or omitted (in which case it refers to the start - or end of the array). + or end of the array). Indices are zero-based. The `.[2]` syntax can be used to return the element at the given index. Negative indices are allowed, with -1 referring @@ -382,7 +381,8 @@ sections: entirely, it will return *all* of the elements of an array. Running `.[]` with the input `[1,2,3]` will produce the numbers as three separate results, rather than as a single - array. + array. A filter of the form `.foo[]` is equivalent to + `.foo | .[]`. You can also use this on an object, and it will return all the values of the object. @@ -398,6 +398,10 @@ sections: input: '[]' output: [] + - program: '.foo[]' + input: '{"foo":[1,2,3]}' + output: ['1','2','3'] + - program: '.[]' input: '{"a": 1, "b": 1}' output: ['1', '1'] @@ -406,7 +410,8 @@ sections: body: | Like `.[]`, but no errors will be output if . is not an array - or object. + or object. A filter of the form `.foo[]?` is equivalent to + `.foo | .[]?`. - title: "`,`" body: | @@ -456,7 +461,7 @@ sections: hashes with only string keys), and "null". Booleans, null, strings and numbers are written the same way as - in javascript. Just like everything else in jq, these simple + in JSON. Just like everything else in jq, these simple values take an input and produce an output - `42` is a valid jq expression that takes an input, ignores it, and returns 42 instead. @@ -843,12 +848,25 @@ sections: input: 'null' output: ['[1,2,3]'] - - title: "`error(message)`" + - title: "`error`, `error(message)`" body: | - Produces an error, just like `.a` applied to values other than - null and objects would, but with the given message as the - error's value. + Produces an error with the input value, or with the message + given as the argument. Errors can be caught with try/catch; + see below. + + When the error value is `null`, it produces nothing and works + just like `empty`. So `[null | error]` and `[error(null)]` both + emit `[]`. + + examples: + - program: 'try error catch .' + input: '"error message"' + output: ['"error message"'] + + - program: 'try error("invalid value: \(.)") catch .' + input: '42' + output: ['"invalid value: 42"'] - title: "`$__loc__`" body: | @@ -1005,7 +1023,7 @@ sections: input: '[{"foo": "bar"}, [{"foo": "baz"}]]' output: ['[{"foo": "bar"}, {"foo": "baz"}]'] - - title: "`range(upto)`, `range(from; upto)` `range(from; upto; by)`" + - title: "`range(upto)`, `range(from; upto)`, `range(from; upto; by)`" body: | The `range` function produces a range of numbers. `range(4; 10)` @@ -1141,19 +1159,25 @@ sections: sorted order), and if their keys are equal then the values are compared key by key. - `sort` may be used to sort by a particular field of an - object, or by applying any jq filter. - - `sort_by(f)` compares two elements by comparing the result of - `f` on each element. + `sort_by` may be used to sort by a particular field of an + object, or by applying any jq filter. `sort_by(f)` compares + two elements by comparing the result of `f` on each element. + When `f` produces multiple values, it firstly compares the + first values, and the second values if the first values are + equal, and so on. examples: - program: 'sort' input: '[8,3,null,6]' output: ['[null,3,6,8]'] + - program: 'sort_by(.foo)' - input: '[{"foo":4, "bar":10}, {"foo":3, "bar":100}, {"foo":2, "bar":1}]' - output: ['[{"foo":2, "bar":1}, {"foo":3, "bar":100}, {"foo":4, "bar":10}]'] + input: '[{"foo":4, "bar":10}, {"foo":3, "bar":10}, {"foo":2, "bar":1}]' + output: ['[{"foo":2, "bar":1}, {"foo":3, "bar":10}, {"foo":4, "bar":10}]'] + + - program: 'sort_by(.foo, .bar)' + input: '[{"foo":4, "bar":10}, {"foo":3, "bar":20}, {"foo":2, "bar":1}, {"foo":3, "bar":10}]' + output: ['[{"foo":2, "bar":1}, {"foo":3, "bar":10}, {"foo":3, "bar":20}, {"foo":4, "bar":10}]'] - title: "`group_by(path_expression)`" body: | @@ -1280,9 +1304,21 @@ sections: - program: 'index(", ")' input: '"a,b, cd, efg, hijk"' output: ['3'] + - program: 'index(1)' + input: '[0,1,2,1,3,1,4]' + output: ['1'] + - program: 'index([1,2])' + input: '[0,1,2,3,1,4,2,5,1,2,6,7]' + output: ['1'] - program: 'rindex(", ")' input: '"a,b, cd, efg, hijk"' output: ['12'] + - program: 'rindex(1)' + input: '[0,1,2,1,3,1,4]' + output: ['5'] + - program: 'rindex([1,2])' + input: '[0,1,2,3,1,4,2,5,1,2,6,7]' + output: ['8'] - title: "`inside`" body: | @@ -2308,29 +2344,6 @@ sections: input: '[[1,2],[10,20]]' output: ['[[1,2,1,2], [10,20,1,2]]'] - - title: Reduce - body: | - - The `reduce` syntax in jq allows you to combine all of the - results of an expression by accumulating them into a single - answer. As an example, we'll pass `[3,2,1]` to this expression: - - reduce .[] as $item (0; . + $item) - - For each result that `.[]` produces, `. + $item` is run to - accumulate a running total, starting from 0. In this - example, `.[]` produces the results 3, 2, and 1, so the - effect is similar to running something like this: - - 0 | (3 as $item | . + $item) | - (2 as $item | . + $item) | - (1 as $item | . + $item) - - examples: - - program: 'reduce .[] as $item (0; . + $item)' - input: '[10,2,5,3]' - output: ['20'] - - title: "`limit(n; exp)`" body: | @@ -2370,32 +2383,79 @@ sections: input: '10' output: ['[0,9,5]'] + - title: "`reduce`" + body: | + + The `reduce` syntax allows you to combine all of the results of + an expression by accumulating them into a single answer. + The form is `reduce EXP as $var (INIT; UPDATE)`. + As an example, we'll pass `[1,2,3]` to this expression: + + reduce .[] as $item (0; . + $item) + + For each result that `.[]` produces, `. + $item` is run to + accumulate a running total, starting from 0 as the input value. + In this example, `.[]` produces the results `1`, `2`, and `3`, + so the effect is similar to running something like this: + + 0 | 1 as $item | . + $item | + 2 as $item | . + $item | + 3 as $item | . + $item + + examples: + - program: 'reduce .[] as $item (0; . + $item)' + input: '[1,2,3,4,5]' + output: ['15'] + + - program: 'reduce .[] as [$i,$j] (0; . + $i * $j)' + input: '[[1,2],[3,4],[5,6]]' + output: ['44'] + + - program: 'reduce .[] as {$x,$y} (null; .x += $x | .y += [$y])' + input: '[{"x":"a","y":1},{"x":"b","y":2},{"x":"c","y":3}]' + output: ['{"x":"abc","y":[1,2,3]}'] + - title: "`foreach`" body: | The `foreach` syntax is similar to `reduce`, but intended to allow the construction of `limit` and reducers that produce - intermediate results (see example). + intermediate results. The form is `foreach EXP as $var (INIT; UPDATE; EXTRACT)`. - Like `reduce`, `INIT` is evaluated once to produce a state - value, then each output of `EXP` is bound to `$var`, `UPDATE` - is evaluated for each output of `EXP` with the current state - and with `$var` visible. Each value output by `UPDATE` - replaces the previous state. Finally, `EXTRACT` is evaluated - for each new state to extract an output of `foreach`. + As an example, we'll pass `[1,2,3]` to this expression: + + foreach .[] as $item (0; . + $item; [$item, . * 2]) + + Like the `reduce` syntax, `. + $item` is run for each result + that `.[]` produces, but `[$item, . * 2]` is run for each + intermediate values. In this example, since the intermediate + values are `1`, `3`, and `6`, the `foreach` expression produces + `[1,2]`, `[2,6]`, and `[3,12]`. So the effect is similar + to running something like this: + + 0 | 1 as $item | . + $item | [$item, . * 2], + 2 as $item | . + $item | [$item, . * 2], + 3 as $item | . + $item | [$item, . * 2] - This is mostly useful only for constructing `reduce`- and - `limit`-like functions. But it is much more general, as it - allows for partial reductions (see the example below). + When `EXTRACT` is omitted, the identity filter is used. + That is, it outputs the intermediate values as they are. examples: - - program: '[foreach .[] as $item - ([[],[]]; - if $item == null then [[],.[0]] else [(.[0] + [$item]),[]] end; - if $item == null then .[1] else empty end)]' - input: '[1,2,3,4,null,"a","b",null]' - output: ['[[1,2,3,4],["a","b"]]'] + - program: 'foreach .[] as $item (0; . + $item)' + input: '[1,2,3,4,5]' + output: ['1','3','6','10','15'] + + - program: 'foreach .[] as $item (0; . + $item; [$item, . * 2])' + input: '[1,2,3,4,5]' + output: ['[1,2]','[2,6]','[3,12]','[4,20]','[5,30]'] + + - program: 'foreach .[] as $item (0; . + 1; {index: ., $item})' + input: '["foo", "bar", "baz"]' + output: + - '{"index":1,"item":"foo"}' + - '{"index":2,"item":"bar"}' + - '{"index":3,"item":"baz"}' - title: Recursion body: | @@ -2511,6 +2571,8 @@ sections: Outputs one new input. + echo 1 2 3 4 | jq '[., input]' # [1,2] [3,4] + - title: "`inputs`" body: | @@ -2519,6 +2581,8 @@ sections: This is primarily useful for reductions over a program's inputs. + echo 1 2 3 | jq -n 'reduce inputs as $i (0; . + $i)' # 6 + - title: "`debug`" body: | @@ -2571,9 +2635,9 @@ sections: given streaming expression. examples: - - program: '[1|truncate_stream([[0],1],[[1,0],2],[[1,0]],[[1]])]' + - program: 'truncate_stream([[0],1],[[1,0],2],[[1,0]],[[1]])' input: '1' - output: ['[[[0],2],[[0]]]'] + output: ['[[0],2]', '[[0]]'] - title: "`fromstream(stream_expression)`" body: | diff --git a/docs/content/manual/v1.6/manual.yml b/docs/content/manual/v1.6/manual.yml index fabf70c97b..a33eff7a01 100644 --- a/docs/content/manual/v1.6/manual.yml +++ b/docs/content/manual/v1.6/manual.yml @@ -78,8 +78,8 @@ sections: jq filters run on a stream of JSON data. The input to jq is parsed as a sequence of whitespace-separated JSON values which are passed through the provided filter one at a time. The - output(s) of the filter are written to standard out, again as a - sequence of whitespace-separated JSON data. + output(s) of the filter are written to standard output, as a + sequence of newline-separated JSON data. Note: it is important to mind the shell's quoting rules. As a general rule it's best to always quote (with single-quote @@ -397,6 +397,7 @@ sections: Either index may be negative (in which case it counts backwards from the end of the array), or omitted (in which case it refers to the start or end of the array). + Indices are zero-based. examples: - program: '.[2:4]' @@ -422,7 +423,8 @@ sections: entirely, it will return *all* of the elements of an array. Running `.[]` with the input `[1,2,3]` will produce the numbers as three separate results, rather than as a single - array. + array. A filter of the form `.foo[]` is equivalent to + `.foo | .[]`. You can also use this on an object, and it will return all the values of the object. @@ -438,6 +440,10 @@ sections: input: '[]' output: [] + - program: '.foo[]' + input: '{"foo":[1,2,3]}' + output: ['1','2','3'] + - program: '.[]' input: '{"a": 1, "b": 1}' output: ['1', '1'] @@ -446,7 +452,8 @@ sections: body: | Like `.[]`, but no errors will be output if . is not an array - or object. + or object. A filter of the form `.foo[]?` is equivalent to + `.foo | .[]?`. - title: "Comma: `,`" body: | @@ -516,7 +523,7 @@ sections: hashes with only string keys), and "null". Booleans, null, strings and numbers are written the same way as - in javascript. Just like everything else in jq, these simple + in JSON. Just like everything else in jq, these simple values take an input and produce an output - `42` is a valid jq expression that takes an input, ignores it, and returns 42 instead. @@ -618,15 +625,15 @@ sections: Recursively descends `.`, producing every value. This is the same as the zero-argument `recurse` builtin (see below). This is intended to resemble the XPath `//` operator. Note that - `..a` does not work; use `..|.a` instead. In the example - below we use `..|.a?` to find all the values of object keys + `..a` does not work; use `.. | .a` instead. In the example + below we use `.. | .a?` to find all the values of object keys "a" in any object found "below" `.`. This is particularly useful in conjunction with `path(EXP)` (also see below) and the `?` operator. examples: - - program: '..|.a?' + - program: '.. | .a?' input: '[[{"a":1}]]' output: ['1'] @@ -1008,12 +1015,25 @@ sections: input: 'null' output: ['[1,2,3]'] - - title: "`error(message)`" + - title: "`error`, `error(message)`" body: | - Produces an error, just like `.a` applied to values other than - null and objects would, but with the given message as the - error's value. Errors can be caught with try/catch; see below. + Produces an error with the input value, or with the message + given as the argument. Errors can be caught with try/catch; + see below. + + When the error value is `null`, it produces nothing and works + just like `empty`. So `[null | error]` and `[error(null)]` both + emit `[]`. + + examples: + - program: 'try error catch .' + input: '"error message"' + output: ['"error message"'] + + - program: 'try error("invalid value: \(.)") catch .' + input: '42' + output: ['"invalid value: 42"'] - title: "`halt`" body: | @@ -1302,19 +1322,25 @@ sections: sorted order), and if their keys are equal then the values are compared key by key. - `sort` may be used to sort by a particular field of an - object, or by applying any jq filter. - - `sort_by(f)` compares two elements by comparing the result of - `f` on each element. + `sort_by` may be used to sort by a particular field of an + object, or by applying any jq filter. `sort_by(f)` compares + two elements by comparing the result of `f` on each element. + When `f` produces multiple values, it firstly compares the + first values, and the second values if the first values are + equal, and so on. examples: - program: 'sort' input: '[8,3,null,6]' output: ['[null,3,6,8]'] + - program: 'sort_by(.foo)' - input: '[{"foo":4, "bar":10}, {"foo":3, "bar":100}, {"foo":2, "bar":1}]' - output: ['[{"foo":2, "bar":1}, {"foo":3, "bar":100}, {"foo":4, "bar":10}]'] + input: '[{"foo":4, "bar":10}, {"foo":3, "bar":10}, {"foo":2, "bar":1}]' + output: ['[{"foo":2, "bar":1}, {"foo":3, "bar":10}, {"foo":4, "bar":10}]'] + + - program: 'sort_by(.foo, .bar)' + input: '[{"foo":4, "bar":10}, {"foo":3, "bar":20}, {"foo":2, "bar":1}, {"foo":3, "bar":10}]' + output: ['[{"foo":2, "bar":1}, {"foo":3, "bar":10}, {"foo":3, "bar":20}, {"foo":4, "bar":10}]'] - title: "`group_by(path_expression)`" body: | @@ -1441,9 +1467,21 @@ sections: - program: 'index(", ")' input: '"a,b, cd, efg, hijk"' output: ['3'] + - program: 'index(1)' + input: '[0,1,2,1,3,1,4]' + output: ['1'] + - program: 'index([1,2])' + input: '[0,1,2,3,1,4,2,5,1,2,6,7]' + output: ['1'] - program: 'rindex(", ")' input: '"a,b, cd, efg, hijk"' output: ['12'] + - program: 'rindex(1)' + input: '[0,1,2,1,3,1,4]' + output: ['5'] + - program: 'rindex([1,2])' + input: '[0,1,2,3,1,4,2,5,1,2,6,7]' + output: ['8'] - title: "`inside`" body: | @@ -1648,7 +1686,7 @@ sections: When called without an argument, `recurse` is equivalent to `recurse(.[]?)`. - `recurse(f)` is identical to `recurse(f; . != null)` and can be + `recurse(f)` is identical to `recurse(f; true)` and can be used without concerns about recursion depth. `recurse(f; condition)` is a generator which begins by @@ -2634,29 +2672,6 @@ sections: $times_three | [. + $times_three]) | ...`: here the binding `$times_three` is _not_ visible past the closing parenthesis. - - title: Reduce - body: | - - The `reduce` syntax in jq allows you to combine all of the - results of an expression by accumulating them into a single - answer. As an example, we'll pass `[3,2,1]` to this expression: - - reduce .[] as $item (0; . + $item) - - For each result that `.[]` produces, `. + $item` is run to - accumulate a running total, starting from 0. In this - example, `.[]` produces the results 3, 2, and 1, so the - effect is similar to running something like this: - - 0 | (3 as $item | . + $item) | - (2 as $item | . + $item) | - (1 as $item | . + $item) - - examples: - - program: 'reduce .[] as $item (0; . + $item)' - input: '[10,2,5,3]' - output: ['20'] - - title: "`isempty(exp)`" body: | @@ -2667,6 +2682,14 @@ sections: input: 'null' output: ['true'] + - program: 'isempty(.[])' + input: '[]' + output: ['true'] + + - program: 'isempty(.[])' + input: '[1,2,3]' + output: ['false'] + - title: "`limit(n; exp)`" body: | @@ -2706,32 +2729,79 @@ sections: input: '10' output: ['[0,9,5]'] + - title: "`reduce`" + body: | + + The `reduce` syntax allows you to combine all of the results of + an expression by accumulating them into a single answer. + The form is `reduce EXP as $var (INIT; UPDATE)`. + As an example, we'll pass `[1,2,3]` to this expression: + + reduce .[] as $item (0; . + $item) + + For each result that `.[]` produces, `. + $item` is run to + accumulate a running total, starting from 0 as the input value. + In this example, `.[]` produces the results `1`, `2`, and `3`, + so the effect is similar to running something like this: + + 0 | 1 as $item | . + $item | + 2 as $item | . + $item | + 3 as $item | . + $item + + examples: + - program: 'reduce .[] as $item (0; . + $item)' + input: '[1,2,3,4,5]' + output: ['15'] + + - program: 'reduce .[] as [$i,$j] (0; . + $i * $j)' + input: '[[1,2],[3,4],[5,6]]' + output: ['44'] + + - program: 'reduce .[] as {$x,$y} (null; .x += $x | .y += [$y])' + input: '[{"x":"a","y":1},{"x":"b","y":2},{"x":"c","y":3}]' + output: ['{"x":"abc","y":[1,2,3]}'] + - title: "`foreach`" body: | The `foreach` syntax is similar to `reduce`, but intended to allow the construction of `limit` and reducers that produce - intermediate results (see example). + intermediate results. The form is `foreach EXP as $var (INIT; UPDATE; EXTRACT)`. - Like `reduce`, `INIT` is evaluated once to produce a state - value, then each output of `EXP` is bound to `$var`, `UPDATE` - is evaluated for each output of `EXP` with the current state - and with `$var` visible. Each value output by `UPDATE` - replaces the previous state. Finally, `EXTRACT` is evaluated - for each new state to extract an output of `foreach`. + As an example, we'll pass `[1,2,3]` to this expression: + + foreach .[] as $item (0; . + $item; [$item, . * 2]) + + Like the `reduce` syntax, `. + $item` is run for each result + that `.[]` produces, but `[$item, . * 2]` is run for each + intermediate values. In this example, since the intermediate + values are `1`, `3`, and `6`, the `foreach` expression produces + `[1,2]`, `[2,6]`, and `[3,12]`. So the effect is similar + to running something like this: - This is mostly useful only for constructing `reduce`- and - `limit`-like functions. But it is much more general, as it - allows for partial reductions (see the example below). + 0 | 1 as $item | . + $item | [$item, . * 2], + 2 as $item | . + $item | [$item, . * 2], + 3 as $item | . + $item | [$item, . * 2] + + When `EXTRACT` is omitted, the identity filter is used. + That is, it outputs the intermediate values as they are. examples: - - program: '[foreach .[] as $item - ([[],[]]; - if $item == null then [[],.[0]] else [(.[0] + [$item]),[]] end; - if $item == null then .[1] else empty end)]' - input: '[1,2,3,4,null,"a","b",null]' - output: ['[[1,2,3,4],["a","b"]]'] + - program: 'foreach .[] as $item (0; . + $item)' + input: '[1,2,3,4,5]' + output: ['1','3','6','10','15'] + + - program: 'foreach .[] as $item (0; . + $item; [$item, . * 2])' + input: '[1,2,3,4,5]' + output: ['[1,2]','[2,6]','[3,12]','[4,20]','[5,30]'] + + - program: 'foreach .[] as $item (0; . + 1; {index: ., $item})' + input: '["foo", "bar", "baz"]' + output: + - '{"index":1,"item":"foo"}' + - '{"index":2,"item":"bar"}' + - '{"index":3,"item":"baz"}' - title: Recursion body: | @@ -2870,6 +2940,8 @@ sections: Outputs one new input. + echo 1 2 3 4 | jq '[., input]' # [1,2] [3,4] + - title: "`inputs`" body: | @@ -2878,6 +2950,8 @@ sections: This is primarily useful for reductions over a program's inputs. + echo 1 2 3 | jq -n 'reduce inputs as $i (0; . + $i)' # 6 + - title: "`debug`" body: | @@ -2936,9 +3010,9 @@ sections: given streaming expression. examples: - - program: '[1|truncate_stream([[0],1],[[1,0],2],[[1,0]],[[1]])]' + - program: 'truncate_stream([[0],1],[[1,0],2],[[1,0]],[[1]])' input: '1' - output: ['[[[0],2],[[0]]]'] + output: ['[[0],2]', '[[0]]'] - title: "`fromstream(stream_expression)`" body: | diff --git a/jq.1.prebuilt b/jq.1.prebuilt index 8df532e324..e7d1a972c3 100644 --- a/jq.1.prebuilt +++ b/jq.1.prebuilt @@ -32,7 +32,7 @@ It\'s important to remember that every filter has an input and an output\. Even But that\'s getting ahead of ourselves\. :) Let\'s start with something simpler: . .SH "INVOKING JQ" -jq filters run on a stream of JSON data\. The input to jq is parsed as a sequence of whitespace\-separated JSON values which are passed through the provided filter one at a time\. The output(s) of the filter are written to standard out, again as a sequence of whitespace\-separated JSON data\. +jq filters run on a stream of JSON data\. The input to jq is parsed as a sequence of whitespace\-separated JSON values which are passed through the provided filter one at a time\. The output(s) of the filter are written to standard output, as a sequence of newline\-separated JSON data\. . .P Note: it is important to mind the shell\'s quoting rules\. As a general rule it\'s best to always quote (with single\-quote characters) the jq program, as too many characters with special meaning to jq are also shell meta\-characters\. For example, \fBjq "foo"\fR will fail on most Unix shells because that will be the same as \fBjq foo\fR, which will generally fail because \fBfoo is not defined\fR\. When using the Windows command shell (cmd\.exe) it\'s best to use double quotes around your jq program when given on the command\-line (instead of the \fB\-f program\-file\fR option), but then double\-quotes in the jq program need backslash escaping\. When using the Powershell (\fBpowershell\.exe\fR) or the Powershell Core (\fBpwsh\fR/\fBpwsh\.exe\fR), use single\-quote characters around the jq program and backslash\-escaped double\-quotes (\fB\e"\fR) inside the jq program\. @@ -76,7 +76,7 @@ This is useful for processing very large inputs\. Use this in conjunction with f \fB\-\-stream\-errors\fR: . .IP -Like \fB\-\-stream\fR, but invalid JSON inputs yield array calues where the first element is the error and the second is a path\. For example, \fB["a",n]\fR produces ["Invalid literal at line 1, column 9",[1]]`\. +Like \fB\-\-stream\fR, but invalid JSON inputs yield array values where the first element is the error and the second is a path\. For example, \fB["a",n]\fR produces ["Invalid literal at line 1, column 9",[1]]`\. . .IP Implies \fB\-\-stream\fR\. Invalid JSON inputs produce no error values when \fB\-\-stream\fR without \fB\-\-stream\-errors\fR\. @@ -327,7 +327,7 @@ jq \'\. as $big | [$big, $big + 1] | map(\. > 10000000000000000000000000000000)\ The simplest \fIuseful\fR filter has the form \fB\.foo\fR\. When given a JSON object (aka dictionary or hash) as input, \fB\.foo\fR produces the value at the key "foo" if the key is present, or null otherwise\. . .P -A filter of the form \fB\.foo\.bar\fR is equivalent to \fB\.foo|\.bar\fR\. +A filter of the form \fB\.foo\.bar\fR is equivalent to \fB\.foo | \.bar\fR\. . .P The \fB\.foo\fR syntax only works for simple, identifier\-like keys, that is, keys that are all made of alphanumeric characters and underscore, and which do not start with a digit\. @@ -415,7 +415,7 @@ jq \'\.[\-2]\' .IP "" 0 . .SS "Array/String Slice: \.[:]" -The \fB\.[:]\fR syntax can be used to return a subarray of an array or substring of a string\. The array returned by \fB\.[10:15]\fR will be of length 5, containing the elements from index 10 (inclusive) to index 15 (exclusive)\. Either index may be negative (in which case it counts backwards from the end of the array), or omitted (in which case it refers to the start or end of the array)\. +The \fB\.[:]\fR syntax can be used to return a subarray of an array or substring of a string\. The array returned by \fB\.[10:15]\fR will be of length 5, containing the elements from index 10 (inclusive) to index 15 (exclusive)\. Either index may be negative (in which case it counts backwards from the end of the array), or omitted (in which case it refers to the start or end of the array)\. Indices are zero\-based\. . .IP "" 4 . @@ -442,7 +442,7 @@ jq \'\.[\-2:]\' .IP "" 0 . .SS "Array/Object Value Iterator: \.[]" -If you use the \fB\.[index]\fR syntax, but omit the index entirely, it will return \fIall\fR of the elements of an array\. Running \fB\.[]\fR with the input \fB[1,2,3]\fR will produce the numbers as three separate results, rather than as a single array\. +If you use the \fB\.[index]\fR syntax, but omit the index entirely, it will return \fIall\fR of the elements of an array\. Running \fB\.[]\fR with the input \fB[1,2,3]\fR will produce the numbers as three separate results, rather than as a single array\. A filter of the form \fB\.foo[]\fR is equivalent to \fB\.foo | \.[]\fR\. . .P You can also use this on an object, and it will return all the values of the object\. @@ -459,6 +459,10 @@ jq \'\.[]\' [] => +jq \'\.foo[]\' + {"foo":[1,2,3]} +=> 1, 2, 3 + jq \'\.[]\' {"a": 1, "b": 1} => 1, 1 @@ -468,7 +472,7 @@ jq \'\.[]\' .IP "" 0 . .SS "\.[]?" -Like \fB\.[]\fR, but no errors will be output if \. is not an array or object\. +Like \fB\.[]\fR, but no errors will be output if \. is not an array or object\. A filter of the form \fB\.foo[]?\fR is equivalent to \fB\.foo | \.[]?\fR\. . .SS "Comma: ," If two filters are separated by a comma, then the same input will be fed into both and the two filters\' output value streams will be concatenated in order: first, all of the outputs produced by the left expression, and then all of the outputs produced by the right\. For instance, filter \fB\.foo, \.bar\fR, produces both the "foo" fields and "bar" fields as separate outputs\. @@ -705,7 +709,7 @@ jq \'{(\.user): \.titles}\' .IP "" 0 . .SS "Recursive Descent: \.\." -Recursively descends \fB\.\fR, producing every value\. This is the same as the zero\-argument \fBrecurse\fR builtin (see below)\. This is intended to resemble the XPath \fB//\fR operator\. Note that \fB\.\.a\fR does not work; use \fB\.\.|\.a\fR instead\. In the example below we use \fB\.\.|\.a?\fR to find all the values of object keys "a" in any object found "below" \fB\.\fR\. +Recursively descends \fB\.\fR, producing every value\. This is the same as the zero\-argument \fBrecurse\fR builtin (see below)\. This is intended to resemble the XPath \fB//\fR operator\. Note that \fB\.\.a\fR does not work; use \fB\.\. | \.a\fR instead\. In the example below we use \fB\.\. | \.a?\fR to find all the values of object keys "a" in any object found "below" \fB\.\fR\. . .P This is particularly useful in conjunction with \fBpath(EXP)\fR (also see below) and the \fB?\fR operator\. @@ -714,7 +718,7 @@ This is particularly useful in conjunction with \fBpath(EXP)\fR (also see below) . .nf -jq \'\.\.|\.a?\' +jq \'\.\. | \.a?\' [[{"a":1}]] => 1 . @@ -1226,8 +1230,27 @@ jq \'[1,2,empty,3]\' . .IP "" 0 . -.SS "error(message)" -Produces an error, just like \fB\.a\fR applied to values other than null and objects would, but with the given message as the error\'s value\. Errors can be caught with try/catch; see below\. +.SS "error, error(message)" +Produces an error with the input value, or with the message given as the argument\. Errors can be caught with try/catch; see below\. +. +.P +When the error value is \fBnull\fR, it produces nothing and works just like \fBempty\fR\. So \fB[null | error]\fR and \fB[error(null)]\fR both emit \fB[]\fR\. +. +.IP "" 4 +. +.nf + +jq \'try error catch \.\' + "error message" +=> "error message" + +jq \'try error("invalid value: \e(\.)") catch \.\' + 42 +=> "invalid value: 42" +. +.fi +. +.IP "" 0 . .SS "halt" Stops the jq program with no further outputs\. jq will exit with exit status \fB0\fR\. @@ -1572,10 +1595,7 @@ objects The ordering for objects is a little complex: first they\'re compared by comparing their sets of keys (as arrays in sorted order), and if their keys are equal then the values are compared key by key\. . .P -\fBsort\fR may be used to sort by a particular field of an object, or by applying any jq filter\. -. -.P -\fBsort_by(f)\fR compares two elements by comparing the result of \fBf\fR on each element\. +\fBsort_by\fR may be used to sort by a particular field of an object, or by applying any jq filter\. \fBsort_by(f)\fR compares two elements by comparing the result of \fBf\fR on each element\. When \fBf\fR produces multiple values, it firstly compares the first values, and the second values if the first values are equal, and so on\. . .IP "" 4 . @@ -1586,8 +1606,12 @@ jq \'sort\' => [null,3,6,8] jq \'sort_by(\.foo)\' - [{"foo":4, "bar":10}, {"foo":3, "bar":100}, {"foo":2, "bar":1}] -=> [{"foo":2, "bar":1}, {"foo":3, "bar":100}, {"foo":4, "bar":10}] + [{"foo":4, "bar":10}, {"foo":3, "bar":10}, {"foo":2, "bar":1}] +=> [{"foo":2, "bar":1}, {"foo":3, "bar":10}, {"foo":4, "bar":10}] + +jq \'sort_by(\.foo, \.bar)\' + [{"foo":4, "bar":10}, {"foo":3, "bar":20}, {"foo":2, "bar":1}, {"foo":3, "bar":10}] +=> [{"foo":2, "bar":1}, {"foo":3, "bar":10}, {"foo":3, "bar":20}, {"foo":4, "bar":10}] . .fi . @@ -1739,9 +1763,25 @@ jq \'index(", ")\' "a,b, cd, efg, hijk" => 3 +jq \'index(1)\' + [0,1,2,1,3,1,4] +=> 1 + +jq \'index([1,2])\' + [0,1,2,3,1,4,2,5,1,2,6,7] +=> 1 + jq \'rindex(", ")\' "a,b, cd, efg, hijk" => 12 + +jq \'rindex(1)\' + [0,1,2,1,3,1,4] +=> 5 + +jq \'rindex([1,2])\' + [0,1,2,3,1,4,2,5,1,2,6,7] +=> 8 . .fi . @@ -2032,7 +2072,7 @@ recurse(\.children[]) | \.name When called without an argument, \fBrecurse\fR is equivalent to \fBrecurse(\.[]?)\fR\. . .P -\fBrecurse(f)\fR is identical to \fBrecurse(f; \. != null)\fR and can be used without concerns about recursion depth\. +\fBrecurse(f)\fR is identical to \fBrecurse(f; true)\fR and can be used without concerns about recursion depth\. . .P \fBrecurse(f; condition)\fR is a generator which begins by emitting \. and then emits in turn \.|f, \.|f|f, \.|f|f|f, \.\.\. so long as the computed value satisfies the condition\. For example, to generate all the integers, at least in principle, one could write \fBrecurse(\.+1; true)\fR\. @@ -3264,38 +3304,6 @@ There are two types of symbols in jq: value bindings (a\.k\.a\., "variables"), a .P For example, in the following expression there is a binding which is visible "to the right" of it, \fB\.\.\. | \.*3 as $times_three | [\. + $times_three] | \.\.\.\fR, but not "to the left"\. Consider this expression now, \fB\.\.\. | (\.*3 as $times_three | [\. + $times_three]) | \.\.\.\fR: here the binding \fB$times_three\fR is \fInot\fR visible past the closing parenthesis\. . -.SS "Reduce" -The \fBreduce\fR syntax in jq allows you to combine all of the results of an expression by accumulating them into a single answer\. As an example, we\'ll pass \fB[3,2,1]\fR to this expression: -. -.IP "" 4 -. -.nf - -reduce \.[] as $item (0; \. + $item) -. -.fi -. -.IP "" 0 -. -.P -For each result that \fB\.[]\fR produces, \fB\. + $item\fR is run to accumulate a running total, starting from 0\. In this example, \fB\.[]\fR produces the results 3, 2, and 1, so the effect is similar to running something like this: -. -.IP "" 4 -. -.nf - -0 | (3 as $item | \. + $item) | - (2 as $item | \. + $item) | - (1 as $item | \. + $item) - -jq \'reduce \.[] as $item (0; \. + $item)\' - [10,2,5,3] -=> 20 -. -.fi -. -.IP "" 0 -. .SS "isempty(exp)" Returns true if \fBexp\fR produces no outputs, false otherwise\. . @@ -3306,6 +3314,14 @@ Returns true if \fBexp\fR produces no outputs, false otherwise\. jq \'isempty(empty)\' null => true + +jq \'isempty(\.[])\' + [] +=> true + +jq \'isempty(\.[])\' + [1,2,3] +=> false . .fi . @@ -3362,22 +3378,95 @@ jq \'[range(\.)]|[first, last, nth(5)]\' . .IP "" 0 . +.SS "reduce" +The \fBreduce\fR syntax allows you to combine all of the results of an expression by accumulating them into a single answer\. The form is \fBreduce EXP as $var (INIT; UPDATE)\fR\. As an example, we\'ll pass \fB[1,2,3]\fR to this expression: +. +.IP "" 4 +. +.nf + +reduce \.[] as $item (0; \. + $item) +. +.fi +. +.IP "" 0 +. +.P +For each result that \fB\.[]\fR produces, \fB\. + $item\fR is run to accumulate a running total, starting from 0 as the input value\. In this example, \fB\.[]\fR produces the results \fB1\fR, \fB2\fR, and \fB3\fR, so the effect is similar to running something like this: +. +.IP "" 4 +. +.nf + +0 | 1 as $item | \. + $item | + 2 as $item | \. + $item | + 3 as $item | \. + $item + +jq \'reduce \.[] as $item (0; \. + $item)\' + [1,2,3,4,5] +=> 15 + +jq \'reduce \.[] as [$i,$j] (0; \. + $i * $j)\' + [[1,2],[3,4],[5,6]] +=> 44 + +jq \'reduce \.[] as {$x,$y} (null; \.x += $x | \.y += [$y])\' + [{"x":"a","y":1},{"x":"b","y":2},{"x":"c","y":3}] +=> {"x":"abc","y":[1,2,3]} +. +.fi +. +.IP "" 0 +. .SS "foreach" -The \fBforeach\fR syntax is similar to \fBreduce\fR, but intended to allow the construction of \fBlimit\fR and reducers that produce intermediate results (see example)\. +The \fBforeach\fR syntax is similar to \fBreduce\fR, but intended to allow the construction of \fBlimit\fR and reducers that produce intermediate results\. . .P -The form is \fBforeach EXP as $var (INIT; UPDATE; EXTRACT)\fR\. Like \fBreduce\fR, \fBINIT\fR is evaluated once to produce a state value, then each output of \fBEXP\fR is bound to \fB$var\fR, \fBUPDATE\fR is evaluated for each output of \fBEXP\fR with the current state and with \fB$var\fR visible\. Each value output by \fBUPDATE\fR replaces the previous state\. Finally, \fBEXTRACT\fR is evaluated for each new state to extract an output of \fBforeach\fR\. +The form is \fBforeach EXP as $var (INIT; UPDATE; EXTRACT)\fR\. As an example, we\'ll pass \fB[1,2,3]\fR to this expression: +. +.IP "" 4 +. +.nf + +foreach \.[] as $item (0; \. + $item; [$item, \. * 2]) +. +.fi +. +.IP "" 0 . .P -This is mostly useful only for constructing \fBreduce\fR\- and \fBlimit\fR\-like functions\. But it is much more general, as it allows for partial reductions (see the example below)\. +Like the \fBreduce\fR syntax, \fB\. + $item\fR is run for each result that \fB\.[]\fR produces, but \fB[$item, \. * 2]\fR is run for each intermediate values\. In this example, since the intermediate values are \fB1\fR, \fB3\fR, and \fB6\fR, the \fBforeach\fR expression produces \fB[1,2]\fR, \fB[2,6]\fR, and \fB[3,12]\fR\. So the effect is similar to running something like this: . .IP "" 4 . .nf -jq \'[foreach \.[] as $item ([[],[]]; if $item == null then [[],\.[0]] else [(\.[0] + [$item]),[]] end; if $item == null then \.[1] else empty end)]\' - [1,2,3,4,null,"a","b",null] -=> [[1,2,3,4],["a","b"]] +0 | 1 as $item | \. + $item | [$item, \. * 2], + 2 as $item | \. + $item | [$item, \. * 2], + 3 as $item | \. + $item | [$item, \. * 2] +. +.fi +. +.IP "" 0 +. +.P +When \fBEXTRACT\fR is omitted, the identity filter is used\. That is, it outputs the intermediate values as they are\. +. +.IP "" 4 +. +.nf + +jq \'foreach \.[] as $item (0; \. + $item)\' + [1,2,3,4,5] +=> 1, 3, 6, 10, 15 + +jq \'foreach \.[] as $item (0; \. + $item; [$item, \. * 2])\' + [1,2,3,4,5] +=> [1,2], [2,6], [3,12], [4,20], [5,30] + +jq \'foreach \.[] as $item (0; \. + 1; {index: \., $item})\' + ["foo", "bar", "baz"] +=> {"index":1,"item":"foo"}, {"index":2,"item":"bar"}, {"index":3,"item":"baz"} . .fi . @@ -3474,13 +3563,33 @@ Most jq builtins are referentially transparent, and yield constant and repeatabl Outputs one new input\. . .P -Note that when using \fBinput\fR it is generally be necessary to invoke jq with the \-n command\-line option, otherwise the first entity will be lost\. +Note that when using \fBinput\fR it is generally be necessary to invoke jq with the \fB\-n\fR command\-line option, otherwise the first entity will be lost\. +. +.IP "" 4 +. +.nf + +echo 1 2 3 4 | jq \'[\., input]\' # [1,2] [3,4] +. +.fi +. +.IP "" 0 . .SS "inputs" Outputs all remaining inputs, one by one\. . .P -This is primarily useful for reductions over a program\'s inputs\. Note that when using \fBinputs\fR it is generally necessary to invoke jq with the \-n command\-line option, otherwise the first entity will be lost\. +This is primarily useful for reductions over a program\'s inputs\. Note that when using \fBinputs\fR it is generally necessary to invoke jq with the \fB\-n\fR command\-line option, otherwise the first entity will be lost\. +. +.IP "" 4 +. +.nf + +echo 1 2 3 | jq \-n \'reduce inputs as $i (0; \. + $i)\' # 6 +. +.fi +. +.IP "" 0 . .SS "debug, debug(msgs)" These two filters are like \fB\.\fR but have as a side\-effect the production of one or more messages on stderr\. @@ -3562,9 +3671,9 @@ Consumes a number as input and truncates the corresponding number of path elemen . .nf -jq \'[1|truncate_stream([[0],1],[[1,0],2],[[1,0]],[[1]])]\' +jq \'truncate_stream([[0],1],[[1,0],2],[[1,0]],[[1]])\' 1 -=> [[[0],2],[[0]]] +=> [[0],2], [[0]] . .fi . diff --git a/tests/jq.test b/tests/jq.test index da35e9a84c..e4b9c78cea 100644 --- a/tests/jq.test +++ b/tests/jq.test @@ -1232,6 +1232,14 @@ null [null,true,{"a":1}] [] +[1,error,2] +null +[1,2] + +[1,error(null),2] +0 +[1,2] + try error("\($__loc__)") catch . null "{\"file\":\"\",\"line\":1}" diff --git a/tests/man.test b/tests/man.test index 00fae382e7..cce2edb3f8 100644 --- a/tests/man.test +++ b/tests/man.test @@ -86,6 +86,12 @@ null .[] [] +.foo[] +{"foo":[1,2,3]} +1 +2 +3 + .[] {"a": 1, "b": 1} 1 @@ -133,7 +139,7 @@ null {"user":"stedolan","titles":["JQ Primer", "More JQ"]} {"stedolan": ["JQ Primer", "More JQ"]} -..|.a? +.. | .a? [[{"a":1}]] 1 @@ -316,6 +322,14 @@ null null [1,2,3] +try error catch . +"error message" +"error message" + +try error("invalid value: \(.)") catch . +42 +"invalid value: 42" + try error("\($__loc__)") catch . null "{\"file\":\"\",\"line\":1}" @@ -443,8 +457,12 @@ sort [null,3,6,8] sort_by(.foo) -[{"foo":4, "bar":10}, {"foo":3, "bar":100}, {"foo":2, "bar":1}] -[{"foo":2, "bar":1}, {"foo":3, "bar":100}, {"foo":4, "bar":10}] +[{"foo":4, "bar":10}, {"foo":3, "bar":10}, {"foo":2, "bar":1}] +[{"foo":2, "bar":1}, {"foo":3, "bar":10}, {"foo":4, "bar":10}] + +sort_by(.foo, .bar) +[{"foo":4, "bar":10}, {"foo":3, "bar":20}, {"foo":2, "bar":1}, {"foo":3, "bar":10}] +[{"foo":2, "bar":1}, {"foo":3, "bar":10}, {"foo":3, "bar":20}, {"foo":4, "bar":10}] group_by(.foo) [{"foo":1, "bar":10}, {"foo":3, "bar":100}, {"foo":1, "bar":1}] @@ -510,10 +528,26 @@ index(", ") "a,b, cd, efg, hijk" 3 +index(1) +[0,1,2,1,3,1,4] +1 + +index([1,2]) +[0,1,2,3,1,4,2,5,1,2,6,7] +1 + rindex(", ") "a,b, cd, efg, hijk" 12 +rindex(1) +[0,1,2,1,3,1,4] +5 + +rindex([1,2]) +[0,1,2,3,1,4,2,5,1,2,6,7] +8 + inside("foobar") "bar" true @@ -799,14 +833,18 @@ def addvalue(f): f as $x | map(. + $x); addvalue(.[0]) [[1,2],[10,20]] [[1,2,1,2], [10,20,1,2]] -reduce .[] as $item (0; . + $item) -[10,2,5,3] -20 - isempty(empty) null true +isempty(.[]) +[] +true + +isempty(.[]) +[1,2,3] +false + [limit(3;.[])] [0,1,2,3,4,5,6,7,8,9] [0,1,2] @@ -819,9 +857,39 @@ true 10 [0,9,5] -[foreach .[] as $item ([[],[]]; if $item == null then [[],.[0]] else [(.[0] + [$item]),[]] end; if $item == null then .[1] else empty end)] -[1,2,3,4,null,"a","b",null] -[[1,2,3,4],["a","b"]] +reduce .[] as $item (0; . + $item) +[1,2,3,4,5] +15 + +reduce .[] as [$i,$j] (0; . + $i * $j) +[[1,2],[3,4],[5,6]] +44 + +reduce .[] as {$x,$y} (null; .x += $x | .y += [$y]) +[{"x":"a","y":1},{"x":"b","y":2},{"x":"c","y":3}] +{"x":"abc","y":[1,2,3]} + +foreach .[] as $item (0; . + $item) +[1,2,3,4,5] +1 +3 +6 +10 +15 + +foreach .[] as $item (0; . + $item; [$item, . * 2]) +[1,2,3,4,5] +[1,2] +[2,6] +[3,12] +[4,20] +[5,30] + +foreach .[] as $item (0; . + 1; {index: ., $item}) +["foo", "bar", "baz"] +{"index":1,"item":"foo"} +{"index":2,"item":"bar"} +{"index":3,"item":"baz"} def range(init; upto; by): def _range: if (by > 0 and . < upto) or (by < 0 and . > upto) then ., ((.+by)|_range) else . end; if by == 0 then init else init|_range end | select((by > 0 and . < upto) or (by < 0 and . > upto)); range(0; 10; 3) null @@ -834,9 +902,10 @@ def while(cond; update): def _while: if cond then ., (update | _while) else empt 1 [1,2,4,8,16,32,64] -[1|truncate_stream([[0],1],[[1,0],2],[[1,0]],[[1]])] +truncate_stream([[0],1],[[1,0],2],[[1,0]],[[1]]) 1 -[[[0],2],[[0]]] +[[0],2] +[[0]] fromstream(1|truncate_stream([[0],1],[[1,0],2],[[1,0]],[[1]])) null