doc, fix tests, and improve errors for '-d DELIM' change in #150

CHANGES.md

@@ -2,7 +2,51 @@
 
 ## not yet released
 
-(nothing yet)
+- **Backward incompatible** and **security-related** change to parsing the
+  `-d DELIM` option. ([#148](https://github.com/trentm/json/issues/148))
+
+  The `-d DELIM` option allows specifying the field delimiter in output:
+
+        % echo '{"name":"trent","age":38}' | json -a name age
+        trent 38
+        % echo '{"name":"trent","age":38}' | json -a name age -d,
+        trent,38
+
+  The given "DELIM" string is parsed to allow escapes. For example:
+
+        % echo '{"name":"trent","age":38}' | json -a name age -d'\t'
+        trent	38
+        % echo '{"name":"trent","age":38}' | json -a name age -d'\n'
+        trent
+        38
+
+  Before this change, that parsing used `eval()`, which allowed for unintended
+  code execution if an untrusted argument to `-d` was provided. The fix for
+  this vulnerability changes to use `JSON.parse()` to support escapes. However
+  that results in a backward incompatible change, because the set
+  [JSON escapes](https://tools.ietf.org/html/rfc7159#section-7) is a subset of
+  [JavaScript escapes](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String#escape_notation).
+
+  The only escape I expect that would affect any current user would be the
+  null byte escape (`\0`) which can be useful for processing values that may
+  have spaces or other likely delimiter characters. For example:
+
+        # BEFORE
+        % echo '{"title":"Monsters, Inc.","year":"2001"}' \
+          | json -a title year -d'\0' \
+          | xargs -0 node -e 'console.log(process.argv)'
+        [ 'node', 'Monsters, Inc.', '2001\n' ]
+
+        # AFTER
+        % echo '{"title":"Monsters, Inc.","year":"2001"}' | json -a title year -d'\0'
+        json: error: Unexpected number in JSON at position 2
+
+  One must now use the JSON unicode escape syntax, '\u0000':
+
+        % echo '{"title":"Monsters, Inc.","year":"2001"}' \
+          | json -a title year -d'\u0000' \
+          | xargs -0 node -e 'console.log(process.argv)'
+        [ 'node', 'Monsters, Inc.', '2001\n' ]
 
 
 ## 10.0.0

README.md

@@ -30,11 +30,11 @@ Features:
 - natural syntax (like JS code) for extracting particular values
 - get details on JSON syntax errors (handy for config files)
 - filter input JSON (see `-e` and `-c` options)
-- fast stream processing
+- fast stream processing (see `-ga`)
 - JSON validation
 - in-place file editing
 
-See <http://trentm.com/json> for full docs and examples as a man page.
+See <https://trentm.com/json> for full docs and examples as a man page.
 
 Follow <a href="https://twitter.com/intent/user?screen_name=trentmick" target="_blank">@trentmick</a>
 for updates to json.

docs/json.1.ronn

@@ -499,7 +499,9 @@ output:
     Process input as an array of separate inputs and output in tabular form.
 
   * `-d DELIM`:
-    Delimiter character for tabular output (default is ' ').
+    Delimiter character for tabular output (default is ' '). This supports
+    [JSON escapes](https://tools.ietf.org/html/rfc7159#section-7), e.g.:
+    '\t' for tab or '\u0000' for the null byte.
 
   * `-A`:
     Process input as a single object, i.e. stop `-e` and `-c` automatically
@@ -786,5 +788,4 @@ MIT License (see <https://github.com/trentm/json/blob/master/LICENSE.txt>)
 
 ## COPYRIGHT
 
-json is Copyright (c) 2014 Trent Mick and Copyright (c) 2014 Joyent Inc.
-All rights reserved.
+json is Copyright 2021 Trent Mick and Copyright 2020 Joyent Inc.

lib/json.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 /**
- * Copyright 2020 Trent Mick.
+ * Copyright 2021 Trent Mick.
  * Copyright 2020 Joyent Inc.
  *
  * json -- JSON love for your command line.
@@ -122,10 +122,12 @@ if (util.format) {
 
 /**
  * Parse the given string into a JS string. Basically: handle escapes.
+ * Note that this only handles JSON escapes, which are a subset of all
+ * JavaScript-supported string escapes.
  */
 function _parseString(s) {
     /* JSSTYLED */
-    var quoted = '"' + s.replace(/\\"/, '"').replace('"', '\\"') + '"';
+    var quoted = '"' + s.replace(/\\"/g, '"').replace(/"/g, '\\"') + '"';
     return JSON.parse(quoted);
 }
 
@@ -409,7 +411,13 @@ function parseArgv(argv) {
             parsed.array = false;
             break;
         case '-d':
-            parsed.delim = _parseString(args.shift());
+            var outputDelim = args.shift()
+            try {
+                parsed.delim = _parseString(outputDelim);
+            } catch (parseErr) {
+                throw new Error(format('could not parse delim "%s": %s',
+                    outputDelim, parseErr));
+            }
             break;
         case '-D':
             parsed.lookupDelim = args.shift();

man/man1/json.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "JSON" "1" "August 2014" "" "json tool manual"
+.TH "JSON" "1" "April 2021" "" "json tool manual"
 .
 .SH "NAME"
 \fBjson\fR \- JSON love for your command line
@@ -766,7 +766,7 @@ Process input as an array of separate inputs and output in tabular form\.
 .
 .TP
 \fB\-d DELIM\fR
-Delimiter character for tabular output (default is \' \')\.
+Delimiter character for tabular output (default is \' \')\. This supports JSON escapes \fIhttps://tools\.ietf\.org/html/rfc7159#section\-7\fR, e\.g\.: \'\et\' for tab or \'\eu0000\' for the null byte\.
 .
 .TP
 \fB\-A\fR
@@ -1148,4 +1148,4 @@ This project lives at \fIhttps://github\.com/trentm/json\fR\. Please report bugs
 MIT License (see \fIhttps://github\.com/trentm/json/blob/master/LICENSE\.txt\fR)
 .
 .SH "COPYRIGHT"
-json is Copyright (c) 2014 Trent Mick and Copyright (c) 2014 Joyent Inc\. All rights reserved\.
+json is Copyright 2021 Trent Mick and Copyright 2020 Joyent Inc\.

test/array-processing/cmd

@@ -18,7 +18,7 @@ echo '{"name":"Trent","id":12,"email":"trent@example.com"}' | $JSON -a name emai
 echo ""
 echo '[{"name":"Trent","id":12,"email":"trent@example.com"},
        {"name":"Mark","id":13,"email":"mark@example.com"}]' \
-  | $JSON -d, -a name email     # TODO: how to spec '\0' as delim (a la -print0)?
+  | $JSON -d, -a name email
 
 # Delims (issue #26)
 echo ''
@@ -29,7 +29,7 @@ echo '[{"name":"trent","age":38},{"name":"ewan", "age":4}]' | $JSON -a name age
 echo '# newline'
 echo '[{"name":"trent","age":38},{"name":"ewan", "age":4}]' | $JSON -a name age -d '\n'
 echo '# nul to xargs'
-echo '[{"name":"trent","age":38},{"name":"ewan", "age":4}]' | $JSON -a name age -d '\0' | xargs -0 && echo 'end-of-xargs-output'
+echo '[{"name":"trent","age":38},{"name":"ewan", "age":4}]' | $JSON -a name age -d '\u0000' | xargs -0 && echo 'end-of-xargs-output'
 
 echo ""
 echo '[{"foo": "bar"}, {"foo": "baz"}]' | $JSON -a

test/output-delim/cmd

@@ -0,0 +1,23 @@
+JSON=../../lib/json.js
+
+echo "# simple -d DELIM"
+echo '{"name":"trent", "age":38}' | $JSON -a name age
+echo '{"name":"trent", "age":38}' | $JSON -a name age -d' '
+echo '{"name":"trent", "age":38}' | $JSON -a name age -d,
+
+echo "# -d DELIM with escapes"
+echo '{"name":"trent", "age":38}' | $JSON -a name age -d'\t'
+echo '{"name":"trent", "age":38}' | $JSON -a name age -d'\n'
+echo '{"title":"Monsters, Inc.","year":"2001"}' \
+    | json -a title year -d'\u0000' \
+    | xargs -0 && echo "eof"
+
+echo "# -d DELIM edge cases"
+echo '{"name":"trent", "age":38}' | $JSON -a name age -d'"'
+echo '{"name":"trent", "age":38}' | $JSON -a name age -d'""' # had a bug in this at one point
+
+echo "# unsupported JavaScript-only escapes in -d DELIM" >&2
+echo '{"title":"Monsters, Inc.","year":"2001"}' \
+    | $JSON -a title year -d'\0' \
+    | xargs -0 node -e 'console.log(process.argv)'
+echo '{"name":"trent", "age":38}' | $JSON -a name age -d'\x2c'

test/output-delim/expected.stderr

@@ -0,0 +1,3 @@
+# unsupported JavaScript-only escapes in -d DELIM
+json: error: could not parse delim "\0": SyntaxError: Unexpected number in JSON at position 2
+json: error: could not parse delim "\x2c": SyntaxError: Unexpected token x in JSON at position 2

test/output-delim/expected.stdout

@@ -0,0 +1,14 @@
+# simple -d DELIM
+trent 38
+trent 38
+trent,38
+# -d DELIM with escapes
+trent	38
+trent
+38
+Monsters, Inc. 2001
+
+eof
+# -d DELIM edge cases
+trent"38
+trent""38