Save literal value of the parsed number to preserve it for the output

.gitignore

@@ -49,3 +49,4 @@ tests/*.trs
 cscope.in.out
 cscope.out
 cscope.po.out
+jq.dSYM

Makefile.am

@@ -170,7 +170,8 @@ EXTRA_DIST = $(DOC_FILES) $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER)     \
         tests/modules/test_bind_order.jq                                \
         tests/modules/test_bind_order0.jq                               \
         tests/modules/test_bind_order1.jq                               \
-        tests/modules/test_bind_order2.jq tests/onig.supp               \
+        tests/modules/test_bind_order2.jq                               \
+        tests/onig.supp tests/local.supp                                \
         tests/onig.test tests/setup tests/torture/input0.json		\
         tests/optional.test tests/optionaltest				\
 	tests/utf8-truncate.jq tests/utf8test				\

docs/content/3.manual/manual.yml

@@ -292,11 +292,21 @@ sections:
           program can be a useful way of formatting JSON output from,
           say, `curl`.
 
+          An important point about the identity filter is that it 
+          guarantees to preserve the literal representation of values. 
+          This is particularly important when dealing with numbers 
+          which otherwise get truncated to an IEEE754 double precision 
+          representation.
+
         examples:
           - program: '.'
             input: '"Hello, world!"'
             output: ['"Hello, world!"']
 
+          - program: '. | tojson'
+            input: '12345678909876543212345'
+            output: ['"12345678909876543212345"']
+
       - title: "Object Identifier-Index: `.foo`, `.foo.bar`"
         body: |
 
@@ -511,6 +521,16 @@ sections:
       expression that takes an input, ignores it, and returns 42
       instead.
 
+      Numbers in jq are internally represented by their IEEE754 double 
+      precision approximation. Any arithmetic operation with numbers, 
+      whether they are literals or results of previous filters, will 
+      produce a double precision floating point result.
+
+      However, when parsing a literal jq will store the original literal
+      string. If no mutation is applied to this value then it will make
+      to the output in its original form, even if conversion to double 
+      would result in a loss. 
+
     entries:
       - title: "Array construction: `[]`"
         body: |
@@ -629,6 +649,18 @@ sections:
       try to add a string to an object you'll get an error message and
       no result.
 
+      Please note that all numbers are converted to IEEE754 double precision 
+      floating point representation. Arithmetic and logical operators are working
+      with these converted doubles. Results of all such operations are also limited 
+      to the double precision. 
+
+      The only exception to this behaviour of number is a snapshot of original number 
+      literal. When a number which originally was provided as a literal is never
+      mutated until the end of the program then it is printed to the output in its 
+      original literal form. This also includes cases when the original literal 
+      would be truncated when converted to the IEEE754 double precision floating point
+      number. See `toliteral` filter for more information on this topic.
+
     entries:
       - title: "Addition: `+`"
         body: |
@@ -1233,6 +1265,46 @@ sections:
             input: '[1, "1", [1]]'
             output: ['"1"', '"1"', '"[1]"']
 
+      - title: "`toliteral`"
+        body: |
+
+          The `toliteral` function can be used to return the original literal
+          representation of a value as a string. This function can only be applied
+          to the simple types [string, null, false, true] and numbers which were provided
+          as literal constants. Calucated numbers don't have an associated literal, and will 
+          produce an error. Trying to get a literal of NaN or infinity will produce 
+          an error as well.
+
+          There is resemblence between this function and `tojson`, `tostring`. 
+          jq will preserve the original literal value of a number constant
+          when converted to string via any of `literal`, `tostring`, `tojson`.
+          However, the `literal` varaint will provide type safety by as per
+          the rules described above.
+
+          Please note that for strings `toliteral` will produce a quoted JSON string,
+          while `tostring` will produce the string itelf. In this, `toliteral` works
+          like `tojson`
+
+          In practice, `toliteral` can be used as a typesefe version of `tojson`.
+
+          `fromjson` can be used to generate a number from its literal representation.
+
+        examples: 
+          - program: '. as $big | [($big | toliteral), ($big | tojson), ($big | tostring)] | map(.=="10000000000000000000000000000001")'
+            input: '10000000000000000000000000000001'
+            output: ['[true, true, true]']
+
+          - program: '(. + 1) as $big | (try ($big | toliteral) catch .), ($big | tojson), ($big | tostring)'
+            input: '10000000000000000000000000000001'
+            output: ['"literal value is not available for a calculated number, infinite number or a NaN"', '"1e+31"', '"1e+31"']
+
+          - program: '(. | toliteral) as $l | [$l, ($l | fromjson)] | tojson'
+            input: '10000000000000000000000000000001'
+            output: ['"[\"10000000000000000000000000000001\",10000000000000000000000000000001]"']
+
+          - program: '(. | toliteral) == (. | tojson)'
+            input: '"Hello literal"'
+            output: [true]
       - title: "`type`"
         body: |
 

src/builtin.c

@@ -87,8 +87,11 @@ static jv f_plus(jq_state *jq, jv input, jv a, jv b) {
     jv_free(b);
     return a;
   } else if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
-    return jv_number(jv_number_value(a) +
+    jv r = jv_number(jv_number_value(a) +
                      jv_number_value(b));
+    jv_free(a);
+    jv_free(b);
+    return r;
   } else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_STRING) {
     return jv_string_concat(a, b);
   } else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) {
@@ -271,7 +274,10 @@ static jv f_rtrimstr(jq_state *jq, jv input, jv right) {
 static jv f_minus(jq_state *jq, jv input, jv a, jv b) {
   jv_free(input);
   if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
-    return jv_number(jv_number_value(a) - jv_number_value(b));
+    jv r = jv_number(jv_number_value(a) - jv_number_value(b));
+    jv_free(a);
+    jv_free(b);
+    return r;
   } else if (jv_get_kind(a) == JV_KIND_ARRAY && jv_get_kind(b) == JV_KIND_ARRAY) {
     jv out = jv_array();
     jv_array_foreach(a, i, x) {
@@ -299,7 +305,10 @@ static jv f_multiply(jq_state *jq, jv input, jv a, jv b) {
   jv_kind bk = jv_get_kind(b);
   jv_free(input);
   if (ak == JV_KIND_NUMBER && bk == JV_KIND_NUMBER) {
-    return jv_number(jv_number_value(a) * jv_number_value(b));
+    jv r = jv_number(jv_number_value(a) * jv_number_value(b));
+    jv_free(a);
+    jv_free(b);
+    return r;
   } else if ((ak == JV_KIND_STRING && bk == JV_KIND_NUMBER) ||
              (ak == JV_KIND_NUMBER && bk == JV_KIND_STRING)) {
     jv str = a;
@@ -333,7 +342,10 @@ static jv f_divide(jq_state *jq, jv input, jv a, jv b) {
   if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
     if (jv_number_value(b) == 0.0)
       return type_error2(a, b, "cannot be divided because the divisor is zero");
-    return jv_number(jv_number_value(a) / jv_number_value(b));
+    jv r = jv_number(jv_number_value(a) / jv_number_value(b));
+    jv_free(a);
+    jv_free(b);
+    return r;
   } else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_STRING) {
     return jv_string_split(a, b);
   } else {
@@ -346,7 +358,10 @@ static jv f_mod(jq_state *jq, jv input, jv a, jv b) {
   if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) {
     if ((intmax_t)jv_number_value(b) == 0)
       return type_error2(a, b, "cannot be divided (remainder) because the divisor is zero");
-    return jv_number((intmax_t)jv_number_value(a) % (intmax_t)jv_number_value(b));
+    jv r = jv_number((intmax_t)jv_number_value(a) % (intmax_t)jv_number_value(b));
+    jv_free(a);
+    jv_free(b);
+    return r;
   } else {
     return type_error2(a, b, "cannot be divided (remainder)");
   }
@@ -437,7 +452,9 @@ static jv f_length(jq_state *jq, jv input) {
   } else if (jv_get_kind(input) == JV_KIND_STRING) {
     return jv_number(jv_string_length_codepoints(input));
   } else if (jv_get_kind(input) == JV_KIND_NUMBER) {
-    return jv_number(fabs(jv_number_value(input)));
+    jv r = jv_number(fabs(jv_number_value(input)));
+    jv_free(input);
+    return r;
   } else if (jv_get_kind(input) == JV_KIND_NULL) {
     jv_free(input);
     return jv_number(0);
@@ -454,6 +471,24 @@ static jv f_tostring(jq_state *jq, jv input) {
   }
 }
 
+static jv f_toliteral(jq_state *jq, jv input) {
+  switch(jv_get_kind(input)) {
+    case JV_KIND_NUMBER:
+      if (!jv_number_has_literal(input)) {
+        return ret_error(input, jv_string("literal value is not available for a calculated number, infinite number or a NaN"));
+      } else {
+        // fallthrough!
+      }
+    case JV_KIND_NULL:
+    case JV_KIND_TRUE:
+    case JV_KIND_FALSE:
+    case JV_KIND_STRING:
+      return jv_dump_string(input, 0);
+    default:
+      return type_error(input, "isn't a simple type; only string, null, true, false and non calculated numbers can produce a literal");
+  }
+}
+
 static jv f_utf8bytelength(jq_state *jq, jv input) {
   if (jv_get_kind(input) != JV_KIND_STRING)
     return type_error(input, "only strings have UTF-8 byte length");
@@ -1594,6 +1629,7 @@ static const struct cfunction function_list[] = {
   {(cfunction_ptr)f_json_parse, "fromjson", 1},
   {(cfunction_ptr)f_tonumber, "tonumber", 1},
   {(cfunction_ptr)f_tostring, "tostring", 1},
+  {(cfunction_ptr)f_toliteral, "toliteral", 1},
   {(cfunction_ptr)f_keys, "keys", 1},
   {(cfunction_ptr)f_keys_unsorted, "keys_unsorted", 1},
   {(cfunction_ptr)f_startswith, "startswith", 2},

src/execute.c

@@ -509,21 +509,25 @@ jv jq_next(jq_state *jq) {
       uint16_t v = *pc++;
       jv* var = frame_local_var(jq, v, level);
       jv max = stack_pop(jq);
-      if (raising) goto do_backtrack;
+      if (raising) {
+        jv_free(max);
+        goto do_backtrack;
+      } 
       if (jv_get_kind(*var) != JV_KIND_NUMBER ||
           jv_get_kind(max) != JV_KIND_NUMBER) {
         set_error(jq, jv_invalid_with_msg(jv_string_fmt("Range bounds must be numeric")));
         jv_free(max);
         goto do_backtrack;
-      } else if (jv_number_value(jv_copy(*var)) >= jv_number_value(jv_copy(max))) {
+      } else if (jv_number_value(*var) >= jv_number_value(max)) {
         /* finished iterating */
+        jv_free(max);
         goto do_backtrack;
       } else {
-        jv curr = jv_copy(*var);
+        jv curr = *var;
         *var = jv_number(jv_number_value(*var) + 1);
 
         struct stack_pos spos = stack_get_pos(jq);
-        stack_push(jq, jv_copy(max));
+        stack_push(jq, max);
         stack_save(jq, pc - 3, spos);
 
         stack_push(jq, curr);
@@ -1010,6 +1014,9 @@ jq_state *jq_init(void) {
   jq->attrs = jv_object();
   jq->path = jv_null();
   jq->value_at_path = jv_null();
+
+  jq->nomem_handler = NULL;
+  jq->nomem_handler_data = NULL;
   return jq;
 }
 

src/jq_test.c

@@ -6,20 +6,32 @@
 #include "jq.h"
 
 static void jv_test();
-static void run_jq_tests(jv, int, FILE *);
+static void run_jq_tests(jv, int, FILE *, int, int);
 
 
 int jq_testsuite(jv libdirs, int verbose, int argc, char* argv[]) {
   FILE *testdata = stdin;
+  int skip = -1;
+  int take = -1;
   jv_test();
   if (argc > 0) {
-    testdata = fopen(argv[0], "r");
-    if (!testdata) {
-      perror("fopen");
-      exit(1);
+    for(int i = 0; i < argc; i++) {
+      if (!strcmp(argv[i], "--skip")) {
+        skip = atoi(argv[i+1]);
+        i++;
+      } else if (!strcmp(argv[i], "--take")) {
+        take = atoi(argv[i+1]);
+        i++;
+      } else {
+        testdata = fopen(argv[i], "r");
+        if (!testdata) {
+          perror("fopen");
+          exit(1);
+        }
+      }
     }
   }
-  run_jq_tests(libdirs, verbose, testdata);
+  run_jq_tests(libdirs, verbose, testdata, skip, take);
   return 0;
 }
 
@@ -53,7 +65,7 @@ static void test_err_cb(void *data, jv e) {
   jv_free(e);
 }
 
-static void run_jq_tests(jv lib_dirs, int verbose, FILE *testdata) {
+static void run_jq_tests(jv lib_dirs, int verbose, FILE *testdata, int skip, int take) {
   char prog[4096];
   char buf[4096];
   struct err_data err_msg;
@@ -63,6 +75,9 @@ static void run_jq_tests(jv lib_dirs, int verbose, FILE *testdata) {
   int check_msg = 0;
   jq_state *jq = NULL;
 
+  int tests_to_skip = skip;
+  int tests_to_take = take;
+
   jq = jq_init();
   assert(jq);
   if (jv_get_kind(lib_dirs) == JV_KIND_NULL)
@@ -80,6 +95,34 @@ static void run_jq_tests(jv lib_dirs, int verbose, FILE *testdata) {
       continue;
     }
     if (prog[strlen(prog)-1] == '\n') prog[strlen(prog)-1] = 0;
+
+    if (skip > 0) {
+      skip--;
+
+      // skip past test data
+      while (fgets(buf, sizeof(buf), testdata)) {
+        lineno++;
+        if (buf[0] == '\n' || (buf[0] == '\r' && buf[1] == '\n'))
+          break;
+      }
+
+      must_fail = 0;
+      check_msg = 0;
+
+      continue;
+    } else if (skip == 0) {
+      printf("Skipped %d tests\n", tests_to_skip);
+      skip = -1;
+    }
+
+    if (take > 0) {
+      take--;
+    } else if (take == 0) {
+      printf("Hit the number of tests limit (%d), breaking\n", tests_to_take);
+      take = -1;
+      break;
+    }
+
     printf("Testing '%s' at line number %u\n", prog, lineno);
     int pass = 1;
     tests++;
@@ -179,7 +222,21 @@ static void run_jq_tests(jv lib_dirs, int verbose, FILE *testdata) {
     passed+=pass;
   }
   jq_teardown(&jq);
-  printf("%d of %d tests passed (%d malformed)\n", passed,tests,invalid);
+
+  int total_skipped = tests_to_skip > 0 ? tests_to_skip : 0;
+
+  if (skip > 0) {
+    total_skipped = tests_to_skip - skip;
+  }
+
+  printf("%d of %d tests passed (%d malformed, %d skipped)\n", 
+    passed, tests, invalid, total_skipped);
+
+  if (skip > 0) {
+    printf("WARN: skipped past the end of file, exiting with status 2\n");
+    exit(2);
+  }
+
   if (passed != tests) exit(1);
 }