Some changes to import for jqlang#659

bytecode.c

@@ -9,6 +9,7 @@
 #define NONE 0, 1
 #define CONSTANT OP_HAS_CONSTANT, 2
 #define VARIABLE (OP_HAS_VARIABLE | OP_HAS_BINDING), 3
+#define GLOBAL (OP_HAS_CONSTANT | OP_HAS_VARIABLE | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4
 #define BRANCH OP_HAS_BRANCH, 2
 #define CFUNC (OP_HAS_CFUNC | OP_HAS_BINDING), 3
 #define UFUNC (OP_HAS_UFUNC | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4

compile.c

@@ -148,6 +148,15 @@ block gen_const(jv constant) {
   return inst_block(i);
 }
 
+block gen_const_global(jv constant, const char *name) {
+  assert((opcode_describe(STORE_GLOBAL)->flags & (OP_HAS_CONSTANT | OP_HAS_VARIABLE | OP_HAS_BINDING)) ==
+         (OP_HAS_CONSTANT | OP_HAS_VARIABLE | OP_HAS_BINDING));
+  inst* i = inst_new(STORE_GLOBAL);
+  i->imm.constant = constant;
+  i->symbol = strdup(name);
+  return inst_block(i);
+}
+
 int block_is_const(block b) {
   return (block_is_single(b) && b.first->op == LOADK);
 }
@@ -239,6 +248,10 @@ int block_has_only_binders_and_imports(block binders, int bindflags) {
   return 1;
 }
 
+static int inst_is_binder(inst *i, int bindflags) {
+  return !((opcode_describe(i->op)->flags & bindflags) != bindflags && i->op != MODULEMETA);
+}
+
 int block_has_only_binders(block binders, int bindflags) {
   bindflags |= OP_HAS_BINDING;
   bindflags &= ~OP_BIND_WILDCARD;
@@ -311,10 +324,6 @@ static int block_bind_subblock(block binder, block body, int bindflags, int brea
          ((bindflags & OP_BIND_WILDCARD) && i->symbol[0] == '*' &&
           break_distance <= 3 && (i->symbol[1] == '1' + break_distance) &&
           i->symbol[2] == '\0'))) {
-      if (bindflags & OP_BIND_WILDCARD) {
-        jv_mem_free(i->symbol);
-        i->symbol = strdup(binder.first->symbol);
-      }
       // bind this instruction
       if (i->op == CALL_JQ && i->nactuals == -1)
         i->nactuals = block_count_actuals(i->arglist);
@@ -353,7 +362,6 @@ block block_bind(block binder, block body, int bindflags) {
 }
 
 block block_bind_library(block binder, block body, int bindflags, const char* libname) {
-  assert(block_has_only_binders(binder, bindflags));
   bindflags |= OP_HAS_BINDING;
   int nrefs = 0;
   int matchlen = strlen(libname);
@@ -363,13 +371,21 @@ block block_bind_library(block binder, block body, int bindflags, const char* li
     strcpy(matchname+matchlen,"::");
     matchlen += 2;
   }
+  assert(block_has_only_binders(binder, bindflags));
   for (inst *curr = binder.first; curr; curr = curr->next) {
+    int bindflags2 = bindflags;
     char* cname = curr->symbol;
     char* tname = malloc(strlen(curr->symbol)+matchlen+1);
     strcpy(tname, matchname);
     strcpy(tname+matchlen,cname);
+
+    // Ew
+    if ((opcode_describe(curr->op)->flags & (OP_HAS_VARIABLE | OP_HAS_CONSTANT)))
+      bindflags2 = OP_HAS_VARIABLE | OP_HAS_BINDING;
+
+    // This mutation is ugly, even if we undo it
     curr->symbol = tname;
-    nrefs += block_bind_subblock(inst_block(curr), body, bindflags, 0);
+    nrefs += block_bind_subblock(inst_block(curr), body, bindflags2, 0);
     curr->symbol = cname;
     free(tname);
   }
@@ -440,15 +456,13 @@ jv block_take_imports(block* body) {
   jv imports = jv_array();
 
   inst* top = NULL;
-  if (body->first->op == TOP) {
+  if (body->first && body->first->op == TOP) {
     top = block_take(body);
   }
   while (body->first && (body->first->op == MODULEMETA || body->first->op == DEPS)) {
     inst* dep = block_take(body);
     if (dep->op == DEPS) {
-      jv opts = jv_copy(dep->imm.constant);
-      opts = jv_object_set(opts,jv_string("name"),jv_string(dep->symbol));
-      imports = jv_array_append(imports, opts);
+      imports = jv_array_append(imports, jv_copy(dep->imm.constant));
     }
     inst_free(dep);
   }
@@ -458,13 +472,11 @@ jv block_take_imports(block* body) {
   return imports;
 }
 
-block gen_module(const char* name, block metadata) {
+block gen_module(block metadata) {
   inst* i = inst_new(MODULEMETA);
-  i->symbol = strdup(name);
   i->imm.constant = block_const(metadata);
   if (jv_get_kind(i->imm.constant) != JV_KIND_OBJECT)
     i->imm.constant = jv_object_set(jv_object(), jv_string("metadata"), i->imm.constant);
-  i->imm.constant = jv_object_set(i->imm.constant, jv_string("name"), jv_string(name));
   block_free(metadata);
   return inst_block(i);
 }
@@ -475,17 +487,17 @@ jv block_module_meta(block b) {
   return jv_null();
 }
 
-block gen_import(const char* name, block metadata, const char* as) {
+block gen_import(const char* name, block metadata, const char* as, int is_data) {
   assert(metadata.first == NULL || block_is_const(metadata));
   inst* i = inst_new(DEPS);
-  i->symbol = strdup(name);
   jv meta;
   if (block_is_const(metadata))
     meta = block_const(metadata);
   else
     meta = jv_object();
-  if (as)
-    meta = jv_object_set(meta, jv_string("as"), jv_string(as));
+  meta = jv_object_set(meta, jv_string("as"), jv_string(as));
+  meta = jv_object_set(meta, jv_string("is_data"), is_data ? jv_true() : jv_false());
+  meta = jv_object_set(meta, jv_string("relpath"), jv_string(name));
   i->imm.constant = meta;
   block_free(metadata);
   return inst_block(i);
@@ -797,7 +809,11 @@ block gen_or(block a, block b) {
 }
 
 block gen_var_binding(block var, const char* name, block body) {
-  return BLOCK(gen_op_simple(DUP), var,
+  // var bindings can be added after coding the program; leave the TOP first.
+  block top = gen_noop();
+  if (body.first && body.first->op == TOP)
+    top = inst_block(block_take(&body));
+  return BLOCK(top, gen_op_simple(DUP), var,
                block_bind(gen_op_unbound(STOREV, name),
                           body, OP_HAS_VARIABLE));
 }
@@ -1084,6 +1100,13 @@ static int compile(struct bytecode* bc, block b) {
         code[pos++] = nesting_level(bc, arg->bound_by);
         code[pos++] = arg->bound_by->imm.intval | ARG_NEWCLOSURE;
       }
+    } else if ((op->flags & OP_HAS_CONSTANT) && (op->flags & OP_HAS_VARIABLE)) {
+      // STORE_GLOBAL: constant global, basically
+      code[pos++] = jv_array_length(jv_copy(constant_pool));
+      constant_pool = jv_array_append(constant_pool, jv_copy(curr->imm.constant));
+      code[pos++] = nesting_level(bc, curr->bound_by);
+      uint16_t var = (uint16_t)curr->bound_by->imm.intval;
+      code[pos++] = var;
     } else if (op->flags & OP_HAS_CONSTANT) {
       code[pos++] = jv_array_length(jv_copy(constant_pool));
       constant_pool = jv_array_append(constant_pool, jv_copy(curr->imm.constant));

compile.h

@@ -20,6 +20,7 @@ block gen_noop();
 int block_is_noop(block b);
 block gen_op_simple(opcode op);
 block gen_const(jv constant);
+block gen_const_global(jv constant, const char *name);
 int block_is_const(block b);
 jv_kind block_const_kind(block b);
 jv block_const(block b);
@@ -28,9 +29,9 @@ block gen_op_unbound(opcode op, const char* name);
 block gen_op_bound(opcode op, block binder);
 block gen_op_var_fresh(opcode op, const char* name);
 
-block gen_module(const char* name, block metadata);
+block gen_module(block metadata);
 jv block_module_meta(block b);
-block gen_import(const char* name, block metadata, const char *as);
+block gen_import(const char* name, block metadata, const char *as, int is_data);
 block gen_function(const char* name, block formals, block body);
 block gen_param_regular(const char* name);
 block gen_param(const char* name);

docs/content/3.manual/manual.yml

@@ -2474,9 +2474,7 @@ sections:
     body: |
 
       jq has a library/module system.  Modules are files whose names end
-      in `.jq`.  Modules should (but don't have to) start with a
-      `module` directive.  Programs and modules must `import` their
-      dependencies, if any.
+      in `.jq`.
 
       Modules imported by a program are searched for in a default search
       path (see below).  The `import` directive allows the importer to
@@ -2499,67 +2497,102 @@ sections:
 
       The default search path is the search path given to the `-L`
       command-line option, else the "$JQ_LIBRARY_PATH", if set in the
-      environment, else `["~/.jq", "$ORIGIN/../lib/jq"]` (on Unix) or
-      `["~/.jq", "$ORIGIN/lib"]` (on Windows).
+      environment, else `["~/.jq", "$ORIGIN/../lib/jq",
+      "$ORIGIN/../lib"]`.
 
       The `-L` argument's and "$JQ_LIBRARY_PATH" environment variable's
       string values are split on ":" on Unix, or ";" on Windows.
 
       Null and empty string path elements terminate search path
       processing.
 
-      Modules may be organized into a hierarchy, with name components
-      separated by double-colons ("::").  For example: "foo::bar".  Each
-      component in the name corresponds to a directory on the
-      filesystem.
-
-      A module named "foo::bar" would be searched for in "foo/bar.jq"
-      and "foo/bar/bar.jq" in the given search path. This is intended to
-      allow modules to be placed in a directory along with, for example,
-      version control files, README files, and so on, but also to allow
-      for single-file modules.
+      A dependency with relative path "foo/bar" would be searched for in
+      "foo/bar.jq" and "foo/bar/bar.jq" in the given search path. This
+      is intended to allow modules to be placed in a directory along
+      with, for example, version control files, README files, and so on,
+      but also to allow for single-file modules.
 
       Consecutive components with the same name are not allowed to avoid
-      ambiguities (e.g., "foo::foo").
+      ambiguities (e.g., "foo/foo").
 
       For example, with `-L$HOME/.jq` a module `foo` can be found in
       `$HOME/.jq/foo.jq` and `$HOME/.jq/foo/foo.jq`.
 
       If "$HOME/.jq" is a file, it is sourced into the main program.
 
     entries:
-      - title: "`module NAME {<metadata>};`"
+      - title: "`import RelativePathString as NAME;`"
         body: |
 
-          Defines a module named `NAME` with the given metadata.  If
-          given it must be the first thing in the module.  Programs also
-          may use this directive.
+          Imports a module found at the given path relative to a
+          directory in a search path.  The module's symbols are prefixed
+          with "NAME::".  A ".jq" suffix will be added to the relative
+          path string.
+
+      - title: "`import RelativePathString as NAME {<metadata>};`"
+        body: |
+
+          Imports a module found at the given path relative to a
+          directory in a search path.  The module's symbols are prefixed
+          with "NAME::".  A ".jq" suffix will be added to the relative
+          path string.
 
           The metadata must be a constant jq expression.  It should be
-          an object with keys like "version", "repo", "URI", and so on.
-          At this time jq doesn't use this metadata, but it is made
-          available to users via the `modulemeta` builtin.
+          an object with keys like "homepage" and so on.  At this time
+          jq only uses the "search" key/value of the metadate.  The
+          metadata is also made available to users via the `modulemeta`
+          builtin.
+          
+          The "search" key in the metadata, if present, should have a
+          string or array value (array of strings); this is the search
+          path to be prefixed to the top-level search path.
 
-      - title: "`import NAME {<metadata>};` `import NAME {<metadata>} as NAME;`"
+      - title: "`import RelativePathString as NAME;`"
         body: |
 
-          Imports a module named `NAME`.  If the second form is used
-          then the module's symbols are prefixed with "NAME::".
+          Imports a JSON file found at the given path relative to a
+          directory in a search path.  The file's data will be available
+          as `$NAME::NAME`.  A ".json" suffix will be added to the
+          relative path string.
+
+      - title: "`import RelativePathString as $NAME {<metadata>};`"
+        body: |
+
+          Imports a JSON file found at the given path relative to a
+          directory in a search path.  The file's data will be available
+          as `$NAME::NAME`.  A ".json" suffix will be added to the
+          relative path string.
 
           The metadata must be a constant jq expression.  It should be
-          an object with keys like "version", "repo", "URI", and so on.
+          an object with keys like "homepage" and so on.  At this time
+          jq only uses the "search" key/value of the metadate.  The
+          metadata is also made available to users via the `modulemeta`
+          builtin.
           
           The "search" key in the metadata, if present, should have a
           string or array value (array of strings); this is the search
           path to be prefixed to the top-level search path.
 
+      - title: "`module {<metadata>};`"
+        body: |
+
+          This directive is entirely optional.  It's not required for
+          proper operation.  It serves only the purpose of providing
+          metadata that can be read with the `modulemeta` builtin.
+
+          The metadata must be a constant jq expression.  It should be
+          an object with keys like "homepage".  At this time jq doesn't
+          use this metadata, but it is made available to users via the
+          `modulemeta` builtin.
+
       - title: "`modulemeta`"
         body: |
 
-          Takes a module name as input and outputs the module's
-          metadata, with the module's imports (including metadata) as
-          the "deps" key.
+          Takes a module name as input and outputs the module's metadata
+          as an object, with the module's imports (including metadata)
+          as an array value for the "deps" key.
 
           Programs can use this to query a module's metadata, which they
           could then use to, for example, search for, download, and
           install missing dependencies.
+

execute.c

@@ -508,6 +508,25 @@ jv jq_next(jq_state *jq) {
       break;
     }
 
+    case STORE_GLOBAL: {
+      // Get the constant
+      jv val = jv_array_get(jv_copy(frame_current(jq)->bc->constants), *pc++);
+      assert(jv_is_valid(val));
+
+      // Store the var
+      uint16_t level = *pc++;
+      uint16_t v = *pc++;
+      jv* var = frame_local_var(jq, v, level);
+      if (jq->debug_trace_enabled) {
+        printf("V%d = ", v);
+        jv_dump(jv_copy(val), 0);
+        printf(" (%d)\n", jv_get_refcnt(val));
+      }
+      jv_free(*var);
+      *var = val;
+      break;
+    }
+
     case PATH_BEGIN: {
       jv v = stack_pop(jq);
       stack_push(jq, jq->path);