Further module system revamp (fix #659)

To import a module now use:

    # Import module.jq file:
    import "relative/path/to/module" as foo;

    # Use the module's defs as foo::<def-name>

To import a JSON file:

    # Read file.json:
    import "relative/path/to/file" as $foo;
    #
    # Use as $foo::foo

Using `-L` now drops the builtin library path and appends the requested
path to the empty array (or the result of an earlier `-L`).

Support for the `$JQ_LIBRARY_PATH` environment variable has been
removed.

builtin.c

@@ -824,6 +824,21 @@ static jv f_env(jq_state *jq, jv input) {
   return env;
 }
 
+static jv f_get_search_list(jq_state *jq, jv input) {
+  jv_free(input);
+  return jq_get_lib_dirs(jq);
+}
+
+static jv f_get_prog_origin(jq_state *jq, jv input) {
+  jv_free(input);
+  return jq_get_prog_origin(jq);
+}
+
+static jv f_get_jq_origin(jq_state *jq, jv input) {
+  jv_free(input);
+  return jq_get_jq_origin(jq);
+}
+
 static jv f_string_split(jq_state *jq, jv a, jv b) {
   if (jv_get_kind(a) != JV_KIND_STRING || jv_get_kind(b) != JV_KIND_STRING) {
     jv_free(a);
@@ -937,6 +952,9 @@ static const struct cfunction function_list[] = {
   {(cfunction_ptr)f_error, "error", 2},
   {(cfunction_ptr)f_format, "format", 2},
   {(cfunction_ptr)f_env, "env", 1},
+  {(cfunction_ptr)f_get_search_list, "get_search_list", 1},
+  {(cfunction_ptr)f_get_prog_origin, "get_prog_origin", 1},
+  {(cfunction_ptr)f_get_jq_origin, "get_jq_origin", 1},
   {(cfunction_ptr)f_match, "_match_impl", 4},
   {(cfunction_ptr)f_modulemeta, "modulemeta", 1},
   {(cfunction_ptr)f_input, "_input", 1},

bytecode.c

@@ -9,6 +9,7 @@
 #define NONE 0, 1
 #define CONSTANT OP_HAS_CONSTANT, 2
 #define VARIABLE (OP_HAS_VARIABLE | OP_HAS_BINDING), 3
+#define GLOBAL (OP_HAS_CONSTANT | OP_HAS_VARIABLE | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4
 #define BRANCH OP_HAS_BRANCH, 2
 #define CFUNC (OP_HAS_CFUNC | OP_HAS_BINDING), 3
 #define UFUNC (OP_HAS_UFUNC | OP_HAS_BINDING | OP_IS_CALL_PSEUDO), 4

compile.c

@@ -148,6 +148,15 @@ block gen_const(jv constant) {
   return inst_block(i);
 }
 
+block gen_const_global(jv constant, const char *name) {
+  assert((opcode_describe(STORE_GLOBAL)->flags & (OP_HAS_CONSTANT | OP_HAS_VARIABLE | OP_HAS_BINDING)) ==
+         (OP_HAS_CONSTANT | OP_HAS_VARIABLE | OP_HAS_BINDING));
+  inst* i = inst_new(STORE_GLOBAL);
+  i->imm.constant = constant;
+  i->symbol = strdup(name);
+  return inst_block(i);
+}
+
 int block_is_const(block b) {
   return (block_is_single(b) && b.first->op == LOADK);
 }
@@ -239,6 +248,10 @@ int block_has_only_binders_and_imports(block binders, int bindflags) {
   return 1;
 }
 
+static int inst_is_binder(inst *i, int bindflags) {
+  return !((opcode_describe(i->op)->flags & bindflags) != bindflags && i->op != MODULEMETA);
+}
+
 int block_has_only_binders(block binders, int bindflags) {
   bindflags |= OP_HAS_BINDING;
   bindflags &= ~OP_BIND_WILDCARD;
@@ -311,10 +324,6 @@ static int block_bind_subblock(block binder, block body, int bindflags, int brea
          ((bindflags & OP_BIND_WILDCARD) && i->symbol[0] == '*' &&
           break_distance <= 3 && (i->symbol[1] == '1' + break_distance) &&
           i->symbol[2] == '\0'))) {
-      if (bindflags & OP_BIND_WILDCARD) {
-        jv_mem_free(i->symbol);
-        i->symbol = strdup(binder.first->symbol);
-      }
       // bind this instruction
       if (i->op == CALL_JQ && i->nactuals == -1)
         i->nactuals = block_count_actuals(i->arglist);
@@ -353,7 +362,6 @@ block block_bind(block binder, block body, int bindflags) {
 }
 
 block block_bind_library(block binder, block body, int bindflags, const char* libname) {
-  assert(block_has_only_binders(binder, bindflags));
   bindflags |= OP_HAS_BINDING;
   int nrefs = 0;
   int matchlen = strlen(libname);
@@ -363,13 +371,21 @@ block block_bind_library(block binder, block body, int bindflags, const char* li
     strcpy(matchname+matchlen,"::");
     matchlen += 2;
   }
+  assert(block_has_only_binders(binder, bindflags));
   for (inst *curr = binder.first; curr; curr = curr->next) {
+    int bindflags2 = bindflags;
     char* cname = curr->symbol;
     char* tname = malloc(strlen(curr->symbol)+matchlen+1);
     strcpy(tname, matchname);
     strcpy(tname+matchlen,cname);
+
+    // Ew
+    if ((opcode_describe(curr->op)->flags & (OP_HAS_VARIABLE | OP_HAS_CONSTANT)))
+      bindflags2 = OP_HAS_VARIABLE | OP_HAS_BINDING;
+
+    // This mutation is ugly, even if we undo it
     curr->symbol = tname;
-    nrefs += block_bind_subblock(inst_block(curr), body, bindflags, 0);
+    nrefs += block_bind_subblock(inst_block(curr), body, bindflags2, 0);
     curr->symbol = cname;
     free(tname);
   }
@@ -440,15 +456,13 @@ jv block_take_imports(block* body) {
   jv imports = jv_array();
   
   inst* top = NULL;
-  if (body->first->op == TOP) {
+  if (body->first && body->first->op == TOP) {
     top = block_take(body);
   }
   while (body->first && (body->first->op == MODULEMETA || body->first->op == DEPS)) {
     inst* dep = block_take(body);
     if (dep->op == DEPS) {
-      jv opts = jv_copy(dep->imm.constant);
-      opts = jv_object_set(opts,jv_string("name"),jv_string(dep->symbol));
-      imports = jv_array_append(imports, opts);
+      imports = jv_array_append(imports, jv_copy(dep->imm.constant));
     }
     inst_free(dep);
   }
@@ -458,13 +472,11 @@ jv block_take_imports(block* body) {
   return imports;
 }
 
-block gen_module(const char* name, block metadata) {
+block gen_module(block metadata) {
   inst* i = inst_new(MODULEMETA);
-  i->symbol = strdup(name);
   i->imm.constant = block_const(metadata);
   if (jv_get_kind(i->imm.constant) != JV_KIND_OBJECT)
     i->imm.constant = jv_object_set(jv_object(), jv_string("metadata"), i->imm.constant);
-  i->imm.constant = jv_object_set(i->imm.constant, jv_string("name"), jv_string(name));
   block_free(metadata);
   return inst_block(i);
 }
@@ -475,17 +487,17 @@ jv block_module_meta(block b) {
   return jv_null();
 }
 
-block gen_import(const char* name, block metadata, const char* as) {
+block gen_import(const char* name, block metadata, const char* as, int is_data) {
   assert(metadata.first == NULL || block_is_const(metadata));
   inst* i = inst_new(DEPS);
-  i->symbol = strdup(name);
   jv meta;
   if (block_is_const(metadata))
     meta = block_const(metadata);
   else
     meta = jv_object();
-  if (as)
-    meta = jv_object_set(meta, jv_string("as"), jv_string(as));
+  meta = jv_object_set(meta, jv_string("as"), jv_string(as));
+  meta = jv_object_set(meta, jv_string("is_data"), is_data ? jv_true() : jv_false());
+  meta = jv_object_set(meta, jv_string("relpath"), jv_string(name));
   i->imm.constant = meta;
   block_free(metadata);
   return inst_block(i);
@@ -797,7 +809,11 @@ block gen_or(block a, block b) {
 }
 
 block gen_var_binding(block var, const char* name, block body) {
-  return BLOCK(gen_op_simple(DUP), var,
+  // var bindings can be added after coding the program; leave the TOP first.
+  block top = gen_noop();
+  if (body.first && body.first->op == TOP)
+    top = inst_block(block_take(&body));
+  return BLOCK(top, gen_op_simple(DUP), var,
                block_bind(gen_op_unbound(STOREV, name),
                           body, OP_HAS_VARIABLE));
 }
@@ -1084,6 +1100,13 @@ static int compile(struct bytecode* bc, block b) {
         code[pos++] = nesting_level(bc, arg->bound_by);
         code[pos++] = arg->bound_by->imm.intval | ARG_NEWCLOSURE;
       }
+    } else if ((op->flags & OP_HAS_CONSTANT) && (op->flags & OP_HAS_VARIABLE)) {
+      // STORE_GLOBAL: constant global, basically
+      code[pos++] = jv_array_length(jv_copy(constant_pool));
+      constant_pool = jv_array_append(constant_pool, jv_copy(curr->imm.constant));
+      code[pos++] = nesting_level(bc, curr->bound_by);
+      uint16_t var = (uint16_t)curr->bound_by->imm.intval;
+      code[pos++] = var;
     } else if (op->flags & OP_HAS_CONSTANT) {
       code[pos++] = jv_array_length(jv_copy(constant_pool));
       constant_pool = jv_array_append(constant_pool, jv_copy(curr->imm.constant));

compile.h

@@ -20,6 +20,7 @@ block gen_noop();
 int block_is_noop(block b);
 block gen_op_simple(opcode op);
 block gen_const(jv constant);
+block gen_const_global(jv constant, const char *name);
 int block_is_const(block b);
 jv_kind block_const_kind(block b);
 jv block_const(block b);
@@ -28,9 +29,9 @@ block gen_op_unbound(opcode op, const char* name);
 block gen_op_bound(opcode op, block binder);
 block gen_op_var_fresh(opcode op, const char* name);
 
-block gen_module(const char* name, block metadata);
+block gen_module(block metadata);
 jv block_module_meta(block b);
-block gen_import(const char* name, block metadata, const char *as);
+block gen_import(const char* name, block metadata, const char *as, int is_data);
 block gen_function(const char* name, block formals, block body);
 block gen_param_regular(const char* name);
 block gen_param(const char* name);

docs/content/3.manual/manual.yml

@@ -185,8 +185,9 @@ sections:
 
       * `-Ldirectory` / `-L directory`:
 
-        Add `directory` to the search list for modules.  See the section
-        on modules below.
+        Prepend `directory` to the search list for modules.  If this
+        option is used then no builtin search list is used.  See the
+        section on modules below.
 
       * `-e` / `--exit-status`:
 
@@ -2474,9 +2475,7 @@ sections:
     body: |
 
       jq has a library/module system.  Modules are files whose names end
-      in `.jq`.  Modules should (but don't have to) start with a
-      `module` directive.  Programs and modules must `import` their
-      dependencies, if any.
+      in `.jq`.
 
       Modules imported by a program are searched for in a default search
       path (see below).  The `import` directive allows the importer to
@@ -2498,29 +2497,20 @@ sections:
       the default is appended.
 
       The default search path is the search path given to the `-L`
-      command-line option, else the "$JQ_LIBRARY_PATH", if set in the
-      environment, else `["~/.jq", "$ORIGIN/../lib/jq"]` (on Unix) or
-      `["~/.jq", "$ORIGIN/lib"]` (on Windows).
-
-      The `-L` argument's and "$JQ_LIBRARY_PATH" environment variable's
-      string values are split on ":" on Unix, or ";" on Windows.
+      command-line option, else `["~/.jq", "$ORIGIN/../lib/jq",
+      "$ORIGIN/../lib"]`.
 
       Null and empty string path elements terminate search path
       processing.
 
-      Modules may be organized into a hierarchy, with name components
-      separated by double-colons ("::").  For example: "foo::bar".  Each
-      component in the name corresponds to a directory on the
-      filesystem.
-
-      A module named "foo::bar" would be searched for in "foo/bar.jq"
-      and "foo/bar/bar.jq" in the given search path. This is intended to
-      allow modules to be placed in a directory along with, for example,
-      version control files, README files, and so on, but also to allow
-      for single-file modules.
+      A dependency with relative path "foo/bar" would be searched for in
+      "foo/bar.jq" and "foo/bar/bar.jq" in the given search path. This
+      is intended to allow modules to be placed in a directory along
+      with, for example, version control files, README files, and so on,
+      but also to allow for single-file modules.
 
       Consecutive components with the same name are not allowed to avoid
-      ambiguities (e.g., "foo::foo").
+      ambiguities (e.g., "foo/foo").
 
       For example, with `-L$HOME/.jq` a module `foo` can be found in
       `$HOME/.jq/foo.jq` and `$HOME/.jq/foo/foo.jq`.
@@ -2528,38 +2518,78 @@ sections:
       If "$HOME/.jq" is a file, it is sourced into the main program.
 
     entries:
-      - title: "`module NAME {<metadata>};`"
+      - title: "`import RelativePathString as NAME;`"
         body: |
 
-          Defines a module named `NAME` with the given metadata.  If
-          given it must be the first thing in the module.  Programs also
-          may use this directive.
+          Imports a module found at the given path relative to a
+          directory in a search path.  The module's symbols are prefixed
+          with "NAME::".  A ".jq" suffix will be added to the relative
+          path string.
 
-          The metadata must be a constant jq expression.  It should be
-          an object with keys like "version", "repo", "URI", and so on.
-          At this time jq doesn't use this metadata, but it is made
-          available to users via the `modulemeta` builtin.
-
-      - title: "`import NAME {<metadata>};` `import NAME {<metadata>} as NAME;`"
+      - title: "`import RelativePathString as NAME {<metadata>};`"
         body: |
 
-          Imports a module named `NAME`.  If the second form is used
-          then the module's symbols are prefixed with "NAME::".
+          Imports a module found at the given path relative to a
+          directory in a search path.  The module's symbols are prefixed
+          with "NAME::".  A ".jq" suffix will be added to the relative
+          path string.
 
           The metadata must be a constant jq expression.  It should be
-          an object with keys like "version", "repo", "URI", and so on.
+          an object with keys like "homepage" and so on.  At this time
+          jq only uses the "search" key/value of the metadate.  The
+          metadata is also made available to users via the `modulemeta`
+          builtin.
           
           The "search" key in the metadata, if present, should have a
           string or array value (array of strings); this is the search
           path to be prefixed to the top-level search path.
 
+      - title: "`import RelativePathString as NAME;`"
+        body: |
+
+          Imports a JSON file found at the given path relative to a
+          directory in a search path.  The file's data will be available
+          as `$NAME::NAME`.  A ".json" suffix will be added to the
+          relative path string.
+
+      - title: "`import RelativePathString as $NAME {<metadata>};`"
+        body: |
+
+          Imports a JSON file found at the given path relative to a
+          directory in a search path.  The file's data will be available
+          as `$NAME::NAME`.  A ".json" suffix will be added to the
+          relative path string.
+
+          The metadata must be a constant jq expression.  It should be
+          an object with keys like "homepage" and so on.  At this time
+          jq only uses the "search" key/value of the metadate.  The
+          metadata is also made available to users via the `modulemeta`
+          builtin.
+          
+          The "search" key in the metadata, if present, should have a
+          string or array value (array of strings); this is the search
+          path to be prefixed to the top-level search path.
+
+      - title: "`module {<metadata>};`"
+        body: |
+
+          This directive is entirely optional.  It's not required for
+          proper operation.  It serves only the purpose of providing
+          metadata that can be read with the `modulemeta` builtin.
+
+          The metadata must be a constant jq expression.  It should be
+          an object with keys like "homepage".  At this time jq doesn't
+          use this metadata, but it is made available to users via the
+          `modulemeta` builtin.
+
       - title: "`modulemeta`"
         body: |
 
-          Takes a module name as input and outputs the module's
-          metadata, with the module's imports (including metadata) as
-          the "deps" key.
+          Takes a module name as input and outputs the module's metadata
+          as an object, with the module's imports (including metadata)
+          as an array value for the "deps" key.
 
           Programs can use this to query a module's metadata, which they
           could then use to, for example, search for, download, and
           install missing dependencies.
+

execute.c

@@ -508,6 +508,25 @@ jv jq_next(jq_state *jq) {
       break;
     }
 
+    case STORE_GLOBAL: {
+      // Get the constant
+      jv val = jv_array_get(jv_copy(frame_current(jq)->bc->constants), *pc++);
+      assert(jv_is_valid(val));
+
+      // Store the var
+      uint16_t level = *pc++;
+      uint16_t v = *pc++;
+      jv* var = frame_local_var(jq, v, level);
+      if (jq->debug_trace_enabled) {
+        printf("V%d = ", v);
+        jv_dump(jv_copy(val), 0);
+        printf(" (%d)\n", jv_get_refcnt(val));
+      }
+      jv_free(*var);
+      *var = val;
+      break;
+    }
+
     case PATH_BEGIN: {
       jv v = stack_pop(jq);
       stack_push(jq, jq->path);

linker.c

@@ -21,33 +21,71 @@ struct lib_loading_state {
   block *defs;
   uint64_t ct;
 };
-static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_loading_state *lib_state);
+static int load_library(jq_state *jq, jv lib_path, int is_data, int raw,
+                        const char *as, block *out_block,
+                        struct lib_loading_state *lib_state);
+
 
 // Given a lib_path to search first, creates a chain of search paths
 // in the following order:
 // 1. lib_path
-// 2. -L paths passed in on the command line (from jq_state*)
-// 3. JQ_LIBRARY_PATH environment variable
-jv build_lib_search_chain(jq_state *jq, jv search_path) {
+// 2. -L paths passed in on the command line (from jq_state*) or builtin list
+static jv build_lib_search_chain(jq_state *jq, jv search_path, jv jq_origin, jv lib_origin) {
   assert(jv_get_kind(search_path) == JV_KIND_ARRAY);
   jv expanded = jv_array();
+  jv expanded_elt;
   jv_array_foreach(search_path, i, path) {
-    path = expand_path(path);
-    if (jv_is_valid(path))
-      expanded = jv_array_append(expanded, expand_path(path));
-    else
-      jv_free(path);
+    if (strcmp(".",jv_string_value(path)) == 0) {
+      expanded_elt = jv_copy(path);
+    } else if (jv_get_kind(lib_origin) == JV_KIND_STRING &&
+               strncmp("./", jv_string_value(path),sizeof("./")-1) == 0) {
+      expanded_elt = jv_string_fmt("%s/%s",
+                               jv_string_value(lib_origin),
+                               jv_string_value(path) + sizeof ("./") - 1);
+    } else if (strncmp("$ORIGIN/",jv_string_value(path),sizeof("$ORIGIN/")-1) == 0) {
+      expanded_elt = jv_string_fmt("%s/%s",
+                               jv_string_value(jq_origin),
+                               jv_string_value(path) + sizeof ("$ORIGIN/") - 1);
+    } else {
+      expanded_elt = expand_path(path);
+      if (!jv_is_valid(expanded_elt)) {
+        jv_free(search_path);
+        jv_free(expanded);
+        jv_free(path);
+        return expanded_elt;
+      }
+      path = jv_invalid();
+    }
+    expanded = jv_array_append(expanded, expanded_elt);
+    jv_free(path);
   }
+  jv_free(jq_origin);
+  jv_free(lib_origin);
   jv_free(search_path);
-  expanded = jv_array_concat(expanded, jq_get_lib_dirs(jq));
   return expanded;
 }
 
-static jv name2relpath(jv name) {
-  jv components = jv_string_split(jv_copy(name), jv_string("::"));
+// Doesn't actually check that name not be an absolute path; we could
+// just always prepend "./"!
+static jv validate_relpath(jv name) {
+  const char *s = jv_string_value(name);
+  if (strchr(s, '\\')) {
+    jv res = jv_invalid_with_msg(jv_string_fmt("Modules must be named by relative paths using '/', not '\\' (%s)", s));
+    jv_free(name);
+    return res;
+  }
+  jv components = jv_string_split(jv_copy(name), jv_string("/"));
   jv rp = jv_array_get(jv_copy(components), 0);
   components = jv_array_slice(components, 1, jv_array_length(jv_copy(components)));
   jv_array_foreach(components, i, x) {
+    if (!strcmp(jv_string_value(x), "..")) {
+      jv_free(x);
+      jv_free(rp);
+      jv_free(components);
+      jv res = jv_invalid_with_msg(jv_string_fmt("Relative paths to modules may not traverse to parent directories (%s)", s));
+      jv_free(name);
+      return res;
+    }
     if (i > 0 && jv_equal(jv_copy(x), jv_array_get(jv_copy(components), i - 1))) {
       jv_free(x);
       jv_free(rp);
@@ -64,22 +102,33 @@ static jv name2relpath(jv name) {
   return rp;
 }
 
-static jv find_lib(jq_state *jq, jv lib_name, jv lib_search_path) {
-  assert(jv_get_kind(lib_search_path) == JV_KIND_ARRAY);
-  assert(jv_get_kind(lib_name) == JV_KIND_STRING);
+// Assumes name has been validated
+static jv jv_basename(jv name) {
+  const char *s = jv_string_value(name);
+  const char *p = strrchr(s, '/');
+  if (!p)
+    return name;
+  jv res = jv_string_fmt("%s", p);
+  jv_free(name);
+  return res;
+}
 
-  jv rel_path = name2relpath(jv_copy(lib_name));
-  if (!jv_is_valid(rel_path)) {
-    jv_free(lib_name);
-    return rel_path;
-  }
+// Asummes validated relative path to module
+static jv find_lib(jq_state *jq, jv rel_path, jv search, const char *suffix, jv jq_origin, jv lib_origin) {
+  if (jv_get_kind(search) != JV_KIND_ARRAY)
+    return jv_invalid_with_msg(jv_string_fmt("Module search path must be an array"));
+  if (jv_get_kind(rel_path) != JV_KIND_STRING)
+    return jv_invalid_with_msg(jv_string_fmt("Module path must be a string"));
 
   struct stat st;
   int ret;
 
-  jv lib_search_paths = build_lib_search_chain(jq, lib_search_path);
+  // Ideally we should cache this somewhere
+  search = build_lib_search_chain(jq, search, jq_origin, lib_origin);
 
-  jv_array_foreach(lib_search_paths, i, spath) {
+  jv bname = jv_basename(jv_copy(rel_path));
+
+  jv_array_foreach(search, i, spath) {
     if (jv_get_kind(spath) == JV_KIND_NULL) {
       jv_free(spath);
       break;
@@ -89,43 +138,44 @@ static jv find_lib(jq_state *jq, jv lib_name, jv lib_search_path) {
       jv_free(spath);
       continue; /* XXX report non-strings in search path?? */
     }
-    jv testpath = jq_realpath(jv_string_fmt("%s/%s.jq",
+    // Try .../module/last/component.jq
+    jv testpath = jq_realpath(jv_string_fmt("%s/%s%s",
                                             jv_string_value(spath),
-                                            jv_string_value(rel_path)));
+                                            jv_string_value(rel_path),
+                                            suffix));
     ret = stat(jv_string_value(testpath),&st);
     if (ret == -1 && errno == ENOENT) {
       jv_free(testpath);
-      testpath = jq_realpath(jv_string_fmt("%s/%s/%s.jq",
+      // Try .../module/last/component/component.jq
+      testpath = jq_realpath(jv_string_fmt("%s/%s/%s%s",
                                            jv_string_value(spath),
                                            jv_string_value(rel_path),
-                                           jv_string_value(lib_name)));
+                                           jv_string_value(bname),
+                                           suffix));
       ret = stat(jv_string_value(testpath),&st);
     }
     if (ret == 0) {
-      jv_free(spath);
       jv_free(rel_path);
-      jv_free(lib_name);
-      jv_free(lib_search_paths);
+      jv_free(search);
+      jv_free(bname);
+      jv_free(spath);
       return testpath;
     }
     jv_free(testpath);
     jv_free(spath);
   }
-  jv output = jv_invalid_with_msg(jv_string_fmt("module not found: %s", jv_string_value(lib_name)));
+  jv output = jv_invalid_with_msg(jv_string_fmt("module not found: %s", jv_string_value(rel_path)));
   jv_free(rel_path);
-  jv_free(lib_name);
-  jv_free(lib_search_paths);
+  jv_free(search);
+  jv_free(bname);
   return output;
 }
 
-static int version_matches(jq_state *jq, block importer, block module) {
-  return 1;
-}
-
-static jv default_search(jv value) {
+static jv default_search(jq_state *jq, jv value) {
   if (!jv_is_valid(value)) {
+    // dependent didn't say; prepend . to system search path listj
     jv_free(value);
-    return JV_ARRAY(jv_string("."), jv_string("$ORIGIN"));
+    return jv_array_concat(JV_ARRAY(jv_string(".")), jq_get_lib_dirs(jq));
   }
   if (jv_get_kind(value) != JV_KIND_ARRAY)
     return JV_ARRAY(value);
@@ -139,38 +189,23 @@ static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block
   int nerrors = 0;
 
   jv_array_foreach(deps, i, dep) {
-    jv name = jv_object_get(jv_copy(dep), jv_string("name"));
+    int is_data = jv_get_kind(jv_object_get(jv_copy(dep), jv_string("is_data"))) == JV_KIND_TRUE;
+    int raw = 0;
+    jv v = jv_object_get(jv_copy(dep), jv_string("raw"));
+    if (jv_get_kind(v) == JV_KIND_TRUE)
+      raw = 1;
+    jv_free(v);
+    jv relpath = validate_relpath(jv_object_get(jv_copy(dep), jv_string("relpath")));
     jv as = jv_object_get(jv_copy(dep), jv_string("as"));
-    if (!jv_is_valid(as)) {
-      jv_free(as);
-      as = jv_string("");
-    }
-    jv search = default_search(jv_object_get(dep, jv_string("search")));
-    jv_array_foreach(search, k, search_elt) {
-      if (strcmp(".",jv_string_value(search_elt)) == 0) {
-        jv tsearch = jv_copy(lib_origin);
-        jv_free(search_elt);
-        search = jv_array_set(search, k, tsearch);
-      } else if (strncmp("./",jv_string_value(search_elt),sizeof("./")-1) == 0) {
-        jv tsearch = jv_string_fmt("%s/%s",
-                                   jv_string_value(lib_origin),
-                                   jv_string_value(search_elt) + sizeof ("./") - 1);
-        jv_free(search_elt);
-        search = jv_array_set(search, k, tsearch);
-      } else if (strncmp("$ORIGIN/",jv_string_value(search_elt),sizeof("$ORIGIN/")-1) == 0) {
-        jv tsearch = jv_string_fmt("%s/%s",
-                                   jv_string_value(jq_origin),
-                                   jv_string_value(search_elt) + sizeof ("$ORIGIN/") - 1);
-        jv_free(search_elt);
-        search = jv_array_set(search, k, tsearch);
-      } else {
-        jv_free(search_elt);
-      }
-    }
-    jv lib_path = find_lib(jq, name, search);
+    assert(jv_is_valid(as) && jv_get_kind(as) == JV_KIND_STRING);
+    jv search = default_search(jq, jv_object_get(dep, jv_string("search")));
+    // dep is now freed; do not reuse
+
+    // find_lib does a lot of work that could be cached...
+    jv resolved = find_lib(jq, relpath, search, is_data ? ".json" : ".jq", jv_copy(jq_origin), jv_copy(lib_origin));
     // XXX ...move the rest of this into a callback.
-    if (!jv_is_valid(lib_path)) {
-      jv emsg = jv_invalid_get_msg(lib_path);
+    if (!jv_is_valid(resolved)) {
+      jv emsg = jv_invalid_get_msg(resolved);
       jq_report_error(jq, jv_string_fmt("jq: error: %s\n",jv_string_value(emsg)));
       jv_free(emsg);
       jv_free(as);
@@ -181,26 +216,20 @@ static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block
     }
     uint64_t state_idx = 0;
     for (; state_idx < lib_state->ct; ++state_idx) {
-      if (strcmp(lib_state->names[state_idx],jv_string_value(lib_path)) == 0)
+      if (strcmp(lib_state->names[state_idx],jv_string_value(resolved)) == 0)
         break;
     }
     if (state_idx < lib_state->ct) { // Found
-      // XXX Check version matching here!
-      if (version_matches(jq, bk, lib_state->defs[state_idx]))
-        bk = block_bind_library(lib_state->defs[state_idx], bk, OP_IS_CALL_PSEUDO, jv_string_value(as));
-      else
-        // XXX Would be nice to have the dependent's name here too
-        jq_report_error(jq, jv_string_fmt("jq: error: version mismatch for %s", jv_string_value(name)));
-      jv_free(lib_path);
+      jv_free(resolved);
+      // Bind the library to the program
+      bk = block_bind_library(lib_state->defs[state_idx], bk, OP_IS_CALL_PSEUDO, jv_string_value(as));
     } else { // Not found.   Add it to the table before binding.
       block dep_def_block = gen_noop();
-      nerrors += load_library(jq, lib_path, &dep_def_block, lib_state);
+      nerrors += load_library(jq, resolved, is_data, raw, jv_string_value(as), &dep_def_block, lib_state);
+      // resolved has been freed
       if (nerrors == 0) {
-        // XXX Check version matching here!
-        if (version_matches(jq, bk, dep_def_block))
-          bk = block_bind_library(dep_def_block, bk, OP_IS_CALL_PSEUDO, jv_string_value(as));
-        else
-          jq_report_error(jq, jv_string_fmt("jq: error: version mismatch for %s", jv_string_value(name)));
+        // Bind the library to the program
+        bk = block_bind_library(dep_def_block, bk, OP_IS_CALL_PSEUDO, jv_string_value(as));
       }
     }
     jv_free(as);
@@ -213,30 +242,48 @@ static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block
 
 // Loads the library at lib_path into lib_state, putting the library's defs
 // into *out_block
-static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_loading_state *lib_state) {
+static int load_library(jq_state *jq, jv lib_path, int is_data, int raw, const char *as, block *out_block, struct lib_loading_state *lib_state) {
   int nerrors = 0;
-  struct locfile* src;
+  struct locfile* src = NULL;
   block program;
-  jv data = jv_load_file(jv_string_value(lib_path), 1);
+  jv data;
+  if (is_data && !raw)
+    data = jv_load_file(jv_string_value(lib_path), 0);
+  else
+    data = jv_load_file(jv_string_value(lib_path), 1);
   int state_idx;
-  if (jv_is_valid(data)) {
+  if (!jv_is_valid(data)) {
+    if (jv_invalid_has_msg(jv_copy(data)))
+      data = jv_invalid_get_msg(data);
+    else
+      data = jv_string("unknown error");
+    jq_report_error(jq, jv_string_fmt("jq: error loading data file %s: %s\n", jv_string_value(lib_path), jv_string_value(data)));
+    nerrors++;
+    goto out;
+  } else if (is_data) {
+    // import "foo" as $bar;
+    program = gen_const_global(jv_copy(data), as);
+  } else {
+    // import "foo" as bar;
     src = locfile_init(jq, jv_string_value(data), jv_string_length_bytes(jv_copy(data)));
     nerrors += jq_parse_library(src, &program);
     if (nerrors == 0) {
-      state_idx = lib_state->ct++;
-      lib_state->names = realloc(lib_state->names, lib_state->ct * sizeof(const char *));
-      lib_state->defs = realloc(lib_state->defs, lib_state->ct * sizeof(block));
-      lib_state->names[state_idx] = strdup(jv_string_value(lib_path));
-      lib_state->defs[state_idx] = program;
       char *lib_origin = strdup(jv_string_value(lib_path));
       nerrors += process_dependencies(jq, jq_get_jq_origin(jq),
                                       jv_string(dirname(lib_origin)),
-                                      &lib_state->defs[state_idx], lib_state);
+                                      &program, lib_state);
       free(lib_origin);
-      *out_block = lib_state->defs[state_idx];
     }
-    locfile_free(src);
   }
+  state_idx = lib_state->ct++;
+  lib_state->names = realloc(lib_state->names, lib_state->ct * sizeof(const char *));
+  lib_state->defs = realloc(lib_state->defs, lib_state->ct * sizeof(block));
+  lib_state->names[state_idx] = strdup(jv_string_value(lib_path));
+  lib_state->defs[state_idx] = program;
+  *out_block = program;
+  if (src)
+    locfile_free(src);
+out:
   jv_free(lib_path);
   jv_free(data);
   return nerrors;
@@ -244,8 +291,11 @@ static int load_library(jq_state *jq, jv lib_path, block *out_block, struct lib_
 
 // FIXME It'd be nice to have an option to search the same search path
 // as we do in process_dependencies.
-jv load_module_meta(jq_state *jq, jv modname) {
-  jv lib_path = find_lib(jq, modname, jv_array());
+jv load_module_meta(jq_state *jq, jv mod_relpath) {
+  // We can't know the caller's origin; we could though, if it was passed in
+  jv lib_path = find_lib(jq, validate_relpath(mod_relpath), jq_get_lib_dirs(jq), ".jq", jq_get_jq_origin(jq), jv_null());
+  if (!jv_is_valid(lib_path))
+    return lib_path;
   jv meta = jv_null();
   jv data = jv_load_file(jv_string_value(lib_path), 1);
   if (jv_is_valid(data)) {
@@ -278,7 +328,7 @@ int load_program(jq_state *jq, struct locfile* src, block *out_block) {
   block libs = gen_noop();
   for (uint64_t i = 0; i < lib_state.ct; ++i) {
     free(lib_state.names[i]);
-    if (nerrors == 0)
+    if (nerrors == 0 && !block_is_const(lib_state.defs[i]))
       libs = block_join(libs, lib_state.defs[i]);
     else
       block_free(lib_state.defs[i]);

main.c

@@ -444,23 +444,11 @@ int main(int argc, char* argv[]) {
   if (options & COLOUR_OUTPUT) dumpopts |= JV_PRINT_COLOUR;
   if (options & NO_COLOUR_OUTPUT) dumpopts &= ~JV_PRINT_COLOUR;
 
-  char *penv = getenv("JQ_LIBRARY_PATH");
-  if (penv && jv_get_kind(lib_search_paths) == JV_KIND_NULL) {
-    // Use $JQ_LIBRARY_PATH
-#ifdef WIN32
-#define PATH_ENV_SEPARATOR ";"
-#else
-#define PATH_ENV_SEPARATOR ":"
-#endif
-    lib_search_paths = jv_array_concat(lib_search_paths,jv_string_split(jv_string(penv),jv_string(PATH_ENV_SEPARATOR)));
-#undef PATH_ENV_SEPARATOR
-  } else if (jv_get_kind(lib_search_paths) == JV_KIND_NULL) {
-    // Use compiled-in default JQ_LIBRARY_PATH
-#ifdef WIN32
-    lib_search_paths = JV_ARRAY(jv_string("~/.jq"), jv_string("$ORIGIN/lib"));
-#else
-    lib_search_paths = JV_ARRAY(jv_string("~/.jq"), jv_string("$ORIGIN/../lib/jq"));
-#endif
+  if (jv_get_kind(lib_search_paths) == JV_KIND_NULL) {
+    // Default search path list
+    lib_search_paths = JV_ARRAY(jv_string("~/.jq"),
+                                jv_string("$ORIGIN/../lib/jq"),
+                                jv_string("$ORIGIN/lib"));
   }
   jq_set_attr(jq, jv_string("JQ_LIBRARY_PATH"), lib_search_paths);
 

opcode_list.h

@@ -6,6 +6,7 @@ OP(POP,   NONE,     1, 0)
 OP(LOADV, VARIABLE, 1, 1)
 OP(LOADVN, VARIABLE, 1, 1)
 OP(STOREV, VARIABLE, 1, 0)
+OP(STORE_GLOBAL, GLOBAL, 0, 0)
 OP(INDEX, NONE,     2, 1)
 OP(INDEX_OPT, NONE,     2, 1)
 OP(EACH,  NONE,     1, 1)

parser.y

@@ -280,14 +280,13 @@ Module:
 %empty {
   $$ = gen_noop();
 } |
-"module" IDENT Exp ';' {
-  if (!block_is_const($3)) {
+"module" Exp ';' {
+  if (!block_is_const($2)) {
     FAIL(@$, "Module metadata must be constant.");
     $$ = gen_noop();
   } else {
-    $$ = gen_module(jv_string_value($2), $3);
+    $$ = gen_module($2);
   }
-  jv_free($2);
 }
 
 Imports:
@@ -479,33 +478,45 @@ Term {
 }
 
 Import:
-"import" IDENT ';' {
-  $$ = gen_import(jv_string_value($2), gen_noop(), NULL);
-  jv_free($2);
+"import" String "as" '$' IDENT ';' {
+  jv v = block_const($2);
+  // XXX Make gen_import take only blocks and the int is_data so we
+  // don't have to free so much stuff here
+  $$ = gen_import(jv_string_value(v), gen_noop(), jv_string_value($5), 1);
+  block_free($2);
+  jv_free($5);
+  jv_free(v);
 } |
-"import" IDENT Exp ';' {
-  if (!block_is_const($3)) {
-    FAIL(@$, "Module metadata must be constant.");
-    $$ = gen_noop();
-  } else {
-    $$ = gen_import(jv_string_value($2), $3, NULL);
-  }
-  jv_free($2);
-} |
-"import" IDENT "as" IDENT ';' {
-  $$ = gen_import(jv_string_value($2), gen_noop(), jv_string_value($4));
-  jv_free($2);
+"import" String "as" IDENT ';' {
+  jv v = block_const($2);
+  $$ = gen_import(jv_string_value(v), gen_noop(), jv_string_value($4), 0);
+  block_free($2);
   jv_free($4);
+  jv_free(v);
 } |
-"import" IDENT "as" IDENT Exp ';' {
+"import" String "as" IDENT Exp ';' {
   if (!block_is_const($5)) {
     FAIL(@$, "Module metadata must be constant.");
     $$ = gen_noop();
   } else {
-    $$ = gen_import(jv_string_value($2), $5, jv_string_value($4));
+    jv v = block_const($2);
+    $$ = gen_import(jv_string_value(v), $5, jv_string_value($4), 0);
+    jv_free(v);
   }
-  jv_free($2);
+  block_free($2);
   jv_free($4);
+} |
+"import" String "as" '$' IDENT Exp ';' {
+  if (!block_is_const($6)) {
+    FAIL(@$, "Module metadata must be constant.");
+    $$ = gen_noop();
+  } else {
+    jv v = block_const($2);
+    $$ = gen_import(jv_string_value(v), $6, jv_string_value($5), 1);
+    jv_free(v);
+  }
+  block_free($2);
+  jv_free($5);
 }
 
 FuncDef:

tests/modules/a.jq

@@ -1,2 +1,2 @@
-module a {version:1.7};
+module {version:1.7};
 def a: "a";

tests/modules/c/c.jq

@@ -1,12 +1,14 @@
-module c {whatever:null};
-import a as foo;
-import d as d {search:"./"};
-import d {search:"./"};
-import e as e {search:"./../lib/jq"};
-import f as f {search:"./../lib/jq"};
+module {whatever:null};
+import "a" as foo;
+import "d" as d {search:"./"};
+import "d" as d2{search:"./"};
+import "e" as e {search:"./../lib/jq"};
+import "f" as f {search:"./../lib/jq"};
+import "data" as $d;
 def a: 0;
 def c:
-  if meh != d::meh then error("import into namespace doesn't work")
+  if $d::d[0] != {this:"is a test",that:"is too"} then error("data import is busted")
+  elif d2::meh != d::meh then error("import twice doesn't work")
   elif foo::a != "a" then error("foo::a didn't work as expected")
   elif d::meh != "meh" then error("d::meh didn't work as expected")
   elif e::bah != "bah" then error("e::bah didn't work as expected")

tests/run

@@ -159,7 +159,7 @@ if which seq > /dev/null 2>&1; then
         fi
 
         dd "if=tests/torture/input0.json" bs=$i count=1 2>/dev/null |
-            $VALGRIND ./jq -cn --stream -L "$mods" 'import streaming; tovalues(inputs)' > $d/out1 2>$d/err || true
+            $VALGRIND ./jq -cn --stream -L "$mods" 'import "streaming" as streaming; streaming::tovalues(inputs)' > $d/out1 2>$d/err || true
         if [ -n "$VALGRIND" ]; then
             grep '^==[0-9][0-9]*== ERROR SUMMARY: 0 errors' $d/err > /dev/null
         else
@@ -200,22 +200,22 @@ if [ `HOME="$mods" $VALGRIND $Q ./jq --debug-dump-disasm -n fg | grep '^[a-z]' |
     exit 1
 fi
 
-if ! $VALGRIND $Q ./jq -ner -L "$mods" 'import a as foo; import b as bar; import a; def fooa: foo::a; [fooa, bar::a, bar::b, foo::a, a] | . == ["a","b","c","a","a"]' > /dev/null; then
+if ! $VALGRIND $Q ./jq -ner -L "$mods" 'import "a" as foo; import "b" as bar; def fooa: foo::a; [fooa, bar::a, bar::b, foo::a] | . == ["a","b","c","a"]' > /dev/null; then
     echo "Module system appears to be broken" 1>&2
     exit 1
 fi
 
-if ! $VALGRIND $Q ./jq -ner -L "$mods" 'import c as foo; [foo::a, foo::c] | . == [0,"acmehbah"]' > /dev/null; then
+if ! $VALGRIND $Q ./jq -ner -L "$mods" 'import "c" as foo; [foo::a, foo::c] | . == [0,"acmehbah"]' > /dev/null; then
     echo "Module system appears to be broken" 1>&2
     exit 1
 fi
 
-if [ "`$VALGRIND $Q ./jq -cner -L "$mods" '\"c\" | modulemeta'`" != '{"whatever":null,"name":"c","deps":[{"as":"foo","name":"a"},{"search":"./","as":"d","name":"d"},{"search":"./","name":"d"},{"search":"./../lib/jq","as":"e","name":"e"},{"search":"./../lib/jq","as":"f","name":"f"}]}' ]; then
+if [ "`$VALGRIND $Q ./jq -cner -L "$mods" '\"c\" | modulemeta'`" != '{"whatever":null,"deps":[{"as":"foo","is_data":false,"relpath":"a"},{"search":"./","as":"d","is_data":false,"relpath":"d"},{"search":"./","as":"d2","is_data":false,"relpath":"d"},{"search":"./../lib/jq","as":"e","is_data":false,"relpath":"e"},{"search":"./../lib/jq","as":"f","is_data":false,"relpath":"f"},{"as":"d","is_data":true,"relpath":"data"}]}' ]; then
     echo "modulemeta builtin appears to be broken" 1>&2
     exit 1
 fi
     
-if $VALGRIND ./jq -ner -L "$mods" 'import syntaxerror; .' > $d/out 2>&1; then
+if $VALGRIND ./jq -ner -L "$mods" 'import "syntaxerror" as e; .' > $d/out 2>&1; then
     echo "Module system appears to be broken" 1>&2
     exit 1
 fi