Refactor moar: move parts of main.c into libjq

This adds utility functions for reading and parsing files that should be
easy to use by other apps, together with jq_start()/jq_next().

jq.h

@@ -37,4 +37,16 @@ jv jq_get_prog_origin(jq_state *);
 jv jq_get_lib_dirs(jq_state *);
 void jq_set_attr(jq_state *, jv, jv);
 jv jq_get_attr(jq_state *, jv);
+
+typedef struct jq_util_input_state *jq_util_input_state;
+
+jq_util_input_state jq_util_input_init(jq_err_cb, void *);
+void jq_util_input_set_parser(jq_util_input_state, jv_parser *, int);
+void jq_util_input_free(jq_util_input_state *);
+void jq_util_input_add_input(jq_util_input_state, jv);
+int jq_util_input_open_errors(jq_util_input_state);
+int jq_util_input_read_more(jq_util_input_state);
+jv jq_util_input_next_input(jq_util_input_state);
+jv jq_util_input_next_input_cb(jq_state *, void *);
+
 #endif /* !_JQ_H_ */

jv.h

@@ -184,12 +184,12 @@ void jv_nomem_handler(jv_nomem_handler_f, void *);
 
 jv jv_load_file(const char *, int);
 
-struct jv_parser;
+typedef struct jv_parser jv_parser;
-struct jv_parser* jv_parser_new(int);
+jv_parser* jv_parser_new(int);
-void jv_parser_set_buf(struct jv_parser*, const char*, int, int);
+void jv_parser_set_buf(jv_parser*, const char*, int, int);
-int jv_parser_remaining(struct jv_parser*);
+int jv_parser_remaining(jv_parser*);
-jv jv_parser_next(struct jv_parser*);
+jv jv_parser_next(jv_parser*);
-void jv_parser_free(struct jv_parser*);
+void jv_parser_free(jv_parser*);
 
 jv jv_get(jv, jv);
 jv jv_set(jv, jv, jv);

main.c

@@ -140,137 +140,6 @@ static int process(jq_state *jq, jv value, int flags, int dumpopts) {
   return ret;
 }
 
-// XXX Move this and related functions into libjq (with a better name), into util.[ch] say
-struct next_input_state {
-  FILE* current_input;
-  const char** input_filenames;
-  int alloced;
-  int ninput_files;
-  int next_input_idx;
-  int open_failures;
-  struct jv_parser *parser;
-  jv slurped;
-  char buf[4096];
-};
-typedef struct next_input_state *next_input_state;
-
-static void input_state_free(next_input_state *state) {
-  next_input_state old_state = *state;
-  *state = NULL;
-  if (old_state == NULL)
-    return;
-  if (old_state->parser != NULL)
-    jv_parser_free(old_state->parser);
-  jv_mem_free(old_state->input_filenames);
-  jv_free(old_state->slurped);
-  jv_mem_free(old_state);
-}
-
-static int input_state_init(next_input_state *state, int max_inputs) {
-  next_input_state new_state = jv_mem_alloc(sizeof(*new_state));
-  new_state->next_input_idx = 0;
-  new_state->open_failures = 0;
-  new_state->ninput_files = 0;
-  new_state->current_input = NULL;
-  new_state->parser = NULL;         // initialized when we know the flags
-  new_state->slurped = jv_invalid();
-  new_state->buf[0] = 0;
-
-  // XXX a jv_mem_calloc() would be nice
-  assert(max_inputs > 0 && (uintmax_t)max_inputs * sizeof(const char*) < SIZE_MAX);
-  new_state->input_filenames = jv_mem_alloc(sizeof(const char*) * max_inputs);
-  new_state->alloced = max_inputs;
-  for (; max_inputs > 0; max_inputs--) 
-    new_state->input_filenames[max_inputs - 1] = NULL;
-  *state = new_state;
-  return 0;
-}
-
-static void input_state_add_input(next_input_state state, const char *input) {
-  assert(state->ninput_files < state->alloced);
-  state->input_filenames[state->ninput_files++] = input;
-}
-
-static int input_state_read_more(next_input_state state) {
-  if (!state->current_input || feof(state->current_input)) {
-    if (state->current_input) {
-      if (state->current_input == stdin) {
-        clearerr(stdin); // perhaps we can read again; anyways, we don't fclose(stdin)
-      } else {
-        fclose(state->current_input);
-      }
-      state->current_input = NULL;
-    }
-    if (state->next_input_idx < state->ninput_files) {
-      if (!strcmp(state->input_filenames[state->next_input_idx], "-")) {
-        state->current_input = stdin;
-      } else {
-        state->current_input = fopen(state->input_filenames[state->next_input_idx], "r");
-        if (!state->current_input) {
-          fprintf(stderr, "%s: Error: could not open %s: %s\n", progname, state->input_filenames[state->next_input_idx], strerror(errno));
-          state->open_failures++;
-        }
-      }
-      state->next_input_idx++;
-    }
-  }
-
-  state->buf[0] = 0;
-  if (state->current_input) {
-    if (!fgets(state->buf, sizeof(state->buf), state->current_input))
-      state->buf[0] = 0;
-  }
-  return state->next_input_idx == state->ninput_files && (!state->current_input || feof(state->current_input));
-}
-
-// Blocks to read one more input from stdin and/or given files
-// When slurping, it returns just one value
-static jv input_state_next_input(jq_state *jq, void *data) {
-  next_input_state state = data;
-  int is_last = 0;
-  jv value = jv_invalid(); // need more input
-  do {
-    if (options & RAW_INPUT) {
-      is_last = input_state_read_more(state);
-      if (state->buf[0] == '\0')
-        continue;
-      int len = strlen(state->buf); // Raw input doesn't support NULs
-      if (len > 0) {
-        if (options & SLURP) {
-          state->slurped = jv_string_concat(state->slurped, jv_string(state->buf));
-        } else {
-          if (!jv_is_valid(value))
-            value = jv_string("");
-          if (state->buf[len-1] == '\n') {
-            // whole line
-            state->buf[len-1] = 0;
-            return jv_string_concat(value, jv_string(state->buf));
-          }
-          value = jv_string_concat(value, jv_string(state->buf));
-          state->buf[0] = '\0';
-        }
-      }
-    } else {
-      if (jv_parser_remaining(state->parser) == 0) {
-        is_last = input_state_read_more(state);
-        jv_parser_set_buf(state->parser, state->buf, strlen(state->buf), !is_last); // NULs also not supported here
-      }
-      value = jv_parser_next(state->parser);
-      if (options & SLURP) {
-        if (jv_is_valid(value)) {
-          state->slurped = jv_array_append(state->slurped, value);
-          value = jv_invalid();
-        } else if (jv_invalid_has_msg(jv_copy(value)))
-          return value;
-      } else if (jv_is_valid(value) || jv_invalid_has_msg(jv_copy(value))) {
-        return value;
-      }
-    }
-  } while (!is_last);
-  return value;
-}
-// XXX End of stuff to move into utils
-
 static void debug_cb(jq_state *jq, void *data, jv input) {
   int dumpopts = *(int *)data;
   jv_dumpf(JV_ARRAY(jv_string("DEBUG:"), input), stderr, dumpopts & ~(JV_PRINT_PRETTY));
@@ -295,23 +164,25 @@ int main(int argc, char* argv[]) {
 
   const char* program = 0;
 
-  next_input_state input_state;
+  jq_util_input_state input_state = jq_util_input_init(NULL, NULL); // XXX add err_cb
-  input_state_init(&input_state, argc);
 
   int further_args_are_files = 0;
   int jq_flags = 0;
   size_t short_opts = 0;
   jv program_arguments = jv_array();
   jv lib_search_paths = jv_null();
+  char *first_file = 0;
   for (int i=1; i<argc; i++, short_opts = 0) {
     if (further_args_are_files) {
-      input_state_add_input(input_state, argv[i]);
+      jq_util_input_add_input(input_state, jv_string(argv[i]));
+      first_file = first_file ? first_file : argv[i];
     } else if (!strcmp(argv[i], "--")) {
       if (!program) usage(2);
       further_args_are_files = 1;
     } else if (!isoptish(argv[i])) {
       if (program) {
-        input_state_add_input(input_state, argv[i]);
+        jq_util_input_add_input(input_state, jv_string(argv[i]));
+        first_file = first_file ? first_file : argv[i];
       } else {
         program = argv[i];
       }
@@ -534,11 +405,11 @@ int main(int argc, char* argv[]) {
 
   if (!program) usage(2);
   if ((options & IN_PLACE)) {
-    if (input_state->ninput_files == 0) usage(2);
+    if (first_file == 0) usage(2);
-    if (strcmp(input_state->input_filenames[0], "-") == 0) usage(2);
+    if (strcmp(first_file, "-") == 0) usage(2);
-    size_t tlen = strlen(input_state->input_filenames[0]) + 7;
+    size_t tlen = strlen(first_file) + 7;
     t = jv_mem_alloc(tlen);
-    int n = snprintf(t, tlen,"%sXXXXXX", input_state->input_filenames[0]);
+    int n = snprintf(t, tlen,"%sXXXXXX", first_file);
     assert(n > 0 && (size_t)n < tlen);
     if (mkstemp(t) == -1) {
       fprintf(stderr, "Error: %s creating temporary file", strerror(errno));
@@ -591,26 +462,23 @@ int main(int argc, char* argv[]) {
   if ((options & SEQ))
     parser_flags |= JV_PARSE_SEQ;
 
-  if (input_state->ninput_files == 0) input_state->current_input = stdin;
+  if (!(options & RAW_INPUT))
-  input_state->parser = jv_parser_new(parser_flags);
+    jq_util_input_set_parser(input_state, jv_parser_new(parser_flags), (options & SLURP) ? 1 : 0);
-  if ((options & RAW_INPUT) && (options & SLURP))
-    input_state->slurped = jv_string("");
-  else if ((options & SLURP))
-    input_state->slurped = jv_array();
-  else
-    input_state->slurped = jv_invalid();
 
   // Let jq program read from inputs
-  jq_set_input_cb(jq, input_state_next_input, input_state);
+  jq_set_input_cb(jq, jq_util_input_next_input_cb, input_state);
 
+  // Let jq program call `debug` builtin and have that go somewhere
   jq_set_debug_cb(jq, debug_cb, &dumpopts);
 
+  if (first_file == 0)
+    jq_util_input_add_input(input_state, jv_string("-"));
   if (options & PROVIDE_NULL) {
     ret = process(jq, jv_null(), jq_flags, dumpopts);
   } else {
     jv value;
-    while (input_state->open_failures == 0 &&
+    while (jq_util_input_open_errors(input_state) == 0 &&
-           (jv_is_valid((value = input_state_next_input(jq, input_state))) || jv_invalid_has_msg(jv_copy(value)))) {
+           (jv_is_valid((value = jq_util_input_next_input(input_state))) || jv_invalid_has_msg(jv_copy(value)))) {
       if (jv_is_valid(value)) {
         ret = process(jq, value, jq_flags, dumpopts);
         continue;
@@ -628,13 +496,9 @@ int main(int argc, char* argv[]) {
       fprintf(stderr, "ignoring parse error: %s\n", jv_string_value(msg));
       jv_free(msg);
     }
-    if (options & SLURP) {
-      ret = process(jq, input_state->slurped, jq_flags, dumpopts);
-      input_state->slurped = jv_invalid();
-    }
   }
 
-  if (ret == 0 && input_state->open_failures != 0)
+  if (jq_util_input_open_errors(input_state) != 0 && ret == 0 && (options & EXIT_STATUS))
     ret = 2;
 
   if (ret != 0)
@@ -651,15 +515,15 @@ int main(int argc, char* argv[]) {
       fprintf(stderr, "Error: %s opening /dev/null\n", strerror(errno));
       exit(3);
     }
-    assert(input_state->ninput_files > 0 && !strcmp(input_state->input_filenames[0], "-"));
+    assert(first_file != 0 && !strcmp(first_file, "-"));
-    if (rename(t, input_state->input_filenames[0]) == -1) {
+    if (rename(t, first_file) == -1) {
       fprintf(stderr, "Error: %s renaming temporary file\n", strerror(errno));
       exit(3);
     }
     jv_mem_free(t);
   }
 out:
-  input_state_free(&input_state);
+  jq_util_input_free(&input_state);
   jq_teardown(&jq);
   if (ret >= 10 && (options & EXIT_STATUS))
     return ret - 10;

util.c

@@ -6,6 +6,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <assert.h>
+#include <errno.h>
 #include <fcntl.h>
 #include <limits.h>
 #include <string.h>
@@ -17,7 +18,8 @@
 
 
 #include "util.h"
-#include "jv.h"
+#include "jq.h"
+#include "jv_alloc.h"
 
 #ifndef HAVE_MKSTEMP
 int mkstemp(char *template) {
@@ -143,3 +145,167 @@ not_this:
 #endif /* !HAVE_MEMMEM */
 }
 
+struct jq_util_input_state {
+  jq_err_cb err_cb;
+  void *err_cb_data;
+  jv_parser *parser;
+  FILE* current_input;
+  jv files;
+  int open_failures;
+  jv slurped;
+  char buf[4096];
+};
+
+static void fprinter(void *data, jv fname) {
+  fprintf((FILE *)data, "jq: error: Could not open file %s: %s\n", jv_string_value(fname), strerror(errno));
+  jv_free(fname);
+}
+
+// If parser == NULL -> RAW
+jq_util_input_state jq_util_input_init(jq_err_cb err_cb, void *err_cb_data) {
+  if (err_cb == NULL) {
+    err_cb = fprinter;
+    err_cb_data = stderr;
+  }
+  jq_util_input_state new_state = jv_mem_alloc(sizeof(*new_state));
+  memset(new_state, 0, sizeof(*new_state));
+  new_state->err_cb = err_cb;
+  new_state->err_cb_data = err_cb_data;
+  new_state->parser = NULL;
+  new_state->current_input = NULL;
+  new_state->files = jv_array();
+  new_state->slurped = jv_invalid();
+  new_state->buf[0] = 0;
+
+  return new_state;
+}
+
+void jq_util_input_set_parser(jq_util_input_state state, jv_parser *parser, int slurp) {
+  assert(!jv_is_valid(state->slurped));
+  state->parser = parser;
+
+  if (parser == NULL && slurp)
+    state->slurped = jv_string("");
+  else if (slurp)
+    state->slurped = jv_array();
+  else
+    state->slurped = jv_invalid();
+}
+
+void jq_util_input_free(jq_util_input_state *state) {
+  jq_util_input_state old_state = *state;
+  *state = NULL;
+  if (old_state == NULL)
+    return;
+
+  if (old_state->parser != NULL)
+    jv_parser_free(old_state->parser);
+  jv_free(old_state->files);
+  jv_free(old_state->slurped);
+  jv_mem_free(old_state);
+}
+
+void jq_util_input_add_input(jq_util_input_state state, jv fname) {
+  state->files = jv_array_append(state->files, fname);
+}
+
+int jq_util_input_open_errors(jq_util_input_state state) {
+  return state->open_failures;
+}
+
+static jv next_file(jq_util_input_state state) {
+  jv next = jv_array_get(jv_copy(state->files), 0);
+  if (jv_array_length(jv_copy(state->files)) > 0)
+    state->files = jv_array_slice(state->files, 1, jv_array_length(jv_copy(state->files)));
+  return next;
+}
+
+int jq_util_input_read_more(jq_util_input_state state) {
+  if (!state->current_input || feof(state->current_input)) {
+    if (state->current_input) {
+      if (state->current_input == stdin) {
+        clearerr(stdin); // perhaps we can read again; anyways, we don't fclose(stdin)
+      } else {
+        fclose(state->current_input);
+      }
+      state->current_input = NULL;
+    }
+    jv f = next_file(state);
+    if (jv_is_valid(f)) {
+      if (!strcmp(jv_string_value(f), "-")) {
+        state->current_input = stdin;
+      } else {
+        state->current_input = fopen(jv_string_value(f), "r");
+        if (!state->current_input) {
+          state->err_cb(state->err_cb_data, jv_copy(f));
+          state->open_failures++;
+        }
+      }
+      jv_free(f);
+    }
+  }
+
+  state->buf[0] = 0;
+  if (state->current_input) {
+    if (!fgets(state->buf, sizeof(state->buf), state->current_input))
+      state->buf[0] = 0;
+  }
+  return jv_array_length(jv_copy(state->files)) == 0 && (!state->current_input || feof(state->current_input));
+}
+
+jv jq_util_input_next_input_cb(jq_state *jq, void *data) {
+  return jq_util_input_next_input((jq_util_input_state)data);
+}
+
+// Blocks to read one more input from stdin and/or given files
+// When slurping, it returns just one value
+jv jq_util_input_next_input(jq_util_input_state state) {
+  int is_last = 0;
+  jv value = jv_invalid(); // need more input
+  do {
+    if (state->parser == NULL) {
+      // Raw input
+      is_last = jq_util_input_read_more(state);
+      if (state->buf[0] == '\0')
+        continue;
+      int len = strlen(state->buf); // Raw input doesn't support NULs
+      if (len > 0) {
+        if (jv_is_valid(state->slurped)) {
+          // Slurped raw input
+          state->slurped = jv_string_concat(state->slurped, jv_string(state->buf));
+        } else {
+          if (!jv_is_valid(value))
+            value = jv_string("");
+          if (state->buf[len-1] == '\n') {
+            // whole line
+            state->buf[len-1] = 0;
+            return jv_string_concat(value, jv_string(state->buf));
+          }
+          value = jv_string_concat(value, jv_string(state->buf));
+          state->buf[0] = '\0';
+        }
+      }
+    } else {
+      if (jv_parser_remaining(state->parser) == 0) {
+        is_last = jq_util_input_read_more(state);
+        jv_parser_set_buf(state->parser, state->buf, strlen(state->buf), !is_last);
+      }
+      value = jv_parser_next(state->parser);
+      if (jv_is_valid(state->slurped)) {
+        if (jv_is_valid(value)) {
+          state->slurped = jv_array_append(state->slurped, value);
+          value = jv_invalid();
+        } else if (jv_invalid_has_msg(jv_copy(value)))
+          return value; // Not slurped parsed input
+      } else if (jv_is_valid(value) || jv_invalid_has_msg(jv_copy(value))) {
+        return value;
+      }
+    }
+  } while (!is_last);
+
+  if (jv_is_valid(state->slurped)) {
+    value = state->slurped;
+    state->slurped = jv_invalid();
+  }
+  return value;
+}