From 1fc56567621bd5ff8e3d5004b12558a285974c98 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Thu, 28 Nov 2013 13:04:29 -0600 Subject: [PATCH 01/32] Support more arguments for defs --- parser.y | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/parser.y b/parser.y index b068b430..6794116b 100644 --- a/parser.y +++ b/parser.y @@ -485,6 +485,26 @@ IDENT '(' Exp ';' Exp ')' { $$ = gen_location(@1, $$); jv_free($1); } | +IDENT '(' Exp ';' Exp ';' Exp ')' { + $$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5), gen_lambda($7))); + $$ = gen_location(@1, $$); + jv_free($1); +} | +IDENT '(' Exp ';' Exp ';' Exp ';' Exp ')' { + $$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5), gen_lambda($7), gen_lambda($9))); + $$ = gen_location(@1, $$); + jv_free($1); +} | +IDENT '(' Exp ';' Exp ';' Exp ';' Exp ';' Exp ')' { + $$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5), gen_lambda($7), gen_lambda($9), gen_lambda($11))); + $$ = gen_location(@1, $$); + jv_free($1); +} | +IDENT '(' Exp ';' Exp ';' Exp ';' Exp ';' Exp ';' Exp ')' { + $$ = gen_call(jv_string_value($1), BLOCK(gen_lambda($3), gen_lambda($5), gen_lambda($7), gen_lambda($9), gen_lambda($11), gen_lambda($13))); + $$ = gen_location(@1, $$); + jv_free($1); +} | '(' error ')' { $$ = gen_noop(); } | '[' error ']' { $$ = gen_noop(); } | Term '[' error ']' { $$ = $1; } | From 08b4a34a8ece5917baf25d2b594f03d43aa1c437 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Thu, 28 Nov 2013 14:48:46 -0600 Subject: [PATCH 02/32] Add jv string utility functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit jv_string_empty() -> return an empty string with given allocated length (for fast appends) jv_string_append_codepoint -> append a single codepoint (int) to the given string jv_string_explode -> return an array of codepoints making up a string jv_string_implode -> return the UTF-8 encoding of an array of codepoint numbers --- jv.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ jv.h | 4 ++++ 2 files changed, 57 insertions(+) diff --git a/jv.c b/jv.c index 71824456..92c48604 100644 --- a/jv.c +++ b/jv.c @@ -417,6 +417,14 @@ static jv_nontrivial jvp_string_new(const char* data, uint32_t length) { return r; } +static jv_nontrivial jvp_string_empty_new(uint32_t length) { + jvp_string* s = jvp_string_alloc(length); + s->length_hashed = 0; + memset(s->data, 0, length); + jv_nontrivial r = {&s->refcnt, {0,0}}; + return r; +} + static void jvp_string_free(jv_nontrivial* s) { if (jvp_refcnt_dec(s)) { jvp_string* str = jvp_string_ptr(s); @@ -560,6 +568,13 @@ jv jv_string_sized(const char* str, int len) { return j; } +jv jv_string_empty(int len) { + jv j; + j.kind = JV_KIND_STRING; + j.val.nontrivial = jvp_string_empty_new(len); + return j; +} + jv jv_string(const char* str) { return jv_string_sized(str, strlen(str)); } @@ -581,6 +596,37 @@ int jv_string_length_codepoints(jv j) { return len; } +jv jv_string_explode(jv j) { + assert(jv_get_kind(j) == JV_KIND_STRING); + const char* i = jv_string_value(j); + int len = jv_string_length_bytes(jv_copy(j)); + const char* end = i + len; + jv a = jv_array_sized(len); + int c; + while ((i = jvp_utf8_next(i, end, &c))) + a = jv_array_append(a, jv_number(c)); + jv_free(j); + return a; +} + +jv jv_string_implode(jv j) { + assert(jv_get_kind(j) == JV_KIND_ARRAY); + int len = jv_array_length(jv_copy(j)); + jv s = jv_string_empty(len); + int i; + + assert(len >= 0); + + for (i = 0; i < len; i++) { + jv n = jv_array_get(j, i); + assert(jv_get_kind(n) == JV_KIND_NUMBER); + s = jv_string_append_codepoint(s, jv_number_value(n)); + } + + jv_free(j); + return s; +} + unsigned long jv_string_hash(jv j) { assert(jv_get_kind(j) == JV_KIND_STRING); uint32_t hash = jvp_string_hash(jvp_string_ptr(&j.val.nontrivial)); @@ -612,6 +658,13 @@ jv jv_string_append_buf(jv a, const char* buf, int len) { return a; } +jv jv_string_append_codepoint(jv a, int c) { + char buf[5]; + int len = jvp_utf8_encode(c, buf); + jvp_string_append(&a.val.nontrivial, buf, len); + return a; +} + jv jv_string_append_str(jv a, const char* str) { return jv_string_append_buf(a, str, strlen(str)); } diff --git a/jv.h b/jv.h index 0f50359a..50efbe60 100644 --- a/jv.h +++ b/jv.h @@ -75,14 +75,18 @@ jv jv_array_slice(jv, int, int); jv jv_string(const char*); jv jv_string_sized(const char*, int); +jv jv_string_empty(int len); int jv_string_length_bytes(jv); int jv_string_length_codepoints(jv); unsigned long jv_string_hash(jv); const char* jv_string_value(jv); jv jv_string_concat(jv, jv); jv jv_string_fmt(const char*, ...); +jv jv_string_append_codepoint(jv a, int c); jv jv_string_append_buf(jv a, const char* buf, int len); jv jv_string_append_str(jv a, const char* str); +jv jv_string_explode(jv j); +jv jv_string_implode(jv j); jv jv_object(); jv jv_object_get(jv object, jv key); From ef41d1cb57b521334d3d8f771c9417881d6e0c94 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Thu, 28 Nov 2013 14:53:05 -0600 Subject: [PATCH 03/32] Use uint32_t for codepoint in jv_string_append_codepoint() --- jv.c | 2 +- jv.h | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/jv.c b/jv.c index 92c48604..a1e98026 100644 --- a/jv.c +++ b/jv.c @@ -658,7 +658,7 @@ jv jv_string_append_buf(jv a, const char* buf, int len) { return a; } -jv jv_string_append_codepoint(jv a, int c) { +jv jv_string_append_codepoint(jv a, uint32_t c) { char buf[5]; int len = jvp_utf8_encode(c, buf); jvp_string_append(&a.val.nontrivial, buf, len); diff --git a/jv.h b/jv.h index 50efbe60..a0bc3d4b 100644 --- a/jv.h +++ b/jv.h @@ -1,6 +1,8 @@ #ifndef JV_H #define JV_H +#include + typedef enum { JV_KIND_INVALID, JV_KIND_NULL, @@ -82,7 +84,7 @@ unsigned long jv_string_hash(jv); const char* jv_string_value(jv); jv jv_string_concat(jv, jv); jv jv_string_fmt(const char*, ...); -jv jv_string_append_codepoint(jv a, int c); +jv jv_string_append_codepoint(jv a, uint32_t c); jv jv_string_append_buf(jv a, const char* buf, int len); jv jv_string_append_str(jv a, const char* str); jv jv_string_explode(jv j); From 6c734cb6098221c6994a854b46c978c8ba526bac Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Thu, 28 Nov 2013 14:53:50 -0600 Subject: [PATCH 04/32] Add explode/implode jq functions to match jv API --- builtin.c | 2 ++ jv.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/builtin.c b/builtin.c index 721cd3fc..371df372 100644 --- a/builtin.c +++ b/builtin.c @@ -493,6 +493,8 @@ static const struct cfunction function_list[] = { {(cfunction_ptr)f_tonumber, "tonumber", 1}, {(cfunction_ptr)f_tostring, "tostring", 1}, {(cfunction_ptr)f_keys, "keys", 1}, + {(cfunction_ptr)jv_string_explode, "explode", 1}, + {(cfunction_ptr)jv_string_implode, "implode", 1}, {(cfunction_ptr)jv_setpath, "setpath", 3}, // FIXME typechecking {(cfunction_ptr)jv_getpath, "getpath", 2}, {(cfunction_ptr)jv_delpaths, "delpaths", 2}, diff --git a/jv.c b/jv.c index a1e98026..d9a908ce 100644 --- a/jv.c +++ b/jv.c @@ -618,7 +618,7 @@ jv jv_string_implode(jv j) { assert(len >= 0); for (i = 0; i < len; i++) { - jv n = jv_array_get(j, i); + jv n = jv_array_get(jv_copy(j), i); assert(jv_get_kind(n) == JV_KIND_NUMBER); s = jv_string_append_codepoint(s, jv_number_value(n)); } From db19a11399f085043ead11b6860089a807fd9ec1 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Thu, 28 Nov 2013 15:17:32 -0600 Subject: [PATCH 05/32] Add startswith/endswith --- builtin.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/builtin.c b/builtin.c index 371df372..0d920abd 100644 --- a/builtin.c +++ b/builtin.c @@ -77,6 +77,37 @@ static jv f_negate(jv input) { return ret; } +static jv f_startswith(jv a, jv b) { + int alen = jv_string_length_bytes(jv_copy(a)); + int blen = jv_string_length_bytes(jv_copy(b)); + jv ret; + + if (blen <= alen && memcmp(jv_string_value(a), jv_string_value(b), blen) == 0) + ret = jv_true(); + else + ret = jv_false(); + jv_free(a); + jv_free(b); + return ret; +} + +static jv f_endswith(jv a, jv b) { + const char *astr = jv_string_value(a); + const char *bstr = jv_string_value(b); + size_t alen = jv_string_length_bytes(jv_copy(a)); + size_t blen = jv_string_length_bytes(jv_copy(b)); + jv ret;; + + if (alen < blen || + memcmp(astr + (alen - blen), bstr, blen) != 0) + ret = jv_false(); + else + ret = jv_true(); + jv_free(a); + jv_free(b); + return ret; +} + static jv f_minus(jv input, jv a, jv b) { jv_free(input); if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) { @@ -493,6 +524,8 @@ static const struct cfunction function_list[] = { {(cfunction_ptr)f_tonumber, "tonumber", 1}, {(cfunction_ptr)f_tostring, "tostring", 1}, {(cfunction_ptr)f_keys, "keys", 1}, + {(cfunction_ptr)f_startswith, "startswith", 2}, + {(cfunction_ptr)f_endswith, "endswith", 2}, {(cfunction_ptr)jv_string_explode, "explode", 1}, {(cfunction_ptr)jv_string_implode, "implode", 1}, {(cfunction_ptr)jv_setpath, "setpath", 3}, // FIXME typechecking From 5989dbdfcfb42aeb6abcb19289f797d9e260f145 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Fri, 29 Nov 2013 15:49:11 -0600 Subject: [PATCH 06/32] Add string multiplication by number --- builtin.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/builtin.c b/builtin.c index 0d920abd..1e38482d 100644 --- a/builtin.c +++ b/builtin.c @@ -138,6 +138,19 @@ static jv f_multiply(jv input, jv a, jv b) { jv_free(input); if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) { return jv_number(jv_number_value(a) * jv_number_value(b)); + } else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_NUMBER) { + int n; + size_t alen = jv_string_length_bytes(jv_copy(a)); + jv res = a; + + for (n = jv_number_value(b) - 1; n > 0; n--) + res = jv_string_append_buf(res, jv_string_value(a), alen); + + if (n < 0) { + jv_free(a); + return jv_null(); + } + return res; } else { return type_error2(a, b, "cannot be multiplied"); } From 63bed9bdf1919ed9da8a6b46e3fbc25ee4e071be Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Fri, 29 Nov 2013 15:51:51 -0600 Subject: [PATCH 07/32] Test starts/endswith and string multiplication --- tests/all.test | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/all.test b/tests/all.test index f8019972..d6231ee8 100644 --- a/tests/all.test +++ b/tests/all.test @@ -574,6 +574,19 @@ def inc(x): x |= .+1; inc(.[].a) {} [true, true, false] +# string operations +[.[]|startswith("foo")] +["fo", "foo", "barfoo", "foobar", "barfoob"] +[false, true, false, true, false] + +[.[]|endswith("foo")] +["fo", "foo", "barfoo", "foobar", "barfoob"] +[false, true, true, false, false] + +[.[] * 3] +["a", "ab", "abc"] +["aaa", "ababab", "abcabcabc"] + map(.[1] as $needle | .[0] | contains($needle)) [[[],[]], [[1,2,3], [1,2]], [[1,2,3], [3,1]], [[1,2,3], [4]], [[1,2,3], [1,4]]] [true, true, true, false, false] From cf562961b7fa35e13fbe8a4ac9a6d7836a0ecde9 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Thu, 28 Nov 2013 22:27:23 -0600 Subject: [PATCH 08/32] Add string division by string (split on separator) --- builtin.c | 3 +++ jv.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ jv.h | 1 + 3 files changed, 52 insertions(+) diff --git a/builtin.c b/builtin.c index 1e38482d..07682266 100644 --- a/builtin.c +++ b/builtin.c @@ -160,6 +160,8 @@ static jv f_divide(jv input, jv a, jv b) { jv_free(input); if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) { return jv_number(jv_number_value(a) / jv_number_value(b)); + } else if (jv_get_kind(a) == JV_KIND_STRING && jv_get_kind(b) == JV_KIND_STRING) { + return jv_string_split(a, b); } else { return type_error2(a, b, "cannot be divided"); } @@ -539,6 +541,7 @@ static const struct cfunction function_list[] = { {(cfunction_ptr)f_keys, "keys", 1}, {(cfunction_ptr)f_startswith, "startswith", 2}, {(cfunction_ptr)f_endswith, "endswith", 2}, + {(cfunction_ptr)jv_string_split, "split", 2}, {(cfunction_ptr)jv_string_explode, "explode", 1}, {(cfunction_ptr)jv_string_implode, "implode", 1}, {(cfunction_ptr)jv_setpath, "setpath", 3}, // FIXME typechecking diff --git a/jv.c b/jv.c index d9a908ce..671f8d2f 100644 --- a/jv.c +++ b/jv.c @@ -596,6 +596,54 @@ int jv_string_length_codepoints(jv j) { return len; } +#ifndef HAVE_MEMMEM +static const void *memmem(const void *haystack, size_t haystacklen, + const void *needle, size_t needlelen) +{ + const char *h = haystack; + const char *n = needle; + size_t hi, hi2, ni; + + if (haystacklen < needlelen || haystacklen == 0) + return NULL; + for (hi = 0; hi < (haystacklen - needlelen + 1); hi++) { + for (ni = 0, hi2 = hi; ni < needlelen; ni++, hi2++) { + if (h[hi2] != n[ni]) + goto not_this; + } + + return &h[hi]; + +not_this: + continue; + } + return NULL; +} +#endif /* HAVE_MEMMEM */ + +jv jv_string_split(jv j, jv sep) { + assert(jv_get_kind(j) == JV_KIND_STRING); + assert(jv_get_kind(sep) == JV_KIND_STRING); + const char *jstr = jv_string_value(j); + const char *sepstr = jv_string_value(sep); + const char *p, *s; + int jlen = jv_string_length_bytes(jv_copy(j)); + int seplen = jv_string_length_bytes(jv_copy(sep)); + jv a = jv_array(); + + assert(jv_get_refcnt(a) == 1); + + for (p = jstr; p < jstr + jlen; p = s + seplen) { + s = memmem(p, (jstr + jlen) - p, sepstr, seplen); + if (s == NULL) + s = jstr + jlen; + a = jv_array_append(a, jv_string_sized(p, s - p)); + } + jv_free(j); + jv_free(sep); + return a; +} + jv jv_string_explode(jv j) { assert(jv_get_kind(j) == JV_KIND_STRING); const char* i = jv_string_value(j); diff --git a/jv.h b/jv.h index a0bc3d4b..921345c4 100644 --- a/jv.h +++ b/jv.h @@ -87,6 +87,7 @@ jv jv_string_fmt(const char*, ...); jv jv_string_append_codepoint(jv a, uint32_t c); jv jv_string_append_buf(jv a, const char* buf, int len); jv jv_string_append_str(jv a, const char* str); +jv jv_string_split(jv j, jv sep); jv jv_string_explode(jv j); jv jv_string_implode(jv j); From 5aadaa79ebd43bfba8d28d1e1487393085b64245 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Thu, 28 Nov 2013 22:36:14 -0600 Subject: [PATCH 09/32] Add tests for string division/splitting --- tests/all.test | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/all.test b/tests/all.test index d6231ee8..72115818 100644 --- a/tests/all.test +++ b/tests/all.test @@ -583,10 +583,26 @@ def inc(x): x |= .+1; inc(.[].a) ["fo", "foo", "barfoo", "foobar", "barfoob"] [false, true, true, false, false] +[.[]|split(",")] +["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"] +[["a"," bc"," def"," ghij"," jklmn"," a","b"," c","d"," e","f"],["a","b","c","d"," e","f","g","h"]] + +[.[]|split(", ")] +["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"] +[["a","bc","def","ghij","jklmn","a,b","c,d","e,f"],["a,b,c,d","e,f,g,h"]] + [.[] * 3] ["a", "ab", "abc"] ["aaa", "ababab", "abcabcabc"] +[.[] / ","] +["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"] +[["a"," bc"," def"," ghij"," jklmn"," a","b"," c","d"," e","f"],["a","b","c","d"," e","f","g","h"]] + +[.[] / ", "] +["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"] +[["a","bc","def","ghij","jklmn","a,b","c,d","e,f"],["a,b,c,d","e,f,g,h"]] + map(.[1] as $needle | .[0] | contains($needle)) [[[],[]], [[1,2,3], [1,2]], [[1,2,3], [3,1]], [[1,2,3], [4]], [[1,2,3], [1,4]]] [true, true, true, false, false] From 884e6c7d8bc2595e9baade62bc1cfbc77a8a9dd3 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Fri, 29 Nov 2013 12:50:02 -0600 Subject: [PATCH 10/32] Add string slicing --- jv.c | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ jv.h | 1 + jv_aux.c | 16 ++++++++++++++-- 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/jv.c b/jv.c index 671f8d2f..999c776b 100644 --- a/jv.c +++ b/jv.c @@ -687,6 +687,63 @@ const char* jv_string_value(jv j) { return jvp_string_ptr(&j.val.nontrivial)->data; } +jv jv_string_slice(jv j, int start, int end) { + assert(jv_get_kind(j) == JV_KIND_STRING); + const char *s = jv_string_value(j); + int len = jv_string_length_bytes(jv_copy(j)); + int i; + const char *p, *e; + int c; + jv res; + + if (start < 0) start = len + start; + if (end < 0) end = len + end; + + if (start < 0) start = 0; + if (start > len) start = len; + if (end > len) end = len; + if (end < start) end = start; + if (start < 0 || start > end || end > len) + return jv_invalid_with_msg(jv_string("Invalid string slice indices")); + assert(0 <= start && start <= end && end <= len); + + /* Look for byte offset corresponding to start codepoints */ + for (p = s, i = 0; i < start; i++) { + p = jvp_utf8_next(p, s + len, &c); + if (p == NULL) { + jv_free(j); + return jv_string_empty(16); + } + if (c == -1) { + jv_free(j); + return jv_invalid_with_msg(jv_string("Invalid UTF-8 string")); + } + } + /* Look for byte offset corresponding to end codepoints */ + for (e = p; e != NULL && i < end; i++) { + e = jvp_utf8_next(e, s + len, &c); + if (e == NULL) { + e = s + len; + break; + } + if (c == -1) { + jv_free(j); + return jv_invalid_with_msg(jv_string("Invalid UTF-8 string")); + } + } + + /* + * NOTE: Ideally we should do here what jvp_array_slice() does instead + * of allocating a new string as we do! However, we assume NUL- + * terminated strings all over, and in the jv API, so for now we waste + * memory like a drunken navy programmer. There's probably nothing we + * can do about it. + */ + res = jv_string_sized(p, e - p); + jv_free(j); + return res; +} + jv jv_string_concat(jv a, jv b) { jvp_string* sb = jvp_string_ptr(&b.val.nontrivial); jvp_string_append(&a.val.nontrivial, sb->data, jvp_string_length(sb)); diff --git a/jv.h b/jv.h index 921345c4..1362acca 100644 --- a/jv.h +++ b/jv.h @@ -82,6 +82,7 @@ int jv_string_length_bytes(jv); int jv_string_length_codepoints(jv); unsigned long jv_string_hash(jv); const char* jv_string_value(jv); +jv jv_string_slice(jv j, int start, int end); jv jv_string_concat(jv, jv); jv jv_string_fmt(const char*, ...); jv jv_string_append_codepoint(jv a, uint32_t c); diff --git a/jv_aux.c b/jv_aux.c index 89f36e69..f3260c9d 100644 --- a/jv_aux.c +++ b/jv_aux.c @@ -3,15 +3,19 @@ #include #include "jv_alloc.h" -static int parse_slice(jv array, jv slice, int* pstart, int* pend) { +static int parse_slice(jv j, jv slice, int* pstart, int* pend) { // Array slices - int len = jv_array_length(jv_copy(array)); jv start_jv = jv_object_get(jv_copy(slice), jv_string("start")); jv end_jv = jv_object_get(slice, jv_string("end")); if (jv_get_kind(start_jv) == JV_KIND_NULL) { jv_free(start_jv); start_jv = jv_number(0); } + int len; + if (jv_get_kind(j) == JV_KIND_ARRAY) + len = jv_array_length(jv_copy(j)); + else + len = jv_string_length_codepoints(jv_copy(j)); if (jv_get_kind(end_jv) == JV_KIND_NULL) { jv_free(end_jv); end_jv = jv_number(len); @@ -61,6 +65,14 @@ jv jv_get(jv t, jv k) { v = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an array slice must be numbers")); jv_free(t); } + } else if (jv_get_kind(t) == JV_KIND_STRING && jv_get_kind(k) == JV_KIND_OBJECT) { + int start, end; + if (parse_slice(t, k, &start, &end)) { + v = jv_string_slice(t, start, end); + } else { + v = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an string slice must be numbers")); + jv_free(t); + } } else if (jv_get_kind(t) == JV_KIND_NULL && (jv_get_kind(k) == JV_KIND_STRING || jv_get_kind(k) == JV_KIND_NUMBER || From da129f666409ed0d44e61c917f06cd09b3099012 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Fri, 29 Nov 2013 12:50:21 -0600 Subject: [PATCH 11/32] Test string slicing --- tests/all.test | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/all.test b/tests/all.test index 72115818..dbf51dfb 100644 --- a/tests/all.test +++ b/tests/all.test @@ -178,6 +178,10 @@ null [0,1,2,3,4,5,6] [[], [2,3], [0,1,2,3,4], [5,6], [], []] +[.[3:2], .[-5:4], .[:-2], .[-2:], .[3:3][1:], .[10:]] +"abcdefghi" +["","","abcdefg","hi","",""] + del(.[2:4],.[0],.[-2:]) [0,1,2,3,4,5,6,7] [1,4,5] From c9e8581f4533bf90d869a162a28398287b60826b Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Fri, 29 Nov 2013 13:19:10 -0600 Subject: [PATCH 12/32] Document string functions and slicing --- docs/content/3.manual/manual.yml | 56 +++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/docs/content/3.manual/manual.yml b/docs/content/3.manual/manual.yml index 467617ac..2d516a4f 100644 --- a/docs/content/3.manual/manual.yml +++ b/docs/content/3.manual/manual.yml @@ -199,11 +199,12 @@ sections: returns the third element of the array. The `.[10:15]` syntax can be used to return a subarray of an - array. The array returned by `.[10:15]` will be of length 5, - containing the elements from index 10 (inclusive) to index - 15 (exclusive). Either index may be negative (in which case - it counts backwards from the end of the array), or omitted - (in which case it refers to the start or end of the array). + array or substring of a string. The array returned by + `.[10:15]` will be of length 5, containing the elements from + index 10 (inclusive) to index 15 (exclusive). Either index may + be negative (in which case it counts backwards from the end of + the array), or omitted (in which case it refers to the start + or end of the array). examples: - program: '.[0]' @@ -217,6 +218,10 @@ sections: - program: '.[2:4]' input: '["a","b","c","d","e"]' output: ['["c", "d"]'] + + - program: '.[2:4]' + input: '"abcdefghi"' + output: ['"cd"'] - program: '.[:3]' input: '["a","b","c","d","e"]' @@ -806,6 +811,47 @@ sections: input: '{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}' output: ['false'] + - title: `startswith` + body: | + + Outputs true if . starts with the given string argument. + + examples: + - program: '[.[]|startswith("foo")]' + input: '["fo", "foo", "barfoo", "foobar", "barfoob"]' + output: ['[false, true, false, true, false]'] + + - title: `endswith` + body: | + + Outputs true if . ends with the given string argument. + + examples: + - program: '[.[]|endswith("foo")]' + input: '["foobar", "barfoo"]' + output: ['[false, true, true, false, false]'] + + - title: `explode` + body: | + + Converts an input string into an array of the string's + codepoint numbers. + + examples: + - program: 'explode' + input: '"foobar"' + output: ['[102,111,111,98,97,114]'] + + - title: `implode` + body: | + + The inverse of explode. + + examples: + - program: 'implode' + input: '[65, 66, 67]' + output: ['"ABC"'] + - title: `recurse` body: | From b634722c8a972311c18888233537280d5d506537 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Fri, 29 Nov 2013 13:26:54 -0600 Subject: [PATCH 13/32] Document string multiplication and division --- docs/content/3.manual/manual.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/content/3.manual/manual.yml b/docs/content/3.manual/manual.yml index 2d516a4f..8832ceb2 100644 --- a/docs/content/3.manual/manual.yml +++ b/docs/content/3.manual/manual.yml @@ -463,10 +463,19 @@ sections: These operators only work on numbers, and do the expected. + Multiplying a string by a number produces the concatenation of + that string that many times. + + Dividing a string by another splits the first using the second + as separators. + examples: - program: '10 / . * 3' input: 5 output: [6] + - program: '. / ", "' + input: '"a, b,c,d, e"' + output: ['["a","b,c,d","e"]'] - title: `length` body: | From 20b32db5af37c322e3116cbc646b8b60974460e6 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Fri, 29 Nov 2013 13:29:25 -0600 Subject: [PATCH 14/32] Document split function --- docs/content/3.manual/manual.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/content/3.manual/manual.yml b/docs/content/3.manual/manual.yml index 8832ceb2..876704c2 100644 --- a/docs/content/3.manual/manual.yml +++ b/docs/content/3.manual/manual.yml @@ -861,6 +861,16 @@ sections: input: '[65, 66, 67]' output: ['"ABC"'] + - title: `split` + body: | + + Splits an input string on the separator argument. + + examples: + - program: 'split(", ")' + input: '"a, b,c,d, e"' + output: ['["a","b,c,d","e"]'] + - title: `recurse` body: | From 255f8ef5be85c211f95932eff821205070a1b22a Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Fri, 29 Nov 2013 16:16:52 -0600 Subject: [PATCH 15/32] Add tojson and fromjson builtins --- builtin.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/builtin.c b/builtin.c index 07682266..0bdc3320 100644 --- a/builtin.c +++ b/builtin.c @@ -226,6 +226,19 @@ static jv f_contains(jv a, jv b) { } } +static jv f_dump(jv input) { + return jv_dump_string(input, 0); +} + +static jv f_json_parse(jv input) { + if (jv_get_kind(input) != JV_KIND_STRING) + return type_error(input, "only strings can be parsed"); + jv res = jv_parse_sized(jv_string_value(input), + jv_string_length_bytes(jv_copy(input))); + jv_free(input); + return res; +} + static jv f_tonumber(jv input) { if (jv_get_kind(input) == JV_KIND_NUMBER) { return input; @@ -536,6 +549,8 @@ static const struct cfunction function_list[] = { {(cfunction_ptr)f_multiply, "_multiply", 3}, {(cfunction_ptr)f_divide, "_divide", 3}, {(cfunction_ptr)f_mod, "_mod", 3}, + {(cfunction_ptr)f_dump, "tojson", 1}, + {(cfunction_ptr)f_json_parse, "fromjson", 1}, {(cfunction_ptr)f_tonumber, "tonumber", 1}, {(cfunction_ptr)f_tostring, "tostring", 1}, {(cfunction_ptr)f_keys, "keys", 1}, From 3c4248192d1c2a2b5cedbd2f5ecc8af381de4e7c Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Fri, 29 Nov 2013 16:20:50 -0600 Subject: [PATCH 16/32] Test tojson and fromjson --- tests/all.test | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/all.test b/tests/all.test index dbf51dfb..836bc75d 100644 --- a/tests/all.test +++ b/tests/all.test @@ -75,6 +75,10 @@ null "" "<script>hax</script>" +[.[]|tojson|fromjson] +["foo", 1, ["a", 1, "b", 2, {"foo":"bar"}]] +["foo",1,["a",1,"b",2,{"foo":"bar"}]] + # # Dictionary construction syntax # From 8cee66e60929f274555f57154b82d8f48a417d50 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Fri, 29 Nov 2013 16:26:52 -0600 Subject: [PATCH 17/32] Document tojson and fromjson builtins --- docs/content/3.manual/manual.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/content/3.manual/manual.yml b/docs/content/3.manual/manual.yml index 876704c2..12920e17 100644 --- a/docs/content/3.manual/manual.yml +++ b/docs/content/3.manual/manual.yml @@ -915,6 +915,24 @@ sections: input: '42' output: ['"The input was 42, which is one less than 43"'] + - title: "Convert to/from JSON" + + The `tojson` and `fromjson` builtins dump values as JSON texts + or parse JSON texts into values, respectively. The tojson + builtin differs from tostring in that tostring returns strings + unmodified, while tojson encodes strings as JSON strings. + + examples: + - program: '[.[]|tostring]' + input: '[1, "foo", ["foo"]]' + output: '["1","foo","[\"foo\"]"]' + - program: '[.[]|tojson]' + input: '[1, "foo", ["foo"]]' + output: '["1","\"foo\"","[\"foo\"]"]' + - program: '[.[]|tojson|fromjson]' + input: '[1, "foo", ["foo"]]' + output: '[1,"foo",["foo"]]' + - title: "Format strings and escaping" body: | From d018e43ddc07a6791188f4b7bf3aa41ad69fd869 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Fri, 29 Nov 2013 16:36:15 -0600 Subject: [PATCH 18/32] Add -e | --exit-status CLI option --- main.c | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/main.c b/main.c index c9e09ce8..a7159219 100644 --- a/main.c +++ b/main.c @@ -59,17 +59,26 @@ enum { FROM_FILE = 512, + EXIT_STATUS = 8192, + /* debugging only */ DUMP_DISASM = 2048, }; static int options = 0; -static void process(jq_state *jq, jv value, int flags) { +static int process(jq_state *jq, jv value, int flags) { + int ret = 0; jq_start(jq, value, flags); jv result; while (jv_is_valid(result = jq_next(jq))) { if ((options & RAW_OUTPUT) && jv_get_kind(result) == JV_KIND_STRING) { fwrite(jv_string_value(result), 1, jv_string_length_bytes(jv_copy(result)), stdout); + if (jv_get_kind(result) == JV_KIND_FALSE) + ret = 10; + else if (jv_get_kind(result) == JV_KIND_NULL) + ret = 11; + else + ret = 0; jv_free(result); } else { int dumpopts; @@ -86,11 +95,18 @@ static void process(jq_state *jq, jv value, int flags) { if (options & COLOUR_OUTPUT) dumpopts |= JV_PRINT_COLOUR; if (options & NO_COLOUR_OUTPUT) dumpopts &= ~JV_PRINT_COLOUR; if (options & UNBUFFERED_OUTPUT) dumpopts |= JV_PRINT_UNBUFFERED; + if (jv_get_kind(result) == JV_KIND_FALSE) + ret = 10; + else if (jv_get_kind(result) == JV_KIND_NULL) + ret = 11; + else + ret = 0; jv_dump(result, dumpopts); } printf("\n"); } jv_free(result); + return ret; } FILE* current_input; @@ -178,6 +194,8 @@ int main(int argc, char* argv[]) { options |= PROVIDE_NULL; } else if (isoption(argv[i], 'f', "from-file")) { options |= FROM_FILE; + } else if (isoption(argv[i], 'e', "exit-status")) { + options |= EXIT_STATUS; } else if (isoption(argv[i], 0, "arg")) { if (i >= argc - 2) { fprintf(stderr, "%s: --arg takes two parameters (e.g. -a varname value)\n", progname); @@ -251,7 +269,7 @@ int main(int argc, char* argv[]) { } if (options & PROVIDE_NULL) { - process(jq, jv_null(), jq_flags); + ret = process(jq, jv_null(), jq_flags); } else { jv slurped; if (options & SLURP) { @@ -271,7 +289,7 @@ int main(int argc, char* argv[]) { slurped = jv_string_concat(slurped, jv_string(buf)); } else { if (buf[len-1] == '\n') buf[len-1] = 0; - process(jq, jv_string(buf), jq_flags); + ret = process(jq, jv_string(buf), jq_flags); } } } else { @@ -281,7 +299,7 @@ int main(int argc, char* argv[]) { if (options & SLURP) { slurped = jv_array_append(slurped, value); } else { - process(jq, value, jq_flags); + ret = process(jq, value, jq_flags); } } if (jv_invalid_has_msg(jv_copy(value))) { @@ -299,11 +317,13 @@ int main(int argc, char* argv[]) { if (ret != 0) goto out; if (options & SLURP) { - process(jq, slurped, jq_flags); + ret = process(jq, slurped, jq_flags); } } out: jv_mem_free(input_filenames); jq_teardown(&jq); + if (ret >= 10 && ret <= 11 && !(options & EXIT_STATUS)) + return 0; return ret; } From 4e07eac35659e681a6c96206d973772171870fe5 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Fri, 29 Nov 2013 16:39:02 -0600 Subject: [PATCH 19/32] Document -e / --exit-status argument --- docs/content/3.manual/manual.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/content/3.manual/manual.yml b/docs/content/3.manual/manual.yml index 12920e17..aac9112d 100644 --- a/docs/content/3.manual/manual.yml +++ b/docs/content/3.manual/manual.yml @@ -136,6 +136,13 @@ sections: Read filter from the file rather than from a command line, like awk's -f option. You can also use '#' to make comments. + * `-e` / `--exit-status`: + + Sets the exit status of jq to 10 if the last output values was + `false`, 11 if the last output value was `null`, or 0 if the + last output value was valid. Normally jq exits with 0, or 1 if + there was any usage problem or other error. + * `--arg name value`: This option passes a value to the jq program as a predefined From 77936a594d797c480f26bfcef3636a74588a6918 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Fri, 29 Nov 2013 17:41:04 -0600 Subject: [PATCH 20/32] Add -I / --online-input for huge top-level arrays --- jv.h | 3 +++ jv_file.c | 2 +- jv_parse.c | 32 +++++++++++++++++++++++--------- main.c | 5 ++++- 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/jv.h b/jv.h index 1362acca..92d0e058 100644 --- a/jv.h +++ b/jv.h @@ -130,6 +130,9 @@ void jv_nomem_handler(jv_nomem_handler_f, void *); jv jv_load_file(const char *, int); +typedef enum { + JV_PARSE_EXPLODE_TOPLEVEL_ARRAY = 1 +} jv_parser_flags; struct jv_parser; struct jv_parser* jv_parser_new(); void jv_parser_set_buf(struct jv_parser*, const char*, int, int); diff --git a/jv_file.c b/jv_file.c index 54ed36be..a6331607 100644 --- a/jv_file.c +++ b/jv_file.c @@ -18,7 +18,7 @@ jv jv_load_file(const char* filename, int raw) { data = jv_string(""); } else { data = jv_array(); - parser = jv_parser_new(); + parser = jv_parser_new(0); } while (!feof(file) && !ferror(file)) { char buf[4096]; diff --git a/jv_parse.c b/jv_parse.c index cc1e7b96..4667f1f9 100644 --- a/jv_parse.c +++ b/jv_parse.c @@ -28,6 +28,8 @@ struct jv_parser { int stackpos; int stacklen; jv next; + + jv_parser_flags flags; char* tokenbuf; int tokenpos; @@ -45,7 +47,8 @@ struct jv_parser { }; -static void parser_init(struct jv_parser* p) { +static void parser_init(struct jv_parser* p, jv_parser_flags flags) { + p->flags = flags; p->stack = 0; p->stacklen = p->stackpos = 0; p->next = jv_invalid(); @@ -110,6 +113,9 @@ static pfunc token(struct jv_parser* p, char ch) { break; case ',': + if (p->stackpos == 1 && (p->flags & JV_PARSE_EXPLODE_TOPLEVEL_ARRAY) && + jv_get_kind(p->stack[0]) == JV_KIND_ARRAY) + return 0; if (!jv_is_valid(p->next)) return "Expected value before ','"; if (p->stackpos == 0) @@ -133,16 +139,22 @@ static pfunc token(struct jv_parser* p, char ch) { if (p->stackpos == 0 || jv_get_kind(p->stack[p->stackpos-1]) != JV_KIND_ARRAY) return "Unmatched ']'"; if (jv_is_valid(p->next)) { - p->stack[p->stackpos-1] = jv_array_append(p->stack[p->stackpos-1], p->next); - p->next = jv_invalid(); + if (p->stackpos != 1 || !(p->flags & JV_PARSE_EXPLODE_TOPLEVEL_ARRAY)) { + p->stack[p->stackpos-1] = jv_array_append(p->stack[p->stackpos-1], p->next); + p->next = jv_invalid(); + } } else { if (jv_array_length(jv_copy(p->stack[p->stackpos-1])) != 0) { // this case hits on input like [1,2,3,] return "Expected another array element"; } } - jv_free(p->next); - p->next = p->stack[--p->stackpos]; + if (p->stackpos == 1 && (p->flags & JV_PARSE_EXPLODE_TOPLEVEL_ARRAY)) { + jv_free(p->stack[--p->stackpos]); + } else { + jv_free(p->next); + p->next = p->stack[--p->stackpos]; + } break; case '}': @@ -315,7 +327,9 @@ static chclass classify(char c) { static const presult OK = "output produced"; static int check_done(struct jv_parser* p, jv* out) { - if (p->stackpos == 0 && jv_is_valid(p->next)) { + if ((p->stackpos == 0 && jv_is_valid(p->next)) || + (p->stackpos == 1 && (p->flags & JV_PARSE_EXPLODE_TOPLEVEL_ARRAY) && + jv_get_kind(p->stack[0]) == JV_KIND_ARRAY && jv_is_valid(p->next))) { *out = p->next; p->next = jv_invalid(); return 1; @@ -370,9 +384,9 @@ static pfunc scan(struct jv_parser* p, char ch, jv* out) { return answer; } -struct jv_parser* jv_parser_new() { +struct jv_parser* jv_parser_new(jv_parser_flags flags) { struct jv_parser* p = jv_mem_alloc(sizeof(struct jv_parser)); - parser_init(p); + parser_init(p, flags); return p; } @@ -444,7 +458,7 @@ jv jv_parser_next(struct jv_parser* p) { jv jv_parse_sized(const char* string, int length) { struct jv_parser parser; - parser_init(&parser); + parser_init(&parser, 0); jv_parser_set_buf(&parser, string, length, 0); jv value = jv_parser_next(&parser); if (jv_is_valid(value)) { diff --git a/main.c b/main.c index a7159219..c8da96a2 100644 --- a/main.c +++ b/main.c @@ -159,6 +159,7 @@ int main(int argc, char* argv[]) { ninput_files = 0; int further_args_are_files = 0; int jq_flags = 0; + jv_parser_flags parser_flags = 0; jv program_arguments = jv_array(); for (int i=1; i= argc - 2) { fprintf(stderr, "%s: --arg takes two parameters (e.g. -a varname value)\n", progname); @@ -279,7 +282,7 @@ int main(int argc, char* argv[]) { slurped = jv_array(); } } - struct jv_parser* parser = jv_parser_new(); + struct jv_parser* parser = jv_parser_new(parser_flags); char buf[4096]; while (read_more(buf, sizeof(buf))) { if (options & RAW_INPUT) { From fa316ac430221cc004bddf23eb5c96882e83a499 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Fri, 29 Nov 2013 17:43:40 -0600 Subject: [PATCH 21/32] Document -u / --unslurp option --- docs/content/3.manual/manual.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/content/3.manual/manual.yml b/docs/content/3.manual/manual.yml index aac9112d..469ccd79 100644 --- a/docs/content/3.manual/manual.yml +++ b/docs/content/3.manual/manual.yml @@ -80,6 +80,12 @@ sections: Instead of running the filter for each JSON object in the input, read the entire input stream into a large array and run the filter just once. + + * `--online-input`/`-I`: + + When the top-level input value is an array produce its elements + instead of the array. This allows on-line processing of + potentially very large top-level arrays' elements. * `--raw-input`/`-R`: From 346da34432f3b8087d2c100e910d68641b1b396f Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Sat, 30 Nov 2013 00:44:14 -0600 Subject: [PATCH 22/32] Add ltrimstr and rtrimstr functions --- builtin.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/builtin.c b/builtin.c index 0bdc3320..e90dd2cf 100644 --- a/builtin.c +++ b/builtin.c @@ -108,6 +108,33 @@ static jv f_endswith(jv a, jv b) { return ret; } +static jv f_ltrimstr(jv input, jv left) { + if (jv_get_kind(f_startswith(jv_copy(input), jv_copy(left))) != JV_KIND_TRUE) { + jv_free(left); + return input; + } + /* + * FIXME It'd be better to share the suffix with the original input -- + * that we could do, we just can't share prefixes. + */ + int prefixlen = jv_string_length_bytes(left); + jv res = jv_string_sized(jv_string_value(input) + prefixlen, + jv_string_length_bytes(jv_copy(input)) - prefixlen); + jv_free(input); + return res; +} + +static jv f_rtrimstr(jv input, jv right) { + if (jv_get_kind(f_endswith(jv_copy(input), jv_copy(right))) == JV_KIND_TRUE) { + jv res = jv_string_sized(jv_string_value(input), + jv_string_length_bytes(jv_copy(input)) - jv_string_length_bytes(right)); + jv_free(input); + return res; + } + jv_free(right); + return input; +} + static jv f_minus(jv input, jv a, jv b) { jv_free(input); if (jv_get_kind(a) == JV_KIND_NUMBER && jv_get_kind(b) == JV_KIND_NUMBER) { @@ -556,6 +583,8 @@ static const struct cfunction function_list[] = { {(cfunction_ptr)f_keys, "keys", 1}, {(cfunction_ptr)f_startswith, "startswith", 2}, {(cfunction_ptr)f_endswith, "endswith", 2}, + {(cfunction_ptr)f_ltrimstr, "ltrimstr", 2}, + {(cfunction_ptr)f_rtrimstr, "rtrimstr", 2}, {(cfunction_ptr)jv_string_split, "split", 2}, {(cfunction_ptr)jv_string_explode, "explode", 1}, {(cfunction_ptr)jv_string_implode, "implode", 1}, From 9fa2173825d14053d5ac2adb62b655c50f96aff0 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Sat, 30 Nov 2013 00:44:30 -0600 Subject: [PATCH 23/32] Test ltrimstr and rtrimstr functions --- tests/all.test | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/all.test b/tests/all.test index 836bc75d..2af1e1c9 100644 --- a/tests/all.test +++ b/tests/all.test @@ -591,6 +591,14 @@ def inc(x): x |= .+1; inc(.[].a) ["fo", "foo", "barfoo", "foobar", "barfoob"] [false, true, true, false, false] +[.[]|ltrimstr("foo")] +["fo", "foo", "barfoo", "foobar", "afoo"] +["fo","","barfoo","bar","afoo"] + +[.[]|rtrimstr("foo")] +["fo", "foo", "barfoo", "foobar", "foob"] +["fo","","bar","foobar","foob"] + [.[]|split(",")] ["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"] [["a"," bc"," def"," ghij"," jklmn"," a","b"," c","d"," e","f"],["a","b","c","d"," e","f","g","h"]] From 09104ce531f2ff2b941592b161bbb10f28b90c09 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Sat, 30 Nov 2013 00:45:55 -0600 Subject: [PATCH 24/32] Document ltrimstr and rtrimstr --- docs/content/3.manual/manual.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/content/3.manual/manual.yml b/docs/content/3.manual/manual.yml index 469ccd79..d114d31e 100644 --- a/docs/content/3.manual/manual.yml +++ b/docs/content/3.manual/manual.yml @@ -853,6 +853,28 @@ sections: input: '["foobar", "barfoo"]' output: ['[false, true, true, false, false]'] + - title: `ltrimstr` + body: | + + Outputs its input with the given prefix string removed, if it + starts with it. + + examples: + - program: '[.[]|ltrimstr("foo")]' + input: ["fo", "foo", "barfoo", "foobar", "afoo"] + output: ['["fo","","barfoo","bar","afoo"]'] + + - title: `rtrimstr` + body: | + + Outputs its input with the given suffix string removed, if it + starts with it. + + examples: + - program: '[.[]|rtrimstr("foo")]' + input: ["fo", "foo", "barfoo", "foobar", "foob"] + output: ['["fo","","bar","foobar","foob"]'] + - title: `explode` body: | From 04bc2ef7cf0b02f434c99239d80f59bbf275a1d2 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Wed, 4 Dec 2013 18:14:10 -0600 Subject: [PATCH 25/32] Add jv_string_vfmt() --- jv.c | 18 +++++++++++++----- jv.h | 2 ++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/jv.c b/jv.c index 999c776b..b66b0738 100644 --- a/jv.c +++ b/jv.c @@ -774,14 +774,14 @@ jv jv_string_append_str(jv a, const char* str) { return jv_string_append_buf(a, str, strlen(str)); } -jv jv_string_fmt(const char* fmt, ...) { +jv jv_string_vfmt(const char* fmt, va_list ap) { int size = 1024; while (1) { char* buf = jv_mem_alloc(size); - va_list args; - va_start(args, fmt); - int n = vsnprintf(buf, size, fmt, args); - va_end(args); + va_list ap2; + va_copy(ap2, ap); + int n = vsnprintf(buf, size, fmt, ap2); + va_end(ap2); if (n < size) { jv ret = jv_string_sized(buf, n); jv_mem_free(buf); @@ -793,6 +793,14 @@ jv jv_string_fmt(const char* fmt, ...) { } } +jv jv_string_fmt(const char* fmt, ...) { + va_list args; + va_start(args, fmt); + jv res = jv_string_vfmt(fmt, args); + va_end(args); + return res; +} + /* * Objects (internal helpers) */ diff --git a/jv.h b/jv.h index 92d0e058..6b06eea3 100644 --- a/jv.h +++ b/jv.h @@ -1,6 +1,7 @@ #ifndef JV_H #define JV_H +#include #include typedef enum { @@ -84,6 +85,7 @@ unsigned long jv_string_hash(jv); const char* jv_string_value(jv); jv jv_string_slice(jv j, int start, int end); jv jv_string_concat(jv, jv); +jv jv_string_vfmt(const char*, va_list); jv jv_string_fmt(const char*, ...); jv jv_string_append_codepoint(jv a, uint32_t c); jv jv_string_append_buf(jv a, const char* buf, int len); From eb165459aae82f0b5d25bc6e5132db501fa9e148 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Sat, 30 Nov 2013 02:05:42 -0600 Subject: [PATCH 26/32] Add callback interface for errors Printing to stderr is not the right answer for a library. --- builtin.c | 14 +++++------ builtin.h | 3 ++- execute.c | 67 ++++++++++++++++++++++++++++++++++++++-------------- jq.h | 5 ++++ locfile.c | 70 ++++++++++++++++++++++++++++++++++++++++++------------- locfile.h | 6 ++++- 6 files changed, 123 insertions(+), 42 deletions(-) diff --git a/builtin.c b/builtin.c index e90dd2cf..3c3ac5bd 100644 --- a/builtin.c +++ b/builtin.c @@ -677,9 +677,9 @@ static const char* const jq_builtins[] = { }; -int builtins_bind_one(block* bb, const char* code) { +static int builtins_bind_one(jq_state *jq, block* bb, const char* code) { struct locfile src; - locfile_init(&src, code, strlen(code)); + locfile_init(&src, jq, code, strlen(code)); block funcs; int nerrors = jq_parse_library(&src, &funcs); if (nerrors == 0) { @@ -689,14 +689,14 @@ int builtins_bind_one(block* bb, const char* code) { return nerrors; } -int slurp_lib(block* bb) { +static int slurp_lib(jq_state *jq, block* bb) { int nerrors = 0; char* home = getenv("HOME"); if (home) { // silently ignore no $HOME jv filename = jv_string_append_str(jv_string(home), "/.jq"); jv data = jv_load_file(jv_string_value(filename), 1); if (jv_is_valid(data)) { - nerrors = builtins_bind_one(bb, jv_string_value(data) ); + nerrors = builtins_bind_one(jq, bb, jv_string_value(data) ); } jv_free(filename); jv_free(data); @@ -704,14 +704,14 @@ int slurp_lib(block* bb) { return nerrors; } -int builtins_bind(block* bb) { - int nerrors = slurp_lib(bb); +int builtins_bind(jq_state *jq, block* bb) { + int nerrors = slurp_lib(jq, bb); if (nerrors) { block_free(*bb); return nerrors; } for (int i=(int)(sizeof(jq_builtins)/sizeof(jq_builtins[0]))-1; i>=0; i--) { - nerrors = builtins_bind_one(bb, jq_builtins[i]); + nerrors = builtins_bind_one(jq, bb, jq_builtins[i]); assert(!nerrors); } *bb = bind_bytecoded_builtins(*bb); diff --git a/builtin.h b/builtin.h index 1aab133f..bca9bae1 100644 --- a/builtin.h +++ b/builtin.h @@ -1,9 +1,10 @@ #ifndef BUILTIN_H #define BUILTIN_H +#include "jq.h" #include "bytecode.h" #include "compile.h" -int builtins_bind(block*); +int builtins_bind(jq_state *, block*); #endif diff --git a/execute.c b/execute.c index 4b05d793..4ed20d8d 100644 --- a/execute.c +++ b/execute.c @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -19,6 +21,9 @@ struct jq_state { void *nomem_handler_data; struct bytecode* bc; + jq_err_cb err_cb; + void *err_cb_data; + struct stack stk; stack_ptr curr_frame; stack_ptr stk_top; @@ -238,15 +243,22 @@ static void jq_reset(jq_state *jq) { jq->subexp_nest = 0; } - - - -void print_error(jv value) { +static void print_error(jq_state *jq, jv value) { assert(!jv_is_valid(value)); jv msg = jv_invalid_get_msg(value); - if (jv_get_kind(msg) == JV_KIND_STRING) { - fprintf(stderr, "jq: error: %s\n", jv_string_value(msg)); - } + jv msg2; + if (jv_get_kind(msg) == JV_KIND_STRING) + msg2 = jv_string_fmt("jq: error: %s", jv_string_value(msg)); + else + msg2 = jv_string_fmt("jq: error: "); + jv_free(msg); + + if (jq->err_cb) + jq->err_cb(jq->err_cb_data, msg2); + else if (jv_get_kind(msg2) == JV_KIND_STRING) + fprintf(stderr, "%s\n", jv_string_value(msg2)); + else + fprintf(stderr, "jq: error: %s\n", strerror(ENOMEM)); jv_free(msg); } #define ON_BACKTRACK(op) ((op)+NUM_OPCODES) @@ -368,8 +380,8 @@ jv jq_next(jq_state *jq) { stack_push(jq, jv_object_set(objv, k, v)); stack_push(jq, stktop); } else { - print_error(jv_invalid_with_msg(jv_string_fmt("Cannot use %s as object key", - jv_kind_name(jv_get_kind(k))))); + print_error(jq, jv_invalid_with_msg(jv_string_fmt("Cannot use %s as object key", + jv_kind_name(jv_get_kind(k))))); jv_free(stktop); jv_free(v); jv_free(k); @@ -387,7 +399,7 @@ jv jq_next(jq_state *jq) { jv max = stack_pop(jq); if (jv_get_kind(*var) != JV_KIND_NUMBER || jv_get_kind(max) != JV_KIND_NUMBER) { - print_error(jv_invalid_with_msg(jv_string_fmt("Range bounds must be numeric"))); + print_error(jq, jv_invalid_with_msg(jv_string_fmt("Range bounds must be numeric"))); jv_free(max); goto do_backtrack; } else if (jv_number_value(jv_copy(*var)) >= jv_number_value(jv_copy(max))) { @@ -499,7 +511,7 @@ jv jq_next(jq_state *jq) { if (jv_is_valid(v)) { stack_push(jq, v); } else { - print_error(v); + print_error(jq, v); goto do_backtrack; } break; @@ -552,8 +564,8 @@ jv jq_next(jq_state *jq) { } } else { assert(opcode == EACH); - print_error(jv_invalid_with_msg(jv_string_fmt("Cannot iterate over %s", - jv_kind_name(jv_get_kind(container))))); + print_error(jq, jv_invalid_with_msg(jv_string_fmt("Cannot iterate over %s", + jv_kind_name(jv_get_kind(container))))); keep_going = 0; } @@ -624,7 +636,7 @@ jv jq_next(jq_state *jq) { if (jv_is_valid(top)) { stack_push(jq, top); } else { - print_error(top); + print_error(jq, top); goto do_backtrack; } break; @@ -682,10 +694,23 @@ jq_state *jq_init(void) { jq->fork_top = 0; jq->curr_frame = 0; + jq->err_cb = NULL; + jq->err_cb_data = NULL; + jq->path = jv_null(); return jq; } +void jq_set_error_cb(jq_state *jq, jq_err_cb cb, void *data) { + jq->err_cb = cb; + jq->err_cb_data = data; +} + +void jq_get_error_cb(jq_state *jq, jq_err_cb *cb, void **data) { + *cb = jq->err_cb; + *data = jq->err_cb_data; +} + void jq_set_nomem_handler(jq_state *jq, void (*nomem_handler)(void *), void *data) { jv_nomem_handler(nomem_handler, data); jq->nomem_handler = nomem_handler; @@ -729,7 +754,7 @@ int jq_compile_args(jq_state *jq, const char* str, jv args) { jv_nomem_handler(jq->nomem_handler, jq->nomem_handler_data); assert(jv_get_kind(args) == JV_KIND_ARRAY); struct locfile locations; - locfile_init(&locations, str, strlen(str)); + locfile_init(&locations, jq, str, strlen(str)); block program; jq_reset(jq); if (jq->bc) { @@ -746,13 +771,21 @@ int jq_compile_args(jq_state *jq, const char* str, jv args) { jv_free(name); } jv_free(args); - nerrors = builtins_bind(&program); + nerrors = builtins_bind(jq, &program); if (nerrors == 0) { nerrors = block_compile(program, &locations, &jq->bc); } } if (nerrors) { - fprintf(stderr, "%d compile %s\n", nerrors, nerrors > 1 ? "errors" : "error"); + jv s = jv_string_fmt("%d compile %s", nerrors, + nerrors > 1 ? "errors" : "error"); + if (jq->err_cb != NULL) + jq->err_cb(jq->err_cb_data, s); + else if (!jv_is_valid(s)) + fprintf(stderr, "Error formatting jq compilation errors: %s\n", strerror(errno)); + else + fprintf(stderr, "%s\n", jv_string_value(s)); + jv_free(s); } locfile_free(&locations); return jq->bc != NULL; diff --git a/jq.h b/jq.h index e35845c6..66b79387 100644 --- a/jq.h +++ b/jq.h @@ -1,12 +1,17 @@ #ifndef _JQ_H_ #define _JQ_H_ +#include #include enum {JQ_DEBUG_TRACE = 1}; typedef struct jq_state jq_state; +typedef void (*jq_err_cb)(void *, jv); + jq_state *jq_init(void); +void jq_set_error_cb(jq_state *, jq_err_cb, void *); +void jq_get_error_cb(jq_state *, jq_err_cb *, void **); void jq_set_nomem_handler(jq_state *, void (*)(void *), void *); int jq_compile(jq_state *, const char* str); int jq_compile_args(jq_state *, const char* str, jv args); diff --git a/locfile.c b/locfile.c index f1f17d0f..48cacd6f 100644 --- a/locfile.c +++ b/locfile.c @@ -1,12 +1,16 @@ +#include +#include +#include #include #include -#include -#include +#include +#include "jq.h" #include "jv_alloc.h" #include "locfile.h" -void locfile_init(struct locfile* l, const char* data, int length) { +void locfile_init(struct locfile* l, jq_state *jq, const char* data, int length) { + l->jq = jq; l->data = data; l->length = length; l->nlines = 1; @@ -43,23 +47,57 @@ static int locfile_line_length(struct locfile* l, int line) { } void locfile_locate(struct locfile* l, location loc, const char* fmt, ...) { + jq_err_cb cb; + void *cb_data; va_list fmtargs; va_start(fmtargs, fmt); - vfprintf(stderr, fmt, fmtargs); - va_end(fmtargs); - fprintf(stderr, "\n"); + int startline; + int offset; + + if (loc.start != -1) { + startline = locfile_get_line(l, loc.start); + offset = l->linemap[startline]; + } + + jq_get_error_cb(l->jq, &cb, &cb_data); + + jv m1 = jv_string_vfmt(fmt, fmtargs); + if (!jv_is_valid(m1)) { + jv_free(m1); + goto enomem; + } + jv m2; if (loc.start == -1) { - fprintf(stderr, "\n"); + m2 = jv_string_fmt("%s\n", jv_string_value(m1)); + if (cb) + cb(cb_data, m2); + else + fprintf(stderr, "%s", jv_string_value(m2)); + jv_free(m1); + jv_free(m2); return; } - int startline = locfile_get_line(l, loc.start); - int offset = l->linemap[startline]; - fprintf(stderr, "%.*s\n", locfile_line_length(l, startline), l->data + offset); - fprintf(stderr, "%*s", loc.start - offset, ""); - for (int i = loc.start; - i < loc.end && i < offset + locfile_line_length(l, startline); - i++){ - fprintf(stderr, "^"); + m2 = jv_string_fmt("%s\n%.*s%*s", jv_string_value(m1), + locfile_line_length(l, startline), l->data + offset, + loc.start - offset, ""); + jv_free(m1); + if (!jv_is_valid(m2)) { + jv_free(m2); + goto enomem; } - fprintf(stderr, "\n"); + if (cb) + cb(cb_data, m2); + else + fprintf(stderr, "%s", jv_string_value(m2)); + jv_free(m2); + return; + +enomem: + if (cb != NULL) + cb(cb_data, jv_invalid()); + else if (errno == ENOMEM || errno == 0) + vfprintf(stderr, "Error formatting jq compilation error: %s", strerror(errno ? errno : ENOMEM)); + else + vfprintf(stderr, "Error formatting jq compilation error: %s", strerror(errno)); + return; } diff --git a/locfile.h b/locfile.h index 35907974..4e20df27 100644 --- a/locfile.h +++ b/locfile.h @@ -1,6 +1,8 @@ #ifndef _LOCFILE_H #define _LOCFILE_H +#include "jq.h" + typedef struct { int start, end; } location; @@ -12,9 +14,11 @@ struct locfile { int length; int* linemap; int nlines; + char *error; + jq_state *jq; }; -void locfile_init(struct locfile* l, const char* data, int length); +void locfile_init(struct locfile* l, jq_state *jq, const char* data, int length); void locfile_free(struct locfile* l); From ae7a042876130c471dce28a4396abc215192eaa9 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Sat, 30 Nov 2013 02:29:21 -0600 Subject: [PATCH 27/32] Make length return abs value of numeric inputs --- builtin.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builtin.c b/builtin.c index 3c3ac5bd..aa3c888a 100644 --- a/builtin.c +++ b/builtin.c @@ -287,6 +287,8 @@ static jv f_length(jv input) { return jv_number(jv_object_length(input)); } else if (jv_get_kind(input) == JV_KIND_STRING) { return jv_number(jv_string_length_codepoints(input)); + } else if (jv_get_kind(input) == JV_KIND_NUMBER) { + return jv_number(fabs(jv_number_value(input))); } else if (jv_get_kind(input) == JV_KIND_NULL) { jv_free(input); return jv_number(0); From cac14a531de733cbffe50eb4e6c479b5ef577e21 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Mon, 2 Dec 2013 12:16:38 -0600 Subject: [PATCH 28/32] Add index strings by string; return string indexes % jq '.[","]' "a,bc,def,ghij,klmno" [1,4,8,13] % --- jv.c | 20 ++++++++++++++++++++ jv.h | 1 + jv_aux.c | 2 ++ 3 files changed, 23 insertions(+) diff --git a/jv.c b/jv.c index b66b0738..0c0bbb76 100644 --- a/jv.c +++ b/jv.c @@ -621,6 +621,26 @@ not_this: } #endif /* HAVE_MEMMEM */ +jv jv_string_indexes(jv j, jv k) { + assert(jv_get_kind(j) == JV_KIND_STRING); + assert(jv_get_kind(k) == JV_KIND_STRING); + const char *jstr = jv_string_value(j); + const char *idxstr = jv_string_value(k); + const char *p; + int jlen = jv_string_length_bytes(jv_copy(j)); + int idxlen = jv_string_length_bytes(jv_copy(k)); + jv a = jv_array(); + + p = jstr; + while ((p = memmem(p, (jstr + jlen) - p, idxstr, idxlen)) != NULL) { + a = jv_array_append(a, jv_number(p - jstr)); + p += idxlen; + } + jv_free(j); + jv_free(k); + return a; +} + jv jv_string_split(jv j, jv sep) { assert(jv_get_kind(j) == JV_KIND_STRING); assert(jv_get_kind(sep) == JV_KIND_STRING); diff --git a/jv.h b/jv.h index 6b06eea3..efc6fecb 100644 --- a/jv.h +++ b/jv.h @@ -83,6 +83,7 @@ int jv_string_length_bytes(jv); int jv_string_length_codepoints(jv); unsigned long jv_string_hash(jv); const char* jv_string_value(jv); +jv jv_string_indexes(jv j, jv k); jv jv_string_slice(jv j, int start, int end); jv jv_string_concat(jv, jv); jv jv_string_vfmt(const char*, va_list); diff --git a/jv_aux.c b/jv_aux.c index f3260c9d..76712aee 100644 --- a/jv_aux.c +++ b/jv_aux.c @@ -73,6 +73,8 @@ jv jv_get(jv t, jv k) { v = jv_invalid_with_msg(jv_string_fmt("Start and end indices of an string slice must be numbers")); jv_free(t); } + } else if (jv_get_kind(t) == JV_KIND_STRING && jv_get_kind(k) == JV_KIND_STRING) { + v = jv_string_indexes(t, k); } else if (jv_get_kind(t) == JV_KIND_NULL && (jv_get_kind(k) == JV_KIND_STRING || jv_get_kind(k) == JV_KIND_NUMBER || From b37ce20c2ad9e742cf2a67ab73a1818ad1c9e254 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Mon, 2 Dec 2013 12:19:23 -0600 Subject: [PATCH 29/32] Add index and rindex builtins --- builtin.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builtin.c b/builtin.c index aa3c888a..087741fa 100644 --- a/builtin.c +++ b/builtin.c @@ -676,6 +676,8 @@ static const char* const jq_builtins[] = { "def from_entries: map({(.key): .value}) | add;", "def with_entries(f): to_entries | map(f) | from_entries;", "def reverse: [.[length - 1 - range(0;length)]];", + "def index(i): .[i][0];", + "def rindex(i): .[i][-1:][0];", }; From 426edff09c0b77866d209ccfc34eb879982aed64 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Mon, 2 Dec 2013 12:19:36 -0600 Subject: [PATCH 30/32] Add tests for string index by string and builtins --- tests/all.test | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/all.test b/tests/all.test index 2af1e1c9..71918a6f 100644 --- a/tests/all.test +++ b/tests/all.test @@ -599,6 +599,22 @@ def inc(x): x |= .+1; inc(.[].a) ["fo", "foo", "barfoo", "foobar", "foob"] ["fo","","bar","foobar","foob"] +.[","] +"a,bc,def,ghij,klmno" +[1,4,8,13] + +.[", "] +"a,bc,def,ghij,klmno" +[] + +.[", "] +"a,b,, c, d,ef, , ghi, jklmn, o" +[4,7,13,15,20,27] + +[(index(","), rindex(","))] +"a,bc,def,ghij,klmno" +[1,13] + [.[]|split(",")] ["a, bc, def, ghij, jklmn, a,b, c,d, e,f", "a,b,c,d, e,f,g,h"] [["a"," bc"," def"," ghij"," jklmn"," a","b"," c","d"," e","f"],["a","b","c","d"," e","f","g","h"]] From 54635000c5a9660affd7a08c9d72d3c350ecb241 Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Wed, 4 Dec 2013 18:14:31 -0600 Subject: [PATCH 31/32] Conditionally #define _GNU_SOURCE in compile.c --- compile.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compile.c b/compile.c index 248d363e..f634284f 100644 --- a/compile.c +++ b/compile.c @@ -1,4 +1,6 @@ +#ifndef _GNU_SOURCE #define _GNU_SOURCE // for strdup +#endif #include #include #include From 3ba7c97b2a897044dc63ecc659830a32ab8db56b Mon Sep 17 00:00:00 2001 From: Nicolas Williams Date: Sat, 30 Nov 2013 02:31:47 -0600 Subject: [PATCH 32/32] Add self to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 98fdf672..2ef6ae09 100644 --- a/AUTHORS +++ b/AUTHORS @@ -4,3 +4,4 @@ Stephen Dolan Contributions by Anthony Shortland - rpmbuild target Lee Thompson - autoconf stuff +Nicolas Williams - library-fication, autoconf stuff, misc