test: check that --run-tests reports failing tests (#3650)

Nothing tested that jq --run-tests exits with status 1 when the
tests fail, so a failure could have been reported as a pass.

Add shell tests with incorrect, missing and extra output that check
both the exit status and the reported message, such as "Insufficient
results", along with a passing control. Under Valgrind, memory errors
exit with status 99 so they are not mistaken for a test failure.

tests/shtest

@@ -70,6 +70,68 @@ $VALGRIND $Q $JQ -Rne '[inputs] == ["[{\"foo\": 1}", ",{\"foo\": 2}", "]", "[{\"
 $VALGRIND $Q $JQ -ne --slurpfile x $d/bom1.json '$x == [[{"foo":1},{"foo":2}]]'
 $VALGRIND $Q $JQ -ne --rawfile x $d/bom1.json '$x == "[{\"foo\": 1}\n,{\"foo\": 2}\n]\n"'
 
+## Test that --run-tests reports failures (#2765)
+cat > $d/passing.test <<'EOF'
+.
+1
+1
+EOF
+cat > $d/passing.expected <<'EOF'
+Test #1: '.' at line number 1
+1 of 1 tests passed (0 malformed, 0 skipped)
+EOF
+$VALGRIND $Q $JQ --run-tests $d/passing.test > $d/out 2>&1
+head -n 2 $d/out | cmp $d/passing.expected -
+
+cat > $d/erroneous-result.test <<'EOF'
+.
+1
+2
+EOF
+cat > $d/erroneous-result.expected <<'EOF'
+Test #1: '.' at line number 1
+*** Expected 2, but got 1 for test at line number 3: .
+0 of 1 tests passed (0 malformed, 0 skipped)
+EOF
+
+cat > $d/insufficient-results.test <<'EOF'
+empty
+null
+{"expected":true}
+EOF
+cat > $d/insufficient-results.expected <<'EOF'
+Test #1: 'empty' at line number 1
+*** Insufficient results for test at line number 3: empty
+0 of 1 tests passed (0 malformed, 0 skipped)
+EOF
+
+cat > $d/superfluous-result.test <<'EOF'
+., .
+1
+1
+EOF
+cat > $d/superfluous-result.expected <<'EOF'
+Test #1: '., .' at line number 1
+*** Superfluous result: 1 for test at line number 3, ., .
+0 of 1 tests passed (1 malformed, 0 skipped)
+EOF
+
+for test in erroneous-result insufficient-results superfluous-result; do
+  ret=0
+  # Keep memory errors distinct from the expected test failure status.
+  if [ -n "$VALGRIND" ]; then
+    $VALGRIND $Q --error-exitcode=99 $JQ --run-tests $d/$test.test > $d/out 2>&1 || ret=$?
+  else
+    $JQ --run-tests $d/$test.test > $d/out 2>&1 || ret=$?
+  fi
+  if [ "$ret" -ne 1 ]; then
+    cat $d/out
+    echo "Expected --run-tests to report $test with status 1, got $ret" >&2
+    exit 1
+  fi
+  cmp $d/out $d/$test.expected
+done
+
 ## Test JSON sequence support
 cat > $d/expected <<EOF
 jq: ignoring parse error: Truncated value at line 2, column 5