Make --continue/-c govern every input shape in validate (#846)

Signed-off-by: Juan Cruz Viotti <jv@jviotti.com>

Author: jviotti

docs/validate.markdown

@@ -37,9 +37,9 @@ To help scripts distinguish validation errors, these are reported using exit
 code 2.
 
 > [!NOTE]
-> When validating JSONL datasets, the command stops at the first entry that
-> fails validation. Pass `--continue`/`-c` to report all failing entries
-> instead.
+> The command stops at the first instance that fails validation, whether that
+> instance is a file, an entry of a JSONL dataset, or one of many arguments.
+> Pass `--continue`/`-c` to report every failing instance instead.
 
 > [!TIP]
 > GZIP-compressed JSONL datasets (`.jsonl.gz`) are transparently decompressed
@@ -174,16 +174,16 @@ jsonschema validate path/to/my/schema.json path/to/my/dataset.jsonl
 jsonschema validate path/to/my/schema.json path/to/my/dataset.jsonl.gz
 ```
 
-### Validate a JSONL dataset reporting all failures
+### Report every failing instance instead of stopping at the first
 
 ```sh
 jsonschema validate path/to/my/schema.json path/to/my/dataset.jsonl --continue
 ```
 
-Note that even with `--continue`, only the first error of each failing entry is
-reported. Continuing validation past the first error within a single entry is a
-gray area not covered by the JSON Schema specification and potentially tricky to
-implement correctly at the evaluation level.
+Note that even with `--continue`, only the first error of each failing instance
+is reported. Continuing validation past the first error within a single instance
+is a gray area not covered by the JSON Schema specification and potentially
+tricky to implement correctly at the evaluation level.
 
 ### Validate a JSON instance enabling HTTP resolution
 

src/command_validate.cc

@@ -171,7 +171,7 @@ auto process_entry(const sourcemeta::jsonschema::InputJSON &entry,
   }
 
   if (benchmark) {
-    return true;
+    return subresult || continue_on_error;
   }
 
   if (trace) {
@@ -192,7 +192,7 @@ auto process_entry(const sourcemeta::jsonschema::InputJSON &entry,
     std::cout << "\n";
     if (!suboutput.at("valid").to_boolean()) {
       result = false;
-      if (entry.multidocument && !continue_on_error) {
+      if (!continue_on_error) {
         return false;
       }
     }
@@ -223,7 +223,7 @@ auto process_entry(const sourcemeta::jsonschema::InputJSON &entry,
     }
     sourcemeta::jsonschema::print(output, entry.positions, std::cerr);
     result = false;
-    if (entry.multidocument && !continue_on_error) {
+    if (!continue_on_error) {
       return false;
     }
   }
@@ -376,6 +376,7 @@ auto sourcemeta::jsonschema::validate(const sourcemeta::core::Options &options)
       }
     }
   } else {
+    bool proceed{true};
     for (const auto &instance_path_view : instance_arguments) {
       const std::filesystem::path instance_path{instance_path_view};
       if (trace && (instance_path.extension() == ".jsonl" ||
@@ -407,6 +408,7 @@ auto sourcemeta::jsonschema::validate(const sourcemeta::core::Options &options)
                              benchmark_loop, trace, fast_mode, json_output,
                              continue_on_error, schema_resolution_base, options,
                              result)) {
+            proceed = false;
             break;
           }
         }
@@ -458,6 +460,7 @@ auto sourcemeta::jsonschema::validate(const sourcemeta::core::Options &options)
           assert(suboutput.at("valid").is_boolean());
           if (!suboutput.at("valid").to_boolean()) {
             result = false;
+            proceed = continue_on_error;
           }
 
           sourcemeta::core::prettify(suboutput, std::cout);
@@ -476,8 +479,13 @@ auto sourcemeta::jsonschema::validate(const sourcemeta::core::Options &options)
                     << "\n";
           print(output, tracker, std::cerr);
           result = false;
+          proceed = continue_on_error;
         }
       }
+
+      if (!proceed) {
+        break;
+      }
     }
   }
 

test/CMakeLists.txt

@@ -134,6 +134,10 @@ add_jsonschema_test(validate/pass_directory)
 add_jsonschema_test(validate/pass_directory_verbose)
 add_jsonschema_test(validate/fail_directory)
 add_jsonschema_test(validate/fail_directory_verbose)
+add_jsonschema_test(validate/fail_directory_stop_verbose)
+add_jsonschema_test(validate/fail_directory_stop_json)
+add_jsonschema_test(validate/fail_directory_continue_verbose)
+add_jsonschema_test(validate/fail_directory_continue_json)
 add_jsonschema_test(validate/fail_directory_json)
 add_jsonschema_test(validate/pass_directory_extension)
 add_jsonschema_test(validate/pass_directory_extension_verbose)
@@ -244,6 +248,7 @@ add_jsonschema_test(validate/pass_many)
 add_jsonschema_test(validate/pass_many_verbose)
 add_jsonschema_test(validate/fail_many)
 add_jsonschema_test(validate/fail_many_verbose)
+add_jsonschema_test(validate/fail_many_continue_verbose)
 add_jsonschema_test(validate/fail_yaml)
 add_jsonschema_test(validate/pass_yaml_multi)
 add_jsonschema_test(validate/pass_yaml_multi_verbose)
@@ -292,6 +297,8 @@ add_jsonschema_test(validate/pass_benchmark)
 add_jsonschema_test(validate/fail_benchmark)
 add_jsonschema_test(validate/fail_benchmark_multiple)
 add_jsonschema_test(validate/fail_benchmark_directory)
+add_jsonschema_test(validate/fail_benchmark_jsonl)
+add_jsonschema_test(validate/fail_benchmark_jsonl_continue)
 add_jsonschema_test(validate/pass_benchmark_loop)
 add_jsonschema_test(validate/pass_benchmark_loop_jsonl)
 add_jsonschema_test(validate/fail_benchmark_zero)

test/validate/fail_benchmark_jsonl.clitest

@@ -0,0 +1,27 @@
+WRITE schema.json UNTIL EOF
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Test",
+  "description": "Test schema",
+  "type": "string"
+}
+EOF
+
+WRITE instance.jsonl UNTIL EOF
+"Hello World!"
+1
+"Bonjour tout le monde!"
+EOF
+
+// Validation failure
+RUN validate schema.json instance.jsonl --benchmark STDIN /dev/null IN . INTO result_0.txt EXPECTING 2
+
+REPLACE $CWD WITH '[CWD]' IN result_0.txt
+REPLACE MATCHING '[0-9]+\.[0-9]+ \+- [0-9]+\.[0-9]+ us \([0-9]+\.[0-9]+\)' WITH '[TIMING]' IN result_0.txt
+
+WRITE expected_0.txt UNTIL EOF
+1> [CWD]/instance.jsonl[1]: PASS [TIMING]
+1> [CWD]/instance.jsonl[2]: FAIL [TIMING]
+EOF
+
+COMPARE result_0.txt AGAINST expected_0.txt

test/validate/fail_benchmark_jsonl_continue.clitest

@@ -0,0 +1,28 @@
+WRITE schema.json UNTIL EOF
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Test",
+  "description": "Test schema",
+  "type": "string"
+}
+EOF
+
+WRITE instance.jsonl UNTIL EOF
+"Hello World!"
+1
+"Bonjour tout le monde!"
+EOF
+
+// Validation failure
+RUN validate schema.json instance.jsonl --benchmark --continue STDIN /dev/null IN . INTO result_0.txt EXPECTING 2
+
+REPLACE $CWD WITH '[CWD]' IN result_0.txt
+REPLACE MATCHING '[0-9]+\.[0-9]+ \+- [0-9]+\.[0-9]+ us \([0-9]+\.[0-9]+\)' WITH '[TIMING]' IN result_0.txt
+
+WRITE expected_0.txt UNTIL EOF
+1> [CWD]/instance.jsonl[1]: PASS [TIMING]
+1> [CWD]/instance.jsonl[2]: FAIL [TIMING]
+1> [CWD]/instance.jsonl[3]: PASS [TIMING]
+EOF
+
+COMPARE result_0.txt AGAINST expected_0.txt

test/validate/fail_directory_continue_json.clitest

@@ -0,0 +1,61 @@
+MAKE DIRECTORY instances
+
+WRITE schema.json UNTIL EOF
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "Test",
+  "description": "Test schema",
+  "type": "object",
+  "properties": {
+    "name": {
+      "type": "string"
+    },
+    "age": {
+      "type": "integer"
+    }
+  }
+}
+EOF
+
+WRITE instances/instance_1.json UNTIL EOF
+{ "name": "Bob", "age": "invalid" }
+EOF
+
+WRITE instances/instance_2.json UNTIL EOF
+{ "name": "Alice", "age": 30 }
+EOF
+
+// Validation failure
+RUN validate schema.json instances --json --continue STDIN /dev/null IN . INTO result_0.txt EXPECTING 2
+
+REPLACE $CWD_URI WITH '[CWD_URI]' IN result_0.txt
+REPLACE $CWD WITH '[CWD]' IN result_0.txt
+
+WRITE expected_0.txt UNTIL EOF
+1> {
+1>   "valid": false,
+1>   "errors": [
+1>     {
+1>       "keywordLocation": "/properties/age/type",
+1>       "absoluteKeywordLocation": "[CWD_URI]/schema.json#/properties/age/type",
+1>       "instanceLocation": "/age",
+1>       "instancePosition": [ 1, 18, 1, 33 ],
+1>       "error": "The value was expected to be of type integer but it was of type string"
+1>     },
+1>     {
+1>       "keywordLocation": "/properties",
+1>       "absoluteKeywordLocation": "[CWD_URI]/schema.json#/properties",
+1>       "instanceLocation": "",
+1>       "instancePosition": [ 1, 1, 1, 35 ],
+1>       "error": "The object value was expected to validate against the defined properties subschemas"
+1>     }
+1>   ]
+1> }
+1> {
+1>   "valid": true
+1> }
+2> [CWD]/instances/instance_1.json
+2> [CWD]/instances/instance_2.json
+EOF
+
+COMPARE result_0.txt AGAINST expected_0.txt

test/validate/fail_directory_continue_verbose.clitest

@@ -0,0 +1,43 @@
+MAKE DIRECTORY instances
+
+WRITE schema.json UNTIL EOF
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Test",
+  "description": "Test schema",
+  "type": "object",
+  "properties": {
+    "foo": {
+      "type": "string"
+    }
+  }
+}
+EOF
+
+WRITE instances/instance_1.json UNTIL EOF
+{ "foo": 1 }
+EOF
+
+WRITE instances/instance_2.json UNTIL EOF
+{ "foo": "bar" }
+EOF
+
+// Validation failure
+RUN validate schema.json instances --continue --verbose STDIN /dev/null IN . INTO result_0.txt EXPECTING 2
+
+REPLACE $CWD WITH '[CWD]' IN result_0.txt
+
+WRITE expected_0.txt UNTIL EOF
+2> fail: [CWD]/instances/instance_1.json
+2> error: Schema validation failure
+2>   The value was expected to be of type string but it was of type integer
+2>     at instance location "/foo" (line 1, column 3)
+2>     at evaluate path "/properties/foo/type"
+2>   The object value was expected to validate against the single defined property subschema
+2>     at instance location "" (line 1, column 1)
+2>     at evaluate path "/properties"
+2> ok: [CWD]/instances/instance_2.json
+2>   matches [CWD]/schema.json
+EOF
+
+COMPARE result_0.txt AGAINST expected_0.txt

test/validate/fail_directory_stop_json.clitest

@@ -0,0 +1,57 @@
+MAKE DIRECTORY instances
+
+WRITE schema.json UNTIL EOF
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "Test",
+  "description": "Test schema",
+  "type": "object",
+  "properties": {
+    "name": {
+      "type": "string"
+    },
+    "age": {
+      "type": "integer"
+    }
+  }
+}
+EOF
+
+WRITE instances/instance_1.json UNTIL EOF
+{ "name": "Bob", "age": "invalid" }
+EOF
+
+WRITE instances/instance_2.json UNTIL EOF
+{ "name": "Alice", "age": 30 }
+EOF
+
+// Validation failure
+RUN validate schema.json instances --json STDIN /dev/null IN . INTO result_0.txt EXPECTING 2
+
+REPLACE $CWD_URI WITH '[CWD_URI]' IN result_0.txt
+REPLACE $CWD WITH '[CWD]' IN result_0.txt
+
+WRITE expected_0.txt UNTIL EOF
+1> {
+1>   "valid": false,
+1>   "errors": [
+1>     {
+1>       "keywordLocation": "/properties/age/type",
+1>       "absoluteKeywordLocation": "[CWD_URI]/schema.json#/properties/age/type",
+1>       "instanceLocation": "/age",
+1>       "instancePosition": [ 1, 18, 1, 33 ],
+1>       "error": "The value was expected to be of type integer but it was of type string"
+1>     },
+1>     {
+1>       "keywordLocation": "/properties",
+1>       "absoluteKeywordLocation": "[CWD_URI]/schema.json#/properties",
+1>       "instanceLocation": "",
+1>       "instancePosition": [ 1, 1, 1, 35 ],
+1>       "error": "The object value was expected to validate against the defined properties subschemas"
+1>     }
+1>   ]
+1> }
+2> [CWD]/instances/instance_1.json
+EOF
+
+COMPARE result_0.txt AGAINST expected_0.txt

test/validate/fail_directory_stop_verbose.clitest

@@ -0,0 +1,41 @@
+MAKE DIRECTORY instances
+
+WRITE schema.json UNTIL EOF
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Test",
+  "description": "Test schema",
+  "type": "object",
+  "properties": {
+    "foo": {
+      "type": "string"
+    }
+  }
+}
+EOF
+
+WRITE instances/instance_1.json UNTIL EOF
+{ "foo": 1 }
+EOF
+
+WRITE instances/instance_2.json UNTIL EOF
+{ "foo": "bar" }
+EOF
+
+// Validation failure
+RUN validate schema.json instances --verbose STDIN /dev/null IN . INTO result_0.txt EXPECTING 2
+
+REPLACE $CWD WITH '[CWD]' IN result_0.txt
+
+WRITE expected_0.txt UNTIL EOF
+2> fail: [CWD]/instances/instance_1.json
+2> error: Schema validation failure
+2>   The value was expected to be of type string but it was of type integer
+2>     at instance location "/foo" (line 1, column 3)
+2>     at evaluate path "/properties/foo/type"
+2>   The object value was expected to validate against the single defined property subschema
+2>     at instance location "" (line 1, column 1)
+2>     at evaluate path "/properties"
+EOF
+
+COMPARE result_0.txt AGAINST expected_0.txt