add option "mode" with value "FAILFAST" to JSON file reader (#350)

teragrep · Oct 8, 2024 · 945fa2b · 945fa2b
1 parent 9297dc5
commit 945fa2b
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/src/test/java/com/teragrep/pth10/StreamingTestUtil.java b/src/test/java/com/teragrep/pth10/StreamingTestUtil.java
@@ -273,15 +273,15 @@ private void internalDPLTest(
             StructType schema;
             if (this.schema == null) {
                 // notice that the schema from a JSON-file might be in an unexpected order
-                schema = spark.read().json(testDirectory).schema();
+                schema = spark.read().option("mode", "FAILFAST").json(testDirectory).schema();
             }
             else {
                 // schema was given in constructor
                 schema = this.schema;
             }
 
             // initialize test dataset
-            Dataset<Row> df = spark.readStream().schema(schema).json(testDirectory);
+            Dataset<Row> df = spark.readStream().option("mode", "FAILFAST").schema(schema).json(testDirectory);
             // apply data customizations
             df = dataCustomizations.apply(df);
             this.ctx.setDs(df);