Fix GH#338: Change file extension to .jsonl, remove trailing commas o…

…f .json files to make them adhere to JSON Lines standard (Spark default) (#351)

Commits:
fix evalTest.testEvalStrftimeOnField() .json->.jsonl
(main rebase)
fix missed trailing commas on some files
rename .json files to .jsonl as they're in json lines format, fix
test json filenames to test classes
remove trailing commas from json files

src/test/java/com/teragrep/pth10/AccumTransformationStreamingTest.java

@@ -63,8 +63,8 @@ public class AccumTransformationStreamingTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(AccumTransformationStreamingTest.class);
 
-    private final String numberDataTestFile = "src/test/resources/numberData_0*.json"; // * to make the path into a directory path
-    private final String numberDataWithMixedStringsTestFile = "src/test/resources/numberData_withMixedStrings*.json";
+    private final String numberDataTestFile = "src/test/resources/numberData_0*.jsonl"; // * to make the path into a directory path
+    private final String numberDataWithMixedStringsTestFile = "src/test/resources/numberData_withMixedStrings*.jsonl";
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),
             new StructField("_raw", DataTypes.StringType, true, new MetadataBuilder().build()),

src/test/java/com/teragrep/pth10/AddtotalsTransformationTest.java

@@ -62,7 +62,7 @@
 public class AddtotalsTransformationTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(AddtotalsTransformationTest.class);
-    private final String testFile = "src/test/resources/numberData_0*.json"; // * to make the path into a directory path
+    private final String testFile = "src/test/resources/numberData_0*.jsonl"; // * to make the path into a directory path
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),
             new StructField("_raw", DataTypes.StringType, true, new MetadataBuilder().build()),

src/test/java/com/teragrep/pth10/AggregateAfterSequentialCommandTest.java

@@ -59,7 +59,7 @@ public class AggregateAfterSequentialCommandTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(AggregateAfterSequentialCommandTest.class);
 
-    private final String testFile = "src/test/resources/rexTransformationTest_data*.json"; // * to make the path into a directory path
+    private final String testFile = "src/test/resources/rexTransformationTest_data*.jsonl"; // * to make the path into a directory path
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),
             new StructField("id", DataTypes.LongType, false, new MetadataBuilder().build()),

src/test/java/com/teragrep/pth10/BloomFilterOperationsTest.java

@@ -62,7 +62,7 @@
 public class BloomFilterOperationsTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(BloomFilterOperationsTest.class);
-    private final String testFile = "src/test/resources/xmlWalkerTestDataStreaming/bloomTeragrepStep_data*.json";
+    private final String testFile = "src/test/resources/xmlWalkerTestDataStreaming/bloomTeragrepStep_data*.jsonl";
 
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),

src/test/java/com/teragrep/pth10/CatalystVisitorTest.java

@@ -242,7 +242,7 @@ void endToEndTest() {
     )
     void endToEnd2Test() {
         // Use this file as the test data
-        String testFile = "src/test/resources/subsearchData*.json";
+        String testFile = "src/test/resources/subsearchData*.jsonl";
 
         this.streamingTestUtil.performDPLTest("index=index_A \"(1)(enTIty)\"", testFile, res -> {
             String e = "StructType(StructField(_raw,StringType,true),StructField(_time,StringType,true),StructField(host,StringType,true),StructField(index,StringType,true),StructField(offset,LongType,true),StructField(origin,StringType,true),StructField(partition,StringType,true),StructField(source,StringType,true),StructField(sourcetype,StringType,true))";

src/test/java/com/teragrep/pth10/ConvertTransformationTest.java

@@ -63,7 +63,7 @@ public class ConvertTransformationTest {
     private static final Logger LOGGER = LoggerFactory.getLogger(ConvertTransformationTest.class);
 
     // Use this file for dataset initialization
-    String testFile = "src/test/resources/convertTfData*.json"; // * to make the path into a directory path
+    String testFile = "src/test/resources/convertTfData*.jsonl"; // * to make the path into a directory path
 
     private StreamingTestUtil streamingTestUtil;
 

src/test/java/com/teragrep/pth10/DedupTransformationTest.java

@@ -67,7 +67,7 @@ public class DedupTransformationTest {
     private static final Logger LOGGER = LoggerFactory.getLogger(DedupTransformationTest.class);
 
     // Use this file for  dataset initialization
-    private String testFile = "src/test/resources/dedup_test_data*.json"; // * to make the path into a directory path
+    private String testFile = "src/test/resources/dedup_test_data*.jsonl"; // * to make the path into a directory path
 
     private StreamingTestUtil streamingTestUtil;
 
@@ -102,7 +102,7 @@ public void dedupTest_NoParams() {
                     Arrays.asList("_time", "id", "_raw", "index", "sourcetype", "host", "source", "partition", "offset")
             );
             List<String> actualColumns = Arrays.asList(res.columns());
-            // Columns should be the same. Order can be different because of .json file readStream might shuffle them
+            // Columns should be the same. Order can be different because of .jsonl file readStream might shuffle them
             Assertions
                     .assertTrue(actualColumns.containsAll(expectedColumns) && expectedColumns.containsAll(actualColumns));
 
@@ -129,7 +129,7 @@ public void dedupTest_Consecutive() {
                     Arrays.asList("_time", "id", "_raw", "index", "sourcetype", "host", "source", "partition", "offset")
             );
             List<String> actualColumns = Arrays.asList(res.columns());
-            // Columns should be the same. Order can be different because of .json file readStream might shuffle them
+            // Columns should be the same. Order can be different because of .jsonl file readStream might shuffle them
             Assertions
                     .assertTrue(actualColumns.containsAll(expectedColumns) && expectedColumns.containsAll(actualColumns));
 
@@ -155,7 +155,7 @@ public void dedupTest_SortNum() {
                     Arrays.asList("_time", "id", "_raw", "index", "sourcetype", "host", "source", "partition", "offset")
             );
             List<String> actualColumns = Arrays.asList(res.columns());
-            // Columns should be the same. Order can be different because of .json file readStream might shuffle them
+            // Columns should be the same. Order can be different because of .jsonl file readStream might shuffle them
             Assertions
                     .assertTrue(actualColumns.containsAll(expectedColumns) && expectedColumns.containsAll(actualColumns));
 
@@ -178,7 +178,7 @@ public void dedupTest_KeepEvents() {
                     Arrays.asList("_time", "id", "_raw", "index", "sourcetype", "host", "source", "partition", "offset")
             );
             List<String> actualColumns = Arrays.asList(res.columns());
-            // Columns should be the same. Order can be different because of .json file readStream might shuffle them
+            // Columns should be the same. Order can be different because of .jsonl file readStream might shuffle them
             Assertions
                     .assertTrue(actualColumns.containsAll(expectedColumns) && expectedColumns.containsAll(actualColumns));
 
@@ -206,7 +206,7 @@ public void dedupTest_KeepEmpty() {
                     Arrays.asList("_time", "id", "_raw", "index", "sourcetype", "host", "source", "partition", "offset")
             );
             List<String> actualColumns = Arrays.asList(res.columns());
-            // Columns should be the same. Order can be different because of .json file readStream might shuffle them
+            // Columns should be the same. Order can be different because of .jsonl file readStream might shuffle them
             Assertions
                     .assertTrue(actualColumns.containsAll(expectedColumns) && expectedColumns.containsAll(actualColumns));
 
@@ -231,7 +231,7 @@ public void dedupTest_MultiColumn() {
                     Arrays.asList("_time", "id", "_raw", "index", "sourcetype", "host", "source", "partition", "offset")
             );
             List<String> actualColumns = Arrays.asList(res.columns());
-            // Columns should be the same. Order can be different because of .json file readStream might shuffle them
+            // Columns should be the same. Order can be different because of .jsonl file readStream might shuffle them
             Assertions
                     .assertTrue(actualColumns.containsAll(expectedColumns) && expectedColumns.containsAll(actualColumns));
 

src/test/java/com/teragrep/pth10/EarliestLatestTest.java

@@ -77,8 +77,8 @@ public class EarliestLatestTest {
     private static final Logger LOGGER = LoggerFactory.getLogger(EarliestLatestTest.class);
 
     // Use this file for dataset initialization
-    private final String testFile = "src/test/resources/earliestLatestTest_data*.json"; // * to make the path into a directory path
-    private final String epochTestFile = "src/test/resources/earliestLatestTest_epoch_data*.json";
+    private final String testFile = "src/test/resources/earliestLatestTest_data*.jsonl"; // * to make the path into a directory path
+    private final String epochTestFile = "src/test/resources/earliestLatestTest_epoch_data*.jsonl";
     private StreamingTestUtil streamingTestUtil;
 
     @BeforeAll

src/test/java/com/teragrep/pth10/EventstatsTransformationTest.java

@@ -67,7 +67,7 @@
 public class EventstatsTransformationTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(EventstatsTransformationTest.class);
-    private final String testFile = "src/test/resources/eventstatsTransformationTest_data*.json"; // * to make the path into a directory path
+    private final String testFile = "src/test/resources/eventstatsTransformationTest_data*.jsonl"; // * to make the path into a directory path
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),
             new StructField("id", DataTypes.LongType, false, new MetadataBuilder().build()),

src/test/java/com/teragrep/pth10/FillnullTransformationTest.java

@@ -61,7 +61,7 @@ public class FillnullTransformationTest {
     private static final Logger LOGGER = LoggerFactory.getLogger(FillnullTransformationTest.class);
 
     // data has 3 empty strings ("") and 1 literal null in _raw column
-    private final String testFile = "src/test/resources/fillnull/fillnull0*.json"; // * to make the path into a directory path
+    private final String testFile = "src/test/resources/fillnull/fillnull0*.jsonl"; // * to make the path into a directory path
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),
             new StructField("_raw", DataTypes.StringType, true, new MetadataBuilder().build()),

src/test/java/com/teragrep/pth10/FormatTransformationTest.java

@@ -63,7 +63,7 @@
 public class FormatTransformationTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(FormatTransformationTest.class);
-    private final String testFile = "src/test/resources/strcatTransformationTest_data*.json"; // * to make the path into a directory path
+    private final String testFile = "src/test/resources/strcatTransformationTest_data*.jsonl"; // * to make the path into a directory path
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),
             new StructField("id", DataTypes.LongType, false, new MetadataBuilder().build()),

src/test/java/com/teragrep/pth10/IplocationTransformationTest.java

@@ -67,7 +67,7 @@
 public class IplocationTransformationTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(IplocationTransformationTest.class);
-    private final String testFile = "src/test/resources/IplocationTransformationTest_data*.json"; // * to make the path into a directory path
+    private final String testFile = "src/test/resources/IplocationTransformationTest_data*.jsonl"; // * to make the path into a directory path
 
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),

src/test/java/com/teragrep/pth10/JoinTransformationTest.java

@@ -68,7 +68,7 @@ public class JoinTransformationTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(JoinTransformationTest.class);
 
-    private String testFile = "src/test/resources/joinTransformationTest_data*.json"; // * to make the path into a directory path
+    private String testFile = "src/test/resources/joinTransformationTest_data*.jsonl"; // * to make the path into a directory path
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),
             new StructField("id", DataTypes.LongType, false, new MetadataBuilder().build()),

src/test/java/com/teragrep/pth10/MultipleAggregationsTest.java

@@ -66,7 +66,7 @@ public class MultipleAggregationsTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(MultipleAggregationsTest.class);
 
-    private final String testFile = "src/test/resources/predictTransformationTest_data*.json"; // * to make the path into a directory path
+    private final String testFile = "src/test/resources/predictTransformationTest_data*.jsonl"; // * to make the path into a directory path
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),
             new StructField("id", DataTypes.LongType, false, new MetadataBuilder().build()),

src/test/java/com/teragrep/pth10/PredictTransformationTest.java

@@ -64,7 +64,7 @@ public class PredictTransformationTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(PredictTransformationTest.class);
 
-    private final String testFile = "src/test/resources/predictTransformationTest_data*.json"; // * to make the path into a directory path
+    private final String testFile = "src/test/resources/predictTransformationTest_data*.jsonl"; // * to make the path into a directory path
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),
             new StructField("id", DataTypes.LongType, false, new MetadataBuilder().build()),

src/test/java/com/teragrep/pth10/RangemapTransformationTest.java

@@ -63,7 +63,7 @@ public class RangemapTransformationTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(RangemapTransformationTest.class);
 
-    private final String testFile = "src/test/resources/numberData_0*.json"; // * to make the path into a directory path
+    private final String testFile = "src/test/resources/numberData_0*.jsonl"; // * to make the path into a directory path
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),
             new StructField("_raw", DataTypes.StringType, true, new MetadataBuilder().build()),

src/test/java/com/teragrep/pth10/RegexTransformationTest.java

@@ -64,7 +64,7 @@ public class RegexTransformationTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(RegexTransformationTest.class);
 
-    private final String testFile = "src/test/resources/regexTransformationTest_data*.json"; // * to make the path into a directory path
+    private final String testFile = "src/test/resources/regexTransformationTest_data*.jsonl"; // * to make the path into a directory path
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),
             new StructField("id", DataTypes.LongType, false, new MetadataBuilder().build()),

src/test/java/com/teragrep/pth10/RenameTransformationTest.java

@@ -66,7 +66,7 @@ public class RenameTransformationTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(RenameTransformationTest.class);
 
-    private final String testFile = "src/test/resources/regexTransformationTest_data*.json"; // * to make the path into a directory path
+    private final String testFile = "src/test/resources/regexTransformationTest_data*.jsonl"; // * to make the path into a directory path
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),
             new StructField("id", DataTypes.LongType, false, new MetadataBuilder().build()),

src/test/java/com/teragrep/pth10/ReplaceTransformationTest.java

@@ -68,7 +68,7 @@ public class ReplaceTransformationTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(ReplaceTransformationTest.class);
 
-    private final String testFile = "src/test/resources/replaceTransformationTest_data*.json"; // * to make the path into a directory path
+    private final String testFile = "src/test/resources/replaceTransformationTest_data*.jsonl"; // * to make the path into a directory path
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),
             new StructField("id", DataTypes.LongType, false, new MetadataBuilder().build()),

src/test/java/com/teragrep/pth10/Rex4jTransformationTest.java

@@ -62,7 +62,7 @@ public class Rex4jTransformationTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(Rex4jTransformationTest.class);
 
-    private final String testFile = "src/test/resources/rex4jTransformationTest_data*.json"; // * to make the path into a directory path
+    private final String testFile = "src/test/resources/rex4jTransformationTest_data*.jsonl"; // * to make the path into a directory path
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),
             new StructField("id", DataTypes.LongType, false, new MetadataBuilder().build()),

src/test/java/com/teragrep/pth10/RexTransformationTest.java

@@ -63,7 +63,7 @@ public class RexTransformationTest {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(RexTransformationTest.class);
 
-    private final String testFile = "src/test/resources/rexTransformationTest_data*.json"; // * to make the path into a directory path
+    private final String testFile = "src/test/resources/rexTransformationTest_data*.jsonl"; // * to make the path into a directory path
     private final StructType testSchema = new StructType(new StructField[] {
             new StructField("_time", DataTypes.TimestampType, false, new MetadataBuilder().build()),
             new StructField("id", DataTypes.LongType, false, new MetadataBuilder().build()),