Merge branch 'master' into Spark4.1.0

Author: ghislainfourny

.github/workflows/maven.yml

@@ -81,7 +81,8 @@ jobs:
           "XMLTests",
           "XQueryTests",
           "DeltaUpdateRuntimeTests",
-          "IcebergUpdateRuntimeTests"
+          "IcebergUpdateRuntimeTests",
+          "org.rumbledb.**.*Test"
         ]
     steps:
       - uses: actions/checkout@v5
@@ -101,7 +102,7 @@ jobs:
           name: Package 
           path: target
       - name: Run Test ${{ matrix.test_class }}
-        run: mvn -Dtest=${{ matrix.test_class }} test
+        run: mvn -Dtest='${{ matrix.test_class }}' test
       - name: Publish Surefire Report ${{ matrix.test_class }}
         if: always()
         uses: ScalableCapital/action-surefire-report@v2

.gitignore

@@ -22,20 +22,8 @@ derby.log
 src/main/java/jsound/
 src/main/resources/log4j.properties
 
-/src/main/java/org/rumbledb/parser/jsoniq/
-!/src/main/java/org/rumbledb/parser/jsoniq/JsoniqParser.g4
-!/src/main/java/org/rumbledb/parser/jsoniq/JsoniqLexer.g4
-
 # ANTLR
 *.tokens
 *.interp
 
-# ANTLR generated files for XQuery parser
-/src/main/java/org/rumbledb/parser/xquery/XQueryParser.java
-/src/main/java/org/rumbledb/parser/xquery/XQueryParserBaseVisitor.java
-/src/main/java/org/rumbledb/parser/xquery/XQueryParserVisitor.java
-
-# ANTLR generated files for XQuery lexer
-/src/main/java/org/rumbledb/parser/xquery/XQueryLexer.java
-
 .vscode/settings.json

pom.xml

@@ -26,7 +26,7 @@
 
     <groupId>com.github.rumbledb</groupId>
     <artifactId>rumbledb</artifactId>
-    <version>2.1.0</version>
+    <version>3.0.0</version>
     <packaging>jar</packaging>
     <name>RumbleDB</name>
     <description>A JSONiq engine to query large-scale JSON datasets stored on HDFS. Spark under the hood.</description>
@@ -274,6 +274,13 @@
                         </eclipse>
                         <removeUnusedImports/>
                     </java>
+                    <antlr4>
+                        <includes>
+                            <include>src/main/java/org/rumbledb/parser/*/*.g4</include>
+                        </includes>
+
+                        <antlr4Formatter />
+                    </antlr4>
                 </configuration>
             </plugin>
             <plugin>
@@ -424,8 +431,8 @@
         </dependency>
         <dependency>
             <groupId>com.esotericsoftware</groupId>
-            <artifactId>kryo</artifactId>
-            <version>5.6.0</version>
+            <artifactId>kryo-shaded</artifactId>
+            <version>4.0.3</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
@@ -525,6 +532,11 @@
             <version>${lombok.version}</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>info.picocli</groupId>
+            <artifactId>picocli</artifactId>
+            <version>4.7.7</version>
+        </dependency>
     </dependencies>
 
     <distributionManagement>

server_tests_manual.txt

@@ -0,0 +1,46 @@
+package org.rumbledb.api;
+
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.rumbledb.bindings.ItemSequenceBinding;
+import org.rumbledb.bindings.DataFrameBinding;
+import org.rumbledb.bindings.LexicalBinding;
+import org.rumbledb.context.Name;
+
+import java.util.List;
+
+public class ExternalBindings {
+    private final org.rumbledb.bindings.ExternalBindings bindings;
+
+    public ExternalBindings() {
+        this.bindings = org.rumbledb.bindings.ExternalBindings.empty();
+    }
+
+    public static ExternalBindings empty() {
+        return new ExternalBindings();
+    }
+
+    public void bindItem(String variableName, Item item) {
+        this.bindItems(variableName, List.of(item));
+    }
+
+    public void bindItems(String variableName, List<Item> items) {
+        this.bindings.bind(Name.createVariableInNoNamespace(variableName), new ItemSequenceBinding(items));
+    }
+
+    public void bindLiteral(String variableName, String value) {
+        this.bindings.bind(Name.createVariableInNoNamespace(variableName), new LexicalBinding(value));
+    }
+
+    public void bindDataFrame(String variableName, Dataset<Row> dataFrame) {
+        this.bindings.bind(Name.createVariableInNoNamespace(variableName), new DataFrameBinding(dataFrame));
+    }
+
+    public void unbind(String variableName) {
+        this.bindings.unbind(Name.createVariableInNoNamespace(variableName));
+    }
+
+    org.rumbledb.bindings.ExternalBindings getInternalBindings() {
+        return this.bindings;
+    }
+}

src/main/java/org/rumbledb/api/ExternalBindings.java

@@ -1,6 +1,5 @@
 package org.rumbledb.api;
 
-import com.esotericsoftware.kryo.KryoSerializable;
 import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.ml.Estimator;
 import org.apache.spark.ml.Transformer;
@@ -12,7 +11,7 @@
 import org.rumbledb.context.Name;
 import org.rumbledb.exceptions.DuplicateObjectKeyException;
 import org.rumbledb.exceptions.OurBadException;
-import org.rumbledb.items.structured.JSoundDataFrame;
+import org.rumbledb.items.structured.HomogeneousItemDataFrame;
 import org.rumbledb.items.xml.XMLDocumentPosition;
 import org.rumbledb.runtime.flwor.NativeClauseContext;
 import org.rumbledb.runtime.RuntimeIterator;
@@ -43,7 +42,7 @@
  *
  * @author Ghislain Fourny, Stefan Irimescu, Can Berker Cikis
  */
-public interface Item extends Serializable, KryoSerializable {
+public interface Item extends Serializable {
 
     /**
      * Makes a copy.
@@ -516,17 +515,6 @@ default boolean isObject() {
         return false;
     }
 
-    /**
-     * Returns the string keys of the item, if it is a map.
-     *
-     * @return the list of the keys.
-     * @deprecated use {@link #getStringKeys()} instead
-     */
-    @Deprecated
-    default List<String> getKeys() {
-        return this.getStringKeys();
-    }
-
     /**
      * Returns the string keys of the item, if it is a map.
      *
@@ -548,17 +536,6 @@ default List<Item> getItemKeys() throws UnsupportedOperationException {
         throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
     }
 
-    /**
-     * Returns the values of the item, if it is a map.
-     *
-     * @return the list of the value items.
-     * @deprecated use {@link #getItemValues()} instead
-     */
-    @Deprecated
-    default List<Item> getValues() {
-        return this.getItemValues();
-    }
-
     /**
      * Returns the values of the item, if it is a map.
      *
@@ -784,17 +761,6 @@ default boolean hasKey(Item key) throws UnsupportedOperationException {
         throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
     }
 
-    /**
-     * Returns the members of the item if it is an array.
-     *
-     * @return the list of the array members.
-     * @deprecated use {@link #getItemMembers()} instead
-     */
-    @Deprecated
-    default List<Item> getItems() {
-        return this.getItemMembers();
-    }
-
     /**
      * Returns the members of the item, if it is an array.
      *
@@ -846,17 +812,6 @@ default List<Item> getSequenceAt(int position)
         throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
     }
 
-    /**
-     * Appends an item to the item, if it is an array.
-     *
-     * @param item the item to append.
-     * @throws UnsupportedOperationException if the item is not an array.
-     * @deprecated use {@link #appendItem(Item)} instead
-     */
-    default void append(Item item) throws UnsupportedOperationException {
-        this.appendItem(item);
-    }
-
     /**
      * Appends an item to the item, if it is an array.
      *
@@ -1032,6 +987,18 @@ default Duration getDurationValue() {
         throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
     }
 
+    /**
+     * Returns the day-time (seconds) component of a duration item, i.e., the value of the
+     * duration excluding its months component. Unlike getDurationValue(), this is not
+     * anchored to any reference date, since days, hours, minutes and seconds have a fixed
+     * length and can be converted to a Duration unambiguously.
+     *
+     * @return the day-time component as a Duration.
+     */
+    default Duration getDayTimeDurationComponent() {
+        throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
+    }
+
     /**
      * Returns the EpochMillis of the item, if it's DateTime or Duration
      * It will collect all the parts of the item and compress it into the EpochMillis
@@ -1137,7 +1104,7 @@ default Map<Name, JavaRDD<Item>> getRDDVariablesInClosure() {
      * 
      * @return the function signature.
      */
-    default Map<Name, JSoundDataFrame> getDFVariablesInClosure() {
+    default Map<Name, HomogeneousItemDataFrame> getDFVariablesInClosure() {
         throw new UnsupportedOperationException("Operation not defined for type " + this.getDynamicType());
     }