diff --git a/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java b/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java index cac30d3e94..68fe40aa80 100644 --- a/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java +++ b/src/main/java/org/rumbledb/compiler/RuntimeIteratorVisitor.java @@ -212,7 +212,8 @@ public RuntimeIterator visitFlowrExpression(FlworExpression expression, RuntimeI argument ), expression.getReturnClause().getHighestExecutionMode(this.visitorConfig), - expression.getReturnClause().getMetadata() + expression.getReturnClause().getMetadata(), + this.config.escapeBackticks() ); runtimeIterator.setStaticContext(expression.getStaticContext()); return runtimeIterator; @@ -236,7 +237,8 @@ private RuntimeTupleIterator visitFlowrClause( forClause.isAllowEmpty(), assignmentIterator, forClause.getHighestExecutionMode(this.visitorConfig), - clause.getMetadata() + clause.getMetadata(), + this.config.escapeBackticks() ); } else if (clause instanceof LetClause) { LetClause letClause = (LetClause) clause; @@ -247,7 +249,8 @@ private RuntimeTupleIterator visitFlowrClause( letClause.getActualSequenceType(), assignmentIterator, letClause.getHighestExecutionMode(this.visitorConfig), - clause.getMetadata() + clause.getMetadata(), + this.config.escapeBackticks() ); } else if (clause instanceof GroupByClause) { List groupingExpressions = new ArrayList<>(); @@ -306,7 +309,8 @@ private RuntimeTupleIterator visitFlowrClause( previousIterator, this.visit(((WhereClause) clause).getWhereExpression(), argument), clause.getHighestExecutionMode(this.visitorConfig), - clause.getMetadata() + clause.getMetadata(), + this.config.escapeBackticks() ); } else if (clause instanceof CountClause) { return new CountClauseSparkIterator( diff --git a/src/main/java/org/rumbledb/runtime/flwor/clauses/ForClauseSparkIterator.java b/src/main/java/org/rumbledb/runtime/flwor/clauses/ForClauseSparkIterator.java index 479c450214..436983ac4d 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/clauses/ForClauseSparkIterator.java +++ b/src/main/java/org/rumbledb/runtime/flwor/clauses/ForClauseSparkIterator.java @@ -82,6 +82,7 @@ public class ForClauseSparkIterator extends RuntimeTupleIterator { private RuntimeIterator assignmentIterator; private boolean allowingEmpty; private DataFrameContext dataFrameContext; + private final boolean escapeBackticks; // Computation state private transient DynamicContext tupleContext; // re-use same DynamicContext object for efficiency @@ -97,7 +98,8 @@ public ForClauseSparkIterator( boolean allowingEmpty, RuntimeIterator assignmentIterator, ExecutionMode executionMode, - ExceptionMetadata iteratorMetadata + ExceptionMetadata iteratorMetadata, + boolean escapeBackticks ) { super(child, executionMode, iteratorMetadata); this.variableName = variableName; @@ -106,6 +108,7 @@ public ForClauseSparkIterator( this.allowingEmpty = allowingEmpty; this.assignmentIterator.getVariableDependencies(); this.dataFrameContext = new DataFrameContext(); + this.escapeBackticks = escapeBackticks; } public Name getVariableName() { @@ -412,7 +415,8 @@ private Dataset getDataFrameFromJoin( this.variableName, this.positionalVariableName, Name.CONTEXT_ITEM, - getMetadata() + getMetadata(), + this.escapeBackticks ); } @@ -427,7 +431,8 @@ public static Dataset joinInputTupleWithSequenceOnPredicate( Name forVariableName, Name positionalVariableName, Name sequenceVariableName, - ExceptionMetadata metadata + ExceptionMetadata metadata, + boolean escapeBackticks ) { String inputDFTableName = "inputTuples"; String expressionDFTableName = "sequenceExpression"; @@ -472,7 +477,8 @@ public static Dataset joinInputTupleWithSequenceOnPredicate( Name.CONTEXT_POSITION, false, context, - startingClauseDependencies + startingClauseDependencies, + escapeBackticks ); variablesInExpressionSideTuple.add(sequenceVariableName); variablesInExpressionSideTuple.add(Name.CONTEXT_POSITION); @@ -485,7 +491,8 @@ public static Dataset joinInputTupleWithSequenceOnPredicate( null, false, context, - startingClauseDependencies + startingClauseDependencies, + escapeBackticks ); variablesInExpressionSideTuple.add(sequenceVariableName); } @@ -1004,7 +1011,8 @@ private Dataset getDataFrameStartingClause( this.positionalVariableName, this.allowingEmpty, context, - parentProjection + parentProjection, + this.escapeBackticks ); } @@ -1026,12 +1034,19 @@ public static Dataset getDataFrameStartingClause( Name positionalVariableName, boolean allowingEmpty, DynamicContext context, - Map outputDependencies + Map outputDependencies, + boolean escapeBackticks ) { Dataset df = null;; if (iterator.isDataFrame()) { Dataset rows = iterator.getDataFrame(context); + // escape backticks (`) + if (escapeBackticks) { + rows = rows.sparkSession() + .createDataFrame(rows.rdd(), FlworDataFrameUtils.escapeSchema(rows.schema(), false)); + } + rows.createOrReplaceTempView("assignment"); if (DataFrameUtils.isSequenceOfObjects(rows)) { String[] fields = rows.schema().fieldNames(); diff --git a/src/main/java/org/rumbledb/runtime/flwor/clauses/LetClauseSparkIterator.java b/src/main/java/org/rumbledb/runtime/flwor/clauses/LetClauseSparkIterator.java index 12ddd26848..b021c2c4cd 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/clauses/LetClauseSparkIterator.java +++ b/src/main/java/org/rumbledb/runtime/flwor/clauses/LetClauseSparkIterator.java @@ -72,6 +72,7 @@ public class LetClauseSparkIterator extends RuntimeTupleIterator { private RuntimeIterator assignmentIterator; private DynamicContext tupleContext; // re-use same DynamicContext object for efficiency private FlworTuple nextLocalTupleResult; + private final boolean escapeBackticks; public LetClauseSparkIterator( RuntimeTupleIterator child, @@ -79,12 +80,14 @@ public LetClauseSparkIterator( SequenceType sequenceType, RuntimeIterator assignmentIterator, ExecutionMode executionMode, - ExceptionMetadata iteratorMetadata + ExceptionMetadata iteratorMetadata, + boolean escapeBackticks ) { super(child, executionMode, iteratorMetadata); this.variableName = variableName; this.sequenceType = sequenceType; this.assignmentIterator = assignmentIterator; + this.escapeBackticks = escapeBackticks; } @Override @@ -319,7 +322,8 @@ public Dataset getDataFrameAsJoin( null, false, context, - sequenceDependencies + sequenceDependencies, + this.escapeBackticks ); System.err.println("[INFO] Rumble detected an equi-join in the left clause."); diff --git a/src/main/java/org/rumbledb/runtime/flwor/clauses/ReturnClauseSparkIterator.java b/src/main/java/org/rumbledb/runtime/flwor/clauses/ReturnClauseSparkIterator.java index cc3474a037..d3ee3bade5 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/clauses/ReturnClauseSparkIterator.java +++ b/src/main/java/org/rumbledb/runtime/flwor/clauses/ReturnClauseSparkIterator.java @@ -57,16 +57,19 @@ public class ReturnClauseSparkIterator extends HybridRuntimeIterator { private DynamicContext tupleContext; // re-use same DynamicContext object for efficiency private RuntimeIterator expression; private Item nextResult; + private final boolean escapeBackticks; public ReturnClauseSparkIterator( RuntimeTupleIterator child, RuntimeIterator expression, ExecutionMode executionMode, - ExceptionMetadata iteratorMetadata + ExceptionMetadata iteratorMetadata, + boolean escapeBackticks ) { super(Collections.singletonList(expression), executionMode, iteratorMetadata); this.child = child; this.expression = expression; + this.escapeBackticks = escapeBackticks; } @Override @@ -109,6 +112,12 @@ public JavaRDD getRDDAux(DynamicContext context) { } } Dataset df = this.child.getDataFrame(context, projection); + + // unescape backticks (`) + if (this.escapeBackticks) { + df = df.sparkSession().createDataFrame(df.rdd(), FlworDataFrameUtils.escapeSchema(df.schema(), true)); + } + StructType oldSchema = df.schema(); List UDFcolumns = FlworDataFrameUtils.getColumnNames( oldSchema, diff --git a/src/main/java/org/rumbledb/runtime/flwor/clauses/WhereClauseSparkIterator.java b/src/main/java/org/rumbledb/runtime/flwor/clauses/WhereClauseSparkIterator.java index 31dc013aa4..e18d26360f 100644 --- a/src/main/java/org/rumbledb/runtime/flwor/clauses/WhereClauseSparkIterator.java +++ b/src/main/java/org/rumbledb/runtime/flwor/clauses/WhereClauseSparkIterator.java @@ -54,16 +54,19 @@ public class WhereClauseSparkIterator extends RuntimeTupleIterator { private RuntimeIterator expression; private DynamicContext tupleContext; // re-use same DynamicContext object for efficiency private FlworTuple nextLocalTupleResult; + private final boolean escapeBackticks; public WhereClauseSparkIterator( RuntimeTupleIterator child, RuntimeIterator whereExpression, ExecutionMode executionMode, - ExceptionMetadata iteratorMetadata + ExceptionMetadata iteratorMetadata, + boolean escapeBackticks ) { super(child, executionMode, iteratorMetadata); this.expression = whereExpression; this.expression.getVariableDependencies(); + this.escapeBackticks = escapeBackticks; } @Override @@ -191,7 +194,8 @@ public Dataset getDataFrame( forVariable, null, forVariable, - getMetadata() + getMetadata(), + this.escapeBackticks ); } } diff --git a/src/main/java/org/rumbledb/runtime/typing/InstanceOfIterator.java b/src/main/java/org/rumbledb/runtime/typing/InstanceOfIterator.java index 2036d5e9d9..fc55a9db2f 100644 --- a/src/main/java/org/rumbledb/runtime/typing/InstanceOfIterator.java +++ b/src/main/java/org/rumbledb/runtime/typing/InstanceOfIterator.java @@ -186,7 +186,7 @@ public static boolean doesItemTypeMatchItem(ItemType itemType, Item itemToMatch) if (itemType.equals(BuiltinTypesCatalogue.base64BinaryItem)) { return itemToMatch.isBase64Binary(); } - if (itemType.isFunctionItemType()) { + if (itemType.equals(BuiltinTypesCatalogue.anyFunctionItem)) { return itemToMatch.isFunction(); } throw new OurBadException("Type unrecognized: " + itemType);