diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java index 2ca0be4301..7e3debf1c3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java @@ -197,6 +197,11 @@ public Map getRecordTypes() { return recordTypes; } + @Nonnull + public Map getUserDefinedFunctionMap() { + return userDefinedFunctionMap; + } + @Nonnull public RecordType getRecordType(@Nonnull String name) { RecordType recordType = recordTypes.get(name); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java index cf08a9473f..79836df6b6 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java @@ -228,9 +228,10 @@ private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDa typeBuilder.setRecordTypeKey(LiteralKeyExpression.fromProtoValue(typeProto.getExplicitKey())); } } + PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, + PlanHashable.CURRENT_FOR_CONTINUATION); for (RecordMetaDataProto.PUserDefinedFunction function: metaDataProto.getUserDefinedFunctionsList()) { - UserDefinedFunction func = (UserDefinedFunction)PlanSerialization.dispatchFromProtoContainer(new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, - PlanHashable.CURRENT_FOR_CONTINUATION), function); + UserDefinedFunction func = (UserDefinedFunction)PlanSerialization.dispatchFromProtoContainer(serializationContext, function); userDefinedFunctionMap.put(func.getFunctionName(), func); } if (metaDataProto.hasSplitLongRecords()) { diff --git a/fdb-relational-core/src/main/antlr/RelationalParser.g4 b/fdb-relational-core/src/main/antlr/RelationalParser.g4 index af53567851..80986af8b2 100644 --- a/fdb-relational-core/src/main/antlr/RelationalParser.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalParser.g4 @@ -88,7 +88,7 @@ utilityStatement templateClause : - CREATE ( structDefinition | tableDefinition | enumDefinition | indexDefinition ) + CREATE ( structDefinition | tableDefinition | enumDefinition | indexDefinition | functionDefinition) ; createStatement @@ -154,6 +154,10 @@ indexDefinition : (UNIQUE)? INDEX indexName=uid AS queryTerm indexAttributes? ; +functionDefinition + : FUNCTION functionName=uid LEFT_ROUND_BRACKET paramName=uid inputTypeName=columnType RIGHT_ROUND_BRACKET RETURNS columnType AS fullId + ; + indexAttributes : WITH ATTRIBUTES indexAttribute (COMMA indexAttribute)* ; @@ -560,6 +564,11 @@ uid | DOUBLE_QUOTE_ID ; +userDefinedFunctionName + : ID + | DOUBLE_QUOTE_ID + ; + // done simpleId : ID @@ -789,6 +798,7 @@ functionCall : aggregateWindowedFunction #aggregateFunctionCall // done (supported) | specificFunction #specificFunctionCall // | scalarFunctionName '(' functionArgs? ')' #scalarFunctionCall // done (unsupported) + | userDefinedFunctionName '(' functionArgs? ')' #userDefinedFunctionCall ; specificFunction @@ -899,7 +909,7 @@ levelInWeightListElement ; aggregateWindowedFunction - : functionName=(AVG | MAX | MIN | SUM | MAX_EVER | MIN_EVER ) + : functionName=(AVG | MAX | MIN | SUM | MAX_EVER | MIN_EVER) '(' aggregator=(ALL | DISTINCT)? functionArg ')' overClause? | functionName=BITMAP_CONSTRUCT_AGG '(' functionArg ')' | functionName=COUNT '(' (starArg='*' | aggregator=ALL? functionArg | aggregator=DISTINCT functionArgs) ')' overClause? diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java index fd9c87ba0b..c0c9f2e8e6 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java @@ -25,6 +25,7 @@ import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.query.combinatorics.TopologicalSort; +import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.exceptions.RelationalException; @@ -50,6 +51,7 @@ import java.util.ArrayList; import java.util.BitSet; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -68,6 +70,9 @@ public final class RecordLayerSchemaTemplate implements SchemaTemplate { @Nonnull private final Set tables; + @Nonnull + private final Map userDefinedFunctionMap; + private final int version; private final boolean enableLongRows; @@ -85,11 +90,13 @@ public final class RecordLayerSchemaTemplate implements SchemaTemplate { private RecordLayerSchemaTemplate(@Nonnull final String name, @Nonnull final Set tables, + @Nonnull final Map userDefinedFunctionMap, int version, boolean enableLongRows, boolean storeRowVersions) { this.name = name; this.tables = tables; + this.userDefinedFunctionMap = userDefinedFunctionMap; this.version = version; this.enableLongRows = enableLongRows; this.storeRowVersions = storeRowVersions; @@ -100,6 +107,7 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, private RecordLayerSchemaTemplate(@Nonnull final String name, @Nonnull final Set tables, + @Nonnull final Map userDefinedFunctionMap, int version, boolean enableLongRows, boolean storeRowVersions, @@ -107,6 +115,7 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, this.name = name; this.version = version; this.tables = tables; + this.userDefinedFunctionMap = userDefinedFunctionMap; this.enableLongRows = enableLongRows; this.storeRowVersions = storeRowVersions; this.metaDataSupplier = Suppliers.memoize(() -> cachedMetadata); @@ -147,6 +156,11 @@ public RecordLayerSchema generateSchema(@Nonnull String databaseId, @Nonnull Str return new RecordLayerSchema(schemaName, databaseId, this); } + @Nonnull + public Collection getAllUserDefinedFunctions() { + return userDefinedFunctionMap.values(); + } + @Nonnull public Descriptors.Descriptor getDescriptor(@Nonnull final String tableName) { return toRecordMetadata().getRecordType(tableName).getDescriptor(); @@ -310,12 +324,15 @@ public static final class Builder { private final Map tables; + private final Map functionMap; + private final Map auxiliaryTypes; // for quick lookup private RecordMetaData cachedMetadata; private Builder() { tables = new LinkedHashMap<>(); + functionMap = new HashMap<>(); auxiliaryTypes = new LinkedHashMap<>(); // enable long rows is TRUE by default enableLongRows = true; @@ -403,6 +420,18 @@ public Builder addAuxiliaryTypes(@Nonnull Collection auxiliaryTy return this; } + @Nonnull + public Builder addUserDefinedFunction(@Nonnull UserDefinedFunction userDefinedFunction) { + functionMap.put(userDefinedFunction.getFunctionName(), userDefinedFunction); + return this; + } + + @Nonnull + public Builder addUserDefinedFunctions(@Nonnull Collection functions) { + functions.forEach(this::addUserDefinedFunction); + return this; + } + @Nonnull Builder setCachedMetadata(@Nonnull final RecordMetaData metadata) { this.cachedMetadata = metadata; @@ -460,11 +489,10 @@ public RecordLayerSchemaTemplate build() { if (needsResolution) { resolveTypes(); } - if (cachedMetadata != null) { - return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), version, enableLongRows, storeRowVersions, cachedMetadata); + return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), functionMap, version, enableLongRows, storeRowVersions, cachedMetadata); } else { - return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), version, enableLongRows, storeRowVersions); + return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), functionMap, version, enableLongRows, storeRowVersions); } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java index c900af303d..1db60804f8 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java @@ -61,6 +61,9 @@ public RecordLayerSchemaTemplate.Builder getSchemaTemplate(@Nonnull final String .setEnableLongRows(recordMetaData.isSplitLongRecords()) .setName(schemaTemplateName) .setIntermingleTables(!recordMetaData.primaryKeyHasRecordTypePrefix()); + for (final var u: recordMetaData.getUserDefinedFunctionMap().values()) { + schemaTemplateBuilder.addUserDefinedFunction(u); + } final var nameToTableBuilder = new HashMap(); for (final var registeredType : registeredTypes) { switch (registeredType.getType()) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java index a668315e41..e31f834517 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java @@ -87,6 +87,7 @@ public void visit(@Nonnull com.apple.foundationdb.relational.api.metadata.Index @Override public void visit(@Nonnull SchemaTemplate schemaTemplate) { Assert.thatUnchecked(schemaTemplate instanceof RecordLayerSchemaTemplate); + getBuilder().addUserDefinedFunctions(((RecordLayerSchemaTemplate) schemaTemplate).getAllUserDefinedFunctions()); getBuilder().setSplitLongRecords(schemaTemplate.isEnableLongRows()); getBuilder().setStoreRecordVersions(schemaTemplate.isStoreRowVersions()); getBuilder().setVersion(schemaTemplate.getVersion()); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Identifier.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Identifier.java index 0987c0394d..f4b5b50eba 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Identifier.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Identifier.java @@ -128,6 +128,13 @@ public boolean prefixedWith(@Nonnull Identifier identifier) { return true; } + @Nonnull + public List removePrefix(@Nonnull Identifier prefix) { + // assume the identifier has the prefix, should call prefixedWith(prefix) to check before calling this method + final var fullName = fullyQualifiedName(); + return fullName.subList(prefix.fullyQualifiedName().size(), fullName.size()); + } + public boolean qualifiedWith(@Nonnull Identifier identifier) { final var identifierFullName = identifier.fullyQualifiedName(); final var fullName = fullyQualifiedName(); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index d41445fc29..b55608aed6 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -28,6 +28,7 @@ import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.IndexAccessHint; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; import com.apple.foundationdb.record.query.plan.cascades.values.AggregateValue; @@ -38,6 +39,7 @@ import com.apple.foundationdb.record.query.plan.cascades.values.IndexableAggregateValue; import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; import com.apple.foundationdb.record.query.plan.cascades.values.NotValue; +import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; import com.apple.foundationdb.record.query.plan.cascades.values.RelOpValue; import com.apple.foundationdb.record.query.plan.cascades.values.StreamableAggregateValue; @@ -51,6 +53,7 @@ import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; import com.apple.foundationdb.relational.recordlayer.query.functions.SqlFunctionCatalog; import com.apple.foundationdb.relational.recordlayer.query.functions.SqlFunctionCatalogImpl; import com.apple.foundationdb.relational.recordlayer.query.visitors.QueryVisitor; @@ -100,6 +103,10 @@ public SemanticAnalyzer(@Nonnull SchemaTemplate metadataCatalog, @Nonnull SqlFunctionCatalog functionCatalog) { this.metadataCatalog = metadataCatalog; this.functionCatalog = functionCatalog; + // add UDFs to functionCatalog + for (UserDefinedFunction function: ((RecordLayerSchemaTemplate) metadataCatalog).getAllUserDefinedFunctions()) { + (this.functionCatalog).addUdfFunction(function); + } } /** @@ -473,6 +480,46 @@ public Optional lookupNestedField(@Nonnull Identifier requestedIdent return Optional.of(nestedAttribute); } + @Nonnull + public Optional lookupNestedField(@Nonnull Identifier requestedIdentifier, + @Nonnull Identifier paramId, + @Nonnull QuantifiedObjectValue existingValue, + @Nonnull DataType targetDataType) { + Assert.thatUnchecked(requestedIdentifier.prefixedWith(paramId), "Invalid function definition"); + + // x -> x + if (requestedIdentifier.fullyQualifiedName().size() == paramId.fullyQualifiedName().size()) { + Assert.thatUnchecked(existingValue.getResultType().equals(DataTypeUtils.toRecordLayerType(targetDataType)), ErrorCode.DATATYPE_MISMATCH, "Result data types don't match!"); + return Optional.of(existingValue); + } + // find nested field path + final var remainingPath = requestedIdentifier.removePrefix(paramId); + final ImmutableList.Builder accessors = ImmutableList.builder(); + DataType existingDataType = DataTypeUtils.toRelationalType(existingValue.getResultType()); + for (String s : remainingPath) { + if (existingDataType.getCode() != DataType.Code.STRUCT) { + return Optional.empty(); + } + final var fields = ((DataType.StructType) existingDataType).getFields(); + var found = false; + for (int j = 0; j < fields.size(); j++) { + if (fields.get(j).getName().equals(s)) { + accessors.add(new FieldValue.Accessor(fields.get(j).getName(), j)); + existingDataType = fields.get(j).getType(); + found = true; + break; + } + } + if (!found) { + return Optional.empty(); + } + } + final var fieldPath = FieldValue.resolveFieldPath(existingValue.getResultType(), accessors.build()); + final var fieldValue = FieldValue.ofFieldsAndFuseIfPossible(existingValue, fieldPath); + Assert.thatUnchecked(fieldValue.getResultType().equals(DataTypeUtils.toRecordLayerType(targetDataType)), ErrorCode.DATATYPE_MISMATCH, "Result data types don't match!"); + return Optional.of(fieldValue); + } + @Nonnull public DataType lookupType(@Nonnull Identifier typeIdentifier, boolean isNullable, boolean isRepeated, @Nonnull Function> dataTypeProvider) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java index 47d09261dc..fcfed9362c 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java @@ -20,17 +20,19 @@ package com.apple.foundationdb.relational.recordlayer.query.functions; -import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; -import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; +import com.apple.foundationdb.record.query.plan.cascades.CatalogedFunction; +import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction; import com.apple.foundationdb.relational.recordlayer.query.Expression; import javax.annotation.Nonnull; public interface SqlFunctionCatalog { @Nonnull - BuiltInFunction lookUpFunction(@Nonnull String name, @Nonnull Expression... expressions); + CatalogedFunction lookUpFunction(@Nonnull String name, @Nonnull Expression... expressions); boolean containsFunction(@Nonnull String name); boolean isUdfFunction(@Nonnull String name); + + void addUdfFunction(@Nonnull UserDefinedFunction function); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java index 31ee33c108..3d148ea353 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java @@ -23,6 +23,8 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; +import com.apple.foundationdb.record.query.plan.cascades.CatalogedFunction; +import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction; import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; import com.apple.foundationdb.record.query.plan.cascades.values.FunctionCatalog; import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; @@ -33,7 +35,9 @@ import com.google.common.collect.ImmutableMap; import javax.annotation.Nonnull; +import java.util.HashMap; import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.function.Function; import java.util.stream.StreamSupport; @@ -47,24 +51,32 @@ public final class SqlFunctionCatalogImpl implements SqlFunctionCatalog { @Nonnull - private static final SqlFunctionCatalogImpl INSTANCE = new SqlFunctionCatalogImpl(); + public static final SqlFunctionCatalogImpl INSTANCE = new SqlFunctionCatalogImpl(); @Nonnull private final ImmutableMap>> synonyms; + @Nonnull + private final Map userDefinedFunctionMap = new HashMap<>(); + + private SqlFunctionCatalogImpl() { this.synonyms = createSynonyms(); } @Nonnull @Override - public BuiltInFunction lookUpFunction(@Nonnull final String name, @Nonnull final Expression... expressions) { - return Assert.notNullUnchecked(Objects.requireNonNull(synonyms.get(name.toLowerCase(Locale.ROOT))).apply(expressions.length)); + public CatalogedFunction lookUpFunction(@Nonnull final String name, @Nonnull final Expression... expressions) { + if (synonyms.get(name.toLowerCase(Locale.ROOT)) != null) { + return Assert.notNullUnchecked(Objects.requireNonNull(synonyms.get(name.toLowerCase(Locale.ROOT))).apply(expressions.length)); + } else { + return userDefinedFunctionMap.get(name.toLowerCase(Locale.ROOT)); + } } @Override public boolean containsFunction(@Nonnull String name) { - return synonyms.containsKey(name.toLowerCase(Locale.ROOT)); + return synonyms.containsKey(name.toLowerCase(Locale.ROOT)) || userDefinedFunctionMap.containsKey(name.toLowerCase(Locale.ROOT)); } @Override @@ -72,6 +84,11 @@ public boolean isUdfFunction(@Nonnull final String name) { return "java_call".equals(name.trim().toLowerCase(Locale.ROOT)); } + @Override + public void addUdfFunction(@Nonnull final UserDefinedFunction function) { + userDefinedFunctionMap.put(function.getFunctionName(), function); + } + @Nonnull private static ImmutableMap>> createSynonyms() { return ImmutableMap.>>builder() diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java index cfbb11fb9e..4429a586ca 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java @@ -391,6 +391,18 @@ public RecordLayerIndex visitIndexDefinition(@Nonnull RelationalParser.IndexDefi return ddlVisitor.visitIndexDefinition(ctx); } + @Nonnull + @Override + public Object visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { + return ddlVisitor.visitFunctionDefinition(ctx); + } + + @Nonnull + @Override + public Expression visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx) { + return expressionVisitor.visitUserDefinedFunctionCall(ctx); + } + @Override public Object visitIndexAttributes(RelationalParser.IndexAttributesContext ctx) { return visitChildren(ctx); @@ -1325,6 +1337,12 @@ public Object visitScalarFunctionName(@Nonnull RelationalParser.ScalarFunctionNa return visitChildren(ctx); } + @Nonnull + @Override + public Object visitUserDefinedFunctionName(@Nonnull RelationalParser.UserDefinedFunctionNameContext ctx) { + return visitChildren(ctx); + } + @Nonnull @Override public Expressions visitFunctionArgs(@Nonnull RelationalParser.FunctionArgsContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index 990a4d960a..048cccd413 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -22,12 +22,18 @@ import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; +import com.apple.foundationdb.record.query.plan.cascades.MacroFunction; +import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalSortExpression; +import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.relational.api.Options; import com.apple.foundationdb.relational.api.ddl.MetadataOperationsFactory; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.generated.RelationalParser; +import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerColumn; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; @@ -178,6 +184,31 @@ public RecordLayerIndex visitIndexDefinition(@Nonnull RelationalParser.IndexDefi return generator.generate(indexId.getName(), isUnique, table.getType(), containsNullableArray); } + @Nonnull + @Override + public UserDefinedFunction visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { + final var ddlCatalog = metadataBuilder.build(); + // parse the function definition using the newly constructed metadata. + getDelegate().replaceCatalog(ddlCatalog); + final var semanticAnalyzer = getDelegate().getSemanticAnalyzer(); + + // argumentValue + final var inputColumnId = ctx.columnType(0).customType != null ? visitUid(ctx.columnType(0).customType) : Identifier.of(ctx.columnType(0).getText()); + final var columnType = semanticAnalyzer.lookupType(inputColumnId, true, false, metadataBuilder::findType); + QuantifiedObjectValue argumentValue = QuantifiedObjectValue.of(CorrelationIdentifier.uniqueID(), DataTypeUtils.toRecordLayerType(columnType)); + + // function return type + final var returnColumnId = ctx.columnType(1).customType != null ? visitUid(ctx.columnType(1).customType) : Identifier.of(ctx.columnType(1).getText()); + final var returnType = semanticAnalyzer.lookupType(returnColumnId, true, false, metadataBuilder::findType); + + final var functionBody = visitFullId(ctx.fullId()); + final var paramNameId = Identifier.of(ctx.paramName.getText().toUpperCase(Locale.ROOT)); + + Optional fieldValue = semanticAnalyzer.lookupNestedField(functionBody, paramNameId, argumentValue, returnType); + Assert.thatUnchecked(fieldValue.isPresent(), "couldn't resolve function definition"); + return new MacroFunction(ctx.functionName.getText(), List.of(argumentValue), fieldValue.get()); + } + @Nonnull @Override public DataType.Named visitEnumDefinition(@Nonnull RelationalParser.EnumDefinitionContext ctx) { @@ -214,6 +245,7 @@ public ProceduralPlan visitCreateSchemaTemplateStatement(@Nonnull RelationalPars final ImmutableSet.Builder structClauses = ImmutableSet.builder(); final ImmutableSet.Builder tableClauses = ImmutableSet.builder(); final ImmutableSet.Builder indexClauses = ImmutableSet.builder(); + final ImmutableSet.Builder functionClauses = ImmutableSet.builder(); for (final var templateClause : ctx.templateClause()) { if (templateClause.enumDefinition() != null) { metadataBuilder.addAuxiliaryType(visitEnumDefinition(templateClause.enumDefinition())); @@ -221,6 +253,8 @@ public ProceduralPlan visitCreateSchemaTemplateStatement(@Nonnull RelationalPars structClauses.add(templateClause.structDefinition()); } else if (templateClause.tableDefinition() != null) { tableClauses.add(templateClause.tableDefinition()); + } else if (templateClause.functionDefinition() != null) { + functionClauses.add(templateClause.functionDefinition()); } else { Assert.thatUnchecked(templateClause.indexDefinition() != null); indexClauses.add(templateClause.indexDefinition()); @@ -234,6 +268,8 @@ public ProceduralPlan visitCreateSchemaTemplateStatement(@Nonnull RelationalPars final var tableWithIndex = RecordLayerTable.Builder.from(table).addIndex(index).build(); metadataBuilder.addTable(tableWithIndex); } + final var udfs = functionClauses.build().stream().map(this::visitFunctionDefinition).collect(ImmutableList.toImmutableList()); + metadataBuilder.addUserDefinedFunctions(udfs); return ProceduralPlan.of(metadataOperationsFactory.getCreateSchemaTemplateConstantAction(metadataBuilder.build(), Options.NONE)); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java index 4671921f8c..8ed47b4842 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java @@ -229,6 +229,18 @@ public RecordLayerIndex visitIndexDefinition(@Nonnull RelationalParser.IndexDefi return getDelegate().visitIndexDefinition(ctx); } + @Nonnull + @Override + public Object visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { + return getDelegate().visitFunctionDefinition(ctx); + } + + @Nonnull + @Override + public Expression visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx) { + return getDelegate().visitUserDefinedFunctionCall(ctx); + } + @Nonnull @Override public Object visitIndexAttributes(@Nonnull RelationalParser.IndexAttributesContext ctx) { @@ -1171,6 +1183,12 @@ public Object visitScalarFunctionName(@Nonnull RelationalParser.ScalarFunctionNa return getDelegate().visitScalarFunctionName(ctx); } + @Nonnull + @Override + public Object visitUserDefinedFunctionName(@Nonnull RelationalParser.UserDefinedFunctionNameContext ctx) { + return getDelegate().visitUserDefinedFunctionName(ctx); + } + @Nonnull @Override public Expressions visitFunctionArgs(@Nonnull RelationalParser.FunctionArgsContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index 42c7011c06..a90172e48e 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -249,6 +249,14 @@ public Expression visitCaseFunctionCall(@Nonnull RelationalParser.CaseFunctionCa return Expression.ofUnnamed(new PickValue(new ConditionSelectorValue(implications.build()), pickerValues.build())); } + @Nonnull + @Override + public Expression visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx) { + final var functionName = ctx.userDefinedFunctionName().getText(); + Expressions arguments = visitFunctionArgs(ctx.functionArgs()); + return getDelegate().resolveFunction(functionName, arguments.asList().toArray(new Expression[0])); + } + @Nonnull @Override public Expression visitFunctionCallExpressionAtom(@Nonnull RelationalParser.FunctionCallExpressionAtomContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java index 55acc9f7c2..65fc12cf56 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java @@ -164,6 +164,10 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override RecordLayerIndex visitIndexDefinition(@Nonnull RelationalParser.IndexDefinitionContext ctx); + @Nonnull + @Override + Object visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx); + @Override Object visitIndexAttributes(RelationalParser.IndexAttributesContext ctx); @@ -702,6 +706,10 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override Expression visitScalarFunctionCall(@Nonnull RelationalParser.ScalarFunctionCallContext ctx); + @Nonnull + @Override + Expression visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx); + @Nonnull @Override Object visitSimpleFunctionCall(@Nonnull RelationalParser.SimpleFunctionCallContext ctx); @@ -786,6 +794,10 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override Object visitScalarFunctionName(@Nonnull RelationalParser.ScalarFunctionNameContext ctx); + @Nonnull + @Override + Object visitUserDefinedFunctionName(@Nonnull RelationalParser.UserDefinedFunctionNameContext ctx); + @Nonnull @Override Expressions visitFunctionArgs(@Nonnull RelationalParser.FunctionArgsContext ctx); diff --git a/fdb-relational-core/src/main/proto/continuation.proto b/fdb-relational-core/src/main/proto/continuation.proto index 4b5dfae0d3..470e9dc7e2 100644 --- a/fdb-relational-core/src/main/proto/continuation.proto +++ b/fdb-relational-core/src/main/proto/continuation.proto @@ -22,7 +22,6 @@ syntax = "proto3"; package com.apple.foundationdb.relational.continuation; import "record_metadata.proto"; import "record_query_plan.proto"; -import "record_query_runtime.proto"; option java_multiple_files = true; option java_package = "com.apple.foundationdb.relational.continuation"; diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java index 61000544f7..39a9bdd2a9 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java @@ -881,6 +881,115 @@ void aliasingTableToResolveAmbiguityWorks() throws Exception { } } + @Test + void testUserDefinedFunction() throws Exception { + // "name" is a reserved keyword, users need to put a double quote around it + final String schemaTemplate = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + + "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + + "CREATE FUNCTION lat(x Location) RETURNS string AS x.coord.latitude\n" + + "CREATE FUNCTION \"name\"(x Location) RETURNS string AS x.name\n" + + "CREATE FUNCTION id(x bigint) RETURNS bigint AS x\n"; + + try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { + try (var s = ddl.setSchemaAndGetConnection().createStatement()) { + insertLocationComplexRecord(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); + } + try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE \"name\"(loc) = ?name")) { + ps.setString("name", "Apple Park Visitor Center"); + try (final RelationalResultSet resultSet = ps.executeQuery()) { + ResultSetAssert.assertThat(resultSet) + .hasNextRow() + .hasColumn("UID", 1L) + .hasNoNextRow(); + } + } + } + } + + @Test + void testUserDefinedFunction2() throws Exception { + final String schemaTemplate = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + + "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + + "CREATE FUNCTION lat(x Location) RETURNS string AS x.coord.latitude\n" + + "CREATE FUNCTION \"name\"(x Location) RETURNS string AS x.name\n" + + "CREATE FUNCTION id(x bigint) RETURNS bigint AS x\n"; + + try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { + try (var s = ddl.setSchemaAndGetConnection().createStatement()) { + insertLocationComplexRecord(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); + } + try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE lat(loc) = ?loc")) { + ps.setString("loc", "37.3"); + try (final RelationalResultSet resultSet = ps.executeQuery()) { + ResultSetAssert.assertThat(resultSet).hasNextRow(); + } + } + } + } + + @Test + void testUserDefinedFunction3() throws Exception { + final String schemaTemplate = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + + "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + + "CREATE FUNCTION lat(x Location) RETURNS string AS x.coord.latitude\n" + + "CREATE FUNCTION \"name\"(x Location) RETURNS string AS x.name\n" + + "CREATE FUNCTION id(x bigint) RETURNS bigint AS x"; + + try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { + try (var s = ddl.setSchemaAndGetConnection().createStatement()) { + insertLocationComplexRecord(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); + } + try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE id(uid) = ?id")) { + ps.setLong("id", 1L); + try (final RelationalResultSet resultSet = ps.executeQuery()) { + ResultSetAssert.assertThat(resultSet) + .hasNextRow() + .hasColumn("UID", 1L) + .hasNoNextRow(); + } + } + } + } + + @Test + void testIncorrectUserDefinedFunction() throws Exception { + final String schemaTemplate1 = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + + "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + + "CREATE FUNCTION lat(x Location) RETURNS string AS x.coor.latitude\n"; // "coor" is wrong + + // fail to build the MacroFunctionValue in the DDL step + final var errorMsg1 = Assertions.assertThrows(SQLException.class, () -> Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate1).build()).getMessage(); + Assertions.assertTrue(errorMsg1.contains("couldn't resolve function")); + + final String schemaTemplate2 = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + + "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + + "CREATE FUNCTION id(x bigint) RETURNS string AS x\n"; // wrong return type + // fail to build the MacroFunctionValue in the DDL step + final var errorMsg2 = Assertions.assertThrows(SQLException.class, () -> Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate2).build()).getMessage(); + Assertions.assertTrue(errorMsg2.contains("Result data types don't match")); + + final String schemaTemplate3 = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + + "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + + "CREATE FUNCTION name(x Location) RETURNS string AS x.name\n"; // no double quote around name + + try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate3).build()) { + try (var s = ddl.setSchemaAndGetConnection().createStatement()) { + insertLocationComplexRecord(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); + } + try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE name(loc) = ?name")) { + ps.setString("name", "Apple Park Visitor Center"); + final var errorMsg3 = Assertions.assertThrows(SQLException.class, ps::executeQuery).getMessage(); + Assertions.assertTrue(errorMsg3.contains("syntax error")); + } + } + } + @Test void testBitmap() throws Exception { final String query = "SELECT BITMAP_CONSTRUCT_AGG(BITMAP_BIT_POSITION(uid)) as bitmap, category, BITMAP_BUCKET_OFFSET(uid) as offset FROM T1\n" + @@ -1531,4 +1640,20 @@ private RelationalStruct insertRestaurantComplexRecord(RelationalStatement s, in Assertions.assertEquals(1, cnt, "Incorrect insertion count"); return struct; } + + private void insertLocationComplexRecord(@Nonnull RelationalStatement s, long uid, @Nonnull final String name, @Nonnull String latitude, @Nonnull String longitude) throws SQLException { + var coord = EmbeddedRelationalStruct.newBuilder() + .addString("LATITUDE", latitude) + .addString("LONGITUDE", longitude) + .build(); + var struct = EmbeddedRelationalStruct.newBuilder() + .addLong("UID", uid) + .addStruct("LOC", EmbeddedRelationalStruct.newBuilder() + .addString("NAME", name) + .addStruct("COORD", coord) + .build()) + .build(); + int cnt = s.executeInsert("T1", struct); + Assertions.assertEquals(1, cnt, "Incorrect insertion count"); + } }