Enhancement #3028: Add verbosity-level option to SQL EXPLAIN

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/explain/ExplainPlanVisitor.java

@@ -722,6 +722,14 @@ public static String toStringForExternalExplain(@Nonnull final RecordQueryPlan p
         return toStringForExternalExplain(plan, ExplainLevel.ALL_DETAILS, Integer.MAX_VALUE);
     }
 
+    @Nonnull
+    public static String toStringForExternalExplain(@Nonnull final RecordQueryPlan plan, final int explainLevel) {
+        final var visitor = new ExplainPlanVisitor(Integer.MAX_VALUE);
+        final var explainTokens = visitor.visit(plan);
+        return explainTokens.render(explainLevel, new DefaultExplainFormatter(ExplainSelfContainedSymbolMap::new), explainTokens.getMaxLength(explainLevel))
+                .toString();
+    }
+
     @Nonnull
     public static String toStringForExternalExplain(@Nonnull final RecordQueryPlan plan, final int maxExplainLevel,
                                                     final int maxSize) {

fdb-relational-core/src/main/antlr/RelationalLexer.g4

@@ -147,6 +147,7 @@ MASTER_BIND:                         'MASTER_BIND';
 MASTER_SSL_VERIFY_SERVER_CERT:       'MASTER_SSL_VERIFY_SERVER_CERT';
 MATCH:                               'MATCH';
 MAXVALUE:                            'MAXVALUE';
+MINIMAL:                             'MINIMAL';
 MODIFIES:                            'MODIFIES';
 NATURAL:                             'NATURAL';
 NOT:                                 'NOT';
@@ -223,6 +224,7 @@ USAGE:                               'USAGE';
 USE:                                 'USE';
 USING:                               'USING';
 VALUES:                              'VALUES';
+VERBOSE:                             'VERBOSE';
 WHEN:                                'WHEN';
 WHERE:                               'WHERE';
 WHILE:                               'WHILE';
@@ -1352,4 +1354,4 @@ fragment DECIMAL_TYPE_MODIFIER:      (INT_TYPE_MODIFIER | LONG_TYPE_MODIFIER);
 
 ERROR_RECOGNITION
     : . { this.notifyListeners(new LexerNoViableAltException(this, _input, _tokenStartCharIndex, null)); }
-    ;
+    ;

fdb-relational-core/src/main/antlr/RelationalParser.g4

@@ -645,7 +645,8 @@ helpStatement
 // details
 
 describeObjectClause
-    : (
+    : level=(VERBOSE | MINIMAL)?
+      (
         query | deleteStatement | insertStatement
         | updateStatement | executeContinuationStatement
       )                                                             #describeStatements

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java

@@ -25,6 +25,7 @@
 import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression;
 import com.apple.foundationdb.record.query.plan.cascades.typing.Type;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
+import com.apple.foundationdb.record.query.plan.explain.ExplainLevel;
 import com.apple.foundationdb.relational.api.Options;
 import com.apple.foundationdb.relational.api.RelationalStruct;
 import com.apple.foundationdb.relational.api.exceptions.ErrorCode;
@@ -244,6 +245,18 @@ public Void visitFullDescribeStatement(@Nonnull RelationalParser.FullDescribeSta
         return visitChildren(ctx);
     }
 
+    @Override
+    public Void visitDescribeStatements(@Nonnull RelationalParser.DescribeStatementsContext ctx) {
+        if (ctx.VERBOSE() != null) {
+            queryHasherContextBuilder.setExplainLevel(ExplainLevel.ALL_DETAILS);
+        } else if (ctx.MINIMAL() != null) {
+            queryHasherContextBuilder.setExplainLevel(ExplainLevel.STRUCTURE);
+        } else {
+            queryHasherContextBuilder.setExplainLevel(ExplainLevel.SOME_DETAILS);
+        }
+        return visitChildren(ctx);
+    }
+
     @Override
     public Void visitLimitClause(@Nonnull RelationalParser.LimitClauseContext ctx) {
         if (ctx.offset != null) {

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/MutablePlanGenerationContext.java

@@ -21,7 +21,6 @@
 package com.apple.foundationdb.relational.recordlayer.query;
 
 import com.apple.foundationdb.annotation.API;
-
 import com.apple.foundationdb.record.ExecuteProperties;
 import com.apple.foundationdb.record.PlanHashable;
 import com.apple.foundationdb.record.query.expressions.Comparisons;
@@ -35,13 +34,13 @@
 import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.OfTypeValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
+import com.apple.foundationdb.record.query.plan.explain.ExplainLevel;
 import com.apple.foundationdb.relational.api.RelationalArray;
 import com.apple.foundationdb.relational.api.RelationalStruct;
 import com.apple.foundationdb.relational.api.exceptions.RelationalException;
 import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils;
 import com.apple.foundationdb.relational.util.Assert;
 import com.apple.foundationdb.relational.util.SpotBugsSuppressWarnings;
-
 import com.google.common.collect.ImmutableList;
 import com.google.protobuf.ZeroCopyByteString;
 
@@ -89,6 +88,8 @@ public class MutablePlanGenerationContext implements QueryExecutionContext {
 
     private boolean forExplain;
 
+    private int explainLevel;
+
     @Nullable
     private byte[] continuation;
 
@@ -261,6 +262,7 @@ public MutablePlanGenerationContext(@Nonnull PreparedParams preparedParams,
         constantObjectValues = new LinkedList<>();
         shouldProcessLiteral = true;
         forExplain = false;
+        this.explainLevel = ExplainLevel.SOME_DETAILS;
         setContinuation(null);
         equalityConstraints = ImmutableList.builder();
     }
@@ -336,6 +338,10 @@ public boolean isForExplain() {
         return forExplain;
     }
 
+    @Override
+    public int getExplainLevel() {
+        return explainLevel;
+    }
 
     @Nonnull
     public QueryPlanConstraint getPlanConstraintsForLiteralReferences() {
@@ -372,6 +378,10 @@ public void setForExplain(boolean forExplain) {
         this.forExplain = forExplain;
     }
 
+    public void setExplainLevel(final int explainLevel) {
+        this.explainLevel = explainLevel;
+    }
+
     @Nonnull
     public Value processQueryLiteral(@Nonnull Type type, @Nullable Object literal, int tokenIndex) {
         return processQueryLiteralOrParameter(type, literal, null, null, tokenIndex);

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/NormalizedQueryExecutionContext.java

@@ -24,6 +24,7 @@
 
 import com.apple.foundationdb.record.ExecuteProperties;
 import com.apple.foundationdb.record.PlanHashable;
+import com.apple.foundationdb.record.query.plan.explain.ExplainLevel;
 import com.apple.foundationdb.relational.util.SpotBugsSuppressWarnings;
 
 import javax.annotation.Nonnull;
@@ -44,6 +45,8 @@ public final class NormalizedQueryExecutionContext implements QueryExecutionCont
 
     private final boolean isForExplain;
 
+    private int explainLevel;
+
     private final int parameterHash;
 
     @Nonnull
@@ -53,10 +56,12 @@ private NormalizedQueryExecutionContext(@Nonnull Literals literals,
                                             @Nullable byte[] continuation,
                                             int parameterHash,
                                             boolean isForExplain,
+                                            int explainLevel,
                                             @Nonnull final PlanHashable.PlanHashMode planHashMode) {
         this.literals = literals;
         this.continuation = continuation;
         this.isForExplain = isForExplain;
+        this.explainLevel = explainLevel;
         this.parameterHash = parameterHash;
         this.planHashMode = planHashMode;
     }
@@ -90,6 +95,11 @@ public boolean isForExplain() {
         return isForExplain;
     }
 
+    @Override
+    public int getExplainLevel() {
+        return explainLevel;
+    }
+
     @Nonnull
     @Override
     public PlanHashable.PlanHashMode getPlanHashMode() {
@@ -107,6 +117,8 @@ public static final class Builder {
 
         private boolean isForExplain;
 
+        private int explainLevel;
+
         @Nullable
         private byte[] continuation;
 
@@ -120,6 +132,7 @@ private Builder() {
             this.isForExplain = false;
             this.continuation = null;
             this.planHashMode = null;
+            this.explainLevel = ExplainLevel.SOME_DETAILS;
         }
 
         @Nonnull
@@ -146,6 +159,12 @@ public Builder setForExplain(boolean isForExplain) {
             return this;
         }
 
+        @Nonnull
+        public Builder setExplainLevel(int level) {
+            this.explainLevel = level;
+            return this;
+        }
+
         @Nonnull
         public Builder setPlanHashMode(@Nonnull PlanHashable.PlanHashMode planHashMode) {
             this.planHashMode = planHashMode;
@@ -155,7 +174,7 @@ public Builder setPlanHashMode(@Nonnull PlanHashable.PlanHashMode planHashMode)
         @Nonnull
         public NormalizedQueryExecutionContext build() {
             return new NormalizedQueryExecutionContext(literalsBuilder.build(), continuation,
-                    parameterHash, isForExplain,
+                    parameterHash, isForExplain, explainLevel,
                     Objects.requireNonNull(planHashMode));
         }
     }

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryExecutionContext.java

@@ -60,6 +60,8 @@ default EvaluationContext getEvaluationContext() {
 
     boolean isForExplain(); // todo (yhatem) remove.
 
+    int getExplainLevel();
+
     @Nonnull
     PlanHashable.PlanHashMode getPlanHashMode();
 }

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java

@@ -200,7 +200,7 @@ public PhysicalQueryPlan withExecutionContext(@Nonnull final QueryExecutionConte
         public String explain() {
             final var executeProperties = queryExecutionContext.getExecutionPropertiesBuilder();
             List<String> explainComponents = new ArrayList<>();
-            explainComponents.add(ExplainPlanVisitor.toStringForExternalExplain(recordQueryPlan));
+            explainComponents.add(ExplainPlanVisitor.toStringForExternalExplain(recordQueryPlan, queryExecutionContext.getExplainLevel()));
             if (executeProperties.getReturnedRowLimit() != ReadTransaction.ROW_LIMIT_UNLIMITED) {
                 explainComponents.add("(limit=" + executeProperties.getReturnedRowLimit() + ")");
             }

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java

@@ -33,6 +33,7 @@
 import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue;
 import com.apple.foundationdb.record.query.plan.cascades.values.Value;
+import com.apple.foundationdb.record.query.plan.explain.ExplainLevel;
 import com.apple.foundationdb.record.util.pair.NonnullPair;
 import com.apple.foundationdb.relational.api.exceptions.ErrorCode;
 import com.apple.foundationdb.relational.generated.RelationalLexer;
@@ -536,6 +537,13 @@ public Object visitExecuteContinuationStatement(@Nonnull RelationalParser.Execut
     @Override
     public QueryPlan.LogicalQueryPlan visitFullDescribeStatement(@Nonnull RelationalParser.FullDescribeStatementContext ctx) {
         getDelegate().getPlanGenerationContext().setForExplain(ctx.EXPLAIN() != null);
+        if (!ctx.describeObjectClause().getTokens(RelationalLexer.VERBOSE).isEmpty()) {
+            getDelegate().getPlanGenerationContext().setExplainLevel(ExplainLevel.ALL_DETAILS);
+        } else if (!ctx.describeObjectClause().getTokens(RelationalLexer.MINIMAL).isEmpty()) {
+            getDelegate().getPlanGenerationContext().setExplainLevel(ExplainLevel.STRUCTURE);
+        } else {
+            getDelegate().getPlanGenerationContext().setExplainLevel(ExplainLevel.SOME_DETAILS);
+        }
         final var logicalOperator = Assert.castUnchecked(ctx.describeObjectClause().accept(this), LogicalOperator.class);
         return QueryPlan.LogicalQueryPlan.of(logicalOperator.getQuantifier().getRangesOver().get(), getDelegate().getPlanGenerationContext(), "TODO");
     }

fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/ExplainTests.java

@@ -73,7 +73,7 @@ void explainResultSetMetadataTest() throws Exception {
         final var expectedContTypes = List.of(Types.BINARY, Types.INTEGER, Types.VARCHAR);
         try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) {
             executeInsert(ddl);
-            try (RelationalPreparedStatement ps = ddl.setSchemaAndGetConnection().prepareStatement("EXPLAIN SELECT * FROM RestaurantComplexRecord")) {
+            try (RelationalPreparedStatement ps = ddl.setSchemaAndGetConnection().prepareStatement("EXPLAIN VERBOSE SELECT * FROM RestaurantComplexRecord")) {
                 try (final RelationalResultSet resultSet = ps.executeQuery()) {
                     final var actualMetadata = resultSet.getMetaData();
                     org.junit.jupiter.api.Assertions.assertEquals(expectedLabels.size(), actualMetadata.getColumnCount());
@@ -205,6 +205,60 @@ void explainExecuteStatementTest() throws Exception {
         }
     }
 
+    @Test
+    void explainVerboseTest() throws Exception {
+        try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) {
+            executeInsert(ddl);
+            try (RelationalPreparedStatement ps = ddl.setSchemaAndGetConnection().prepareStatement("EXPLAIN VERBOSE SELECT * FROM RestaurantComplexRecord where COUNT(reviews) > 3 and COUNT(reviews) < 100")) {
+                ps.setMaxRows(2);
+                try (final RelationalResultSet resultSet = ps.executeQuery()) {
+                    final var assertResult = ResultSetAssert.assertThat(resultSet);
+                    assertResult.hasNextRow()
+                            .hasColumn("PLAN", "ISCAN(RECORD_NAME_IDX <,>) | FILTER count_star(*) GREATER_THAN promote(@c12 AS LONG) AND count_star(*) LESS_THAN promote(@c19 AS LONG)")
+                            .hasColumn("PLAN_HASH", -1697137247L)
+                            .hasColumn("PLAN_CONTINUATION", null);
+                    assertResult.hasNoNextRow();
+                }
+            }
+        }
+    }
+
+    @Test
+    void explainMinimalTest() throws Exception {
+        try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) {
+            executeInsert(ddl);
+            try (RelationalPreparedStatement ps = ddl.setSchemaAndGetConnection().prepareStatement("EXPLAIN MINIMAL SELECT * FROM RestaurantComplexRecord where COUNT(reviews) > 3 and COUNT(reviews) < 100")) {
+                ps.setMaxRows(2);
+                try (final RelationalResultSet resultSet = ps.executeQuery()) {
+                    final var assertResult = ResultSetAssert.assertThat(resultSet);
+                    assertResult.hasNextRow()
+                            .hasColumn("PLAN", "ISCAN(RECORD_NAME_IDX) | FILTER count_star(*) GREATER_THAN promote(@c12 AS LONG) AND count_star(*) LESS_THAN promote(@c19 AS LONG)")
+                            .hasColumn("PLAN_HASH", -1697137247L)
+                            .hasColumn("PLAN_CONTINUATION", null);
+                    assertResult.hasNoNextRow();
+                }
+            }
+        }
+    }
+
+    @Test
+    void explainDefaultTest() throws Exception {
+        try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) {
+            executeInsert(ddl);
+            try (RelationalPreparedStatement ps = ddl.setSchemaAndGetConnection().prepareStatement("EXPLAIN SELECT * FROM RestaurantComplexRecord where COUNT(reviews) > 3 and COUNT(reviews) < 100")) {
+                ps.setMaxRows(2);
+                try (final RelationalResultSet resultSet = ps.executeQuery()) {
+                    final var assertResult = ResultSetAssert.assertThat(resultSet);
+                    assertResult.hasNextRow()
+                            .hasColumn("PLAN", "ISCAN(RECORD_NAME_IDX <,>) | FILTER count_star(*) GREATER_THAN promote(@c11 AS LONG) AND count_star(*) LESS_THAN promote(@c18 AS LONG)")
+                            .hasColumn("PLAN_HASH", -1697137247L)
+                            .hasColumn("PLAN_CONTINUATION", null);
+                    assertResult.hasNoNextRow();
+                }
+            }
+        }
+    }
+
     private Continuation consumeResultAndGetContinuation(RelationalPreparedStatement ps, int numRows) throws SQLException {
         Continuation continuation;
         try (final RelationalResultSet resultSet = ps.executeQuery()) {