#135: Allow queries with no particular sort order

Author: nvamelichev

databind/src/main/java/tech/ydb/yoj/databind/expression/OrderExpression.java

@@ -3,6 +3,7 @@
 import com.google.common.base.Preconditions;
 import lombok.NonNull;
 import lombok.Value;
+import tech.ydb.yoj.ExperimentalApi;
 import tech.ydb.yoj.databind.schema.Schema;
 
 import java.util.List;
@@ -26,11 +27,58 @@ public <U> OrderExpression<U> forSchema(@NonNull Schema<U> dstSchema,
                 .collect(toList()));
     }
 
+    /**
+     * Returns an {@code OrderExpression} that applies <em>no particular sort order</em> for the query results.
+     * The results will be returned in an implementation-defined order which is subject to change at any time,
+     * <em>potentially even giving a different ordering for repeated executions of the same query</em>.
+     * <p>This is different from the {@code OrderExpression} being {@code null} or not specified,
+     * which YOJ interprets as "order query results by entity ID ascending" to ensure maximum
+     * predictability of query results.
+     * <p><strong>BEWARE!</strong> For small queries that return results entirely from a single YDB table partition
+     * (<em>data shard</em>), the <em>no particular sort order</em> imposed by {@code OrderExpression.unordered()}
+     * on a real YDB database will <strong>most likely be the same</strong> as "order by entity ID ascending",
+     * but this will quickly and unpredictably change if the table and/or the result set grow bigger.
+     *
+     * @param schema schema to use
+     * @return an {@code OrderExpression} representing <em>no particular sort order</em>
+     *
+     * @param <U> schema type
+     */
+    @NonNull
+    @ExperimentalApi(issue = "https://github.com/ydb-platform/yoj-project/issues/115")
+    public static <U> OrderExpression<U> unordered(@NonNull Schema<U> schema) {
+        return new OrderExpression<>(schema);
+    }
+
+    /**
+     * @return A fresh {@code Stream} of {@link SortKey sort keys} that this expression has.
+     * Will be empty if this expression represents an {@link #isUnordered() arbitrary sort order}.
+     */
     @NonNull
     public Stream<SortKey> keyStream() {
         return keys.stream();
     }
 
+    /**
+     * @return {@code true} if this {@code OrderExpression} represents a well-defined sort order;
+     * {@code false} if the sort order is arbitrary
+     *
+     * @see #isUnordered()
+     */
+    @ExperimentalApi(issue = "https://github.com/ydb-platform/yoj-project/issues/115")
+    public boolean isOrdered() {
+        return !keys.isEmpty();
+    }
+
+    /**
+     * @return {@code true} if this {@code OrderExpression} represents arbitrary sort order (whatever the database returns);
+     * {@code false} otherwise
+     */
+    @ExperimentalApi(issue = "https://github.com/ydb-platform/yoj-project/issues/115")
+    public boolean isUnordered() {
+        return !isOrdered();
+    }
+
     @Override
     public String toString() {
         return keys.stream().map(Object::toString).collect(joining(", "));
@@ -42,6 +90,11 @@ public OrderExpression(@NonNull Schema<T> schema, @NonNull List<SortKey> keys) {
         this.keys = keys;
     }
 
+    private OrderExpression(@NonNull Schema<T> schema) {
+        this.schema = schema;
+        this.keys = List.of();
+    }
+
     @Value
     public static class SortKey {
         Schema.JavaField field;

repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/yql/YqlListingQuery.java

@@ -165,7 +165,9 @@ private static void escapeLikePatternToSb(@NonNull String str, StringBuilder sb)
     }
 
     public static <T extends Entity<T>> YqlOrderBy toYqlOrderBy(@NonNull OrderExpression<T> orderBy) {
-        return YqlOrderBy.orderBy(orderBy.getKeys().stream().map(YqlListingQuery::toSortKey).collect(toList()));
+        return orderBy.isUnordered()
+                ? YqlOrderBy.unordered()
+                : YqlOrderBy.orderBy(orderBy.getKeys().stream().map(YqlListingQuery::toSortKey).collect(toList()));
     }
 
     private static YqlOrderBy.SortKey toSortKey(SortKey k) {

repository-ydb-v2/src/main/java/tech/ydb/yoj/repository/ydb/yql/YqlOrderBy.java

@@ -6,6 +6,7 @@
 import lombok.Getter;
 import lombok.NonNull;
 import lombok.Value;
+import tech.ydb.yoj.ExperimentalApi;
 import tech.ydb.yoj.repository.db.Entity;
 import tech.ydb.yoj.repository.db.EntitySchema;
 
@@ -25,6 +26,8 @@
 @Getter
 @EqualsAndHashCode
 public final class YqlOrderBy implements YqlStatementPart<YqlOrderBy> {
+    private static final YqlOrderBy UNORDERED = new YqlOrderBy(List.of());
+
     public static final String TYPE = "OrderBy";
     private final List<SortKey> keys;
 
@@ -95,6 +98,22 @@ public static YqlOrderBy orderBy(Collection<SortKey> keys) {
         return new Builder().by(keys).build();
     }
 
+    /**
+     * Returns an {@code YqlOrderBy} that applies <em>no particular sort order</em> to the query results.
+     * The results will be returned in an implementation-defined order which is subject to change at any time,
+     * <em>potentially even giving a different ordering for repeated executions of the same query</em>.
+     * <p><strong>BEWARE!</strong> For small queries that return results entirely from a single table partition
+     * (<em>data shard</em>), the <em>no particular sort order</em> imposed by {@code YqlOrderBy.unordered()}
+     * on a real YDB database will <strong>most likely be the same</strong> as "order by entity ID ascending",
+     * but this will quickly and unpredictably change if the table and/or the result set grow bigger.
+     *
+     * @return an {@code YqlOrderBy} representing <em>no particular sort order</em>
+     */
+    @ExperimentalApi(issue = "https://github.com/ydb-platform/yoj-project/issues/115")
+    public static YqlOrderBy unordered() {
+        return UNORDERED;
+    }
+
     /**
      * @return builder for {@code YqlOrderBy}
      */
@@ -104,7 +123,7 @@ public static Builder order() {
 
     @Override
     public <T extends Entity<T>> String toYql(@NonNull EntitySchema<T> schema) {
-        return keys.stream().map(k -> k.toYql(schema)).collect(joining(", "));
+        return keys.isEmpty() ? "" : keys.stream().map(k -> k.toYql(schema)).collect(joining(", "));
     }
 
     @Override
@@ -114,7 +133,7 @@ public String getType() {
 
     @Override
     public String getYqlPrefix() {
-        return "ORDER BY ";
+        return keys.isEmpty() ? "" : "ORDER BY ";
     }
 
     @Override
@@ -124,7 +143,7 @@ public int getPriority() {
 
     @Override
     public String toString() {
-        return format("order by %s", keys.stream().map(Object::toString).collect(joining(", ")));
+        return keys.isEmpty() ? "unordered" : format("order by %s", keys.stream().map(Object::toString).collect(joining(", ")));
     }
 
     public enum SortOrder {
@@ -147,7 +166,7 @@ public String toString() {
      * Sort key: entity field plus {@link SortOrder sort order} (either ascending or descending).
      */
     @Value
-    public static final class SortKey {
+    public static class SortKey {
         String fieldPath;
         SortOrder order;
 

repository-ydb-v2/src/test/java/tech/ydb/yoj/repository/ydb/YdbRepositoryIntegrationTest.java

@@ -931,6 +931,22 @@ public void ydbTransactionCompatibility() {
         }
     }
 
+    @Test
+    public void unordered() {
+        // YDB tends to return data in index-order, not "by PK ascending" order, if we don't force the result order
+        IndexedEntity ie1 = new IndexedEntity(new IndexedEntity.Id("abc"), "z", "v1-1", "v1-2");
+        IndexedEntity ie2 = new IndexedEntity(new IndexedEntity.Id("def"), "y", "v2-1", "v2-2");
+        db.tx(() -> db.indexedTable().insert(ie1, ie2));
+
+        var results = db.tx(() -> db.indexedTable().query()
+                .where("keyId").gte("a")
+                .limit(2)
+                .index(IndexedEntity.KEY_INDEX)
+                .unordered()
+                .find());
+        assertThat(results).containsExactly(ie2, ie1);
+    }
+
     @AllArgsConstructor
     private static class DelegateSchemeServiceImplBase extends SchemeServiceGrpc.SchemeServiceImplBase {
         @Delegate

repository/src/main/java/tech/ydb/yoj/repository/db/EntityExpressions.java

@@ -1,9 +1,11 @@
 package tech.ydb.yoj.repository.db;
 
 import lombok.NonNull;
+import tech.ydb.yoj.ExperimentalApi;
 import tech.ydb.yoj.databind.expression.FilterBuilder;
 import tech.ydb.yoj.databind.expression.OrderBuilder;
 import tech.ydb.yoj.databind.expression.OrderExpression;
+import tech.ydb.yoj.databind.schema.Schema;
 
 import static tech.ydb.yoj.databind.expression.OrderExpression.SortOrder.ASCENDING;
 
@@ -19,6 +21,14 @@ public static <T extends Entity<T>> OrderBuilder<T> newOrderBuilder(@NonNull Cla
         return OrderBuilder.forSchema(schema(entityType));
     }
 
+    /**
+     * @see OrderExpression#unordered(Schema)
+     */
+    @ExperimentalApi(issue = "https://github.com/ydb-platform/yoj-project/issues/115")
+    public static <T extends Entity<T>> OrderExpression<T> unordered(@NonNull Class<T> entityType) {
+        return OrderExpression.unordered(schema(entityType));
+    }
+
     private static <T extends Entity<T>> EntitySchema<T> schema(@NonNull Class<T> entityType) {
         return EntitySchema.of(entityType);
     }

repository/src/main/java/tech/ydb/yoj/repository/db/TableQueryBuilder.java

@@ -3,10 +3,12 @@
 import com.google.common.base.Preconditions;
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
+import tech.ydb.yoj.ExperimentalApi;
 import tech.ydb.yoj.databind.expression.FilterBuilder;
 import tech.ydb.yoj.databind.expression.FilterExpression;
 import tech.ydb.yoj.databind.expression.OrderBuilder;
 import tech.ydb.yoj.databind.expression.OrderExpression;
+import tech.ydb.yoj.databind.schema.Schema;
 
 import javax.annotation.Nullable;
 import java.util.Collection;
@@ -197,6 +199,15 @@ private FilterExpression<T> getFinalFilter() {
         }
     }
 
+    /**
+     * @see OrderExpression#unordered(Schema)
+     */
+    @NonNull
+    @ExperimentalApi(issue = "https://github.com/ydb-platform/yoj-project/issues/115")
+    public TableQueryBuilder<T> unordered() {
+        return orderBy(EntityExpressions.unordered(table.getType()));
+    }
+
     @NonNull
     public TableQueryBuilder<T> orderBy(@Nullable OrderExpression<T> orderBy) {
         this.orderBy = orderBy;

repository/src/main/java/tech/ydb/yoj/repository/db/list/InMemoryQueries.java

@@ -17,8 +17,10 @@
 
 import javax.annotation.Nullable;
 import java.util.Comparator;
+import java.util.IdentityHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Stream;
@@ -144,12 +146,18 @@ public Predicate<T> visitNotExpr(@NonNull NotExpr<T> notExpr) {
     }
 
     public static <T extends Entity<T>> Comparator<T> toComparator(@NonNull OrderExpression<T> orderBy) {
+        if (orderBy.isUnordered()) {
+            // Produces a randomly-ordering, but stable Comparator. UUID.randomUUID() collisions are extremely unlikely, so we ignore them.
+            Map<Object, UUID> randomIds = new IdentityHashMap<>();
+            return Comparator.comparing(e -> randomIds.computeIfAbsent(e, __ -> UUID.randomUUID()));
+        }
+
         Schema<T> schema = orderBy.getSchema();
         return (a, b) -> {
             Map<String, Object> mapA = schema.flatten(a);
             Map<String, Object> mapB = schema.flatten(b);
             for (OrderExpression.SortKey sortKey : orderBy.getKeys()) {
-                for (JavaField field : sortKey.getField().flatten().collect(toList())) {
+                for (JavaField field : sortKey.getField().flatten().toList()) {
                     int res = compare(FieldValue.getComparable(mapA, field), FieldValue.getComparable(mapB, field));
                     if (res != 0) {
                         return sortKey.getOrder() == ASCENDING ? res : -res;