Add new KeySpacePath.exportAllData (#3566)

This introduces a new KeySpacePath.exportAllData to export all the data
stored in the path. This can eventually be used to import into another
cluster, or back into the same cluster, after clearing.

Other than the path information, the data exported is raw bytes, with no
transformation or indicated semantics.

Resolves:
[3572](#3572)

Author: ScottDugas

fdb-extensions/src/main/java/com/apple/foundationdb/tuple/TupleHelpers.java

@@ -141,17 +141,17 @@ public static int packedSizeAsTupleItem(Object item) {
 
     /**
      * Get whether one tuple is a prefix of another.
-     * @param t1 the potential prefix
-     * @param t2 the whole tuple
-     * @return {@code true} if {@code t1} is a prefix of {@code t2}
+     * @param potentialPrefix the potential prefix
+     * @param wholeTuple the whole tuple
+     * @return {@code true} if {@code potentialPrefix} is a prefix of {@code wholeTuple}
      */
-    public static boolean isPrefix(@Nonnull Tuple t1, @Nonnull Tuple t2) {
-        final int len = t1.size();
-        if (t2.size() < len) {
+    public static boolean isPrefix(@Nonnull Tuple potentialPrefix, @Nonnull Tuple wholeTuple) {
+        final int len = potentialPrefix.size();
+        if (wholeTuple.size() < len) {
             return false;
         }
         for (int i = 0; i < len; i++) {
-            int rc = TupleUtil.compareItems(t1.get(i), t2.get(i));
+            int rc = TupleUtil.compareItems(potentialPrefix.get(i), wholeTuple.get(i));
             if (rc != 0) {
                 return false;
             }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/DataInKeySpacePath.java

@@ -0,0 +1,60 @@
+/*
+ * DataInKeySpacePath.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb.record.provider.foundationdb.keyspace;
+
+import com.apple.foundationdb.KeyValue;
+import com.apple.foundationdb.annotation.API;
+import com.apple.foundationdb.record.RecordCoreArgumentException;
+import com.apple.foundationdb.record.logging.LogMessageKeys;
+import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext;
+import com.apple.foundationdb.tuple.ByteArrayUtil2;
+
+import javax.annotation.Nonnull;
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * Class representing a {@link KeyValue} pair within in {@link KeySpacePath}.
+ */
+@API(API.Status.EXPERIMENTAL)
+public class DataInKeySpacePath {
+
+    @Nonnull
+    private final CompletableFuture<ResolvedKeySpacePath> resolvedPath;
+    @Nonnull
+    private final byte[] value;
+
+    public DataInKeySpacePath(KeySpacePath path, KeyValue rawKeyValue, FDBRecordContext context) {
+        this.resolvedPath = path.toResolvedPathAsync(context, rawKeyValue.getKey());
+        this.value = rawKeyValue.getValue();
+        if (this.value == null) {
+            throw new RecordCoreArgumentException("Value cannot be null")
+                    .addLogInfo(LogMessageKeys.KEY, ByteArrayUtil2.loggable(rawKeyValue.getKey()));
+        }
+    }
+
+    public CompletableFuture<ResolvedKeySpacePath> getResolvedPath() {
+        return resolvedPath;
+    }
+
+    public byte[] getValue() {
+        return this.value;
+    }
+}

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpace.java

@@ -227,9 +227,12 @@ public KeySpacePath pathFromKey(@Nonnull FDBRecordContext context, @Nonnull Tupl
     /**
      * Given a tuple from an FDB key, attempts to determine what path through this directory the tuple
      * represents, returning a <code>ResolvedKeySpacePath</code> representing the leaf-most directory in the path.
-     * If entries remained in the tuple beyond the leaf directory, then {@link KeySpacePath#getRemainder()}
-     * can be used to fetch the remaining portion.
-     *
+     * <p>
+     *     If entries remained in the tuple beyond the leaf directory, then {@link KeySpacePath#getRemainder()} can be
+     *     used to fetch the remaining portion.
+     *     See also {@link KeySpacePath#toResolvedPathAsync(FDBRecordContext, byte[])} if you need to resolve and you
+     *     know that it is part of a given path.
+     * </p>
      * @param context context used, if needed, for any database operations
      * @param key the tuple to be decoded
      * @return a path entry representing the leaf directory entry that corresponds to a value in the

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePath.java

@@ -198,6 +198,25 @@ default Tuple toTuple(@Nonnull FDBRecordContext context) {
     @Nonnull
     CompletableFuture<ResolvedKeySpacePath> toResolvedPathAsync(@Nonnull FDBRecordContext context);
 
+    /**
+     * Given a tuple from an FDB key, attempts to determine what sub-path through this directory the tuple
+     * represents, returning a <code>ResolvedKeySpacePath</code> representing the leaf-most directory in the path.
+     * <p>
+     *     If entries remained in the tuple beyond the leaf directory, then {@link KeySpacePath#getRemainder()}
+     *     can be used to fetch the remaining portion.
+     *     See also {@link KeySpace#resolveFromKeyAsync(FDBRecordContext, Tuple)} if you need to resolve from the root.
+     * </p>
+     * @param context  context used, if needed, for any database operations
+     * @param key a raw key from the database
+     * @return the {@link ResolvedKeySpacePath} corresponding to that key, with a potential remainder.
+     * @throws com.apple.foundationdb.record.RecordCoreArgumentException if the key provided is not part of this path
+     */
+    @API(API.Status.EXPERIMENTAL)
+    @Nonnull
+    default CompletableFuture<ResolvedKeySpacePath> toResolvedPathAsync(@Nonnull FDBRecordContext context, byte[] key) {
+        throw new UnsupportedOperationException("toResolvedPathAsync is not supported");
+    }
+
     /**
      * Resolves the path into a {@link ResolvedKeySpacePath}, a form the retains all of the information about
      * the path itself along with the value to which each path entry is resolved.
@@ -566,4 +585,22 @@ default List<ResolvedKeySpacePath> listSubdirectory(@Nonnull FDBRecordContext co
      */
     @API(API.Status.UNSTABLE)
     String toString(@Nonnull Tuple tuple);
+
+    /**
+     * Export all data stored under this KeySpacePath and return it in a RecordCursor.
+     * This method scans all keys that have this path as a prefix and returns the key-value pairs.
+     * Supports continuation to resume scanning from a previous position.
+     *
+     * @param context the transaction context in which to perform the data export
+     * @param continuation optional continuation from a previous export operation, or null to start from the beginning
+     * @param scanProperties properties controlling how the scan should be performed
+     * @return a RecordCursor that iterates over all KeyValue pairs under this path
+     */
+    @API(API.Status.EXPERIMENTAL)
+    @Nonnull
+    default RecordCursor<DataInKeySpacePath> exportAllData(@Nonnull FDBRecordContext context,
+                                                           @Nullable byte[] continuation,
+                                                           @Nonnull ScanProperties scanProperties) {
+        throw new UnsupportedOperationException("exportAllData is not supported");
+    }
 }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathImpl.java

@@ -26,10 +26,16 @@
 import com.apple.foundationdb.record.RecordCursor;
 import com.apple.foundationdb.record.ScanProperties;
 import com.apple.foundationdb.record.ValueRange;
+import com.apple.foundationdb.record.cursors.LazyCursor;
+import com.apple.foundationdb.record.logging.LogMessageKeys;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext;
+import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursor;
+import com.apple.foundationdb.subspace.Subspace;
 import com.apple.foundationdb.tuple.ByteArrayUtil;
 import com.apple.foundationdb.tuple.Tuple;
+import com.apple.foundationdb.tuple.TupleHelpers;
 import com.google.common.collect.Lists;
+
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.util.ArrayList;
@@ -242,6 +248,40 @@ public CompletableFuture<ResolvedKeySpacePath> toResolvedPathAsync(@Nonnull FDBR
         });
     }
 
+    @Nonnull
+    @Override
+    public CompletableFuture<ResolvedKeySpacePath> toResolvedPathAsync(@Nonnull final FDBRecordContext context, final byte[] key) {
+        final Tuple keyTuple = Tuple.fromBytes(key);
+        return toResolvedPathAsync(context).thenCompose(resolvedPath -> {
+            // Now use the resolved path to find the child for the key
+            // We need to figure out how much of the key corresponds to the resolved path
+            Tuple pathTuple = resolvedPath.toTuple();
+            int pathLength = pathTuple.size();
+
+            if (!TupleHelpers.isPrefix(pathTuple, keyTuple)) {
+                throw new RecordCoreArgumentException("Key is not under this path")
+                        .addLogInfo(LogMessageKeys.EXPECTED, pathTuple,
+                                LogMessageKeys.ACTUAL, keyTuple);
+            }
+
+            // The remaining part of the key should be resolved from the resolved path's directory
+            if (keyTuple.size() > pathLength) {
+                // There's more in the key than just the path, so resolve the rest
+                if (resolvedPath.getDirectory().getSubdirectories().isEmpty()) {
+                    return CompletableFuture.completedFuture(
+                            new ResolvedKeySpacePath(resolvedPath.getParent(), resolvedPath.toPath(),
+                                    resolvedPath.getResolvedPathValue(),
+                                    TupleHelpers.subTuple(keyTuple, pathTuple.size(), keyTuple.size())));
+                } else {
+                    return resolvedPath.getDirectory().findChildForKey(context, resolvedPath, keyTuple, keyTuple.size(), pathLength);
+                }
+            } else {
+                // The key exactly matches the path
+                return CompletableFuture.completedFuture(resolvedPath);
+            }
+        });
+    }
+
     @Nonnull
     @Override
     public CompletableFuture<Boolean> hasDataAsync(@Nonnull FDBRecordContext context) {
@@ -331,6 +371,21 @@ public String toString() {
         return toString(null);
     }
 
+    @Nonnull
+    @Override
+    public RecordCursor<DataInKeySpacePath> exportAllData(@Nonnull FDBRecordContext context,
+                                                          @Nullable byte[] continuation,
+                                                          @Nonnull ScanProperties scanProperties) {
+        return new LazyCursor<>(toTupleAsync(context)
+                .thenApply(tuple -> KeyValueCursor.Builder.withSubspace(new Subspace(tuple))
+                        .setContext(context)
+                        .setContinuation(continuation)
+                        .setScanProperties(scanProperties)
+                        .build()),
+                context.getExecutor())
+                .map(keyValue -> new DataInKeySpacePath(this, keyValue, context));
+    }
+
     /**
      * Returns this path properly wrapped in whatever implementation the directory the path is contained in dictates.
      */

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/KeySpacePathWrapper.java

@@ -206,6 +206,11 @@ public CompletableFuture<ResolvedKeySpacePath> toResolvedPathAsync(@Nonnull FDBR
         return inner.toResolvedPathAsync(context);
     }
 
+    @Nonnull
+    @Override
+    public CompletableFuture<ResolvedKeySpacePath> toResolvedPathAsync(@Nonnull final FDBRecordContext context, final byte[] key) {
+        return inner.toResolvedPathAsync(context, key);
+    }
 
     @Override
     public boolean equals(Object obj) {
@@ -226,4 +231,12 @@ public String toString() {
     public String toString(@Nonnull Tuple t) {
         return inner.toString(t);
     }
+
+    @Nonnull
+    @Override
+    public RecordCursor<DataInKeySpacePath> exportAllData(@Nonnull FDBRecordContext context,
+                                                          @Nullable byte[] continuation,
+                                                          @Nonnull ScanProperties scanProperties) {
+        return inner.exportAllData(context, continuation, scanProperties);
+    }
 }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/keyspace/ResolvedKeySpacePath.java

@@ -24,6 +24,7 @@
 import com.apple.foundationdb.subspace.Subspace;
 import com.apple.foundationdb.tuple.ByteArrayUtil2;
 import com.apple.foundationdb.tuple.Tuple;
+import com.google.common.annotations.VisibleForTesting;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -271,4 +272,16 @@ public static void appendValue(StringBuilder sb, Object value) {
             sb.append(value);
         }
     }
+
+    /**
+     * Returns a new {@code ResolvedKeySpacePath} that is the same, except with the provided {@link #getRemainder()}.
+     * @param newRemainder a new remainder. This can be {@code null} to remove the remainder entirely.
+     * @return a new {@code ResolvedKeySpacePath} that is the same as this, except with a different {@link #getRemainder()}.
+     */
+    @Nonnull
+    @VisibleForTesting
+    ResolvedKeySpacePath withRemainder(@Nullable final Tuple newRemainder) {
+        // this could probably copy the cachedTuple & cachedSubspace
+        return new ResolvedKeySpacePath(parent, inner, value, newRemainder);
+    }
 }