YDB C++ SDK Import 6

CMakeLists.txt

@@ -16,7 +16,10 @@ set(YDB_SDK_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(YDB_SDK_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
 set(YDB-CPP-SDK_AVAILABLE_COMPONENTS "" CACHE INTERNAL "")
 set(YDB-CPP-SDK_COMPONENT_TARGETS "" CACHE INTERNAL "")
-file(READ "src/client/resources/ydb_sdk_version.txt" YDB_SDK_VERSION)
+file(READ "src/version.h" YDB_SDK_VERSION_FILE_RAW)
+string(REGEX MATCH "YDB_SDK_VERSION = \"([0-9]+\\.[0-9]+\\.[0-9]+)\"" _ ${YDB_SDK_VERSION_FILE_RAW})
+set(YDB_SDK_VERSION ${CMAKE_MATCH_1})
+message(STATUS "YDB С++ SDK version: ${YDB_SDK_VERSION}")
 
 #[=============================================================================[
   NOTE: if `ccache` is used with the environment variable `CCACHE_BASEDIR`,

examples/secondary_index/secondary_index_list.cpp

@@ -246,7 +246,7 @@ int RunListSeries(TDriver& driver, const std::string& prefix, int argc, char** a
     uint64_t lastSeriesId = -1;
     uint64_t lastViews = -1;
 
-    opts.AddLongOption("by-views", "Sort by views").NoArgument().SetFlag(&byViews);
+    opts.AddLongOption("by-views", "Sort by views").StoreTrue(&byViews);
     opts.AddLongOption("limit", "Maximum number of rows").Optional().RequiredArgument("NUM")
         .StoreResult(&limit);
     opts.AddLongOption("last-id", "Resume from this last series id").Optional().RequiredArgument("NUM")

include/ydb-cpp-sdk/client/coordination/coordination.h

@@ -198,7 +198,9 @@ struct TCreateNodeSettings : public TNodeSettings<TCreateNodeSettings> {
     TCreateNodeSettings(const Ydb::Coordination::Config& config);
 };
 struct TAlterNodeSettings : public TNodeSettings<TAlterNodeSettings> { };
-struct TDropNodeSettings : public TOperationRequestSettings<TDropNodeSettings> { };
+struct TDropNodeSettings : public TOperationRequestSettings<TDropNodeSettings> {
+    using TOperationRequestSettings<TDropNodeSettings>::TOperationRequestSettings;
+};
 struct TDescribeNodeSettings : public TOperationRequestSettings<TDescribeNodeSettings> { };
 
 ////////////////////////////////////////////////////////////////////////////////

include/ydb-cpp-sdk/client/draft/ydb_replication.h

@@ -100,6 +100,8 @@ class TRunningState {
 
 struct TDoneState {};
 
+struct TPausedState {};
+
 class TErrorState {
     class TImpl;
 
@@ -131,6 +133,7 @@ class TReplicationDescription {
         Running,
         Error,
         Done,
+        Paused,
     };
 
     explicit TReplicationDescription(const Ydb::Replication::DescribeReplicationResult& desc);
@@ -145,6 +148,7 @@ class TReplicationDescription {
     const TRunningState& GetRunningState() const;
     const TErrorState& GetErrorState() const;
     const TDoneState& GetDoneState() const;
+    const TPausedState& GetPausedState() const;
 
 private:
     TConnectionParams ConnectionParams_;
@@ -158,7 +162,8 @@ class TReplicationDescription {
     std::variant<
         TRunningState,
         TErrorState,
-        TDoneState
+        TDoneState,
+        TPausedState
     > State_;
 };
 

include/ydb-cpp-sdk/client/extensions/solomon_stats/README.md

@@ -32,8 +32,8 @@ After creating NYdb::TDriver you need to add Solomon Monitoring extension. If yo
 > **Important**: you must plug in monitoring before driver creation.
 
 ```cl
-#include <src/client/ydb_driver/driver.h>
-#include <src/client/extensions/solomon_stats/pull_client.h>
+#include <ydb-cpp-sdk/client/driver/driver.h>
+#include <ydb-cpp-sdk/client/extensions/solomon_stats/pull_client.h>
 
 ...
 
@@ -65,7 +65,7 @@ Implementing NMonitoring::IMetricRegistry provides more flexibility. You can del
 
 Select a method which is right for you:
 ```cl
-#include <src/client/extensions/solomon_stats/pull_connector.h>
+#include <ydb-cpp-sdk/client/extensions/solomon_stats/pull_connector.h>
 
 ...
 

include/ydb-cpp-sdk/client/federated_topic/federated_topic.h

@@ -15,6 +15,8 @@ using TDbInfo = Ydb::FederationDiscovery::DatabaseInfo;
 
 using TSessionClosedEvent = NTopic::TSessionClosedEvent;
 
+using TAsyncDescribeTopicResult = NTopic::TAsyncDescribeTopicResult;
+
 //! Federated partition session.
 struct TFederatedPartitionSession : public TThrRefBase, public TPrintable<TFederatedPartitionSession> {
     using TPtr = TIntrusivePtr<TFederatedPartitionSession>;
@@ -516,6 +518,32 @@ class TFederatedTopicClient {
     // std::shared_ptr<NTopic::ISimpleBlockingWriteSession> CreateSimpleBlockingWriteSession(const TFederatedWriteSessionSettings& settings);
     std::shared_ptr<NTopic::IWriteSession> CreateWriteSession(const TFederatedWriteSessionSettings& settings);
 
+    struct TClusterInfo {
+        enum class EStatus : int {
+            STATUS_UNSPECIFIED,
+            AVAILABLE,
+            READ_ONLY,
+            UNAVAILABLE,
+        };
+        std::string Name;
+        std::string Endpoint;
+        std::string Path;
+        EStatus Status;
+        // TODO: Id, Weight, ...?
+        //! Replaces Endpoint and Database for federated clusters
+        void AdjustTopicClientSettings(NTopic::TTopicClientSettings& settings) const;
+        //! Prepend Database for federated clusters
+        void AdjustTopicPath(std::string& path) const;
+        //! Usable for at least read operations
+        bool IsAvailableForRead() const;
+        bool IsAvailableForWrite() const;
+    };
+
+    //! Discover all clusters for federated topic.
+    // Will return single cluster with empty name for non-federated clusters.
+    // May return empty list if FederatedTopicClient was destroyed when future fired.
+    NThreading::TFuture<std::vector<TClusterInfo>> GetAllClusterInfo();
+
 protected:
     void OverrideCodec(NTopic::ECodec codecId, std::unique_ptr<NTopic::ICodec>&& codecImpl);
 

include/ydb-cpp-sdk/client/table/table.h

@@ -1955,6 +1955,7 @@ class TDataQuery {
     const std::string& GetId() const;
     const std::optional<std::string>& GetText() const;
     TParamsBuilder GetParamsBuilder() const;
+    std::map<std::string, TType> GetParameterTypes() const;
 
     TAsyncDataQueryResult Execute(const TTxControl& txControl,
         const TExecDataQuerySettings& settings = TExecDataQuerySettings());

include/ydb-cpp-sdk/client/types/ydb.h

@@ -16,7 +16,10 @@ enum class EDiscoveryMode {
     //! we got endpoint list. The error will be returned if the endpoint list
     //! is empty and discovery failed
     //! This method is a bit more "user friendly" but can produce additional hidden latency
-    Async
+    Async,
+    //! Do not perform discovery.
+    //! This option disables database discovery and allow to use user provided endpoint for grpc connections
+    Off
 };
 
 enum class EBalancingPolicy {

include/ydb-cpp-sdk/type_switcher.h

@@ -3,7 +3,7 @@
 #include <string>
 
 namespace NYdb {
-inline namespace Dev {
+inline namespace V3 {
 
 using TStringType = std::string;
 

src/api/protos/draft/fq.proto

@@ -431,11 +431,16 @@ message ServiceAccountAuth {
     string id = 1 [(Ydb.length).le = 1024];
 }
 
+message TokenAuth {
+    string token = 1 [(Ydb.length).le = 1024, (Ydb.sensitive) = true];
+}
+
 message IamAuth {
     oneof identity {
         CurrentIAMTokenAuth current_iam = 1;
         ServiceAccountAuth service_account = 2;
         NoneAuth none = 3;
+        TokenAuth token = 4;
     }
 }
 

src/api/protos/draft/ydb_maintenance.proto

@@ -96,9 +96,24 @@ message MaintenanceTaskOptions {
 
 // Used to describe the scope of a single action.
 message ActionScope {
+    message PDiskId {
+        uint32 node_id = 1;
+        uint32 pdisk_id = 2;
+    }
+    message PDiskLocation {
+        string host = 1 [(length).le = 255];
+        string path = 2 [(length).le = 255];
+    }
+    message PDisk {
+        oneof pdisk {
+            PDiskId pdisk_id = 1;
+            PDiskLocation pdisk_location = 2;
+        }
+    }
     oneof scope {
         uint32 node_id = 1;
         string host = 2 [(length).le = 255];
+        PDisk pdisk = 3;
     }
 }
 
@@ -182,6 +197,10 @@ message MaintenanceTaskResult {
     repeated ActionGroupStates action_group_states = 2;
     // Try again after this deadline. Specified if there are no performed actions.
     optional google.protobuf.Timestamp retry_after = 3;
+    // The time when the mainteance task was created.
+    google.protobuf.Timestamp create_time = 4;
+    // The last time when the mainteance task was refreshed. Initially equals to create_time.
+    google.protobuf.Timestamp last_refresh_time = 5;
 }
 
 message MaintenanceTaskResponse {
@@ -197,6 +216,10 @@ message GetMaintenanceTaskRequest {
 message GetMaintenanceTaskResult {
     MaintenanceTaskOptions task_options = 1;
     repeated ActionGroupStates action_group_states = 2;
+    // The time when the mainteance task was created.
+    google.protobuf.Timestamp create_time = 3;
+    // The last time when the mainteance task was refreshed. Initially equals to create_time.
+    google.protobuf.Timestamp last_refresh_time = 4;
 }
 
 message GetMaintenanceTaskResponse {

src/api/protos/draft/ydb_replication.proto

@@ -77,6 +77,9 @@ message DescribeReplicationResult {
     message DoneState {
     }
 
+    message PausedState {
+    }
+
     // Description of scheme object.
     Ydb.Scheme.Entry self = 1;
 
@@ -90,6 +93,7 @@ message DescribeReplicationResult {
         RunningState running = 4;
         ErrorState error = 5;
         DoneState done = 6;
+        PausedState paused = 9;
     }
 }
 

src/api/protos/ydb_export.proto

@@ -1,6 +1,7 @@
 syntax = "proto3";
 option cc_enable_arenas = true;
 
+import "src/api/protos/annotations/sensitive.proto";
 import "src/api/protos/annotations/validation.proto";
 import "src/api/protos/ydb_operation.proto";
 
@@ -86,23 +87,27 @@ message ExportToS3Settings {
     };
 
     message Item {
-        // Database path to a table to be exported
+        // Database path to a table/directory to be exported
         string source_path = 1 [(required) = true];
 
         /* Tables are exported to one or more S3 objects.
            The object name begins with 'destination_prefix'.
            This prefix will be followed by '/data_PartNumber', where 'PartNumber'
            represents the index of the part, starting at zero.
+           Not required if the default `destination_prefix` is set.
+           If not specified, actual S3 path is the default `destination_prefix` concatenated with:
+            * The object path relative to the global `source_path` for a non-encrypted export
+            * The anonymized path for an encrypted export
         */
-        string destination_prefix = 2 [(required) = true];
+        string destination_prefix = 2;
     }
 
     string endpoint = 1 [(required) = true];
     Scheme scheme = 2; // HTTPS if not specified
     string bucket = 3 [(required) = true];
     string access_key = 4 [(required) = true];
     string secret_key = 5 [(required) = true];
-    repeated Item items = 6 [(size).ge = 1];
+    repeated Item items = 6;
     string description = 7 [(length).le = 128];
     uint32 number_of_retries = 8;
     StorageClass storage_class = 9;
@@ -120,6 +125,20 @@ message ExportToS3Settings {
     // details: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html
     // it is especially useful for custom s3 implementations
     bool disable_virtual_addressing = 12;
+
+    // Database root if not provided.
+    // All object names are calculated and written relative to this path.
+    string source_path = 13;
+
+    // A default S3 path prefix for all export items.
+    // When specified, export writes SchemaMapping file with the list of objects.
+    // Must be provided for an encrypted backup.
+    string destination_prefix = 14;
+
+    // Settings for data encryption.
+    // If encryption_settings field is not specified,
+    // the resulting data will not be encrypted.
+    EncryptionSettings encryption_settings = 15;
 }
 
 message ExportToS3Result {
@@ -141,3 +160,24 @@ message ExportToS3Response {
     // operation.metadata = ExportToS3Metadata
     Ydb.Operations.Operation operation = 1;
 }
+
+// Export encryption settings
+// Don't specify this struct for unencrypted exports
+message EncryptionSettings {
+    // Algorithm for export encryption.
+    // Not required in case of import/list operation.
+    // Currently the following algorithms are supported:
+    //  AES-128-GCM
+    //  AES-256-GCM
+    //  ChaCha20-Poly1305
+    string encryption_algorithm = 1;
+
+    oneof Key {
+        SymmetricKey symmetric_key = 2;
+    }
+
+    message SymmetricKey {
+        // This key will be used for data encryption
+        bytes key = 1 [(Ydb.sensitive) = true];
+    }
+}

src/api/protos/ydb_import.proto

@@ -2,6 +2,7 @@ syntax = "proto3";
 option cc_enable_arenas = true;
 
 import "src/api/protos/annotations/validation.proto";
+import "src/api/protos/ydb_export.proto";
 import "src/api/protos/ydb_operation.proto";
 
 import "google/protobuf/timestamp.proto";
@@ -39,25 +40,33 @@ message ImportFromS3Settings {
     }
 
     message Item {
-        /* YDB tables in S3 are stored in one or more objects (see ydb_export.proto).
-           The object name begins with 'source_prefix'.
-           This prefix is followed by:
-             * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero;
-             * '/scheme.pb' - object with information about scheme, indexes, etc;
-             * '/permissions.pb' - object with information about ACL and owner.
-        */
-        string source_prefix = 1 [(required) = true];
-
-        // Database path to a table to import to.
-        string destination_path = 2 [(required) = true];
+        oneof Source {
+            /* YDB database objects in S3 are stored in one or more S3 objects (see ydb_export.proto).
+            The S3 object name begins with a prefix, followed by:
+                * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero;
+                * '/scheme.pb' - object with information about scheme, indexes, etc;
+                * '/permissions.pb' - object with information about ACL and owner.
+            */
+
+            // The S3 object prefix can be either provided explicitly
+            string source_prefix = 1;
+
+            // Or, if the export contains the database objects list, you may specify the database object name, and the S3 prefix will be looked up in the database objects list by the import procedure
+            string source_path = 3;
+        }
+
+        // Database path to a database object to import the item to
+        // Resolved relative to the default destination_path
+        // May be omitted if the item's source_path is specified, in this case will be taken equal to it
+        string destination_path = 2;
     }
 
     string endpoint = 1 [(required) = true];
     Scheme scheme = 2; // HTTPS if not specified
     string bucket = 3 [(required) = true];
     string access_key = 4 [(required) = true];
     string secret_key = 5 [(required) = true];
-    repeated Item items = 6 [(size).ge = 1];
+    repeated Item items = 6; // Empty collection means import of all export objects
     string description = 7 [(length).le = 128];
     uint32 number_of_retries = 8;
 
@@ -76,6 +85,20 @@ message ImportFromS3Settings {
 
     // Skip checksum validation during import
     bool skip_checksum_validation = 12;
+
+    // A default path prefix for all items,
+    // determines that the import works with the list of objects in the SchemaMapping file.
+    // Must be provided to import an encrypted export.
+    string source_prefix = 13;
+
+    // Destination path to restore paths inside database
+    // Default value is database root
+    string destination_path = 14;
+
+    // Settings how data is encrypted.
+    // If encryption_settings field is not specified,
+    // the resulting data is considered not encrypted.
+    Ydb.Export.EncryptionSettings encryption_settings = 15;
 }
 
 message ImportFromS3Result {