add private API to list/acquire/release local NVMe devices

Author: sharpeye

cloud/blockstore/config/disk.proto

@@ -356,6 +356,9 @@ message TDiskAgentConfig
 
     // Allows to control data integrity validation for DR-based disks.
     optional EDataIntegrityValidationPolicy DataIntegrityValidationPolicyForDrBasedDisks = 46;
+
+    // The path where Disk Agent will store the cached NVMe device list.
+    optional string CachedNVMeDevicesPath = 47;
 }
 
 ////////////////////////////////////////////////////////////////////////////////

cloud/blockstore/config/local_nvme.proto

@@ -11,7 +11,6 @@ SRCS(
     disk.proto
     grpc_client.proto
     http_proxy.proto
-    local_nvme.proto
     logbroker.proto
     notify.proto
     plugin.proto

cloud/blockstore/config/ya.make

@@ -41,6 +41,7 @@
 #include <cloud/blockstore/libs/endpoints_spdk/spdk_server.h>
 #include <cloud/blockstore/libs/endpoints_vhost/external_vhost_server.h>
 #include <cloud/blockstore/libs/endpoints_vhost/vhost_server.h>
+#include <cloud/blockstore/libs/local_nvme/service.h>
 #include <cloud/blockstore/libs/nbd/device.h>
 #include <cloud/blockstore/libs/nbd/error_handler.h>
 #include <cloud/blockstore/libs/nbd/netlink_device.h>
@@ -97,10 +98,10 @@
 #include <cloud/storage/core/libs/diagnostics/trace_processor.h>
 #include <cloud/storage/core/libs/diagnostics/trace_processor_mon.h>
 #include <cloud/storage/core/libs/diagnostics/trace_serializer.h>
-#include <cloud/storage/core/libs/grpc/init.h>
-#include <cloud/storage/core/libs/grpc/threadpool.h>
 #include <cloud/storage/core/libs/endpoints/fs/fs_endpoints.h>
 #include <cloud/storage/core/libs/endpoints/keyring/keyring_endpoints.h>
+#include <cloud/storage/core/libs/grpc/init.h>
+#include <cloud/storage/core/libs/grpc/threadpool.h>
 #include <cloud/storage/core/libs/opentelemetry/iface/trace_service_client.h>
 #include <cloud/storage/core/libs/opentelemetry/impl/trace_reader.h>
 #include <cloud/storage/core/libs/version/version.h>
@@ -942,6 +943,7 @@ void TBootstrapBase::Start()
     START_KIKIMR_COMPONENT(StatsUploader);
     START_COMMON_COMPONENT(Spdk);
     START_COMMON_COMPONENT(FileIOServiceProvider);
+    START_COMMON_COMPONENT(LocalNVMeService);
     START_KIKIMR_COMPONENT(ActorSystem);
     START_COMMON_COMPONENT(EndpointManager);
     START_COMMON_COMPONENT(Service);
@@ -1026,6 +1028,8 @@ void TBootstrapBase::Stop()
 
     STOP_KIKIMR_COMPONENT(ActorSystem);
 
+    STOP_COMMON_COMPONENT(LocalNVMeService);
+
     // stop FileIOServiceProvider after ActorSystem to ensure that there are no
     // in-flight I/O requests from TDiskAgentActor
     STOP_COMMON_COMPONENT(FileIOServiceProvider);

cloud/blockstore/libs/daemon/common/bootstrap.cpp

@@ -9,6 +9,7 @@
 #include <cloud/blockstore/libs/encryption/public.h>
 #include <cloud/blockstore/libs/endpoints/public.h>
 #include <cloud/blockstore/libs/endpoints_grpc/public.h>
+#include <cloud/blockstore/libs/local_nvme/public.h>
 #include <cloud/blockstore/libs/nbd/public.h>
 #include <cloud/blockstore/libs/nvme/public.h>
 #include <cloud/blockstore/libs/rdma/iface/public.h>
@@ -83,6 +84,7 @@ class TBootstrapBase
     NBD::IErrorHandlerMapPtr NbdErrorHandlerMap;
     NRdma::IServerPtr RdmaRequestServer;
     IStartablePtr RdmaTarget;
+    NStorage::ILocalNVMeServicePtr LocalNVMeService;
 
     TProgramShouldContinue ShouldContinue;
     TVector<TString> PostponedCriticalEvents;

cloud/blockstore/libs/daemon/common/bootstrap.h

@@ -22,6 +22,7 @@
 #include <cloud/blockstore/libs/kms/iface/compute_client.h>
 #include <cloud/blockstore/libs/kms/iface/key_provider.h>
 #include <cloud/blockstore/libs/kms/iface/kms_client.h>
+#include <cloud/blockstore/libs/local_nvme/service.h>
 #include <cloud/blockstore/libs/logbroker/iface/config.h>
 #include <cloud/blockstore/libs/logbroker/iface/logbroker.h>
 #include <cloud/blockstore/libs/notify/iface/config.h>
@@ -801,6 +802,18 @@ void TBootstrapYdb::InitKikimrService()
 
     STORAGE_INFO("NotifyService initialized");
 
+    if (Configs->DiskAgentConfig->GetEnabled() &&
+        Configs->DiskAgentConfig->GetCachedNVMeDevicesPath())
+    {
+        LocalNVMeService = NStorage::CreateLocalNVMeService(
+            Configs->DiskAgentConfig->GetCachedNVMeDevicesPath(),
+            logging);
+    } else {
+        LocalNVMeService = NStorage::CreateLocalNVMeServiceStub(logging);
+    }
+
+    STORAGE_INFO("Local NVMe service initialized");
+
     NStorage::TServerActorSystemArgs args;
     args.ModuleFactories = ModuleFactories;
     args.NodeId = nodeId;
@@ -847,6 +860,7 @@ void TBootstrapYdb::InitKikimrService()
     args.RootKmsKeyProvider = RootKmsKeyProvider;
     args.TemporaryServer = Configs->Options->TemporaryServer;
     args.BackgroundThreadPool = BackgroundThreadPool;
+    args.LocalNVMeService = LocalNVMeService;
 
     ActorSystem = NStorage::CreateActorSystem(args);
 

cloud/blockstore/libs/daemon/ydb/bootstrap.cpp

@@ -16,6 +16,7 @@ PEERDIR(
     cloud/blockstore/libs/endpoints
     cloud/blockstore/libs/kms/iface
     cloud/blockstore/libs/kikimr
+    cloud/blockstore/libs/local_nvme
     cloud/blockstore/libs/logbroker/iface
     cloud/blockstore/libs/notify/impl
     cloud/blockstore/libs/nvme

cloud/blockstore/libs/daemon/ydb/ya.make

@@ -14,6 +14,7 @@
 #include <cloud/blockstore/libs/diagnostics/server_stats.h>
 #include <cloud/blockstore/libs/diagnostics/stats_aggregator.h>
 #include <cloud/blockstore/libs/diagnostics/volume_stats.h>
+#include <cloud/blockstore/libs/local_nvme/service.h>
 #include <cloud/blockstore/libs/nvme/nvme.h>
 #include <cloud/blockstore/libs/rdma/iface/config.h>
 #include <cloud/blockstore/libs/rdma/iface/probes.h>
@@ -524,6 +525,10 @@ bool TBootstrap::InitKikimrService()
 
     STORAGE_INFO("DigestGenerator initialized");
 
+    LocalNVMeService = NStorage::CreateLocalNVMeServiceStub(logging);
+
+    STORAGE_INFO("Local NVMe service initialized");
+
     NStorage::TDiskAgentActorSystemArgs args;
     args.ModuleFactories = ModuleFactories;
     args.NodeId = nodeId;
@@ -544,6 +549,7 @@ bool TBootstrap::InitKikimrService()
     args.NvmeManager = NvmeManager;
     args.StatsFetcher = StatsFetcher;
     args.BackgroundThreadPool = BackgroundThreadPool;
+    args.LocalNVMeService = LocalNVMeService;
 
     ActorSystem = NStorage::CreateDiskAgentActorSystem(args);
 
@@ -658,6 +664,7 @@ void TBootstrap::Start()
     START_COMPONENT(Spdk);
     START_COMPONENT(RdmaServer);
     START_COMPONENT(FileIOServiceProvider);
+    START_COMPONENT(LocalNVMeService);
     START_COMPONENT(ActorSystem);
 
     // we need to start scheduler after all other components for 2 reasons:
@@ -707,6 +714,7 @@ void TBootstrap::Stop()
     STOP_COMPONENT(Monitoring);
     STOP_COMPONENT(Logging);
     STOP_COMPONENT(AsyncLogger);
+    STOP_COMPONENT(LocalNVMeService);
     STOP_COMPONENT(BackgroundThreadPool);
 
 #undef STOP_COMPONENT

cloud/blockstore/libs/disk_agent/bootstrap.cpp

@@ -5,6 +5,7 @@
 #include <cloud/blockstore/libs/common/public.h>
 #include <cloud/blockstore/libs/diagnostics/public.h>
 #include <cloud/blockstore/libs/kikimr/public.h>
+#include <cloud/blockstore/libs/local_nvme/public.h>
 #include <cloud/blockstore/libs/nvme/public.h>
 #include <cloud/blockstore/libs/rdma/iface/public.h>
 #include <cloud/blockstore/libs/service/public.h>
@@ -75,6 +76,7 @@ class TBootstrap
     IStorageProviderPtr LocalStorageProvider;
     NNvme::INvmeManagerPtr NvmeManager;
     NRdma::IServerPtr RdmaServer;
+    NStorage::ILocalNVMeServicePtr LocalNVMeService;
 
     TProgramShouldContinue ShouldContinue;
     TVector<TString> PostponedCriticalEvents;

cloud/blockstore/libs/disk_agent/bootstrap.h

@@ -108,16 +108,42 @@ class TLocalNVMeService final: public ILocalNVMeService
         }
 
         Devices = std::move(devices);
+
+        STORAGE_INFO(
+            "Discovered: " <<
+            [&]
+            {
+                TStringStream ss;
+                for (size_t i = 0; i != Devices.size(); ++i) {
+                    if (i) {
+                        ss << ", ";
+                    }
+                    ss << Devices[i];
+                }
+                return ss.Str();
+            }() << " ...");
     }
 };
 
 ////////////////////////////////////////////////////////////////////////////////
 
-struct TLocalNVMeServiceStub final: public ILocalNVMeService
+class TLocalNVMeServiceStub final: public ILocalNVMeService
 {
-    void Start() final
+private:
+    const ILoggingServicePtr Logging;
+
+    TLog Log;
+
+public:
+    explicit TLocalNVMeServiceStub(ILoggingServicePtr logging)
+        : Logging(std::move(logging))
     {}
 
+    void Start() final
+    {
+        Log = Logging->CreateLog("BLOCKSTORE_LOCAL_NVME");
+    }
+
     void Stop() final
     {}
 
@@ -129,9 +155,12 @@ struct TLocalNVMeServiceStub final: public ILocalNVMeService
     [[nodiscard]] TFuture<NCloud::NProto::TError> ResetNVMeDevice(
         const TString& serialNumber) const final
     {
-        Y_UNUSED(serialNumber);
+        STORAGE_INFO("Reset NVMe device " << serialNumber.Quote() << " ...");
 
-        return MakeFuture(NProto::TError());
+        return MakeFuture(MakeError(
+            E_NOT_FOUND,
+            TStringBuilder()
+                << "Device " << serialNumber.Quote() << " not found"));
     }
 };
 
@@ -148,9 +177,9 @@ ILocalNVMeServicePtr CreateLocalNVMeService(
         std::move(logging));
 }
 
-ILocalNVMeServicePtr CreateLocalNVMeServiceStub()
+ILocalNVMeServicePtr CreateLocalNVMeServiceStub(ILoggingServicePtr logging)
 {
-    return std::make_shared<TLocalNVMeServiceStub>();
+    return std::make_shared<TLocalNVMeServiceStub>(std::move(logging));
 }
 
 }   // namespace NCloud::NBlockStore::NStorage