Skip to content

Commit 9165c76

Browse files
sharpeyesharpeye
committed
add private API to list/acquire/release local NVMe devices
1 parent f14282f commit 9165c76

40 files changed

+939
-58
lines changed

‎cloud/blockstore/config/disk.proto‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,9 @@ message TDiskAgentConfig
356356

357357
// Allows to control data integrity validation for DR-based disks.
358358
optional EDataIntegrityValidationPolicy DataIntegrityValidationPolicyForDrBasedDisks = 46;
359+
360+
// The path where Disk Agent will store the cached NVMe device list.
361+
optional string CachedNVMeDevicesPath = 47;
359362
}
360363

361364
////////////////////////////////////////////////////////////////////////////////

‎cloud/blockstore/config/local_nvme.proto‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎cloud/blockstore/config/ya.make‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ SRCS(
1111
disk.proto
1212
grpc_client.proto
1313
http_proxy.proto
14-
local_nvme.proto
1514
logbroker.proto
1615
notify.proto
1716
plugin.proto

‎cloud/blockstore/libs/daemon/common/bootstrap.cpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
#include <cloud/blockstore/libs/endpoints_spdk/spdk_server.h>
4242
#include <cloud/blockstore/libs/endpoints_vhost/external_vhost_server.h>
4343
#include <cloud/blockstore/libs/endpoints_vhost/vhost_server.h>
44+
#include <cloud/blockstore/libs/local_nvme/service.h>
4445
#include <cloud/blockstore/libs/nbd/device.h>
4546
#include <cloud/blockstore/libs/nbd/error_handler.h>
4647
#include <cloud/blockstore/libs/nbd/netlink_device.h>
@@ -97,10 +98,10 @@
9798
#include <cloud/storage/core/libs/diagnostics/trace_processor.h>
9899
#include <cloud/storage/core/libs/diagnostics/trace_processor_mon.h>
99100
#include <cloud/storage/core/libs/diagnostics/trace_serializer.h>
100-
#include <cloud/storage/core/libs/grpc/init.h>
101-
#include <cloud/storage/core/libs/grpc/threadpool.h>
102101
#include <cloud/storage/core/libs/endpoints/fs/fs_endpoints.h>
103102
#include <cloud/storage/core/libs/endpoints/keyring/keyring_endpoints.h>
103+
#include <cloud/storage/core/libs/grpc/init.h>
104+
#include <cloud/storage/core/libs/grpc/threadpool.h>
104105
#include <cloud/storage/core/libs/opentelemetry/iface/trace_service_client.h>
105106
#include <cloud/storage/core/libs/opentelemetry/impl/trace_reader.h>
106107
#include <cloud/storage/core/libs/version/version.h>
@@ -942,6 +943,7 @@ void TBootstrapBase::Start()
942943
START_KIKIMR_COMPONENT(StatsUploader);
943944
START_COMMON_COMPONENT(Spdk);
944945
START_COMMON_COMPONENT(FileIOServiceProvider);
946+
START_COMMON_COMPONENT(LocalNVMeService);
945947
START_KIKIMR_COMPONENT(ActorSystem);
946948
START_COMMON_COMPONENT(EndpointManager);
947949
START_COMMON_COMPONENT(Service);
@@ -1026,6 +1028,8 @@ void TBootstrapBase::Stop()
10261028

10271029
STOP_KIKIMR_COMPONENT(ActorSystem);
10281030

1031+
STOP_COMMON_COMPONENT(LocalNVMeService);
1032+
10291033
// stop FileIOServiceProvider after ActorSystem to ensure that there are no
10301034
// in-flight I/O requests from TDiskAgentActor
10311035
STOP_COMMON_COMPONENT(FileIOServiceProvider);

‎cloud/blockstore/libs/daemon/common/bootstrap.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include <cloud/blockstore/libs/encryption/public.h>
1010
#include <cloud/blockstore/libs/endpoints/public.h>
1111
#include <cloud/blockstore/libs/endpoints_grpc/public.h>
12+
#include <cloud/blockstore/libs/local_nvme/public.h>
1213
#include <cloud/blockstore/libs/nbd/public.h>
1314
#include <cloud/blockstore/libs/nvme/public.h>
1415
#include <cloud/blockstore/libs/rdma/iface/public.h>
@@ -83,6 +84,7 @@ class TBootstrapBase
8384
NBD::IErrorHandlerMapPtr NbdErrorHandlerMap;
8485
NRdma::IServerPtr RdmaRequestServer;
8586
IStartablePtr RdmaTarget;
87+
NStorage::ILocalNVMeServicePtr LocalNVMeService;
8688

8789
TProgramShouldContinue ShouldContinue;
8890
TVector<TString> PostponedCriticalEvents;

‎cloud/blockstore/libs/daemon/ydb/bootstrap.cpp‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <cloud/blockstore/libs/kms/iface/compute_client.h>
2323
#include <cloud/blockstore/libs/kms/iface/key_provider.h>
2424
#include <cloud/blockstore/libs/kms/iface/kms_client.h>
25+
#include <cloud/blockstore/libs/local_nvme/service.h>
2526
#include <cloud/blockstore/libs/logbroker/iface/config.h>
2627
#include <cloud/blockstore/libs/logbroker/iface/logbroker.h>
2728
#include <cloud/blockstore/libs/notify/iface/config.h>
@@ -801,6 +802,18 @@ void TBootstrapYdb::InitKikimrService()
801802

802803
STORAGE_INFO("NotifyService initialized");
803804

805+
if (Configs->DiskAgentConfig->GetEnabled() &&
806+
Configs->DiskAgentConfig->GetCachedNVMeDevicesPath())
807+
{
808+
LocalNVMeService = NStorage::CreateLocalNVMeService(
809+
Configs->DiskAgentConfig->GetCachedNVMeDevicesPath(),
810+
logging);
811+
} else {
812+
LocalNVMeService = NStorage::CreateLocalNVMeServiceStub(logging);
813+
}
814+
815+
STORAGE_INFO("Local NVMe service initialized");
816+
804817
NStorage::TServerActorSystemArgs args;
805818
args.ModuleFactories = ModuleFactories;
806819
args.NodeId = nodeId;
@@ -847,6 +860,7 @@ void TBootstrapYdb::InitKikimrService()
847860
args.RootKmsKeyProvider = RootKmsKeyProvider;
848861
args.TemporaryServer = Configs->Options->TemporaryServer;
849862
args.BackgroundThreadPool = BackgroundThreadPool;
863+
args.LocalNVMeService = LocalNVMeService;
850864

851865
ActorSystem = NStorage::CreateActorSystem(args);
852866

‎cloud/blockstore/libs/daemon/ydb/ya.make‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ PEERDIR(
1616
cloud/blockstore/libs/endpoints
1717
cloud/blockstore/libs/kms/iface
1818
cloud/blockstore/libs/kikimr
19+
cloud/blockstore/libs/local_nvme
1920
cloud/blockstore/libs/logbroker/iface
2021
cloud/blockstore/libs/notify/impl
2122
cloud/blockstore/libs/nvme

‎cloud/blockstore/libs/disk_agent/bootstrap.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include <cloud/blockstore/libs/diagnostics/server_stats.h>
1515
#include <cloud/blockstore/libs/diagnostics/stats_aggregator.h>
1616
#include <cloud/blockstore/libs/diagnostics/volume_stats.h>
17+
#include <cloud/blockstore/libs/local_nvme/service.h>
1718
#include <cloud/blockstore/libs/nvme/nvme.h>
1819
#include <cloud/blockstore/libs/rdma/iface/config.h>
1920
#include <cloud/blockstore/libs/rdma/iface/probes.h>
@@ -524,6 +525,10 @@ bool TBootstrap::InitKikimrService()
524525

525526
STORAGE_INFO("DigestGenerator initialized");
526527

528+
LocalNVMeService = NStorage::CreateLocalNVMeServiceStub(logging);
529+
530+
STORAGE_INFO("Local NVMe service initialized");
531+
527532
NStorage::TDiskAgentActorSystemArgs args;
528533
args.ModuleFactories = ModuleFactories;
529534
args.NodeId = nodeId;
@@ -544,6 +549,7 @@ bool TBootstrap::InitKikimrService()
544549
args.NvmeManager = NvmeManager;
545550
args.StatsFetcher = StatsFetcher;
546551
args.BackgroundThreadPool = BackgroundThreadPool;
552+
args.LocalNVMeService = LocalNVMeService;
547553

548554
ActorSystem = NStorage::CreateDiskAgentActorSystem(args);
549555

@@ -658,6 +664,7 @@ void TBootstrap::Start()
658664
START_COMPONENT(Spdk);
659665
START_COMPONENT(RdmaServer);
660666
START_COMPONENT(FileIOServiceProvider);
667+
START_COMPONENT(LocalNVMeService);
661668
START_COMPONENT(ActorSystem);
662669

663670
// we need to start scheduler after all other components for 2 reasons:
@@ -707,6 +714,7 @@ void TBootstrap::Stop()
707714
STOP_COMPONENT(Monitoring);
708715
STOP_COMPONENT(Logging);
709716
STOP_COMPONENT(AsyncLogger);
717+
STOP_COMPONENT(LocalNVMeService);
710718
STOP_COMPONENT(BackgroundThreadPool);
711719

712720
#undef STOP_COMPONENT

‎cloud/blockstore/libs/disk_agent/bootstrap.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include <cloud/blockstore/libs/common/public.h>
66
#include <cloud/blockstore/libs/diagnostics/public.h>
77
#include <cloud/blockstore/libs/kikimr/public.h>
8+
#include <cloud/blockstore/libs/local_nvme/public.h>
89
#include <cloud/blockstore/libs/nvme/public.h>
910
#include <cloud/blockstore/libs/rdma/iface/public.h>
1011
#include <cloud/blockstore/libs/service/public.h>
@@ -75,6 +76,7 @@ class TBootstrap
7576
IStorageProviderPtr LocalStorageProvider;
7677
NNvme::INvmeManagerPtr NvmeManager;
7778
NRdma::IServerPtr RdmaServer;
79+
NStorage::ILocalNVMeServicePtr LocalNVMeService;
7880

7981
TProgramShouldContinue ShouldContinue;
8082
TVector<TString> PostponedCriticalEvents;

‎cloud/blockstore/libs/disk_agent/ya.make‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ PEERDIR(
1010
cloud/blockstore/libs/common
1111
cloud/blockstore/libs/diagnostics
1212
cloud/blockstore/libs/kikimr
13+
cloud/blockstore/libs/local_nvme
1314
cloud/blockstore/libs/rdma/iface
1415
cloud/blockstore/libs/server
1516
cloud/blockstore/libs/service

0 commit comments

Comments
 (0)