Skip to content

Commit 033f7f0

Browse files
alexvrualexvru
authored
Correct distconf enabling sequence (#15578)
1 parent e6c8de4 commit 033f7f0

11 files changed

+191
-88
lines changed

‎ydb/core/blobstorage/base/blobstorage_console_events.h

+2-1
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,10 @@ namespace NKikimr {
1010
NKikimrBlobStorage::TEvControllerProposeConfigRequest, EvControllerProposeConfigRequest> {
1111
TEvControllerProposeConfigRequest() = default;
1212

13-
TEvControllerProposeConfigRequest(ui64 configHash, ui64 configVersion) {
13+
TEvControllerProposeConfigRequest(ui64 configHash, ui64 configVersion, bool distconf) {
1414
Record.SetConfigHash(configHash);
1515
Record.SetConfigVersion(configVersion);
16+
Record.SetDistconf(distconf);
1617
}
1718

1819
TString ToString() const override {

‎ydb/core/blobstorage/nodewarden/distconf.h

+7-2
Original file line numberDiff line numberDiff line change
@@ -351,8 +351,13 @@ namespace NKikimr::NStorage {
351351
bool HasQuorum() const;
352352
void ProcessCollectConfigs(TEvGather::TCollectConfigs *res);
353353

354-
using TProcessCollectConfigsResult = std::variant<std::monostate, TString, NKikimrBlobStorage::TStorageConfig>;
355-
TProcessCollectConfigsResult ProcessCollectConfigs(TEvGather::TCollectConfigs *res, const TString *selfAssemblyUUID);
354+
struct TProcessCollectConfigsResult {
355+
std::variant<std::monostate, TString, NKikimrBlobStorage::TStorageConfig> Outcome;
356+
bool IsDistconfDisabledQuorum = false;
357+
};
358+
TProcessCollectConfigsResult ProcessCollectConfigs(TEvGather::TCollectConfigs *res,
359+
std::optional<TStringBuf> selfAssemblyUUID);
360+
356361
std::optional<TString> ProcessProposeStorageConfig(TEvGather::TProposeStorageConfig *res);
357362

358363
struct TExConfigError : yexception {};

‎ydb/core/blobstorage/nodewarden/distconf_console.cpp

+13-4
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,14 @@ namespace NKikimr::NStorage {
3535
return; // still waiting for previous one
3636
}
3737

38+
ProposeRequestInFlight = true;
39+
3840
if (!StorageConfig || !StorageConfig->HasConfigComposite()) {
39-
return; // no config yet
41+
// send empty proposition just to connect to console
42+
auto ev = std::make_unique<TEvBlobStorage::TEvControllerProposeConfigRequest>();
43+
ev->Record.SetDistconf(true);
44+
NTabletPipe::SendData(SelfId(), ConsolePipeId, ev.release(), ++ProposeRequestCookie);
45+
return;
4046
}
4147

4248
Y_ABORT_UNLESS(MainConfigYamlVersion);
@@ -51,11 +57,14 @@ namespace NKikimr::NStorage {
5157
MainConfigFetchYamlHash, *MainConfigYamlVersion));
5258
ProposedConfigHashVersion.emplace(MainConfigFetchYamlHash, *MainConfigYamlVersion);
5359
NTabletPipe::SendData(SelfId(), ConsolePipeId, new TEvBlobStorage::TEvControllerProposeConfigRequest(
54-
MainConfigFetchYamlHash, *MainConfigYamlVersion), ++ProposeRequestCookie);
55-
ProposeRequestInFlight = true;
60+
MainConfigFetchYamlHash, *MainConfigYamlVersion, true), ++ProposeRequestCookie);
5661
}
5762

5863
void TDistributedConfigKeeper::Handle(TEvBlobStorage::TEvControllerValidateConfigResponse::TPtr ev) {
64+
STLOG(PRI_DEBUG, BS_NODE, NWDC10, "received TEvControllerValidateConfigResponse",
65+
(Sender, ev->Sender), (Cookie, ev->Cookie), (Record, ev->Get()->Record),
66+
(ConsoleConfigValidationQ.size, ConsoleConfigValidationQ.size()));
67+
5968
auto& q = ConsoleConfigValidationQ;
6069
auto pred = [&](const auto& item) {
6170
const auto& [actorId, yaml, cookie] = item;
@@ -114,7 +123,7 @@ namespace NKikimr::NStorage {
114123
break;
115124

116125
case NKikimrBlobStorage::TEvControllerProposeConfigResponse::ReverseCommit:
117-
Y_DEBUG_ABORT();
126+
// just do nothing, we didn't have the config in distconf, possibly it is being enabled
118127
break;
119128
}
120129
}

‎ydb/core/blobstorage/nodewarden/distconf_fsm.cpp

+32-18
Original file line numberDiff line numberDiff line change
@@ -110,16 +110,17 @@ namespace NKikimr::NStorage {
110110
}
111111

112112
void TDistributedConfigKeeper::ProcessCollectConfigs(TEvGather::TCollectConfigs *res) {
113-
TOverloaded handler{
114-
[&](std::monostate&&) {
113+
auto r = ProcessCollectConfigs(res, std::nullopt);
114+
std::visit(TOverloaded{
115+
[&](std::monostate&) {
115116
STLOG(PRI_DEBUG, BS_NODE, NWDC61, "ProcessCollectConfigs: monostate");
116117
RootState = ERootState::RELAX;
117118
},
118-
[&](TString&& error) {
119+
[&](TString& error) {
119120
STLOG(PRI_DEBUG, BS_NODE, NWDC63, "ProcessCollectConfigs: error", (Error, error));
120121
SwitchToError(error);
121122
},
122-
[&](NKikimrBlobStorage::TStorageConfig&& proposedConfig) {
123+
[&](NKikimrBlobStorage::TStorageConfig& proposedConfig) {
123124
STLOG(PRI_DEBUG, BS_NODE, NWDC64, "ProcessCollectConfigs: proposed new config",
124125
(ProposedConfig, proposedConfig));
125126
TEvScatter task;
@@ -130,12 +131,11 @@ namespace NKikimr::NStorage {
130131
propose->MutableConfig()->Swap(&proposedConfig);
131132
IssueScatterTask(TActorId(), std::move(task));
132133
}
133-
};
134-
std::visit(handler, ProcessCollectConfigs(res, nullptr));
134+
}, r.Outcome);
135135
}
136136

137137
TDistributedConfigKeeper::TProcessCollectConfigsResult TDistributedConfigKeeper::ProcessCollectConfigs(
138-
TEvGather::TCollectConfigs *res, const TString *selfAssemblyUUID) {
138+
TEvGather::TCollectConfigs *res, std::optional<TStringBuf> selfAssemblyUUID) {
139139
auto generateSuccessful = [&](auto&& callback) {
140140
for (const auto& item : res->GetNodes()) {
141141
for (const auto& node : item.GetNodeIds()) {
@@ -171,10 +171,24 @@ namespace NKikimr::NStorage {
171171

172172
if (nodeQuorum && !configQuorum) {
173173
// check if there is quorum of no-distconf config along the cluster
174+
auto generateNodesWithoutDistconf = [&](auto&& callback) {
175+
for (const auto& item : res->GetNodes()) {
176+
if (item.GetBaseConfig().GetSelfManagementConfig().GetEnabled()) {
177+
continue;
178+
}
179+
for (const auto& node : item.GetNodeIds()) {
180+
callback(node);
181+
}
182+
}
183+
};
184+
if (HasNodeQuorum(*StorageConfig, generateNodesWithoutDistconf)) {
185+
// yes, distconf is disabled on the majority of the nodes, so we can't do anything about it
186+
return {.IsDistconfDisabledQuorum = true};
187+
}
174188
}
175189

176190
if (!nodeQuorum || !configQuorum) {
177-
return "no quorum for CollectConfigs";
191+
return {"no quorum for CollectConfigs"};
178192
}
179193

180194
// TODO: validate self-assembly UUID
@@ -223,7 +237,7 @@ namespace NKikimr::NStorage {
223237
(BaseConfigs.size, baseConfigs.size()));
224238
Y_DEBUG_ABORT("Multiple nonintersecting node sets have quorum of BaseConfig");
225239
Halt();
226-
return "Multiple nonintersecting node sets have quorum of BaseConfig";
240+
return {"Multiple nonintersecting node sets have quorum of BaseConfig"};
227241
}
228242
NKikimrBlobStorage::TStorageConfig *baseConfig = nullptr;
229243
for (auto& [meta, info] : baseConfigs) {
@@ -289,13 +303,13 @@ namespace NKikimr::NStorage {
289303
(Generation, generation), (Configs, configs));
290304
Y_DEBUG_ABORT("Multiple nonintersecting node sets have quorum of persistent config");
291305
Halt();
292-
return "Multiple nonintersecting node sets have quorum of persistent config";
306+
return {"Multiple nonintersecting node sets have quorum of persistent config"};
293307
}
294308
Y_ABORT_UNLESS(configs.size() == 1);
295309
persistedConfig = configs.front();
296310
}
297311
if (maxSeenGeneration && (!persistedConfig || persistedConfig->GetGeneration() < maxSeenGeneration)) {
298-
return "couldn't obtain quorum for configuration that was seen in effect";
312+
return {"couldn't obtain quorum for configuration that was seen in effect"};
299313
}
300314

301315
// let's try to find possibly proposed config, but without a quorum, and try to reconstruct it
@@ -311,7 +325,7 @@ namespace NKikimr::NStorage {
311325
(PersistentConfig, *persistedConfig), (ProposedConfig, config));
312326
Y_DEBUG_ABORT("persistently proposed config has too big generation");
313327
Halt();
314-
return "persistently proposed config has too big generation";
328+
return {"persistently proposed config has too big generation"};
315329
}
316330
}
317331
if (proposedConfig && (proposedConfig->GetGeneration() != config.GetGeneration() ||
@@ -361,11 +375,11 @@ namespace NKikimr::NStorage {
361375
if (!CurrentSelfAssemblyUUID) {
362376
CurrentSelfAssemblyUUID.emplace(CreateGuidAsString());
363377
}
364-
selfAssemblyUUID = &CurrentSelfAssemblyUUID.value();
378+
selfAssemblyUUID.emplace(CurrentSelfAssemblyUUID.value());
365379
}
366380
propositionBase.emplace(*baseConfig);
367-
if (auto error = GenerateFirstConfig(baseConfig, *selfAssemblyUUID)) {
368-
return *error;
381+
if (auto error = GenerateFirstConfig(baseConfig, TString(*selfAssemblyUUID))) {
382+
return {*error};
369383
}
370384
configToPropose = baseConfig;
371385
}
@@ -392,12 +406,12 @@ namespace NKikimr::NStorage {
392406

393407
if (error) {
394408
Y_DEBUG_ABORT("incorrect config proposition");
395-
return "incorrect config proposition";
409+
return {"incorrect config proposition"};
396410
}
397411

398412
if (propositionBase) {
399413
if (auto error = ValidateConfig(*propositionBase)) {
400-
return TStringBuilder() << "failed to propose configuration, base config contains errors: " << *error;
414+
return {TStringBuilder() << "failed to propose configuration, base config contains errors: " << *error};
401415
}
402416
if (auto error = ValidateConfigUpdate(*propositionBase, *configToPropose)) {
403417
Y_FAIL_S("incorrect config proposed: " << *error);
@@ -408,7 +422,7 @@ namespace NKikimr::NStorage {
408422
}
409423
}
410424

411-
return std::move(*configToPropose);
425+
return {std::move(*configToPropose)};
412426
}
413427

414428
return {};

0 commit comments

Comments
 (0)