From bf272d9d75687f3146ccc27df18709bb6af085d3 Mon Sep 17 00:00:00 2001 From: alexey-ivanov-es Date: Fri, 21 Mar 2025 19:01:45 +0000 Subject: [PATCH 1/5] [main] Move system indices migration to migrate plugin It seems the best way to fix #122949 is to use existing data stream reindex API. However, this API is located in the migrate x-pack plugin. This commit moves the system indices migration logic (REST handlers, transport actions, and task) to the migrate plugin. Port of #123551 --- modules/reindex/build.gradle | 3 - .../plugin-metadata/plugin-security.policy | 3 - .../upgrades/FeatureUpgradeIT.java | 4 +- server/src/main/java/module-info.java | 2 - .../elasticsearch/action/ActionModule.java | 11 - .../create/TransportCreateIndexAction.java | 50 ++- .../elasticsearch/cluster/ClusterModule.java | 2 - .../indices/SystemIndexDescriptor.java | 3 +- .../elasticsearch/indices/SystemIndices.java | 19 +- .../elasticsearch/node/NodeConstruction.java | 25 +- .../plugins/SystemIndexPlugin.java | 2 +- .../TransportCreateIndexActionTests.java | 30 ++ .../cluster/metadata/MetadataTests.java | 100 +----- .../metadata/ProjectMetadataTests.java | 3 - .../authz/privilege/SystemPrivilege.java | 4 + x-pack/plugin/migrate/build.gradle | 4 + .../AbstractFeatureMigrationIntegTest.java | 29 +- .../action}/FeatureMigrationIT.java | 25 +- .../action}/MultiFeatureMigrationIT.java | 21 +- .../action}/SystemIndexMigrationIT.java | 30 +- .../GetFeatureUpgradeStatusAction.java | 10 +- .../GetFeatureUpgradeStatusRequest.java | 10 +- .../GetFeatureUpgradeStatusResponse.java | 10 +- .../action}/PostFeatureUpgradeAction.java | 10 +- .../action}/PostFeatureUpgradeRequest.java | 10 +- .../action}/PostFeatureUpgradeResponse.java | 10 +- ...ransportGetFeatureUpgradeStatusAction.java | 39 +-- .../TransportPostFeatureUpgradeAction.java | 16 +- .../RestGetFeatureUpgradeStatusAction.java | 14 +- .../rest}/RestPostFeatureUpgradeAction.java | 14 +- .../task}/FeatureMigrationResults.java | 10 +- .../task}/MigrationResultsUpdateTask.java | 10 +- .../task}/SingleFeatureMigrationResult.java | 10 +- .../task}/SystemIndexMigrationExecutor.java | 35 +- .../task}/SystemIndexMigrationInfo.java | 10 +- .../task}/SystemIndexMigrationTaskParams.java | 12 +- .../task}/SystemIndexMigrationTaskState.java | 12 +- .../task}/SystemIndexMigrator.java | 79 ++--- .../xpack/migrate/MigratePlugin.java | 80 +++-- .../plugin-metadata/plugin-security.policy | 13 + .../GetFeatureUpgradeStatusResponseTests.java | 27 +- .../PostFeatureUpgradeResponseTests.java | 10 +- ...ortGetFeatureUpgradeStatusActionTests.java | 12 +- .../task}/FeatureMigrationResultsTests.java | 10 +- .../SystemIndexMigrationMetadataTests.java | 320 ++++++++++++++++++ .../SystemIndexMigrationTaskParamsTests.java | 10 +- ...IndexMigrationTaskParamsXContentTests.java | 10 +- .../SystemIndexMigrationTaskStateTests.java | 10 +- ...mIndexMigrationTaskStateXContentTests.java | 10 +- ...tsBlobStoreCacheMaintenanceIntegTests.java | 2 +- .../10_get_feature_upgrade_status.yml | 0 .../migration/20_post_feature_upgrade.yml | 5 +- 52 files changed, 690 insertions(+), 520 deletions(-) rename {modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration => x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action}/AbstractFeatureMigrationIntegTest.java (94%) rename {modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration => x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action}/FeatureMigrationIT.java (95%) rename {modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration => x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action}/MultiFeatureMigrationIT.java (92%) rename {modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration => x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action}/SystemIndexMigrationIT.java (74%) rename {server/src/main/java/org/elasticsearch/action/admin/cluster/migration => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action}/GetFeatureUpgradeStatusAction.java (55%) rename {server/src/main/java/org/elasticsearch/action/admin/cluster/migration => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action}/GetFeatureUpgradeStatusRequest.java (65%) rename {server/src/main/java/org/elasticsearch/action/admin/cluster/migration => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action}/GetFeatureUpgradeStatusResponse.java (96%) rename {server/src/main/java/org/elasticsearch/action/admin/cluster/migration => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action}/PostFeatureUpgradeAction.java (54%) rename {server/src/main/java/org/elasticsearch/action/admin/cluster/migration => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action}/PostFeatureUpgradeRequest.java (64%) rename {server/src/main/java/org/elasticsearch/action/admin/cluster/migration => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action}/PostFeatureUpgradeResponse.java (93%) rename {server/src/main/java/org/elasticsearch/action/admin/cluster/migration => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action}/TransportGetFeatureUpgradeStatusAction.java (79%) rename {server/src/main/java/org/elasticsearch/action/admin/cluster/migration => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action}/TransportPostFeatureUpgradeAction.java (85%) rename {server/src/main/java/org/elasticsearch/rest/action/admin/cluster => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/rest}/RestGetFeatureUpgradeStatusAction.java (67%) rename {server/src/main/java/org/elasticsearch/rest/action/admin/cluster => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/rest}/RestPostFeatureUpgradeAction.java (67%) rename {server/src/main/java/org/elasticsearch/upgrades => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task}/FeatureMigrationResults.java (93%) rename {server/src/main/java/org/elasticsearch/upgrades => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task}/MigrationResultsUpdateTask.java (90%) rename {server/src/main/java/org/elasticsearch/upgrades => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task}/SingleFeatureMigrationResult.java (92%) rename {server/src/main/java/org/elasticsearch/upgrades => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task}/SystemIndexMigrationExecutor.java (76%) rename {server/src/main/java/org/elasticsearch/upgrades => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task}/SystemIndexMigrationInfo.java (96%) rename {server/src/main/java/org/elasticsearch/upgrades => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task}/SystemIndexMigrationTaskParams.java (81%) rename {server/src/main/java/org/elasticsearch/upgrades => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task}/SystemIndexMigrationTaskState.java (91%) rename {server/src/main/java/org/elasticsearch/upgrades => x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task}/SystemIndexMigrator.java (92%) create mode 100644 x-pack/plugin/migrate/src/main/plugin-metadata/plugin-security.policy rename {server/src/test/java/org/elasticsearch/action/admin/cluster/migration => x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action}/GetFeatureUpgradeStatusResponseTests.java (76%) rename {server/src/test/java/org/elasticsearch/action/admin/cluster/migration => x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action}/PostFeatureUpgradeResponseTests.java (85%) rename {server/src/test/java/org/elasticsearch/action/admin/cluster/migration => x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action}/TransportGetFeatureUpgradeStatusActionTests.java (87%) rename {server/src/test/java/org/elasticsearch/upgrades => x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task}/FeatureMigrationResultsTests.java (91%) create mode 100644 x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationMetadataTests.java rename {server/src/test/java/org/elasticsearch/upgrades => x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task}/SystemIndexMigrationTaskParamsTests.java (78%) rename {server/src/test/java/org/elasticsearch/upgrades => x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task}/SystemIndexMigrationTaskParamsXContentTests.java (70%) rename {server/src/test/java/org/elasticsearch/upgrades => x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task}/SystemIndexMigrationTaskStateTests.java (89%) rename {server/src/test/java/org/elasticsearch/upgrades => x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task}/SystemIndexMigrationTaskStateXContentTests.java (76%) rename {rest-api-spec => x-pack/plugin}/src/yamlRestTest/resources/rest-api-spec/test/migration/10_get_feature_upgrade_status.yml (100%) rename {rest-api-spec => x-pack/plugin}/src/yamlRestTest/resources/rest-api-spec/test/migration/20_post_feature_upgrade.yml (56%) diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 29d4eb31b1cc0..93333579e6f8a 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -44,9 +44,6 @@ dependencies { clusterModules project(':modules:lang-painless') clusterModules project(':modules:parent-join') clusterModules project(":modules:rest-root") - - internalClusterTestImplementation project(':modules:lang-painless') - internalClusterTestImplementation project(':modules:lang-painless:spi') } restResources { diff --git a/modules/reindex/src/main/plugin-metadata/plugin-security.policy b/modules/reindex/src/main/plugin-metadata/plugin-security.policy index 2b6d821c4d1a8..016cc6365b6ee 100644 --- a/modules/reindex/src/main/plugin-metadata/plugin-security.policy +++ b/modules/reindex/src/main/plugin-metadata/plugin-security.policy @@ -10,9 +10,6 @@ grant { // reindex opens socket connections using the rest client permission java.net.SocketPermission "*", "connect"; - - // needed for Painless to generate runtime classes - permission java.lang.RuntimePermission "createClassLoader"; }; grant codeBase "${codebase.elasticsearch-rest-client}" { diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java index 6e936e9b1601d..faa012adcc139 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java @@ -11,9 +11,9 @@ import com.carrotsearch.randomizedtesting.annotations.Name; -import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction; import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.test.XContentTestUtils; import org.junit.BeforeClass; @@ -112,7 +112,7 @@ public void testGetFeatureUpgradeStatus() throws Exception { // for the next major version upgrade (see e.g. #93666). Trying to express this with features may be problematic, so we // want to keep using versions here. We also assume that for non-semantic version migrations are not required. boolean migrationNeeded = parseLegacyVersion(getOldClusterVersion()).map( - v -> v.before(TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION) + v -> v.before(SystemIndices.NO_UPGRADE_REQUIRED_VERSION) ).orElse(false); if (migrationNeeded) { assertThat(feature.get("migration_status"), equalTo("MIGRATION_NEEDED")); diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 86de9b9d8bc13..04df2c55ad52d 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -63,7 +63,6 @@ exports org.elasticsearch.action.admin.cluster.coordination; exports org.elasticsearch.action.admin.cluster.desirednodes; exports org.elasticsearch.action.admin.cluster.health; - exports org.elasticsearch.action.admin.cluster.migration; exports org.elasticsearch.action.admin.cluster.node.capabilities; exports org.elasticsearch.action.admin.cluster.node.hotthreads; exports org.elasticsearch.action.admin.cluster.node.info; @@ -385,7 +384,6 @@ exports org.elasticsearch.tasks; exports org.elasticsearch.threadpool; exports org.elasticsearch.transport; - exports org.elasticsearch.upgrades; exports org.elasticsearch.usage; exports org.elasticsearch.watcher; diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 7c6abbe7c788f..92eeed741126e 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -26,10 +26,6 @@ import org.elasticsearch.action.admin.cluster.desirednodes.TransportUpdateDesiredNodesAction; import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesAction; import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction; -import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction; -import org.elasticsearch.action.admin.cluster.migration.TransportPostFeatureUpgradeAction; import org.elasticsearch.action.admin.cluster.node.capabilities.TransportNodesCapabilitiesAction; import org.elasticsearch.action.admin.cluster.node.features.TransportNodesFeaturesAction; import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; @@ -286,7 +282,6 @@ import org.elasticsearch.rest.action.admin.cluster.RestDeleteStoredScriptAction; import org.elasticsearch.rest.action.admin.cluster.RestGetDesiredBalanceAction; import org.elasticsearch.rest.action.admin.cluster.RestGetDesiredNodesAction; -import org.elasticsearch.rest.action.admin.cluster.RestGetFeatureUpgradeStatusAction; import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction; import org.elasticsearch.rest.action.admin.cluster.RestGetScriptContextAction; import org.elasticsearch.rest.action.admin.cluster.RestGetScriptLanguageAction; @@ -300,7 +295,6 @@ import org.elasticsearch.rest.action.admin.cluster.RestNodesStatsAction; import org.elasticsearch.rest.action.admin.cluster.RestNodesUsageAction; import org.elasticsearch.rest.action.admin.cluster.RestPendingClusterTasksAction; -import org.elasticsearch.rest.action.admin.cluster.RestPostFeatureUpgradeAction; import org.elasticsearch.rest.action.admin.cluster.RestPrevalidateNodeRemovalAction; import org.elasticsearch.rest.action.admin.cluster.RestPutRepositoryAction; import org.elasticsearch.rest.action.admin.cluster.RestPutStoredScriptAction; @@ -681,10 +675,7 @@ public void reg actions.register(TransportSnapshotsStatusAction.TYPE, TransportSnapshotsStatusAction.class); actions.register(SnapshottableFeaturesAction.INSTANCE, TransportSnapshottableFeaturesAction.class); actions.register(ResetFeatureStateAction.INSTANCE, TransportResetFeatureStateAction.class); - actions.register(GetFeatureUpgradeStatusAction.INSTANCE, TransportGetFeatureUpgradeStatusAction.class); - actions.register(PostFeatureUpgradeAction.INSTANCE, TransportPostFeatureUpgradeAction.class); actions.register(TransportGetShardSnapshotAction.TYPE, TransportGetShardSnapshotAction.class); - actions.register(IndicesStatsAction.INSTANCE, TransportIndicesStatsAction.class); actions.register(IndicesSegmentsAction.INSTANCE, TransportIndicesSegmentsAction.class); actions.register(TransportIndicesShardStoresAction.TYPE, TransportIndicesShardStoresAction.class); @@ -882,8 +873,6 @@ public void initRestHandlers(Supplier nodesInCluster, Predicate< registerHandler.accept(new RestSnapshotsStatusAction()); registerHandler.accept(new RestSnapshottableFeaturesAction()); registerHandler.accept(new RestResetFeatureStateAction()); - registerHandler.accept(new RestGetFeatureUpgradeStatusAction()); - registerHandler.accept(new RestPostFeatureUpgradeAction()); registerHandler.accept(new RestGetIndicesAction()); registerHandler.accept(new RestIndicesStatsAction()); registerHandler.accept(new RestIndicesSegmentsAction()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java index ad3d3a8daeef6..a32b7949454c6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java @@ -148,7 +148,7 @@ protected void masterOperation( listener.onFailure(new IllegalStateException(message)); return; } - updateRequest = buildSystemIndexUpdateRequest(request, cause, descriptor, projectId); + updateRequest = buildManagedSystemIndexUpdateRequest(request, cause, descriptor, projectId); } else { updateRequest = buildUpdateRequest(request, cause, indexName, resolvedAt, projectId); } @@ -181,31 +181,55 @@ private CreateIndexClusterStateUpdateRequest buildUpdateRequest( .waitForActiveShards(request.waitForActiveShards()); } - private static CreateIndexClusterStateUpdateRequest buildSystemIndexUpdateRequest( + private static CreateIndexClusterStateUpdateRequest buildManagedSystemIndexUpdateRequest( CreateIndexRequest request, String cause, SystemIndexDescriptor descriptor, ProjectId projectId ) { - final Settings settings = Objects.requireNonNullElse(descriptor.getSettings(), Settings.EMPTY); + boolean indexMigrationInProgress = cause.equals(SystemIndices.MIGRATE_SYSTEM_INDEX_CAUSE) + && request.index().endsWith(SystemIndices.UPGRADED_INDEX_SUFFIX); + final Settings settings; + final String mappings; final Set aliases; - if (descriptor.getAliasName() == null) { + final String indexName; + + // if we are migrating a system index to a new index, we use settings/mappings/index name from the request, + // since it was created by SystemIndexMigrator + if (indexMigrationInProgress) { + settings = request.settings(); + mappings = request.mappings(); + indexName = request.index(); + // we will update alias later on aliases = Set.of(); } else { - aliases = Set.of(new Alias(descriptor.getAliasName()).isHidden(true).writeIndex(true)); - } + settings = Objects.requireNonNullElse(descriptor.getSettings(), Settings.EMPTY); + mappings = descriptor.getMappings(); + + if (descriptor.getAliasName() == null) { + aliases = Set.of(); + } else { + aliases = Set.of(new Alias(descriptor.getAliasName()).isHidden(true).writeIndex(true)); + } - // Throw an error if we are trying to directly create a system index other than the primary system index (or the alias) - if (request.index().equals(descriptor.getPrimaryIndex()) == false && request.index().equals(descriptor.getAliasName()) == false) { - throw new IllegalArgumentException( - "Cannot create system index with name " + request.index() + "; descriptor primary index is " + descriptor.getPrimaryIndex() - ); + // Throw an error if we are trying to directly create a system index other + // than the primary system index (or the alias, or we are migrating the index) + if (request.index().equals(descriptor.getPrimaryIndex()) == false + && request.index().equals(descriptor.getAliasName()) == false) { + throw new IllegalArgumentException( + "Cannot create system index with name " + + request.index() + + "; descriptor primary index is " + + descriptor.getPrimaryIndex() + ); + } + indexName = descriptor.getPrimaryIndex(); } - return new CreateIndexClusterStateUpdateRequest(cause, projectId, descriptor.getPrimaryIndex(), request.index()).aliases(aliases) + return new CreateIndexClusterStateUpdateRequest(cause, projectId, indexName, request.index()).aliases(aliases) .waitForActiveShards(ActiveShardCount.ALL) - .mappings(descriptor.getMappings()) + .mappings(mappings) .settings(settings); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java index 1e982ca25e1a4..1bbea6e9f1b63 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -87,7 +87,6 @@ import org.elasticsearch.tasks.TaskResultsService; import org.elasticsearch.telemetry.TelemetryProvider; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.upgrades.FeatureMigrationResults; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; @@ -255,7 +254,6 @@ public static List getNamedWriteables() { ComposableIndexTemplateMetadata::readDiffFrom ); registerProjectCustom(entries, DataStreamMetadata.TYPE, DataStreamMetadata::new, DataStreamMetadata::readDiffFrom); - registerProjectCustom(entries, FeatureMigrationResults.TYPE, FeatureMigrationResults::new, FeatureMigrationResults::readDiffFrom); registerMetadataCustom(entries, NodesShutdownMetadata.TYPE, NodesShutdownMetadata::new, NodesShutdownMetadata::readDiffFrom); registerMetadataCustom(entries, DesiredNodesMetadata.TYPE, DesiredNodesMetadata::new, DesiredNodesMetadata::readDiffFrom); registerProjectCustom( diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index 689eb6775cc9b..2b555553b03fc 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -13,7 +13,6 @@ import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; -import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction; import org.elasticsearch.action.admin.indices.create.AutoCreateAction; import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -148,7 +147,7 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java index 5684140b8728f..2a49afdbe39ad 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndices.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndices.java @@ -15,8 +15,8 @@ import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.Operations; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction; import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateResponse.ResetFeatureStateStatus; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction; @@ -36,7 +36,10 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Predicates; import org.elasticsearch.core.Tuple; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.plugins.SystemIndexPlugin; import org.elasticsearch.snapshots.SnapshotsService; @@ -111,7 +114,16 @@ public class SystemIndices { public static final String SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_system_index_access_allowed"; public static final String EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_external_system_index_access_origin"; - private static final int UPGRADED_TO_VERSION = TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION.major + 1; + + /** + * These versions should be set to current major and current major's index version + */ + @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) + public static final Version NO_UPGRADE_REQUIRED_VERSION = Version.V_9_0_0; + public static final IndexVersion NO_UPGRADE_REQUIRED_INDEX_VERSION = IndexVersions.UPGRADE_TO_LUCENE_10_0_0; + + public static final String MIGRATE_SYSTEM_INDEX_CAUSE = "migrate-system-index"; + private static final int UPGRADED_TO_VERSION = NO_UPGRADE_REQUIRED_VERSION.major + 1; public static final String UPGRADED_INDEX_SUFFIX = "-reindexed-for-" + UPGRADED_TO_VERSION; private static final Automaton EMPTY = Automata.makeEmpty(); @@ -346,6 +358,7 @@ public ExecutorSelector getExecutorSelector() { /** * Finds a single matching {@link SystemIndexDescriptor}, if any, for the given index name. + * Does not take into account system data streams and their backing indices. * @param name the name of the index * @return The matching {@link SystemIndexDescriptor} or {@code null} if no descriptor is found */ @@ -354,7 +367,7 @@ public ExecutorSelector getExecutorSelector() { } @Nullable - static SystemIndexDescriptor findMatchingDescriptor(SystemIndexDescriptor[] indexDescriptors, String name) { + private static SystemIndexDescriptor findMatchingDescriptor(SystemIndexDescriptor[] indexDescriptors, String name) { SystemIndexDescriptor matchingDescriptor = null; for (SystemIndexDescriptor systemIndexDescriptor : indexDescriptors) { if (systemIndexDescriptor.matchesIndexPattern(name)) { diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index ee21ffdc2d48b..639ae2e4095e3 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -218,7 +218,6 @@ import org.elasticsearch.threadpool.internal.BuiltInExecutorBuilders; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.upgrades.SystemIndexMigrationExecutor; import org.elasticsearch.usage.UsageService; import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -576,8 +575,7 @@ private void createClientAndRegistries(Settings settings, ThreadPool threadPool, IndicesModule.getNamedWriteables().stream(), searchModule.getNamedWriteables().stream(), pluginsService.flatMap(Plugin::getNamedWriteables), - ClusterModule.getNamedWriteables().stream(), - SystemIndexMigrationExecutor.getNamedWriteables().stream() + ClusterModule.getNamedWriteables().stream() ).flatMap(Function.identity()).toList() ); xContentRegistry = new NamedXContentRegistry( @@ -587,7 +585,6 @@ private void createClientAndRegistries(Settings settings, ThreadPool threadPool, searchModule.getNamedXContents().stream(), pluginsService.flatMap(Plugin::getNamedXContent), ClusterModule.getNamedXWriteables().stream(), - SystemIndexMigrationExecutor.getNamedXContentParsers().stream(), HealthNodeTaskExecutor.getNamedXContentParsers().stream() ).flatMap(Function.identity()).toList() ); @@ -1147,10 +1144,7 @@ public Map searchFields() { settingsModule, clusterService, threadPool, - systemIndices, - clusterModule.getIndexNameExpressionResolver(), - metadataUpdateSettingsService, - metadataCreateIndexService + clusterModule.getIndexNameExpressionResolver() ) ); @@ -1655,27 +1649,16 @@ private Module loadPersistentTasksService( SettingsModule settingsModule, ClusterService clusterService, ThreadPool threadPool, - SystemIndices systemIndices, - IndexNameExpressionResolver indexNameExpressionResolver, - MetadataUpdateSettingsService metadataUpdateSettingsService, - MetadataCreateIndexService metadataCreateIndexService + IndexNameExpressionResolver indexNameExpressionResolver ) { PersistentTasksService persistentTasksService = new PersistentTasksService(clusterService, threadPool, client); - SystemIndexMigrationExecutor systemIndexMigrationExecutor = new SystemIndexMigrationExecutor( - client, - clusterService, - systemIndices, - metadataUpdateSettingsService, - metadataCreateIndexService, - settingsModule.getIndexScopedSettings() - ); HealthNodeTaskExecutor healthNodeTaskExecutor = HealthNodeTaskExecutor.create( clusterService, persistentTasksService, settingsModule.getSettings(), clusterService.getClusterSettings() ); - Stream> builtinTaskExecutors = Stream.of(systemIndexMigrationExecutor, healthNodeTaskExecutor); + Stream> builtinTaskExecutors = Stream.of(healthNodeTaskExecutor); Stream> pluginTaskExecutors = pluginsService.filterPlugins(PersistentTaskPlugin.class) .map(p -> p.getPersistentTasksExecutor(clusterService, threadPool, client, settingsModule, indexNameExpressionResolver)) diff --git a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java index 13e4d9aab4771..4efea68683635 100644 --- a/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/SystemIndexPlugin.java @@ -48,7 +48,7 @@ *

An implementation may also override {@link #prepareForIndicesMigration(ClusterService, Client, ActionListener)} and * {@link #indicesMigrationComplete(Map, ClusterService, Client, ActionListener)} in order to take special action before and after a * feature migration, which will temporarily block access to system indices. For example, a plugin might want to enter a safe mode and - * reject certain requests while the migration is in progress. See {@link org.elasticsearch.upgrades.SystemIndexMigrationExecutor} for + * reject certain requests while the migration is in progress. See org.elasticsearch.upgrades.SystemIndexMigrationExecutor for * more details. * *

After plugins are loaded, the {@link SystemIndices} class will provide the rest of the system with access to the feature's diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java index 10a086675fa6a..82455be68e154 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java @@ -276,4 +276,34 @@ public void testCreateIndexUnderMissingProject() { ); assertThat(e.getMessage(), containsString("Could not find project with id [unknown_project_id]")); } + + public void testCreatingSystemIndexForMigration() { + CreateIndexRequest request = new CreateIndexRequest(); + String path = "/test"; // just to test that we pass settings + Settings settings = Settings.builder().put(SETTING_INDEX_HIDDEN, true).put(IndexMetadata.SETTING_DATA_PATH, path).build(); + request.index(MANAGED_SYSTEM_INDEX_NAME + SystemIndices.UPGRADED_INDEX_SUFFIX) + .cause(SystemIndices.MIGRATE_SYSTEM_INDEX_CAUSE) + .settings(settings); + + @SuppressWarnings("unchecked") + ActionListener mockListener = mock(ActionListener.class); + + action.masterOperation(mock(Task.class), request, CLUSTER_STATE, mockListener); + + ArgumentCaptor createRequestArgumentCaptor = ArgumentCaptor.forClass( + CreateIndexClusterStateUpdateRequest.class + ); + verify(mockListener, times(0)).onFailure(any()); + verify(metadataCreateIndexService, times(1)).createIndex( + any(TimeValue.class), + any(TimeValue.class), + any(TimeValue.class), + createRequestArgumentCaptor.capture(), + any() + ); + + CreateIndexClusterStateUpdateRequest processedRequest = createRequestArgumentCaptor.getValue(); + assertTrue(processedRequest.settings().getAsBoolean(SETTING_INDEX_HIDDEN, false)); + assertThat(processedRequest.settings().get(IndexMetadata.SETTING_DATA_PATH, ""), is(path)); + } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index c885a9d0b6f93..2162e6280d978 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; -import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.Diff; @@ -31,7 +30,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.Maps; @@ -53,7 +51,6 @@ import org.elasticsearch.index.alias.RandomAliasActionsGenerator; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.ingest.IngestMetadata; import org.elasticsearch.persistent.ClusterPersistentTasksCustomMetadata; import org.elasticsearch.persistent.PersistentTasks; @@ -68,8 +65,6 @@ import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.upgrades.SystemIndexMigrationExecutor; -import org.elasticsearch.upgrades.SystemIndexMigrationTaskParams; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -770,10 +765,6 @@ public void testParseXContentFormatBeforeMultiProject() throws IOException { { "id": "health-node", "task":{ "health-node": {"params":{}} } - }, - { - "id": "upgrade-system-indices", - "task":{ "upgrade-system-indices": {"params":{}} } } ] }, @@ -832,12 +823,6 @@ public void testParseXContentFormatBeforeMultiProject() throws IOException { metadata.getProject().customs().keySet(), containsInAnyOrder("persistent_tasks", "index-graveyard", "component_template") ); - final var projectTasks = PersistentTasksCustomMetadata.get(metadata.getProject()); - assertThat( - projectTasks.tasks().stream().map(PersistentTasksCustomMetadata.PersistentTask::getTaskName).toList(), - containsInAnyOrder("upgrade-system-indices") - ); - assertThat(clusterTasks.getLastAllocationId(), equalTo(projectTasks.getLastAllocationId())); } private Metadata fromJsonXContentStringWithPersistentTasks(String json) throws IOException { @@ -845,7 +830,6 @@ private Metadata fromJsonXContentStringWithPersistentTasks(String json) throws I registry.addAll(ClusterModule.getNamedXWriteables()); registry.addAll(IndicesModule.getNamedXContents()); registry.addAll(HealthNodeTaskExecutor.getNamedXContentParsers()); - registry.addAll(SystemIndexMigrationExecutor.getNamedXContentParsers()); final var clusterService = mock(ClusterService.class); when(clusterService.threadPool()).thenReturn(mock(ThreadPool.class)); @@ -855,15 +839,7 @@ private Metadata fromJsonXContentStringWithPersistentTasks(String json) throws I Settings.EMPTY, ClusterSettings.createBuiltInClusterSettings() ); - final var systemIndexMigrationExecutor = new SystemIndexMigrationExecutor( - mock(Client.class), - clusterService, - mock(SystemIndices.class), - mock(MetadataUpdateSettingsService.class), - mock(MetadataCreateIndexService.class), - IndexScopedSettings.DEFAULT_SCOPED_SETTINGS - ); - new PersistentTasksExecutorRegistry(List.of(healthNodeTaskExecutor, systemIndexMigrationExecutor)); + new PersistentTasksExecutorRegistry(List.of(healthNodeTaskExecutor)); XContentParserConfiguration config = XContentParserConfiguration.EMPTY.withRegistry(new NamedXContentRegistry(registry)); try (XContentParser parser = JsonXContent.jsonXContent.createParser(config, json)) { @@ -2595,29 +2571,7 @@ public void testEmptyDiffReturnsSameInstance() throws IOException { public void testMultiProjectXContent() throws IOException { final long lastAllocationId = randomNonNegativeLong(); - final List projects = randomList(1, 5, () -> randomProject(randomUniqueProjectId(), randomIntBetween(1, 3))) - .stream() - .map( - project -> ProjectMetadata.builder(project) - .putCustom( - PersistentTasksCustomMetadata.TYPE, - new PersistentTasksCustomMetadata( - lastAllocationId, - Map.of( - SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, - new PersistentTasksCustomMetadata.PersistentTask<>( - SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, - SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, - new SystemIndexMigrationTaskParams(), - lastAllocationId, - PersistentTasks.INITIAL_ASSIGNMENT - ) - ) - ) - ) - .build() - ) - .toList(); + final List projects = randomList(1, 5, () -> randomProject(randomUniqueProjectId(), randomIntBetween(1, 3))); final Metadata originalMeta = Metadata.builder(randomMetadata(projects)) .putCustom( @@ -2661,23 +2615,8 @@ public void testMultiProjectXContent() throws IOException { equalTo(projects.stream().map(pp -> pp.id().id()).collect(Collectors.toUnmodifiableSet())) ); - for (int i = 0; i < projects.size(); i++) { - assertThat(objectPath.evaluate("meta-data.projects." + i + ".persistent_tasks"), notNullValue()); - assertThat(objectPath.evaluate("meta-data.projects." + i + ".persistent_tasks.last_allocation_id"), equalTo(lastAllocationId)); - assertThat(objectPath.evaluate("meta-data.projects." + i + ".persistent_tasks.tasks"), hasSize(1)); - assertThat( - objectPath.evaluate("meta-data.projects." + i + ".persistent_tasks.tasks.0.id"), - equalTo(SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME) - ); - } - Metadata fromXContentMeta = fromJsonXContentStringWithPersistentTasks(bytes.utf8ToString()); assertThat(fromXContentMeta.projects().keySet(), equalTo(originalMeta.projects().keySet())); - for (var project : fromXContentMeta.projects().values()) { - final var projectTasks = PersistentTasksCustomMetadata.get(project); - assertThat(projectTasks.getLastAllocationId(), equalTo(lastAllocationId)); - assertThat(projectTasks.taskMap().keySet(), equalTo(Set.of(SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME))); - } final var clusterTasks = ClusterPersistentTasksCustomMetadata.get(fromXContentMeta); assertThat(clusterTasks.getLastAllocationId(), equalTo(lastAllocationId + 1)); assertThat(clusterTasks.taskMap().keySet(), equalTo(Set.of(HealthNode.TASK_NAME))); @@ -2688,25 +2627,6 @@ public void testDefaultProjectXContentWithPersistentTasks() throws IOException { final var originalMeta = Metadata.builder() .clusterUUID(randomUUID()) .clusterUUIDCommitted(true) - .put( - ProjectMetadata.builder(ProjectId.DEFAULT) - .putCustom( - PersistentTasksCustomMetadata.TYPE, - new PersistentTasksCustomMetadata( - lastAllocationId, - Map.of( - SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, - new PersistentTasksCustomMetadata.PersistentTask<>( - SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, - SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, - new SystemIndexMigrationTaskParams(), - lastAllocationId, - PersistentTasks.INITIAL_ASSIGNMENT - ) - ) - ) - ) - ) .putCustom( ClusterPersistentTasksCustomMetadata.TYPE, new ClusterPersistentTasksCustomMetadata( @@ -2736,14 +2656,7 @@ public void testDefaultProjectXContentWithPersistentTasks() throws IOException { assertThat(objectPath.evaluate("meta-data.cluster_persistent_tasks"), nullValue()); // The combined lastAllocationId is the max between cluster and project tasks assertThat(objectPath.evaluate("meta-data.persistent_tasks.last_allocation_id"), equalTo(lastAllocationId + 1)); - assertThat(objectPath.evaluate("meta-data.persistent_tasks.tasks"), hasSize(2)); - assertThat( - Set.of( - objectPath.evaluate("meta-data.persistent_tasks.tasks.0.id"), - objectPath.evaluate("meta-data.persistent_tasks.tasks.1.id") - ), - equalTo(Set.of(HealthNode.TASK_NAME, SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME)) - ); + assertThat(objectPath.evaluate("meta-data.persistent_tasks.tasks"), hasSize(1)); // Deserialize from the XContent should separate cluster and project tasks final Metadata fromXContentMeta = fromJsonXContentStringWithPersistentTasks(bytes.utf8ToString()); @@ -2755,13 +2668,6 @@ public void testDefaultProjectXContentWithPersistentTasks() throws IOException { clusterTasks.tasks().stream().map(PersistentTasksCustomMetadata.PersistentTask::getId).toList(), contains(HealthNode.TASK_NAME) ); - final var projectTasks = PersistentTasksCustomMetadata.get(fromXContentMeta.getProject(ProjectId.DEFAULT)); - assertThat(projectTasks, notNullValue()); - assertThat(projectTasks.getLastAllocationId(), equalTo(lastAllocationId + 1)); - assertThat( - projectTasks.tasks().stream().map(PersistentTasksCustomMetadata.PersistentTask::getId).toList(), - contains(SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME) - ); } public void testSingleNonDefaultProjectXContent() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java index ee84c63a0cc37..79c900b421f9c 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.test.AbstractChunkedSerializingTestCase; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.upgrades.FeatureMigrationResults; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentType; @@ -363,8 +362,6 @@ static int expectedChunkCount(ToXContent.Params params, ProjectMetadata project) params, 4 + dataStreamMetadata.dataStreams().size() + dataStreamMetadata.getDataStreamAliases().size() ); - } else if (custom instanceof FeatureMigrationResults featureMigrationResults) { - chunkCount += checkChunkSize(custom, params, 2 + featureMigrationResults.getFeatureStatuses().size()); } else if (custom instanceof IndexGraveyard indexGraveyard) { chunkCount += checkChunkSize(custom, params, 2 + indexGraveyard.getTombstones().size()); } else if (custom instanceof IngestMetadata ingestMetadata) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java index 98c12930e188e..a7580ea22182b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/SystemPrivilege.java @@ -6,7 +6,9 @@ */ package org.elasticsearch.xpack.core.security.authz.privilege; +import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; import org.elasticsearch.action.admin.indices.readonly.TransportAddIndexBlockAction; +import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction; import org.elasticsearch.action.search.TransportSearchShardsAction; import org.elasticsearch.index.seqno.RetentionLeaseActions; import org.elasticsearch.index.seqno.RetentionLeaseBackgroundSyncAction; @@ -45,7 +47,9 @@ public final class SystemPrivilege extends Privilege { "indices:data/read/*", // needed for SystemIndexMigrator "indices:admin/refresh", // needed for SystemIndexMigrator "indices:admin/aliases", // needed for SystemIndexMigrator + TransportCreateIndexAction.TYPE.name() + "*", // needed for SystemIndexMigrator TransportAddIndexBlockAction.TYPE.name() + "*", // needed for SystemIndexMigrator + TransportUpdateSettingsAction.TYPE.name() + "*", // needed for SystemIndexMigrator TransportSearchShardsAction.TYPE.name(), // added so this API can be called with the system user by other APIs ActionTypes.RELOAD_REMOTE_CLUSTER_CREDENTIALS_ACTION.name() // needed for Security plugin reload of remote cluster credentials ); diff --git a/x-pack/plugin/migrate/build.gradle b/x-pack/plugin/migrate/build.gradle index 796263846859d..9908b2d01a021 100644 --- a/x-pack/plugin/migrate/build.gradle +++ b/x-pack/plugin/migrate/build.gradle @@ -21,6 +21,10 @@ dependencies { testImplementation project(':modules:data-streams') testImplementation project(path: ':modules:reindex') testImplementation project(path: ':modules:ingest-common') + testImplementation project(path: ':modules:lang-painless') + + internalClusterTestImplementation project(path: ':modules:lang-painless') + internalClusterTestImplementation project(path: ':modules:lang-painless:spi') } addQaCheckDependencies(project) diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/AbstractFeatureMigrationIntegTest.java similarity index 94% rename from modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java rename to x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/AbstractFeatureMigrationIntegTest.java index 314d41a6c75ef..efe186953466c 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/AbstractFeatureMigrationIntegTest.java @@ -1,19 +1,16 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.stats.IndexStats; @@ -31,14 +28,18 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.indices.AssociatedIndexDescriptor; import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.ingest.common.IngestCommonPlugin; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.plugins.SystemIndexPlugin; +import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.migrate.MigratePlugin; import org.junit.Assert; import org.junit.Before; @@ -78,9 +79,9 @@ public abstract class AbstractFeatureMigrationIntegTest extends ESIntegTestCase static final int INTERNAL_MANAGED_FLAG_VALUE = 1; static final String FIELD_NAME = "some_field"; protected static final IndexVersion NEEDS_UPGRADE_INDEX_VERSION = IndexVersionUtils.getPreviousMajorVersion( - TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_INDEX_VERSION + SystemIndices.NO_UPGRADE_REQUIRED_INDEX_VERSION ); - protected static final int UPGRADED_TO_VERSION = TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION.major + 1; + protected static final int UPGRADED_TO_VERSION = SystemIndices.NO_UPGRADE_REQUIRED_VERSION.major + 1; static final SystemIndexDescriptor EXTERNAL_UNMANAGED = SystemIndexDescriptor.builder() .setIndexPattern(".ext-unman-*") @@ -154,6 +155,16 @@ public T getPlugin(Class type) { return pluginsService.filterPlugins(type).findFirst().get(); } + @Override + protected Collection> nodePlugins() { + List> plugins = new ArrayList<>(super.nodePlugins()); + plugins.add(MigratePlugin.class); + plugins.add(ReindexPlugin.class); + plugins.add(TestPlugin.class); + plugins.add(IngestCommonPlugin.class); + return plugins; + } + protected void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) { assertThat( "the strategy used below to create index names for descriptors without a primary index name only works for simple patterns", diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/FeatureMigrationIT.java similarity index 95% rename from modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java rename to x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/FeatureMigrationIT.java index 9730ef698af99..81bdb9e1c6e72 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/FeatureMigrationIT.java @@ -1,21 +1,13 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.migration; +package org.elasticsearch.system_indices.action; import org.apache.lucene.util.SetOnce; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusRequest; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeRequest; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeResponse; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.alias.TransportIndicesAliasesAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; @@ -39,14 +31,13 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.SystemIndexDescriptor; -import org.elasticsearch.migration.AbstractFeatureMigrationIntegTest.TestPlugin.BlockingActionFilter; import org.elasticsearch.painless.PainlessPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; -import org.elasticsearch.reindex.ReindexPlugin; +import org.elasticsearch.system_indices.action.AbstractFeatureMigrationIntegTest.TestPlugin.BlockingActionFilter; +import org.elasticsearch.system_indices.task.FeatureMigrationResults; +import org.elasticsearch.system_indices.task.SingleFeatureMigrationResult; import org.elasticsearch.test.InternalTestCluster; -import org.elasticsearch.upgrades.FeatureMigrationResults; -import org.elasticsearch.upgrades.SingleFeatureMigrationResult; import java.util.ArrayList; import java.util.Arrays; @@ -105,9 +96,7 @@ protected boolean forbidPrivateIndexSettings() { @Override protected Collection> nodePlugins() { List> plugins = new ArrayList<>(super.nodePlugins()); - plugins.add(TestPlugin.class); plugins.add(SecondTestPlugin.class); - plugins.add(ReindexPlugin.class); plugins.add(PainlessPlugin.class); return plugins; } diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/MultiFeatureMigrationIT.java similarity index 92% rename from modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java rename to x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/MultiFeatureMigrationIT.java index 90503ffc8fb65..cbbc2c23871b2 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/MultiFeatureMigrationIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/MultiFeatureMigrationIT.java @@ -1,22 +1,14 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.migration; +package org.elasticsearch.system_indices.action; import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusRequest; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeRequest; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.Metadata; @@ -27,8 +19,7 @@ import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.SystemIndexPlugin; -import org.elasticsearch.reindex.ReindexPlugin; -import org.elasticsearch.upgrades.FeatureMigrationResults; +import org.elasticsearch.system_indices.task.FeatureMigrationResults; import java.util.ArrayList; import java.util.Arrays; @@ -71,9 +62,7 @@ protected boolean forbidPrivateIndexSettings() { @Override protected Collection> nodePlugins() { List> plugins = new ArrayList<>(super.nodePlugins()); - plugins.add(TestPlugin.class); plugins.add(SecondPlugin.class); - plugins.add(ReindexPlugin.class); return plugins; } diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/SystemIndexMigrationIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/SystemIndexMigrationIT.java similarity index 74% rename from modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/SystemIndexMigrationIT.java rename to x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/SystemIndexMigrationIT.java index 018b5e5361711..c5589d705d3e4 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/SystemIndexMigrationIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/system_indices/action/SystemIndexMigrationIT.java @@ -1,37 +1,25 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.migration; +package org.elasticsearch.system_indices.action; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusRequest; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeRequest; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.test.InternalTestCluster; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; -import static org.elasticsearch.upgrades.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; +import static org.elasticsearch.system_indices.task.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; import static org.hamcrest.Matchers.equalTo; /** @@ -51,14 +39,6 @@ protected boolean forbidPrivateIndexSettings() { return false; } - @Override - protected Collection> nodePlugins() { - List> plugins = new ArrayList<>(super.nodePlugins()); - plugins.add(TestPlugin.class); - plugins.add(ReindexPlugin.class); - return plugins; - } - public void testSystemIndexMigrationCanBeInterruptedWithShutdown() throws Exception { CyclicBarrier taskCreated = new CyclicBarrier(2); CyclicBarrier shutdownCompleted = new CyclicBarrier(2); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusAction.java similarity index 55% rename from server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusAction.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusAction.java index 23b094fc72b6f..31511d6a752cc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusAction.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.action.ActionType; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusRequest.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusRequest.java similarity index 65% rename from server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusRequest.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusRequest.java index 959197f1e12b6..18287dd280630 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusRequest.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusRequest.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.MasterNodeRequest; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusResponse.java similarity index 96% rename from server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusResponse.java index a84d44712187a..41bcecb3a02dc 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponse.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusResponse.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionResponse; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeAction.java similarity index 54% rename from server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeAction.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeAction.java index d5f881cef9e5c..525c0b78f9be3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeAction.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.action.ActionType; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeRequest.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeRequest.java similarity index 64% rename from server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeRequest.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeRequest.java index 5681043148821..9b8a4000a0e44 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeRequest.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeRequest.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.MasterNodeRequest; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponse.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeResponse.java similarity index 93% rename from server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponse.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeResponse.java index 191939f55d416..67e49d6128c7b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponse.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeResponse.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionResponse; diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/TransportGetFeatureUpgradeStatusAction.java similarity index 79% rename from server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/TransportGetFeatureUpgradeStatusAction.java index 3a0f92270cbbd..26e23a152709a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/TransportGetFeatureUpgradeStatusAction.java @@ -1,15 +1,12 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeAction; @@ -18,31 +15,30 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.IndexVersions; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.system_indices.task.FeatureMigrationResults; +import org.elasticsearch.system_indices.task.SingleFeatureMigrationResult; +import org.elasticsearch.system_indices.task.SystemIndexMigrationTaskParams; +import org.elasticsearch.system_indices.task.SystemIndexMigrationTaskState; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.upgrades.FeatureMigrationResults; -import org.elasticsearch.upgrades.SingleFeatureMigrationResult; -import org.elasticsearch.upgrades.SystemIndexMigrationTaskParams; -import org.elasticsearch.upgrades.SystemIndexMigrationTaskState; import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.stream.Stream; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.ERROR; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.IN_PROGRESS; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.MIGRATION_NEEDED; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED; +import static org.elasticsearch.indices.SystemIndices.NO_UPGRADE_REQUIRED_INDEX_VERSION; import static org.elasticsearch.indices.SystemIndices.UPGRADED_INDEX_SUFFIX; -import static org.elasticsearch.upgrades.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.ERROR; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.IN_PROGRESS; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.MIGRATION_NEEDED; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED; +import static org.elasticsearch.system_indices.task.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; /** * Transport class for the get feature upgrade status action @@ -51,13 +47,6 @@ public class TransportGetFeatureUpgradeStatusAction extends TransportMasterNodeA GetFeatureUpgradeStatusRequest, GetFeatureUpgradeStatusResponse> { - /** - * These versions should be set to current major and current major's index version - */ - @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) - public static final Version NO_UPGRADE_REQUIRED_VERSION = Version.V_9_0_0; - public static final IndexVersion NO_UPGRADE_REQUIRED_INDEX_VERSION = IndexVersions.UPGRADE_TO_LUCENE_10_0_0; - private final SystemIndices systemIndices; @Inject diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/TransportPostFeatureUpgradeAction.java similarity index 85% rename from server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/TransportPostFeatureUpgradeAction.java index 08438de4bfea1..ee4ee93568aa6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportPostFeatureUpgradeAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/action/TransportPostFeatureUpgradeAction.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -24,18 +22,18 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.persistent.PersistentTasksService; +import org.elasticsearch.system_indices.task.SystemIndexMigrationTaskParams; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.upgrades.SystemIndexMigrationTaskParams; import java.util.Comparator; import java.util.EnumSet; import java.util.List; import java.util.Set; -import static org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction.getFeatureUpgradeStatus; -import static org.elasticsearch.upgrades.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; +import static org.elasticsearch.system_indices.action.TransportGetFeatureUpgradeStatusAction.getFeatureUpgradeStatus; +import static org.elasticsearch.system_indices.task.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; /** * Transport action for post feature upgrade action diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetFeatureUpgradeStatusAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/rest/RestGetFeatureUpgradeStatusAction.java similarity index 67% rename from server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetFeatureUpgradeStatusAction.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/rest/RestGetFeatureUpgradeStatusAction.java index 0a2fc9cc5c92a..47d6ff3b1e10c 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestGetFeatureUpgradeStatusAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/rest/RestGetFeatureUpgradeStatusAction.java @@ -1,20 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.rest.action.admin.cluster; +package org.elasticsearch.system_indices.rest; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction; -import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusAction; +import org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusRequest; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPostFeatureUpgradeAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/rest/RestPostFeatureUpgradeAction.java similarity index 67% rename from server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPostFeatureUpgradeAction.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/rest/RestPostFeatureUpgradeAction.java index 38e81668dd0eb..97d6e11cc9fa3 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestPostFeatureUpgradeAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/rest/RestPostFeatureUpgradeAction.java @@ -1,20 +1,18 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.rest.action.admin.cluster; +package org.elasticsearch.system_indices.rest; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction; -import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.system_indices.action.PostFeatureUpgradeAction; +import org.elasticsearch.system_indices.action.PostFeatureUpgradeRequest; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/FeatureMigrationResults.java similarity index 93% rename from server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/FeatureMigrationResults.java index f6c272b7d3cc1..6653d6ce73cc7 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/FeatureMigrationResults.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/FeatureMigrationResults.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; diff --git a/server/src/main/java/org/elasticsearch/upgrades/MigrationResultsUpdateTask.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/MigrationResultsUpdateTask.java similarity index 90% rename from server/src/main/java/org/elasticsearch/upgrades/MigrationResultsUpdateTask.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/MigrationResultsUpdateTask.java index f4abb8f46416e..5158ec1cd9fb7 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/MigrationResultsUpdateTask.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/MigrationResultsUpdateTask.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/server/src/main/java/org/elasticsearch/upgrades/SingleFeatureMigrationResult.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SingleFeatureMigrationResult.java similarity index 92% rename from server/src/main/java/org/elasticsearch/upgrades/SingleFeatureMigrationResult.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SingleFeatureMigrationResult.java index 5dac57fdf8d41..266ebff4239fc 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/SingleFeatureMigrationResult.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SingleFeatureMigrationResult.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.SimpleDiffable; diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationExecutor.java similarity index 76% rename from server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationExecutor.java index 99e739919effc..b37a7a15a8310 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationExecutor.java @@ -1,18 +1,14 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; -import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -31,7 +27,7 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.upgrades.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; +import static org.elasticsearch.system_indices.task.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; /** * Starts the process of migrating system indices. See {@link SystemIndexMigrator} for the actual migration logic. @@ -40,24 +36,18 @@ public class SystemIndexMigrationExecutor extends PersistentTasksExecutor taskInProgress, Map headers ) { - return new SystemIndexMigrator( - client, - id, - type, - action, - parentTaskId, - taskInProgress.getParams(), - headers, - clusterService, - systemIndices, - metadataUpdateSettingsService, - metadataCreateIndexService, - indexScopedSettings - ); + return new SystemIndexMigrator(client, id, type, action, parentTaskId, headers, clusterService, systemIndices, indexScopedSettings); } @Override diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationInfo.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationInfo.java similarity index 96% rename from server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationInfo.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationInfo.java index aa6f09aa1ac54..579e13cd827d3 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationInfo.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationInfo.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParams.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParams.java similarity index 81% rename from server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParams.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParams.java index acdacac748a63..b6be87b25d762 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParams.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParams.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.StreamInput; @@ -19,7 +17,7 @@ import java.io.IOException; -import static org.elasticsearch.upgrades.FeatureMigrationResults.MIGRATION_ADDED_VERSION; +import static org.elasticsearch.system_indices.task.FeatureMigrationResults.MIGRATION_ADDED_VERSION; /** * The params used to initialize {@link SystemIndexMigrator} when it's initially kicked off. diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskState.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskState.java similarity index 91% rename from server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskState.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskState.java index 7bd9a43eca36c..cb7ac812bb8b1 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskState.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskState.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; @@ -25,7 +23,7 @@ import java.util.Map; import java.util.Objects; -import static org.elasticsearch.upgrades.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; +import static org.elasticsearch.system_indices.task.SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; /** diff --git a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrator.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrator.java similarity index 92% rename from server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrator.java rename to x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrator.java index 64b063803481f..9969da914c2eb 100644 --- a/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrator.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/system_indices/task/SystemIndexMigrator.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -16,13 +14,14 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; -import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.readonly.AddIndexBlockRequest; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsClusterStateUpdateRequest; +import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.MasterNodeRequest; -import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.ParentTaskAssigningClient; import org.elasticsearch.cluster.ClusterState; @@ -30,9 +29,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; -import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; @@ -61,10 +58,10 @@ import java.util.function.Consumer; import java.util.stream.Collectors; -import static org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_INDEX_VERSION; import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.WRITE; import static org.elasticsearch.cluster.metadata.IndexMetadata.State.CLOSE; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.indices.SystemIndices.NO_UPGRADE_REQUIRED_INDEX_VERSION; /** * This is where the logic to actually perform the migration lives - {@link SystemIndexMigrator#run(SystemIndexMigrationTaskState)} will @@ -79,8 +76,6 @@ public class SystemIndexMigrator extends AllocatedPersistentTask { private final ParentTaskAssigningClient baseClient; private final ClusterService clusterService; private final SystemIndices systemIndices; - private final MetadataUpdateSettingsService metadataUpdateSettingsService; - private final MetadataCreateIndexService metadataCreateIndexService; private final IndexScopedSettings indexScopedSettings; // In-memory state @@ -96,20 +91,15 @@ public SystemIndexMigrator( String type, String action, TaskId parentTask, - SystemIndexMigrationTaskParams params, Map headers, ClusterService clusterService, SystemIndices systemIndices, - MetadataUpdateSettingsService metadataUpdateSettingsService, - MetadataCreateIndexService metadataCreateIndexService, IndexScopedSettings indexScopedSettings ) { super(id, type, action, "system-index-migrator", parentTask, headers); this.baseClient = new ParentTaskAssigningClient(client, parentTask); this.clusterService = clusterService; this.systemIndices = systemIndices; - this.metadataUpdateSettingsService = metadataUpdateSettingsService; - this.metadataCreateIndexService = metadataCreateIndexService; this.indexScopedSettings = indexScopedSettings; } @@ -471,15 +461,10 @@ private void migrateSingleIndex(ClusterState clusterState, Consumer listener) { + private void createIndex(SystemIndexMigrationInfo migrationInfo, ActionListener listener) { logger.info("creating new system index [{}] from feature [{}]", migrationInfo.getNextIndexName(), migrationInfo.getFeatureName()); - final CreateIndexClusterStateUpdateRequest createRequest = new CreateIndexClusterStateUpdateRequest( - "migrate-system-index", - migrationInfo.getNextIndexName(), - migrationInfo.getNextIndexName() - ); - + CreateIndexRequest createIndexRequest = new CreateIndexRequest(migrationInfo.getNextIndexName()); Settings.Builder settingsBuilder = Settings.builder(); if (Objects.nonNull(migrationInfo.getSettings())) { settingsBuilder.put(migrationInfo.getSettings()); @@ -487,28 +472,32 @@ private void createIndex(SystemIndexMigrationInfo migrationInfo, ActionListener< settingsBuilder.remove("index.blocks.read"); settingsBuilder.remove("index.blocks.metadata"); } - createRequest.waitForActiveShards(ActiveShardCount.ALL) - .mappings(migrationInfo.getMappings()) + createIndexRequest.cause(SystemIndices.MIGRATE_SYSTEM_INDEX_CAUSE) + .ackTimeout(TimeValue.ZERO) + .masterNodeTimeout(MasterNodeRequest.INFINITE_MASTER_NODE_TIMEOUT) + .waitForActiveShards(ActiveShardCount.ALL) + .mapping(migrationInfo.getMappings()) .settings(Objects.requireNonNullElse(settingsBuilder.build(), Settings.EMPTY)); - metadataCreateIndexService.createIndex( - MasterNodeRequest.INFINITE_MASTER_NODE_TIMEOUT, - TimeValue.ZERO, - null, - createRequest, - listener - ); + + baseClient.admin().indices().create(createIndexRequest, listener); } - private void createIndexRetryOnFailure(SystemIndexMigrationInfo migrationInfo, ActionListener listener) { + private void createIndexRetryOnFailure(SystemIndexMigrationInfo migrationInfo, ActionListener listener) { createIndex(migrationInfo, listener.delegateResponse((l, e) -> { - logger.warn("createIndex failed, retrying after removing index [{}] from previous attempt", migrationInfo.getNextIndexName()); + logger.warn( + "createIndex failed with \"{}\", retrying after removing index [{}] from previous attempt", + e.getMessage(), + migrationInfo.getNextIndexName() + ); deleteIndex(migrationInfo, ActionListener.wrap(cleanupResponse -> createIndex(migrationInfo, l.delegateResponse((l3, e3) -> { + e3.addSuppressed(e); logger.error( "createIndex failed after retrying, aborting system index migration. index: " + migrationInfo.getNextIndexName(), e3 ); l.onFailure(e3); })), e2 -> { + e2.addSuppressed(e); logger.error("deleteIndex failed, aborting system index migration. index: " + migrationInfo.getNextIndexName(), e2); l.onFailure(e2); })); @@ -575,18 +564,10 @@ private void setWriteBlock(Index index, boolean readOnlyValue, ActionListener systemIndices = new SetOnce<>(); + @Override public Collection createComponents(PluginServices services) { + systemIndices.set(services.systemIndices()); + var registry = new MigrateTemplateRegistry( services.environment().settings(), services.clusterService(), @@ -97,6 +114,9 @@ public List getRestHandlers( handlers.add(new RestGetMigrationReindexStatusAction()); handlers.add(new RestCancelReindexDataStreamAction()); handlers.add(new RestCreateIndexFromSourceAction()); + + handlers.add(new RestGetFeatureUpgradeStatusAction()); + handlers.add(new RestPostFeatureUpgradeAction()); return handlers; } @@ -109,40 +129,51 @@ public List getRestHandlers( actions.add(new ActionHandler<>(ReindexDataStreamIndexAction.INSTANCE, ReindexDataStreamIndexTransportAction.class)); actions.add(new ActionHandler<>(CreateIndexFromSourceAction.INSTANCE, CreateIndexFromSourceTransportAction.class)); actions.add(new ActionHandler<>(CopyLifecycleIndexMetadataAction.INSTANCE, CopyLifecycleIndexMetadataTransportAction.class)); + + actions.add(new ActionHandler<>(GetFeatureUpgradeStatusAction.INSTANCE, TransportGetFeatureUpgradeStatusAction.class)); + actions.add(new ActionHandler<>(PostFeatureUpgradeAction.INSTANCE, TransportPostFeatureUpgradeAction.class)); return actions; } @Override public List getNamedXContent() { - return List.of( - new NamedXContentRegistry.Entry( - PersistentTaskState.class, - new ParseField(ReindexDataStreamPersistentTaskState.NAME), - ReindexDataStreamPersistentTaskState::fromXContent - ), - new NamedXContentRegistry.Entry( - PersistentTaskParams.class, - new ParseField(ReindexDataStreamTaskParams.NAME), - ReindexDataStreamTaskParams::fromXContent + return Stream.concat( + SystemIndexMigrationExecutor.getNamedXContentParsers().stream(), + Stream.of( + new NamedXContentRegistry.Entry( + PersistentTaskState.class, + new ParseField(ReindexDataStreamPersistentTaskState.NAME), + ReindexDataStreamPersistentTaskState::fromXContent + ), + new NamedXContentRegistry.Entry( + PersistentTaskParams.class, + new ParseField(ReindexDataStreamTaskParams.NAME), + ReindexDataStreamTaskParams::fromXContent + ) ) - ); + ).toList(); } @Override public List getNamedWriteables() { - return List.of( - new NamedWriteableRegistry.Entry( - PersistentTaskState.class, - ReindexDataStreamPersistentTaskState.NAME, - ReindexDataStreamPersistentTaskState::new - ), - new NamedWriteableRegistry.Entry( - PersistentTaskParams.class, - ReindexDataStreamTaskParams.NAME, - ReindexDataStreamTaskParams::new - ), - new NamedWriteableRegistry.Entry(Task.Status.class, ReindexDataStreamStatus.NAME, ReindexDataStreamStatus::new) - ); + return Stream.concat( + SystemIndexMigrationExecutor.getNamedWriteables().stream(), + Stream.of( + new NamedWriteableRegistry.Entry(Metadata.ProjectCustom.class, FeatureMigrationResults.TYPE, FeatureMigrationResults::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, FeatureMigrationResults.TYPE, FeatureMigrationResults::readDiffFrom), + new NamedWriteableRegistry.Entry( + PersistentTaskState.class, + ReindexDataStreamPersistentTaskState.NAME, + ReindexDataStreamPersistentTaskState::new + ), + new NamedWriteableRegistry.Entry( + PersistentTaskParams.class, + ReindexDataStreamTaskParams.NAME, + ReindexDataStreamTaskParams::new + ), + new NamedWriteableRegistry.Entry(Task.Status.class, ReindexDataStreamStatus.NAME, ReindexDataStreamStatus::new) + ) + ).toList(); } @Override @@ -154,6 +185,7 @@ public List> getPersistentTasksExecutor( IndexNameExpressionResolver expressionResolver ) { return List.of( + new SystemIndexMigrationExecutor(client, clusterService, systemIndices.get(), settingsModule.getIndexScopedSettings()), new ReindexDataStreamPersistentTaskExecutor( new OriginSettingClient(client, REINDEX_DATA_STREAM_ORIGIN), clusterService, diff --git a/x-pack/plugin/migrate/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/migrate/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000000000..db02e9267218a --- /dev/null +++ b/x-pack/plugin/migrate/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +grant { + // needed for Painless to generate runtime classes + permission java.lang.RuntimePermission "createClassLoader"; +}; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusResponseTests.java similarity index 76% rename from server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java rename to x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusResponseTests.java index ea538e5b085cc..781cf277a26dd 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/GetFeatureUpgradeStatusResponseTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/GetFeatureUpgradeStatusResponseTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.index.IndexVersion; @@ -16,10 +14,10 @@ import java.util.Collections; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.ERROR; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.IN_PROGRESS; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.MIGRATION_NEEDED; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.ERROR; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.IN_PROGRESS; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.MIGRATION_NEEDED; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; @@ -37,7 +35,7 @@ protected Writeable.Reader instanceReader() { protected GetFeatureUpgradeStatusResponse createTestInstance() { return new GetFeatureUpgradeStatusResponse( randomList(8, GetFeatureUpgradeStatusResponseTests::createFeatureStatus), - randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values()) + randomFrom(GetFeatureUpgradeStatusResponse.UpgradeStatus.values()) ); } @@ -51,10 +49,7 @@ protected GetFeatureUpgradeStatusResponse mutateInstance(GetFeatureUpgradeStatus GetFeatureUpgradeStatusResponseTests::createFeatureStatus ) ), - randomValueOtherThan( - instance.getUpgradeStatus(), - () -> randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values()) - ) + randomValueOtherThan(instance.getUpgradeStatus(), () -> randomFrom(GetFeatureUpgradeStatusResponse.UpgradeStatus.values())) ); } @@ -92,7 +87,7 @@ private static GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus createFeatur return new GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus( randomAlphaOfLengthBetween(3, 20), randomFrom(IndexVersion.current(), IndexVersions.MINIMUM_COMPATIBLE), - randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values()), + randomFrom(GetFeatureUpgradeStatusResponse.UpgradeStatus.values()), randomList(4, GetFeatureUpgradeStatusResponseTests::getIndexInfo) ); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponseTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeResponseTests.java similarity index 85% rename from server/src/test/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponseTests.java rename to x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeResponseTests.java index 13b6a268ab21f..33ee910b70179 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/PostFeatureUpgradeResponseTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/PostFeatureUpgradeResponseTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.stream.Writeable; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/TransportGetFeatureUpgradeStatusActionTests.java similarity index 87% rename from server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java rename to x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/TransportGetFeatureUpgradeStatusActionTests.java index 9fef4c4ed328f..28aa0bdfae46a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/action/TransportGetFeatureUpgradeStatusActionTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.action.admin.cluster.migration; +package org.elasticsearch.system_indices.action; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -24,7 +22,7 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.MIGRATION_NEEDED; +import static org.elasticsearch.system_indices.action.GetFeatureUpgradeStatusResponse.UpgradeStatus.MIGRATION_NEEDED; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; diff --git a/server/src/test/java/org/elasticsearch/upgrades/FeatureMigrationResultsTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/FeatureMigrationResultsTests.java similarity index 91% rename from server/src/test/java/org/elasticsearch/upgrades/FeatureMigrationResultsTests.java rename to x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/FeatureMigrationResultsTests.java index 0045137f73697..fa9e9b7f8fb94 100644 --- a/server/src/test/java/org/elasticsearch/upgrades/FeatureMigrationResultsTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/FeatureMigrationResultsTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.Diff; diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationMetadataTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationMetadataTests.java new file mode 100644 index 0000000000000..125a79d9393f8 --- /dev/null +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationMetadataTests.java @@ -0,0 +1,320 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.system_indices.task; + +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterModule; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; +import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ProjectMetadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ChunkedToXContent; +import org.elasticsearch.health.node.selection.HealthNode; +import org.elasticsearch.health.node.selection.HealthNodeTaskExecutor; +import org.elasticsearch.health.node.selection.HealthNodeTaskParams; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.SystemIndices; +import org.elasticsearch.persistent.ClusterPersistentTasksCustomMetadata; +import org.elasticsearch.persistent.PersistentTasks; +import org.elasticsearch.persistent.PersistentTasksCustomMetadata; +import org.elasticsearch.persistent.PersistentTasksExecutorRegistry; +import org.elasticsearch.persistent.PersistentTasksService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.ObjectPath; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.json.JsonXContent; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.elasticsearch.cluster.metadata.Metadata.CONTEXT_MODE_SNAPSHOT; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class SystemIndexMigrationMetadataTests extends ESTestCase { + public void testParseXContentFormatBeforeMultiProject() throws IOException { + final String json = org.elasticsearch.core.Strings.format(""" + { + "meta-data": { + "version": 54321, + "cluster_uuid":"aba1aa1ababbbaabaabaab", + "cluster_uuid_committed":false, + "persistent_tasks": { + "last_allocation_id": 1, + "tasks": [ + { + "id": "health-node", + "task":{ "health-node": {"params":{}} } + }, + { + "id": "upgrade-system-indices", + "task":{ "upgrade-system-indices": {"params":{}} } + } + ] + }, + "reserved_state":{ } + } + } + """, IndexVersion.current(), IndexVersion.current()); + + final var metadata = fromJsonXContentStringWithPersistentTasks(json); + + assertThat(metadata, notNullValue()); + assertThat(metadata.clusterUUID(), is("aba1aa1ababbbaabaabaab")); + assertThat(metadata.customs().keySet(), contains("cluster_persistent_tasks")); + final var clusterTasks = ClusterPersistentTasksCustomMetadata.get(metadata); + assertThat(clusterTasks.tasks(), hasSize(1)); + assertThat( + clusterTasks.tasks().stream().map(PersistentTasksCustomMetadata.PersistentTask::getTaskName).toList(), + containsInAnyOrder("health-node") + ); + assertThat( + metadata.getProject().customs().keySet(), + containsInAnyOrder("persistent_tasks", "index-graveyard") + ); + final var projectTasks = PersistentTasksCustomMetadata.get(metadata.getProject()); + assertThat( + projectTasks.tasks().stream().map(PersistentTasksCustomMetadata.PersistentTask::getTaskName).toList(), + containsInAnyOrder("upgrade-system-indices") + ); + assertThat(clusterTasks.getLastAllocationId(), equalTo(projectTasks.getLastAllocationId())); + } + + private Metadata fromJsonXContentStringWithPersistentTasks(String json) throws IOException { + List registry = new ArrayList<>(); + registry.addAll(ClusterModule.getNamedXWriteables()); + registry.addAll(IndicesModule.getNamedXContents()); + registry.addAll(HealthNodeTaskExecutor.getNamedXContentParsers()); + registry.addAll(SystemIndexMigrationExecutor.getNamedXContentParsers()); + + final var clusterService = mock(ClusterService.class); + when(clusterService.threadPool()).thenReturn(mock(ThreadPool.class)); + final var healthNodeTaskExecutor = HealthNodeTaskExecutor.create( + clusterService, + mock(PersistentTasksService.class), + Settings.EMPTY, + ClusterSettings.createBuiltInClusterSettings() + ); + final var systemIndexMigrationExecutor = new SystemIndexMigrationExecutor( + mock(Client.class), + clusterService, + mock(SystemIndices.class), + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS + ); + new PersistentTasksExecutorRegistry(List.of(healthNodeTaskExecutor, systemIndexMigrationExecutor)); + + XContentParserConfiguration config = XContentParserConfiguration.EMPTY.withRegistry(new NamedXContentRegistry(registry)); + try (XContentParser parser = JsonXContent.jsonXContent.createParser(config, json)) { + return Metadata.fromXContent(parser); + } + } + + public void testMultiProjectXContent() throws IOException { + final long lastAllocationId = randomNonNegativeLong(); + final List projects = randomList(1, 5, () -> ProjectMetadata.builder(randomUniqueProjectId()) + .putCustom( + PersistentTasksCustomMetadata.TYPE, + new PersistentTasksCustomMetadata( + lastAllocationId, + Map.of( + SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, + new PersistentTasksCustomMetadata.PersistentTask<>( + SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, + SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, + new SystemIndexMigrationTaskParams(), + lastAllocationId, + PersistentTasks.INITIAL_ASSIGNMENT + ) + ) + ) + ) + .build() + ); + + Metadata.Builder metadataBuilder = Metadata.builder() + .putCustom( + ClusterPersistentTasksCustomMetadata.TYPE, + new ClusterPersistentTasksCustomMetadata( + lastAllocationId + 1, + Map.of( + HealthNode.TASK_NAME, + new PersistentTasksCustomMetadata.PersistentTask<>( + HealthNode.TASK_NAME, + HealthNode.TASK_NAME, + HealthNodeTaskParams.INSTANCE, + lastAllocationId + 1, + PersistentTasks.INITIAL_ASSIGNMENT + ) + ) + ) + ); + for (ProjectMetadata project : projects) { + metadataBuilder.put(project); + } + final Metadata originalMeta = metadataBuilder + .build(); + + ToXContent.Params p = new ToXContent.MapParams( + Map.of("multi-project", "true", Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY) + ); + + final BytesReference bytes = toXContentBytes(originalMeta, p); + + // XContent with multi-project=true has separate cluster and project tasks + final var objectPath = ObjectPath.createFromXContent(JsonXContent.jsonXContent, bytes); + assertThat(objectPath.evaluate("meta-data.cluster_persistent_tasks"), notNullValue()); + assertThat(objectPath.evaluate("meta-data.cluster_persistent_tasks.last_allocation_id"), equalTo(lastAllocationId + 1)); + assertThat(objectPath.evaluate("meta-data.cluster_persistent_tasks.tasks"), hasSize(1)); + assertThat(objectPath.evaluate("meta-data.cluster_persistent_tasks.tasks.0.id"), equalTo(HealthNode.TASK_NAME)); + assertThat(objectPath.evaluate("meta-data.projects"), hasSize(projects.size())); + assertThat(IntStream.range(0, projects.size()).mapToObj(i -> { + try { + return (String) objectPath.evaluate("meta-data.projects." + i + ".id"); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }).collect(Collectors.toUnmodifiableSet()), + equalTo(projects.stream().map(pp -> pp.id().id()).collect(Collectors.toUnmodifiableSet())) + ); + + for (int i = 0; i < projects.size(); i++) { + assertThat(objectPath.evaluate("meta-data.projects." + i + ".persistent_tasks"), notNullValue()); + assertThat(objectPath.evaluate("meta-data.projects." + i + ".persistent_tasks.last_allocation_id"), equalTo(lastAllocationId)); + assertThat(objectPath.evaluate("meta-data.projects." + i + ".persistent_tasks.tasks"), hasSize(1)); + assertThat( + objectPath.evaluate("meta-data.projects." + i + ".persistent_tasks.tasks.0.id"), + equalTo(SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME) + ); + } + + Metadata fromXContentMeta = fromJsonXContentStringWithPersistentTasks(bytes.utf8ToString()); + assertThat(fromXContentMeta.projects().keySet(), equalTo(originalMeta.projects().keySet())); + for (var project : fromXContentMeta.projects().values()) { + final var projectTasks = PersistentTasksCustomMetadata.get(project); + assertThat(projectTasks.getLastAllocationId(), equalTo(lastAllocationId)); + assertThat(projectTasks.taskMap().keySet(), equalTo(Set.of(SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME))); + } + final var clusterTasks = ClusterPersistentTasksCustomMetadata.get(fromXContentMeta); + assertThat(clusterTasks.getLastAllocationId(), equalTo(lastAllocationId + 1)); + assertThat(clusterTasks.taskMap().keySet(), equalTo(Set.of(HealthNode.TASK_NAME))); + } + + public void testDefaultProjectXContentWithPersistentTasks() throws IOException { + final long lastAllocationId = randomNonNegativeLong(); + final var originalMeta = Metadata.builder() + .clusterUUID(randomUUID()) + .clusterUUIDCommitted(true) + .put( + ProjectMetadata.builder(ProjectId.DEFAULT) + .putCustom( + PersistentTasksCustomMetadata.TYPE, + new PersistentTasksCustomMetadata( + lastAllocationId, + Map.of( + SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, + new PersistentTasksCustomMetadata.PersistentTask<>( + SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, + SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, + new SystemIndexMigrationTaskParams(), + lastAllocationId, + PersistentTasks.INITIAL_ASSIGNMENT + ) + ) + ) + ) + ) + .putCustom( + ClusterPersistentTasksCustomMetadata.TYPE, + new ClusterPersistentTasksCustomMetadata( + lastAllocationId + 1, + Map.of( + HealthNode.TASK_NAME, + new PersistentTasksCustomMetadata.PersistentTask<>( + HealthNode.TASK_NAME, + HealthNode.TASK_NAME, + HealthNodeTaskParams.INSTANCE, + lastAllocationId + 1, + PersistentTasks.INITIAL_ASSIGNMENT + ) + ) + ) + ) + .build(); + + // For single project metadata, XContent output should combine the cluster and project tasks + final ToXContent.Params p = new ToXContent.MapParams( + Map.ofEntries(Map.entry("multi-project", "false"), Map.entry(Metadata.CONTEXT_MODE_PARAM, CONTEXT_MODE_SNAPSHOT)) + ); + final BytesReference bytes = toXContentBytes(originalMeta, p); + + final var objectPath = ObjectPath.createFromXContent(JsonXContent.jsonXContent, bytes); + // No cluster_persistent_tasks for single project output, it is combined with persistent_tasks + assertThat(objectPath.evaluate("meta-data.cluster_persistent_tasks"), nullValue()); + // The combined lastAllocationId is the max between cluster and project tasks + assertThat(objectPath.evaluate("meta-data.persistent_tasks.last_allocation_id"), equalTo(lastAllocationId + 1)); + assertThat(objectPath.evaluate("meta-data.persistent_tasks.tasks"), hasSize(2)); + assertThat( + Set.of( + objectPath.evaluate("meta-data.persistent_tasks.tasks.0.id"), + objectPath.evaluate("meta-data.persistent_tasks.tasks.1.id") + ), + equalTo(Set.of(HealthNode.TASK_NAME, SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME)) + ); + + // Deserialize from the XContent should separate cluster and project tasks + final Metadata fromXContentMeta = fromJsonXContentStringWithPersistentTasks(bytes.utf8ToString()); + assertThat(fromXContentMeta.projects().keySet(), equalTo(Set.of(ProjectId.DEFAULT))); + final var clusterTasks = ClusterPersistentTasksCustomMetadata.get(fromXContentMeta); + assertThat(clusterTasks, notNullValue()); + assertThat(clusterTasks.getLastAllocationId(), equalTo(lastAllocationId + 1)); + assertThat( + clusterTasks.tasks().stream().map(PersistentTasksCustomMetadata.PersistentTask::getId).toList(), + contains(HealthNode.TASK_NAME) + ); + final var projectTasks = PersistentTasksCustomMetadata.get(fromXContentMeta.getProject(ProjectId.DEFAULT)); + assertThat(projectTasks, notNullValue()); + assertThat(projectTasks.getLastAllocationId(), equalTo(lastAllocationId + 1)); + assertThat( + projectTasks.tasks().stream().map(PersistentTasksCustomMetadata.PersistentTask::getId).toList(), + contains(SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME) + ); + } + + private static BytesReference toXContentBytes(Metadata metadata, ToXContent.Params params) throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + ChunkedToXContent.wrapAsToXContent(metadata).toXContent(builder, params); + builder.endObject(); + return BytesReference.bytes(builder); + } +} diff --git a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParamsTests.java similarity index 78% rename from server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsTests.java rename to x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParamsTests.java index c9f2102dd446c..bc111b44aaa9f 100644 --- a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParamsTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; diff --git a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsXContentTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParamsXContentTests.java similarity index 70% rename from server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsXContentTests.java rename to x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParamsXContentTests.java index b66ba693d9aad..bedda2b7b617a 100644 --- a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskParamsXContentTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskParamsXContentTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.test.AbstractXContentTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; diff --git a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskStateTests.java similarity index 89% rename from server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateTests.java rename to x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskStateTests.java index 427d4d805fea0..dff6cfcdd19ac 100644 --- a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskStateTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; diff --git a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateXContentTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskStateXContentTests.java similarity index 76% rename from server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateXContentTests.java rename to x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskStateXContentTests.java index a5be76f9a18a3..40f635ac4b09d 100644 --- a/server/src/test/java/org/elasticsearch/upgrades/SystemIndexMigrationTaskStateXContentTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationTaskStateXContentTests.java @@ -1,13 +1,11 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ -package org.elasticsearch.upgrades; +package org.elasticsearch.system_indices.task; import org.elasticsearch.test.AbstractXContentTestCase; import org.elasticsearch.xcontent.NamedXContentRegistry; diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java index 0c9334c12161c..f842af6434923 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/cache/blob/SearchableSnapshotsBlobStoreCacheMaintenanceIntegTests.java @@ -339,7 +339,7 @@ public void testCleanUpMigratedSystemIndexAfterIndicesAreDeleted() throws Except /** * Mimics migration of the {@link SearchableSnapshots#SNAPSHOT_BLOB_CACHE_INDEX} as done in - * {@link org.elasticsearch.upgrades.SystemIndexMigrator}, where the index is re-indexed, and replaced by an alias. + * org.elasticsearch.upgrades.SystemIndexMigrator, where the index is re-indexed, and replaced by an alias. */ private void migrateTheSystemIndex() { final var migratedSnapshotBlobCache = SNAPSHOT_BLOB_CACHE_INDEX + SystemIndices.UPGRADED_INDEX_SUFFIX; diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/migration/10_get_feature_upgrade_status.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migration/10_get_feature_upgrade_status.yml similarity index 100% rename from rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/migration/10_get_feature_upgrade_status.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migration/10_get_feature_upgrade_status.yml diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/migration/20_post_feature_upgrade.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migration/20_post_feature_upgrade.yml similarity index 56% rename from rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/migration/20_post_feature_upgrade.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migration/20_post_feature_upgrade.yml index b1d6b0630d0a7..0f29389953c38 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/migration/20_post_feature_upgrade.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migration/20_post_feature_upgrade.yml @@ -1,11 +1,10 @@ -"Get feature upgrade status": +"Start feature upgrade": - requires: cluster_features: ["gte_v7.16.0"] reason: "Endpoint added in 7.16.0" - do: - migration.get_feature_upgrade_status: {} + migration.post_feature_upgrade: {} - is_false: accepted - - is_true: features From 305c2af3a29516c9257701284ea5ada1913a5e23 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 21 Mar 2025 19:10:29 +0000 Subject: [PATCH 2/5] [CI] Auto commit changes from spotless --- .../elasticsearch/node/NodeConstruction.java | 9 +-- .../SystemIndexMigrationMetadataTests.java | 55 +++++++++---------- 2 files changed, 27 insertions(+), 37 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 639ae2e4095e3..b7c189c7fdaf2 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -1139,14 +1139,7 @@ public Map searchFields() { final ShutdownPrepareService shutdownPrepareService = new ShutdownPrepareService(settings, httpServerTransport, terminationHandler); - modules.add( - loadPersistentTasksService( - settingsModule, - clusterService, - threadPool, - clusterModule.getIndexNameExpressionResolver() - ) - ); + modules.add(loadPersistentTasksService(settingsModule, clusterService, threadPool, clusterModule.getIndexNameExpressionResolver())); modules.add( loadPluginShutdownService(clusterService), diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationMetadataTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationMetadataTests.java index 125a79d9393f8..362819ac11544 100644 --- a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationMetadataTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/system_indices/task/SystemIndexMigrationMetadataTests.java @@ -10,8 +10,6 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; -import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService; import org.elasticsearch.cluster.metadata.ProjectId; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.service.ClusterService; @@ -98,10 +96,7 @@ public void testParseXContentFormatBeforeMultiProject() throws IOException { clusterTasks.tasks().stream().map(PersistentTasksCustomMetadata.PersistentTask::getTaskName).toList(), containsInAnyOrder("health-node") ); - assertThat( - metadata.getProject().customs().keySet(), - containsInAnyOrder("persistent_tasks", "index-graveyard") - ); + assertThat(metadata.getProject().customs().keySet(), containsInAnyOrder("persistent_tasks", "index-graveyard")); final var projectTasks = PersistentTasksCustomMetadata.get(metadata.getProject()); assertThat( projectTasks.tasks().stream().map(PersistentTasksCustomMetadata.PersistentTask::getTaskName).toList(), @@ -141,25 +136,28 @@ private Metadata fromJsonXContentStringWithPersistentTasks(String json) throws I public void testMultiProjectXContent() throws IOException { final long lastAllocationId = randomNonNegativeLong(); - final List projects = randomList(1, 5, () -> ProjectMetadata.builder(randomUniqueProjectId()) - .putCustom( - PersistentTasksCustomMetadata.TYPE, - new PersistentTasksCustomMetadata( - lastAllocationId, - Map.of( + final List projects = randomList( + 1, + 5, + () -> ProjectMetadata.builder(randomUniqueProjectId()) + .putCustom( + PersistentTasksCustomMetadata.TYPE, + new PersistentTasksCustomMetadata( + lastAllocationId, + Map.of( + SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, + new PersistentTasksCustomMetadata.PersistentTask<>( SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, - new PersistentTasksCustomMetadata.PersistentTask<>( - SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, - SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, - new SystemIndexMigrationTaskParams(), - lastAllocationId, - PersistentTasks.INITIAL_ASSIGNMENT - ) + SystemIndexMigrationTaskParams.SYSTEM_INDEX_UPGRADE_TASK_NAME, + new SystemIndexMigrationTaskParams(), + lastAllocationId, + PersistentTasks.INITIAL_ASSIGNMENT ) ) ) - .build() - ); + ) + .build() + ); Metadata.Builder metadataBuilder = Metadata.builder() .putCustom( @@ -181,8 +179,7 @@ public void testMultiProjectXContent() throws IOException { for (ProjectMetadata project : projects) { metadataBuilder.put(project); } - final Metadata originalMeta = metadataBuilder - .build(); + final Metadata originalMeta = metadataBuilder.build(); ToXContent.Params p = new ToXContent.MapParams( Map.of("multi-project", "true", Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY) @@ -198,12 +195,12 @@ public void testMultiProjectXContent() throws IOException { assertThat(objectPath.evaluate("meta-data.cluster_persistent_tasks.tasks.0.id"), equalTo(HealthNode.TASK_NAME)); assertThat(objectPath.evaluate("meta-data.projects"), hasSize(projects.size())); assertThat(IntStream.range(0, projects.size()).mapToObj(i -> { - try { - return (String) objectPath.evaluate("meta-data.projects." + i + ".id"); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }).collect(Collectors.toUnmodifiableSet()), + try { + return (String) objectPath.evaluate("meta-data.projects." + i + ".id"); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }).collect(Collectors.toUnmodifiableSet()), equalTo(projects.stream().map(pp -> pp.id().id()).collect(Collectors.toUnmodifiableSet())) ); From 2cab76469e2a545faeb2055bc522035196e2fbe8 Mon Sep 17 00:00:00 2001 From: alexey-ivanov-es Date: Thu, 3 Apr 2025 16:40:59 +0100 Subject: [PATCH 3/5] Fix compilation --- .../java/org/elasticsearch/cluster/metadata/MetadataTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index f61ce9eb4f99f..ef9483165ce54 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -2681,7 +2681,7 @@ public void testEmptyDiffReturnsSameInstance() throws IOException { public void testMultiProjectXContent() throws IOException { final long lastAllocationId = randomNonNegativeLong(); - final List projects = randomList(1, 5, () -> randomProject(randomUniqueProjectId(), randomIntBetween(1, 3))); + final List projects = randomList(1, 5, () -> randomProject(randomUniqueProjectId(), randomIntBetween(1, 3))) .stream() .map( project -> ProjectMetadata.builder(project) From 2003e6909d8eaf0f7bc4aa9799b8c21b6e839eef Mon Sep 17 00:00:00 2001 From: alexey-ivanov-es Date: Fri, 4 Apr 2025 14:54:42 +0100 Subject: [PATCH 4/5] Fix tests --- qa/system-indices/build.gradle | 5 +++++ .../system/indices/AbstractSystemIndicesIT.java | 2 ++ .../xpack-rest-tests-with-multiple-projects/build.gradle | 2 ++ 3 files changed, 9 insertions(+) diff --git a/qa/system-indices/build.gradle b/qa/system-indices/build.gradle index e856bde590f38..dd07a7409f136 100644 --- a/qa/system-indices/build.gradle +++ b/qa/system-indices/build.gradle @@ -17,3 +17,8 @@ esplugin { licenseFile = layout.settingsDirectory.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile } + +dependencies { + clusterModules project(':modules:ingest-common') + clusterModules project(':x-pack:plugin:migrate') +} diff --git a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/AbstractSystemIndicesIT.java b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/AbstractSystemIndicesIT.java index eaa21fe57a7c3..8db36ff370b40 100644 --- a/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/AbstractSystemIndicesIT.java +++ b/qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/AbstractSystemIndicesIT.java @@ -25,6 +25,8 @@ public abstract class AbstractSystemIndicesIT extends ESRestTestCase { @ClassRule public static ElasticsearchCluster cluster = ElasticsearchCluster.local() .plugin("system-indices-qa") + .module("ingest-common") + .module("x-pack-migrate") .setting("xpack.security.enabled", "true") .setting("xpack.security.autoconfiguration.enabled", "false") .user("rest_user", "rest-user-password") diff --git a/x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/build.gradle b/x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/build.gradle index 12657c0a289c6..5e2a172d93123 100644 --- a/x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/build.gradle +++ b/x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/build.gradle @@ -58,6 +58,8 @@ tasks.named("yamlRestTest").configure { '^migrate/10_reindex/*', '^migrate/20_reindex_status/*', '^migrate/30_create_from/*', + '^migration/10_get_feature_upgrade/*', + '^migration/20_post_feature_upgrade/*', '^ml/3rd_party_deployment/*', '^ml/bucket_correlation_agg/*', '^ml/bucket_count_ks_test_agg/*', From ffed0ae2becbab63cc1216d7e2f5fbe8a4cf241b Mon Sep 17 00:00:00 2001 From: alexey-ivanov-es Date: Fri, 4 Apr 2025 16:36:16 +0100 Subject: [PATCH 5/5] Fix test --- .../xpack-rest-tests-with-multiple-projects/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/build.gradle b/x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/build.gradle index 5e2a172d93123..b7c61ceef6dae 100644 --- a/x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/build.gradle +++ b/x-pack/qa/multi-project/xpack-rest-tests-with-multiple-projects/build.gradle @@ -58,7 +58,7 @@ tasks.named("yamlRestTest").configure { '^migrate/10_reindex/*', '^migrate/20_reindex_status/*', '^migrate/30_create_from/*', - '^migration/10_get_feature_upgrade/*', + '^migration/10_get_feature_upgrade_status/*', '^migration/20_post_feature_upgrade/*', '^ml/3rd_party_deployment/*', '^ml/bucket_correlation_agg/*',