[main] Move system indices migration to migrate plugin (#125437)

* [main] Move system indices migration to migrate plugin

It seems the best way to fix #122949 is to use existing data stream reindex API. However, this API is located in the migrate x-pack plugin. This commit moves the system indices migration logic (REST handlers, transport actions, and task) to the migrate plugin.

Port of #123551

* [CI] Auto commit changes from spotless

* Fix compilation

* Fix tests

* Fix test

---------

Co-authored-by: elasticsearchmachine <[email protected]>

Author: alexey-ivanov-es

modules/reindex/build.gradle

@@ -44,9 +44,6 @@ dependencies {
   clusterModules project(':modules:lang-painless')
   clusterModules project(':modules:parent-join')
   clusterModules project(":modules:rest-root")
-
-  internalClusterTestImplementation project(':modules:lang-painless')
-  internalClusterTestImplementation project(':modules:lang-painless:spi')
 }
 
 restResources {

modules/reindex/src/main/plugin-metadata/plugin-security.policy

@@ -10,9 +10,6 @@
 grant {
    // reindex opens socket connections using the rest client
    permission java.net.SocketPermission "*", "connect";
-
-  // needed for Painless to generate runtime classes
-  permission java.lang.RuntimePermission "createClassLoader";
 };
 
 grant codeBase "${codebase.elasticsearch-rest-client}" {

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java

@@ -11,9 +11,9 @@
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 
-import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.test.XContentTestUtils;
 import org.junit.BeforeClass;
 
@@ -112,7 +112,7 @@ public void testGetFeatureUpgradeStatus() throws Exception {
                 // for the next major version upgrade (see e.g. #93666). Trying to express this with features may be problematic, so we
                 // want to keep using versions here. We also assume that for non-semantic version migrations are not required.
                 boolean migrationNeeded = parseLegacyVersion(getOldClusterVersion()).map(
-                    v -> v.before(TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION)
+                    v -> v.before(SystemIndices.NO_UPGRADE_REQUIRED_VERSION)
                 ).orElse(false);
                 if (migrationNeeded) {
                     assertThat(feature.get("migration_status"), equalTo("MIGRATION_NEEDED"));

qa/system-indices/build.gradle

@@ -17,3 +17,8 @@ esplugin {
   licenseFile = layout.settingsDirectory.file('licenses/AGPL-3.0+SSPL-1.0+ELASTIC-LICENSE-2.0.txt').asFile
   noticeFile = layout.settingsDirectory.file('NOTICE.txt').asFile
 }
+
+dependencies {
+  clusterModules project(':modules:ingest-common')
+  clusterModules project(':x-pack:plugin:migrate')
+}

qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/AbstractSystemIndicesIT.java

@@ -25,6 +25,8 @@ public abstract class AbstractSystemIndicesIT extends ESRestTestCase {
     @ClassRule
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .plugin("system-indices-qa")
+        .module("ingest-common")
+        .module("x-pack-migrate")
         .setting("xpack.security.enabled", "true")
         .setting("xpack.security.autoconfiguration.enabled", "false")
         .user("rest_user", "rest-user-password")

server/src/main/java/module-info.java

@@ -62,7 +62,6 @@
     exports org.elasticsearch.action.admin.cluster.coordination;
     exports org.elasticsearch.action.admin.cluster.desirednodes;
     exports org.elasticsearch.action.admin.cluster.health;
-    exports org.elasticsearch.action.admin.cluster.migration;
     exports org.elasticsearch.action.admin.cluster.node.capabilities;
     exports org.elasticsearch.action.admin.cluster.node.hotthreads;
     exports org.elasticsearch.action.admin.cluster.node.info;
@@ -382,7 +381,6 @@
     exports org.elasticsearch.tasks;
     exports org.elasticsearch.threadpool;
     exports org.elasticsearch.transport;
-    exports org.elasticsearch.upgrades;
     exports org.elasticsearch.usage;
     exports org.elasticsearch.watcher;
 

server/src/main/java/org/elasticsearch/action/ActionModule.java

@@ -26,10 +26,6 @@
 import org.elasticsearch.action.admin.cluster.desirednodes.TransportUpdateDesiredNodesAction;
 import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesAction;
 import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction;
-import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction;
-import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction;
-import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
-import org.elasticsearch.action.admin.cluster.migration.TransportPostFeatureUpgradeAction;
 import org.elasticsearch.action.admin.cluster.node.capabilities.TransportNodesCapabilitiesAction;
 import org.elasticsearch.action.admin.cluster.node.features.TransportNodesFeaturesAction;
 import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction;
@@ -286,7 +282,6 @@
 import org.elasticsearch.rest.action.admin.cluster.RestDeleteStoredScriptAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetDesiredBalanceAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetDesiredNodesAction;
-import org.elasticsearch.rest.action.admin.cluster.RestGetFeatureUpgradeStatusAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetScriptContextAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetScriptLanguageAction;
@@ -300,7 +295,6 @@
 import org.elasticsearch.rest.action.admin.cluster.RestNodesStatsAction;
 import org.elasticsearch.rest.action.admin.cluster.RestNodesUsageAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPendingClusterTasksAction;
-import org.elasticsearch.rest.action.admin.cluster.RestPostFeatureUpgradeAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPrevalidateNodeRemovalAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPutRepositoryAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPutStoredScriptAction;
@@ -681,10 +675,7 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
         actions.register(TransportSnapshotsStatusAction.TYPE, TransportSnapshotsStatusAction.class);
         actions.register(SnapshottableFeaturesAction.INSTANCE, TransportSnapshottableFeaturesAction.class);
         actions.register(ResetFeatureStateAction.INSTANCE, TransportResetFeatureStateAction.class);
-        actions.register(GetFeatureUpgradeStatusAction.INSTANCE, TransportGetFeatureUpgradeStatusAction.class);
-        actions.register(PostFeatureUpgradeAction.INSTANCE, TransportPostFeatureUpgradeAction.class);
         actions.register(TransportGetShardSnapshotAction.TYPE, TransportGetShardSnapshotAction.class);
-
         actions.register(IndicesStatsAction.INSTANCE, TransportIndicesStatsAction.class);
         actions.register(IndicesSegmentsAction.INSTANCE, TransportIndicesSegmentsAction.class);
         actions.register(TransportIndicesShardStoresAction.TYPE, TransportIndicesShardStoresAction.class);
@@ -882,8 +873,6 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster, Predicate<
         registerHandler.accept(new RestSnapshotsStatusAction());
         registerHandler.accept(new RestSnapshottableFeaturesAction());
         registerHandler.accept(new RestResetFeatureStateAction());
-        registerHandler.accept(new RestGetFeatureUpgradeStatusAction());
-        registerHandler.accept(new RestPostFeatureUpgradeAction());
         registerHandler.accept(new RestGetIndicesAction());
         registerHandler.accept(new RestIndicesStatsAction());
         registerHandler.accept(new RestIndicesSegmentsAction());

server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java

@@ -148,7 +148,7 @@ protected void masterOperation(
                 listener.onFailure(new IllegalStateException(message));
                 return;
             }
-            updateRequest = buildSystemIndexUpdateRequest(request, cause, descriptor, projectId);
+            updateRequest = buildManagedSystemIndexUpdateRequest(request, cause, descriptor, projectId);
         } else {
             updateRequest = buildUpdateRequest(request, cause, indexName, resolvedAt, projectId);
         }
@@ -181,31 +181,55 @@ private CreateIndexClusterStateUpdateRequest buildUpdateRequest(
             .waitForActiveShards(request.waitForActiveShards());
     }
 
-    private static CreateIndexClusterStateUpdateRequest buildSystemIndexUpdateRequest(
+    private static CreateIndexClusterStateUpdateRequest buildManagedSystemIndexUpdateRequest(
         CreateIndexRequest request,
         String cause,
         SystemIndexDescriptor descriptor,
         ProjectId projectId
     ) {
-        final Settings settings = Objects.requireNonNullElse(descriptor.getSettings(), Settings.EMPTY);
+        boolean indexMigrationInProgress = cause.equals(SystemIndices.MIGRATE_SYSTEM_INDEX_CAUSE)
+            && request.index().endsWith(SystemIndices.UPGRADED_INDEX_SUFFIX);
 
+        final Settings settings;
+        final String mappings;
         final Set<Alias> aliases;
-        if (descriptor.getAliasName() == null) {
+        final String indexName;
+
+        // if we are migrating a system index to a new index, we use settings/mappings/index name from the request,
+        // since it was created by SystemIndexMigrator
+        if (indexMigrationInProgress) {
+            settings = request.settings();
+            mappings = request.mappings();
+            indexName = request.index();
+            // we will update alias later on
             aliases = Set.of();
         } else {
-            aliases = Set.of(new Alias(descriptor.getAliasName()).isHidden(true).writeIndex(true));
-        }
+            settings = Objects.requireNonNullElse(descriptor.getSettings(), Settings.EMPTY);
+            mappings = descriptor.getMappings();
+
+            if (descriptor.getAliasName() == null) {
+                aliases = Set.of();
+            } else {
+                aliases = Set.of(new Alias(descriptor.getAliasName()).isHidden(true).writeIndex(true));
+            }
 
-        // Throw an error if we are trying to directly create a system index other than the primary system index (or the alias)
-        if (request.index().equals(descriptor.getPrimaryIndex()) == false && request.index().equals(descriptor.getAliasName()) == false) {
-            throw new IllegalArgumentException(
-                "Cannot create system index with name " + request.index() + "; descriptor primary index is " + descriptor.getPrimaryIndex()
-            );
+            // Throw an error if we are trying to directly create a system index other
+            // than the primary system index (or the alias, or we are migrating the index)
+            if (request.index().equals(descriptor.getPrimaryIndex()) == false
+                && request.index().equals(descriptor.getAliasName()) == false) {
+                throw new IllegalArgumentException(
+                    "Cannot create system index with name "
+                        + request.index()
+                        + "; descriptor primary index is "
+                        + descriptor.getPrimaryIndex()
+                );
+            }
+            indexName = descriptor.getPrimaryIndex();
         }
 
-        return new CreateIndexClusterStateUpdateRequest(cause, projectId, descriptor.getPrimaryIndex(), request.index()).aliases(aliases)
+        return new CreateIndexClusterStateUpdateRequest(cause, projectId, indexName, request.index()).aliases(aliases)
             .waitForActiveShards(ActiveShardCount.ALL)
-            .mappings(descriptor.getMappings())
+            .mappings(mappings)
             .settings(settings);
     }
 }

server/src/main/java/org/elasticsearch/cluster/ClusterModule.java

@@ -89,7 +89,6 @@
 import org.elasticsearch.tasks.TaskResultsService;
 import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.upgrades.FeatureMigrationResults;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
 
@@ -257,7 +256,6 @@ public static List<Entry> getNamedWriteables() {
             ComposableIndexTemplateMetadata::readDiffFrom
         );
         registerProjectCustom(entries, DataStreamMetadata.TYPE, DataStreamMetadata::new, DataStreamMetadata::readDiffFrom);
-        registerProjectCustom(entries, FeatureMigrationResults.TYPE, FeatureMigrationResults::new, FeatureMigrationResults::readDiffFrom);
         registerMetadataCustom(entries, NodesShutdownMetadata.TYPE, NodesShutdownMetadata::new, NodesShutdownMetadata::readDiffFrom);
         registerMetadataCustom(entries, DesiredNodesMetadata.TYPE, DesiredNodesMetadata::new, DesiredNodesMetadata::readDiffFrom);
         registerProjectCustom(

server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java

@@ -13,7 +13,6 @@
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
-import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
 import org.elasticsearch.action.admin.indices.create.AutoCreateAction;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -148,7 +147,7 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable<Sy
 
     /**
      * An optional reindexing script to use when migrating an index created
-     * before {@link TransportGetFeatureUpgradeStatusAction#NO_UPGRADE_REQUIRED_INDEX_VERSION}.
+     * before {@link SystemIndices#NO_UPGRADE_REQUIRED_INDEX_VERSION}.
      * This script can be used to modify documents before they are added to the new index.
      * For example, it can be used to remove deprecated fields from the index.
      * <br>