[wip] CI test optimizations

.buildkite/pipelines/lucene-snapshot/run-tests.yml

@@ -32,6 +32,14 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: custom-32-98304
       buildDirectory: /dev/shm/bk
+  - label: part4
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk
   - group: bwc-snapshots
     steps:
       - label: "{{matrix.BWC_VERSION}} / bwc-snapshots"

.buildkite/pipelines/periodic-platform-support.yml

@@ -1,8 +1,8 @@
 steps:
   - group: platform-support-unix
     steps:
-      - label: "{{matrix.image}} / platform-support-unix"
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true functionalTests
+      - label: "{{matrix.image}} / platform-support-unix-1"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true {{matrix.GRADLE_TASK}}
         timeout_in_minutes: 420
         matrix:
           setup:
@@ -14,86 +14,70 @@ steps:
               - oraclelinux-7
               - oraclelinux-8
               - sles-12
-              - sles-15
-              - ubuntu-1804
-              - ubuntu-2004
-              - ubuntu-2204
-              - rocky-8
-              - rocky-9
-              - rhel-7
-              - rhel-8
-              - rhel-9
-              - almalinux-8
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - checkPart4
+              - checkPart5
+              - checkRestCompat
         agents:
           provider: gcp
           image: family/elasticsearch-{{matrix.image}}
           localSsds: 1
           localSsdInterface: nvme
           machineType: custom-32-98304
-        env: {}
-  - group: platform-support-windows
-    steps:
-      - label: "{{matrix.image}} / {{matrix.GRADLE_TASK}} / platform-support-windows"
-        command: |
-          .\.buildkite\scripts\run-script.ps1 bash .buildkite/scripts/windows-run-gradle.sh
+        env:
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+      - label: "{{matrix.image}} / platform-support-unix-2"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true {{matrix.GRADLE_TASK}}
         timeout_in_minutes: 420
         matrix:
           setup:
             image:
-              - windows-2016
-              - windows-2019
-              - windows-2022
+              - sles-15
+              - ubuntu-1804
+              - ubuntu-2004
+              - ubuntu-2204
+              - rocky-8
+              - rocky-9
+              - rhel-7
             GRADLE_TASK:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
+              - checkPart5
               - checkRestCompat
         agents:
           provider: gcp
           image: family/elasticsearch-{{matrix.image}}
-          machineType: n1-standard-32
-          diskType: pd-ssd
-          diskSizeGb: 350
+          localSsds: 1
+          localSsdInterface: nvme
+          machineType: custom-32-98304
         env:
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
-  - group: platform-support-arm
-    steps:
-      - label: "{{matrix.image}} / {{matrix.GRADLE_TASK}} / platform-support-arm"
+      - label: "{{matrix.image}} / platform-support-unix-3"
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true {{matrix.GRADLE_TASK}}
         timeout_in_minutes: 420
         matrix:
           setup:
             image:
-              - almalinux-8-aarch64
-              - ubuntu-2004-aarch64
+              - rhel-8
+              - rhel-9
+              - almalinux-8
             GRADLE_TASK:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
+              - checkPart5
               - checkRestCompat
         agents:
-          provider: aws
-          imagePrefix: elasticsearch-{{matrix.image}}
-          instanceType: m6g.8xlarge
-          diskSizeGb: 350
-          diskType: gp3
-          diskName: /dev/sda1
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          localSsds: 1
+          localSsdInterface: nvme
+          machineType: custom-32-98304
         env:
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
-  - group: platform-support-unix-aws
-    steps:
-      - label: "{{matrix.image}} / platform-support-aws"
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true functionalTests
-        timeout_in_minutes: 420
-        matrix:
-          setup:
-            image:
-              - amazonlinux-2023
-              - amazonlinux-2
-        agents:
-          provider: aws
-          imagePrefix: elasticsearch-{{matrix.image}}
-          instanceType: m6a.8xlarge
-          diskSizeGb: 350
-          diskType: gp3
-          diskName: /dev/sda1

.buildkite/pipelines/periodic.template.yml

@@ -49,6 +49,7 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
               - checkRestCompat
         agents:
           provider: gcp
@@ -89,6 +90,7 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
               - checkRestCompat
         agents:
           provider: gcp

.buildkite/pipelines/periodic.yml

@@ -380,6 +380,7 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
               - checkRestCompat
         agents:
           provider: gcp
@@ -420,6 +421,7 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
+              - checkPart4
               - checkRestCompat
         agents:
           provider: gcp

.buildkite/pipelines/pull-request/part-5.yml

@@ -0,0 +1,11 @@
+config:
+  skip-target-branches: "7.17"
+steps:
+  - label: part-5
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed checkPart5
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.ci/init.gradle

@@ -102,7 +102,7 @@ gradle.settingsEvaluated { settings ->
 final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.url')
 final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
 
-if (buildCacheUrl) {
+if (buildCacheUrl && false) {
   final Map<String, String> buildCacheCredentials = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ? [:] : vault.logical()
     .read("${vaultPathPrefix}/gradle-build-cache")
     .getData()

.ci/scripts/run-gradle.sh

@@ -22,10 +22,10 @@ if ! uname -a | grep -q MING; then
         exit 1
     fi
   fi
-  if pwd | grep -v -q ^/dev/shm ; then
-    echo "Not running on a ramdisk, reducing number of workers"
-    MAX_WORKERS=$(($MAX_WORKERS*2/3))
-  fi
+  # if pwd | grep -v -q ^/dev/shm ; then
+  #   echo "Not running on a ramdisk, reducing number of workers"
+  #   MAX_WORKERS=$(($MAX_WORKERS*2/3))
+  # fi
 
   # Export glibc version as environment variable since some BWC tests are incompatible with later versions
   export GLIBC_VERSION=$(ldd --version | grep '^ldd' | sed 's/.* \([1-9]\.[0-9]*\).*/\1/')
@@ -36,5 +36,11 @@ if [[ "$MAX_WORKERS" == "0" ]]; then
   MAX_WORKERS=1
 fi
 
+# if [[ "$MAX_WORKERS" -gt 12 ]]; then
+#   MAX_WORKERS=12
+# fi
+
+# MAX_WORKERS=$(($MAX_WORKERS*3/4))
+
 set -e
 $GRADLEW -S --max-workers=$MAX_WORKERS $@

build.gradle

@@ -287,6 +287,8 @@ allprojects {
         tasks.register('checkPart4') { dependsOn 'check' }
       } else if (project.path == ":x-pack:plugin" || project.path.contains("ql") ||  project.path.contains("smoke-test")) {
         tasks.register('checkPart3') { dependsOn 'check' }
+      } else if (project.path.contains("multi-node")) {
+        tasks.register('checkPart5') { dependsOn 'check' }
       } else {
         tasks.register('checkPart2') { dependsOn 'check' }
       }
@@ -435,3 +437,35 @@ tasks.register("spotlessApply").configure {
   dependsOn gradle.includedBuild('build-tools').task(':reaper:spotlessApply')
   dependsOn gradle.includedBuild('build-tools-internal').task(':spotlessApply')
 }
+
+subprojects { subProject ->
+  subProject.tasks.withType(Test).configureEach { task ->
+    def slowTasks = [
+      ":docs:yamlRestTest",
+      ":qa:smoke-test-multinode:yamlRestTest",
+      ":rest-api-spec:yamlRestTest",
+      ":server:internalClusterTest",
+      ":test:external-modules:test-esql-heap-attack:javaRestTest",
+      ":x-pack:plugin:esql:compute:test",
+      ":x-pack:plugin:esql:qa:server:single-node:javaRestTest",
+      ":x-pack:plugin:esql:test",
+      ":x-pack:plugin:ml:qa:ml-with-security:yamlRestTest",
+      ":x-pack:plugin:ml:qa:native-multi-node-tests:javaRestTest",
+      ":x-pack:plugin:security:internalClusterTest",
+      ":x-pack:plugin:security:qa:multi-cluster:javaRestTest",
+      ":x-pack:plugin:security:qa:security-basic:javaRestTest",
+      ":x-pack:plugin:security:qa:security-trial:javaRestTest",
+      ":x-pack:plugin:sql:qa:server:multi-node:javaRestTest",
+      ":x-pack:plugin:sql:qa:server:security:with-ssl:javaRestTest",
+      ":x-pack:plugin:sql:qa:server:single-node:javaRestTest",
+      ":x-pack:plugin:transform:qa:multi-node-tests:javaRestTest",
+      ":x-pack:plugin:yamlRestTest",
+      ":x-pack:qa:core-rest-tests-with-security:yamlRestTest",
+      ":x-pack:qa:runtime-fields:core-with-mapped:yamlRestTest",
+    ]
+
+    slowTasks.each { slowTask ->
+      task.shouldRunAfter(slowTask)
+    }
+  }
+}