Partition build jobs

Author: smola

.gitlab-ci.yml

@@ -143,22 +143,25 @@ default:
     CACHE_TYPE: lib #default
     RUNTIME_AVAILABLE_PROCESSORS_OVERRIDE: 4 # Runtime.getRuntime().availableProcessors() returns incorrect or very high values in Kubernetes
   cache:
-    - key: '$CI_SERVER_VERSION-$CACHE_TYPE' # Dependencies cache. Reset the cache every time gitlab is upgraded.  ~Every couple months
+    - key: '$CI_SERVER_VERSION-$CACHE_TYPE-$NORMALIZED_NODE_TOTAL-$NORMALIZED_NODE_INDEX' # Dependencies cache. Reset the cache every time gitlab is upgraded.  ~Every couple months
       paths:
         # Cached dependencies and wrappers for gradle
         - .gradle/wrapper
         - .gradle/caches
         - .gradle/notifications
       policy: $DEPENDENCY_CACHE_POLICY
       fallback_keys: # Use fallback keys because all cache types are not populated. See note under: populate_dep_cache
+        #- '$CI_SERVER_VERSION-$CACHE_TYPE'
         - '$CI_SERVER_VERSION-base'
         - '$CI_SERVER_VERSION-lib'
-    - key: $CI_PIPELINE_ID-$CACHE_TYPE # Incremental build cache. Shared by all jobs in the pipeline of the same type
+    - key: $CI_PIPELINE_ID-$CACHE_TYPE-$NORMALIZED_NODE_TOTAL-$NORMALIZED_NODE_INDEX # Incremental build cache. Shared by all jobs in the pipeline of the same type
       paths:
         - .gradle/caches/$GRADLE_VERSION
         - .gradle/$GRADLE_VERSION/executionHistory
         - workspace
       policy: $BUILD_CACHE_POLICY
+      #fallback_keys:
+      #  - '$CI_PIPELINE_ID-$CACHE_TYPE'
   before_script:
     - source .gitlab/gitlab-utils.sh
     - mkdir -p .gradle
@@ -210,7 +213,7 @@ build:
     reports:
       dotenv: build.env
 
-build_tests:
+.build_tests:
   extends: .gradle_build
   variables:
     BUILD_CACHE_POLICY: push
@@ -232,13 +235,62 @@ build_tests:
       - GRADLE_TARGET: ":smokeTest"
         CACHE_TYPE: "smoke"
         MAVEN_OPTS: "-Xms64M -Xmx512M -Dorg.slf4j.simpleLogger.defaultLogLevel=debug" # FIXME: Build :smokeTest build fails unless mvn debug logging is on
-
   script:
     - *gitlab_base_ref_params
-    - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS
+    - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS
+
+build_tests_base:
+  extends: .build_tests
+  needs: [ build_tests_base ]
+  variables:
+    GRADLE_TARGET: ":baseTest"
+    CACHE_TYPE: "base"
+  parallel:
+    matrix:
+      - CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]
+
+build_tests_profiling:
+  extends: .build_tests
+  needs: [ build_tests_profiling ]
+  variables:
+    GRADLE_TARGET: ":profilingTest"
+    CACHE_TYPE: "profiling"
+  parallel:
+    matrix:
+      - CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]
+
+build_tests_inst:
+  extends: .build_tests
+  needs: [ build_tests_inst ]
+  variables:
+    GRADLE_TARGET: ":instrumentationTest"
+    CACHE_TYPE: "inst"
+  parallel:
+    matrix:
+      - CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]
+
+build_tests_latestdep:
+  extends: .build_tests
+  needs: [ build_tests_latestdep ]
+  variables:
+    GRADLE_TARGET: ":instrumentationLatestDepTest"
+    CACHE_TYPE: "latestdep"
+  parallel:
+    matrix:
+      - CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]
+
+build_tests_smoke:
+  extends: .build_tests
+  needs: [ build_tests_smoke ]
+  variables:
+    GRADLE_TARGET: ":smokeTest"
+    CACHE_TYPE: "smoke"
+  parallel:
+    matrix:
+      - CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]
 
 populate_dep_cache:
-  extends: build_tests
+  extends: .build_tests
   variables:
     BUILD_CACHE_POLICY: pull
     DEPENDENCY_CACHE_POLICY: push
@@ -253,16 +305,19 @@ populate_dep_cache:
         CACHE_TYPE: "lib"
       - GRADLE_TARGET: ":baseTest"
         CACHE_TYPE: "base"
+        CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]
       - GRADLE_TARGET: ":profilingTest"
         CACHE_TYPE: "profiling"
-#      FIXME: Gitlab doesn't support s3 based caches >5GB. Fixed in Gitlab 17.5
-#      See: https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26921#note_2132307223
-#      - GRADLE_TARGET: ":instrumentationTest"
-#        CACHE_TYPE: "inst"
-#      - GRADLE_TARGET: ":instrumentationLatestDepTest"
-#        CACHE_TYPE: "latestdep"
-#      - GRADLE_TARGET: ":smokeTest"
-#        CACHE_TYPE: "smoke"
+        CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]
+      - GRADLE_TARGET: ":instrumentationTest"
+        CACHE_TYPE: "inst"
+        CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]
+      - GRADLE_TARGET: ":instrumentationLatestDepTest"
+        CACHE_TYPE: "latestdep"
+        CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]
+      - GRADLE_TARGET: ":smokeTest"
+        CACHE_TYPE: "smoke"
+        CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]
 
 publish-artifacts-to-s3:
   image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29
@@ -391,7 +446,7 @@ check_debugger:
 
 muzzle:
   extends: .gradle_build
-  needs: [ build_tests ]
+  needs: [ build_tests_inst ]
   stage: tests
   parallel: 8
   variables:
@@ -416,7 +471,7 @@ muzzle:
 
 muzzle-dep-report:
   extends: .gradle_build
-  needs: [ build_tests ]
+  needs: [ build_tests_inst ]
   stage: tests
   variables:
     CACHE_TYPE: inst
@@ -451,7 +506,6 @@ muzzle-dep-report:
   extends: .gradle_build
   image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}$testJvm
   tags: [ "docker-in-docker:amd64" ] # use docker-in-docker runner for testcontainers
-  needs: [ build_tests ]
   stage: tests
   variables:
     KUBERNETES_MEMORY_REQUEST: 17Gi