[CRE] mixed-env nightly full-matrix sweep + special-topology variants

Nightly full sweep that runs the entire CRE matrix under mixed-env, built on top
of the dedicated cre-mixed-env-tests.yaml workflow.

- cre-mixed-env-tests.yaml: add a full_matrix input. Off (per-PR default) it runs
  the base EVM set; on (nightly) it also runs the special-topology tests under
  their mixed-env variant. Matrix entries now carry per-test topology + config,
  and the render step / ctf-configs / TOPOLOGY_NAME use them.
- cre-mixed-env-nightly.yaml: scheduled workflow calling cre-mixed-env-tests.yaml
  with full_matrix: true, comparing today's develop nightly against a previous one.
- Special-topology mixed-env templates (2-2 split, verified via `topology show`):
  solana, aptos, stellar, sharded (preserves shard0 custom_ports), cache-test
  (preserves ModuleCache override), multi-gateway (preserves org override + both
  gateway DONs).
- render-mixed-env.sh generalized to render any mixed-env-*-don.toml.tmpl.
- gitignore covers rendered configs/mixed-env-*.toml; docs updated.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: prashantkumar1982

.github/workflows/cre-mixed-env-nightly.yaml

@@ -0,0 +1,60 @@
+name: CRE Mixed-Env Nightly Sweep
+
+# Runs the full CRE system-test matrix under the mixed-env topology (2 nodes on
+# today's develop nightly image, 2 on yesterday's) to catch non-determinism /
+# cross-version incompatibilities introduced into develop between nightlies —
+# across the whole suite (EVM + Solana/Aptos/Stellar), not just the per-PR subset.
+#
+# It calls cre-system-tests.yaml directly with mixed_env_all: true, so no image
+# build is needed: both images are the cached nightly develop builds.
+
+on:
+  schedule:
+    # After the nightly image build (docker-build.yml runs at 03:00 UTC).
+    - cron: "0 5 * * *"
+  workflow_dispatch:
+    inputs:
+      baseline_date:
+        description: "Baseline nightly date (YYYYMMDD). Defaults to yesterday."
+        required: false
+        type: string
+        default: ""
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  set-dates:
+    runs-on: ubuntu-latest
+    outputs:
+      today: ${{ steps.d.outputs.today }}
+      baseline: ${{ steps.d.outputs.baseline }}
+    steps:
+      - id: d
+        shell: bash
+        run: |
+          today="$(date -u +'%Y%m%d')"
+          baseline="${{ inputs.baseline_date }}"
+          if [[ -z "${baseline}" ]]; then
+            baseline="$(date -u -d 'yesterday' +'%Y%m%d')"
+          fi
+          echo "today=${today}" | tee -a "$GITHUB_OUTPUT"
+          echo "baseline=${baseline}" | tee -a "$GITHUB_OUTPUT"
+
+  mixed-env-sweep:
+    needs: set-dates
+    permissions:
+      contents: read
+      id-token: write
+    uses: ./.github/workflows/cre-mixed-env-tests.yaml
+    secrets: inherit
+    with:
+      ecr: "sdlc"
+      chainlink_image_repository_path: "chainlink"
+      # "PR" side = today's develop nightly.
+      chainlink_image_tag: "nightly-${{ needs.set-dates.outputs.today }}"
+      # Baseline side = a previous develop nightly, so the 2-2 split compares two
+      # develop builds and surfaces non-determinism introduced since then.
+      mixed_env_baseline_image: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:nightly-${{ needs.set-dates.outputs.baseline }}"
+      full_matrix: true

.github/workflows/cre-mixed-env-tests.yaml

@@ -38,6 +38,13 @@ on:
         description:
           "Full baseline (develop) chainlink image ref. If empty, defaults to the
           develop nightly image."
+      full_matrix:
+        required: false
+        type: boolean
+        default: false
+        description:
+          "Full sweep: also include chain-specific and other special-topology
+          tests (nightly). Per-PR runs leave this false."
   workflow_call:
     inputs:
       chainlink_image_repository_path:
@@ -65,6 +72,13 @@ on:
         description:
           "Full baseline (develop) chainlink image ref. If empty, defaults to the
           develop nightly image."
+      full_matrix:
+        required: false
+        type: boolean
+        default: false
+        description:
+          "Full sweep: also include chain-specific and other special-topology
+          tests (nightly). Per-PR runs leave this false."
 
 jobs:
   define-test-matrix:
@@ -78,27 +92,42 @@ jobs:
         id: define-matrix
         shell: bash
         run: |
-          # Tests worth running under mixed-env: the OCR3/DON2DON-heavy ones.
-          TESTS='[
-            "Test_CRE_V2_Suite_Bucket_A",
-            "Test_CRE_V2_Suite_Bucket_B",
-            "Test_CRE_V2_EVM_Read_HeavyCalls",
-            "Test_CRE_V2_EVM_Read_StateQueries",
-            "Test_CRE_V2_EVM_Read_TxArtifacts"
+          # Per-PR set: the OCR3/DON2DON-heavy EVM tests, on the base mixed-env topology.
+          ENTRIES='[
+            {"test_name":"Test_CRE_V2_Suite_Bucket_A","topology":"mixed-env","configs":"configs/mixed-env-don.toml"},
+            {"test_name":"Test_CRE_V2_Suite_Bucket_B","topology":"mixed-env","configs":"configs/mixed-env-don.toml"},
+            {"test_name":"Test_CRE_V2_EVM_Read_HeavyCalls","topology":"mixed-env","configs":"configs/mixed-env-don.toml"},
+            {"test_name":"Test_CRE_V2_EVM_Read_StateQueries","topology":"mixed-env","configs":"configs/mixed-env-don.toml"},
+            {"test_name":"Test_CRE_V2_EVM_Read_TxArtifacts","topology":"mixed-env","configs":"configs/mixed-env-don.toml"}
           ]'
+
+          # Nightly full sweep: add the special-topology tests under their mixed-env variant.
+          if [[ "${{ inputs.full_matrix }}" == "true" ]]; then
+            FULL='[
+              {"test_name":"Test_CRE_V2_Aptos_Suite","topology":"mixed-env-aptos","configs":"configs/mixed-env-aptos-don.toml"},
+              {"test_name":"Test_CRE_V2_Stellar_Suite","topology":"mixed-env-stellar","configs":"configs/mixed-env-stellar-don.toml"},
+              {"test_name":"Test_CRE_V2_Solana_Write","topology":"mixed-env-solana","configs":"configs/mixed-env-solana-don.toml"},
+              {"test_name":"Test_CRE_V2_Solana_LogTrigger","topology":"mixed-env-solana","configs":"configs/mixed-env-solana-don.toml"},
+              {"test_name":"Test_CRE_V2_Solana_Read_Accounts","topology":"mixed-env-solana","configs":"configs/mixed-env-solana-don.toml"},
+              {"test_name":"Test_CRE_V2_Sharding","topology":"mixed-env-sharded","configs":"configs/mixed-env-sharded-don.toml"},
+              {"test_name":"Test_CRE_V2_Module_Cache","topology":"mixed-env-cache-test","configs":"configs/mixed-env-cache-test-don.toml"},
+              {"test_name":"Test_CRE_V2_HTTP_Action_Multi_Gateway","topology":"mixed-env-multi-gateway","configs":"configs/mixed-env-multi-gateway-don.toml"}
+            ]'
+            ENTRIES=$(jq -c -n --argjson a "$ENTRIES" --argjson b "$FULL" '$a + $b')
+          fi
+
           matrix=$(jq -c -n \
-            --argjson tests "$TESTS" \
+            --argjson entries "$ENTRIES" \
             --argjson run_id "${{ github.run_id }}" \
             --arg run_attempt "${{ github.run_attempt }}" '
-            $tests | to_entries | map({
-              test_name: .value,
+            $entries | to_entries | map(.value + {
               test_id: .key,
               runs_on: "runs-on=\($run_id)-\(.key)-\($run_attempt)/cpu=16/ram=64/family=m7i+m8i/spot=co/image=ubuntu24-full-x64/extras=s3-cache+tmpfs"
             })')
           echo "matrix=$matrix" | tee -a "${GITHUB_OUTPUT}"
 
   run-mixed-env-tests:
-    name: ${{ matrix.tests.test_name }} (mixed-env)
+    name: ${{ matrix.tests.test_name }} (${{ matrix.tests.topology }})
     permissions:
       contents: read
       id-token: write
@@ -207,7 +236,8 @@ jobs:
           export CRE_PR_IMAGE CRE_BASELINE_IMAGE
           echo "PR image:       ${CRE_PR_IMAGE}"
           echo "Baseline image: ${CRE_BASELINE_IMAGE}"
-          bash configs/render-mixed-env.sh
+          echo "Config:         ${{ matrix.tests.configs }}"
+          bash configs/render-mixed-env.sh "${{ matrix.tests.configs }}"
 
       - name: Start local CRE
         id: start-local-cre
@@ -221,7 +251,7 @@ jobs:
           chainlink-image: ""
           chip-router-image: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{
             secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"
-          ctf-configs: configs/mixed-env-don.toml
+          ctf-configs: ${{ matrix.tests.configs }}
           retry-count: "3"
           retry-delay-seconds: "15"
           cleanup-on-error: "false"
@@ -246,7 +276,7 @@ jobs:
           TEST_NAME: ${{ matrix.tests.test_name }}
           TEST_TIMEOUT: 7m
           RUN_QUARANTINED_TESTS: "true"
-          TOPOLOGY_NAME: mixed-env
+          TOPOLOGY_NAME: ${{ matrix.tests.topology }}
           GITHUB_TOKEN: ${{ steps.github-token.outputs.access-token || '' }}
           PARALLEL_COUNT: "10"
           CRE_TEST_PARALLEL_ENABLED: "true"

core/scripts/cre/environment/.gitignore

@@ -15,8 +15,8 @@ cre_v*
 configs/*-cache.toml
 cache/
 
-# rendered mixed-env topology (generated from configs/mixed-env-don.toml.tmpl)
-configs/mixed-env-don.toml
+# rendered mixed-env topologies (generated from configs/mixed-env-*.toml.tmpl); .tmpl are tracked
+configs/mixed-env-*.toml
 
 logs/
 

core/scripts/cre/environment/configs/mixed-env-aptos-don.toml.tmpl

@@ -0,0 +1,95 @@
+# Mixed-env Aptos topology (template) — derived from workflow-gateway-don-aptos.toml.
+#
+# Same as workflow-gateway-don-aptos, except the workflow DON runs 2 nodes on the PR
+# image and 2 on the develop image, to surface non-determinism between the two code
+# versions (see docs/mixed-env.md). Render with configs/render-mixed-env.sh:
+#
+#   CRE_PR_IMAGE=<pr> CRE_BASELINE_IMAGE=<develop> \
+#     ./configs/render-mixed-env.sh configs/mixed-env-aptos-don.toml
+#
+# Keep in sync with workflow-gateway-don-aptos.toml if that base topology changes.
+# Do NOT set CTF_CHAINLINK_IMAGE with this topology (it would force one image on all nodes).
+
+[chip_router]
+  image = "local-cre-chip-router:v1.0.1"
+
+[[blockchains]]
+  type = "anvil"
+  chain_id = "1337"
+  container_name = "anvil-1337"
+  docker_cmd_params = ["-b", "0.5", "--mixed-mining"]
+
+[[blockchains]]
+  type = "aptos"
+  chain_id = "4"
+
+[jd]
+  csa_encryption_key = "d1093c0060d50a3c89c189b2e485da5a3ce57f3dcb38ab7e2c0d5f0bb2314a44"
+  image = "job-distributor:0.28.0"
+
+[infra]
+  type = "docker"
+
+[[nodesets]]
+  nodes = 4
+  name = "workflow"
+  don_family = "test-don-family"
+  don_types = ["workflow"]
+  override_mode = "each"
+  http_port_range_start = 10100
+
+  supported_evm_chains = [1337]
+  env_vars = { CL_CRE_SETTINGS_DEFAULT = '{"PerWorkflow":{"CapabilityCallTimeout":"5m0s","ChainAllowed":{"Default":"false","Values":{"1337":"true","4457093679053095497":"true"}},"ExecutionTimestampsEnabled":"true","FeatureAptosWriteReportBlockTimestampActivePeriod":"[2020-01-01 00:00:00 +0000 UTC,2030-01-01 00:00:00 +0000 UTC]"}}' }
+  capabilities = ["cron", "consensus", "aptos-4", "don-time"]
+  registry_based_launch_allowlist = ["cron-trigger@1.0.0"]
+
+  [nodesets.db]
+    image = "postgres:12.0"
+    port = 13000
+
+  [[nodesets.node_specs]] # node 0 - PR
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      image = "${CRE_PR_IMAGE}"
+      pull_image = true
+      user_config_overrides = ""
+  [[nodesets.node_specs]] # node 1 - PR
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      image = "${CRE_PR_IMAGE}"
+      pull_image = true
+      user_config_overrides = ""
+  [[nodesets.node_specs]] # node 2 - develop
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      image = "${CRE_BASELINE_IMAGE}"
+      pull_image = true
+      user_config_overrides = ""
+  [[nodesets.node_specs]] # node 3 - develop
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      image = "${CRE_BASELINE_IMAGE}"
+      pull_image = true
+      user_config_overrides = ""
+
+[[nodesets]]
+  nodes = 1
+  name = "bootstrap-gateway"
+  don_family = "test-don-family"
+  don_types = ["bootstrap", "gateway"]
+  override_mode = "each"
+  http_port_range_start = 10300
+
+  supported_evm_chains = [1337]
+
+  [nodesets.db]
+    image = "postgres:12.0"
+    port = 13200
+
+  [[nodesets.node_specs]]
+    roles = ["bootstrap", "gateway"]
+    [nodesets.node_specs.node]
+      image = "${CRE_BASELINE_IMAGE}"
+      pull_image = true
+      custom_ports = ["5002:5002", "15002:15002"]
+      user_config_overrides = ""