[CRE] mixed-env nightly full-matrix sweep + chain-specific variants

Extends mixed-env coverage beyond the per-PR OCR3/DON2DON subset.

- Nightly full sweep (cre-mixed-env-nightly.yaml): runs the entire CRE matrix
  under mixed-env (mixed_env_all: true), comparing today's develop nightly image
  against a previous one — catching non-determinism introduced into develop
  between nightlies, across the whole suite. No image build (reuses nightlies).
- cre-system-tests.yaml: adds a mixed_env_all input + matrix logic that appends
  mixed-env to the default (EVM) topology list and the chain-specific mixed-env
  variants to the Solana/Aptos/Stellar tests; generalizes the render step and the
  chainlink-image gating to any `mixed-env*` topology.
- Chain-specific mixed-env templates: mixed-env-solana-don, mixed-env-aptos-don,
  mixed-env-stellar-don (2-2 PR/develop split; verified via `topology show`).
- render-mixed-env.sh generalized to render any mixed-env-*-don.toml.tmpl.
- gitignore covers rendered configs/mixed-env-*.toml; docs updated.

Sharding / Module_Cache / Multi_Gateway need their own mixed variants (follow-up).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: prashantkumar1982

.github/workflows/cre-mixed-env-nightly.yaml

@@ -0,0 +1,60 @@
+name: CRE Mixed-Env Nightly Sweep
+
+# Runs the full CRE system-test matrix under the mixed-env topology (2 nodes on
+# today's develop nightly image, 2 on yesterday's) to catch non-determinism /
+# cross-version incompatibilities introduced into develop between nightlies —
+# across the whole suite (EVM + Solana/Aptos/Stellar), not just the per-PR subset.
+#
+# It calls cre-system-tests.yaml directly with mixed_env_all: true, so no image
+# build is needed: both images are the cached nightly develop builds.
+
+on:
+  schedule:
+    # After the nightly image build (docker-build.yml runs at 03:00 UTC).
+    - cron: "0 5 * * *"
+  workflow_dispatch:
+    inputs:
+      baseline_date:
+        description: "Baseline nightly date (YYYYMMDD). Defaults to yesterday."
+        required: false
+        type: string
+        default: ""
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  set-dates:
+    runs-on: ubuntu-latest
+    outputs:
+      today: ${{ steps.d.outputs.today }}
+      baseline: ${{ steps.d.outputs.baseline }}
+    steps:
+      - id: d
+        shell: bash
+        run: |
+          today="$(date -u +'%Y%m%d')"
+          baseline="${{ inputs.baseline_date }}"
+          if [[ -z "${baseline}" ]]; then
+            baseline="$(date -u -d 'yesterday' +'%Y%m%d')"
+          fi
+          echo "today=${today}" | tee -a "$GITHUB_OUTPUT"
+          echo "baseline=${baseline}" | tee -a "$GITHUB_OUTPUT"
+
+  mixed-env-sweep:
+    needs: set-dates
+    permissions:
+      contents: read
+      id-token: write
+    uses: ./.github/workflows/cre-system-tests.yaml
+    secrets: inherit
+    with:
+      ecr: "sdlc"
+      chainlink_image_repository_path: "chainlink"
+      # "PR" side = today's develop nightly.
+      chainlink_image_tag: "nightly-${{ needs.set-dates.outputs.today }}"
+      # Baseline side = a previous develop nightly, so the 2-2 split compares two
+      # develop builds and surfaces non-determinism introduced since then.
+      mixed_env_baseline_image: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:nightly-${{ needs.set-dates.outputs.baseline }}"
+      mixed_env_all: true

.github/workflows/cre-system-tests.yaml

@@ -38,6 +38,13 @@ on:
         description:
           "Full baseline (develop) chainlink image ref for the mixed-env
           topology. If empty, defaults to the develop nightly image."
+      mixed_env_all:
+        required: false
+        type: boolean
+        default: false
+        description:
+          "Nightly full sweep: also run every test under a mixed-env topology
+          (EVM base + chain-specific variants)."
   workflow_call:
     inputs:
       chainlink_image_repository_path:
@@ -71,6 +78,13 @@ on:
         description:
           "Full baseline (develop) chainlink image ref for the mixed-env
           topology. If empty, defaults to the develop nightly image."
+      mixed_env_all:
+        required: false
+        type: boolean
+        default: false
+        description:
+          "Nightly full sweep: also run every test under a mixed-env topology
+          (EVM base + chain-specific variants)."
 
 jobs:
   define-test-matrix:
@@ -143,6 +157,22 @@ jobs:
              ]
           }'
 
+          # Nightly full sweep: when mixed_env_all is set, also run every EVM-based test
+          # under mixed-env, and the chain-specific tests under their chain mixed-env variant.
+          # (Sharding / Module_Cache / Multi_Gateway need their own mixed variants — follow-up.)
+          if [[ "${{ inputs.mixed_env_all }}" == "true" ]]; then
+            TOPOLOGIES_JSON=$(jq -c '. + [{"topology":"mixed-env","configs":"configs/mixed-env-don.toml"}]' <<<"$TOPOLOGIES_JSON")
+            MIXED_PER_TEST='{
+              "Test_CRE_V2_Aptos_Suite":[{"topology":"mixed-env-aptos","configs":"configs/mixed-env-aptos-don.toml"}],
+              "Test_CRE_V2_Stellar_Suite":[{"topology":"mixed-env-stellar","configs":"configs/mixed-env-stellar-don.toml"}],
+              "Test_CRE_V2_Solana_Write":[{"topology":"mixed-env-solana","configs":"configs/mixed-env-solana-don.toml"}],
+              "Test_CRE_V2_Solana_LogTrigger":[{"topology":"mixed-env-solana","configs":"configs/mixed-env-solana-don.toml"}],
+              "Test_CRE_V2_Solana_Read_Accounts":[{"topology":"mixed-env-solana","configs":"configs/mixed-env-solana-don.toml"}]
+            }'
+            PER_TEST_TOPOLOGIES_JSON=$(jq -c -n --argjson a "$PER_TEST_TOPOLOGIES_JSON" --argjson b "$MIXED_PER_TEST" \
+              'reduce ($b|to_entries[]) as $e ($a; .[$e.key] = ((.[$e.key] // []) + $e.value))')
+          fi
+
           test_names=$(grep -rh -oP '^func \K(Test|Example)[^(]+' system-tests/tests/smoke/cre/*_test.go | grep -v Test_Upgrade | grep -v '^TestMain$')
 
           tests=$(echo "$test_names" | jq -c -R -s \
@@ -276,7 +306,7 @@ jobs:
           working-directory: system-tests/lib
 
       - name: Render mixed-env topology
-        if: ${{ matrix.tests.topology == 'mixed-env' }}
+        if: ${{ startsWith(matrix.tests.topology, 'mixed-env') }}
         working-directory: core/scripts/cre/environment
         shell: bash
         env:
@@ -296,7 +326,8 @@ jobs:
           export CRE_PR_IMAGE CRE_BASELINE_IMAGE
           echo "PR image:       ${CRE_PR_IMAGE}"
           echo "Baseline image: ${CRE_BASELINE_IMAGE}"
-          bash configs/render-mixed-env.sh
+          echo "Config:         ${{ matrix.tests.configs }}"
+          bash configs/render-mixed-env.sh "${{ matrix.tests.configs }}"
 
       - name: Start observability stack (Beholder suite)
         if: contains(matrix.tests.test_name, 'Beholder_Suite')
@@ -386,7 +417,7 @@ jobs:
           # CTF_CHAINLINK_IMAGE empty (a non-empty value forces every node onto one image).
           # NB: the non-empty value must sit on the `&&` side — `cond && '' || X` always
           # yields X in GitHub Actions because '' is falsy.
-          chainlink-image: "${{ matrix.tests.topology != 'mixed-env' && env.CHAINLINK_IMAGE_FULL || '' }}"
+          chainlink-image: "${{ !startsWith(matrix.tests.topology, 'mixed-env') && env.CHAINLINK_IMAGE_FULL || '' }}"
           chip-router-image: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{
             secrets.QA_AWS_REGION }}.amazonaws.com/local-cre-chip-router:v1.0.1"
           ctf-configs: ${{ matrix.tests.configs }}
@@ -451,7 +482,7 @@ jobs:
         # Dedicated hard gate. The smoke step above is continue-on-error
         # (auto-quarantine), which would swallow the in-test TestMain scan, so we
         # re-scan the live node containers here and fail the job on any marker.
-        if: ${{ always() && matrix.tests.topology == 'mixed-env' }}
+        if: ${{ always() && startsWith(matrix.tests.topology, 'mixed-env') }}
         shell: bash
         run: |
           set -uo pipefail

core/scripts/cre/environment/.gitignore

@@ -15,8 +15,8 @@ cre_v*
 configs/*-cache.toml
 cache/
 
-# rendered mixed-env topology (generated from configs/mixed-env-don.toml.tmpl)
-configs/mixed-env-don.toml
+# rendered mixed-env topologies (generated from configs/mixed-env-*.toml.tmpl); .tmpl are tracked
+configs/mixed-env-*.toml
 
 logs/
 

core/scripts/cre/environment/configs/mixed-env-aptos-don.toml.tmpl

@@ -0,0 +1,95 @@
+# Mixed-env Aptos topology (template) — derived from workflow-gateway-don-aptos.toml.
+#
+# Same as workflow-gateway-don-aptos, except the workflow DON runs 2 nodes on the PR
+# image and 2 on the develop image, to surface non-determinism between the two code
+# versions (see docs/mixed-env.md). Render with configs/render-mixed-env.sh:
+#
+#   CRE_PR_IMAGE=<pr> CRE_BASELINE_IMAGE=<develop> \
+#     ./configs/render-mixed-env.sh configs/mixed-env-aptos-don.toml
+#
+# Keep in sync with workflow-gateway-don-aptos.toml if that base topology changes.
+# Do NOT set CTF_CHAINLINK_IMAGE with this topology (it would force one image on all nodes).
+
+[chip_router]
+  image = "local-cre-chip-router:v1.0.1"
+
+[[blockchains]]
+  type = "anvil"
+  chain_id = "1337"
+  container_name = "anvil-1337"
+  docker_cmd_params = ["-b", "0.5", "--mixed-mining"]
+
+[[blockchains]]
+  type = "aptos"
+  chain_id = "4"
+
+[jd]
+  csa_encryption_key = "d1093c0060d50a3c89c189b2e485da5a3ce57f3dcb38ab7e2c0d5f0bb2314a44"
+  image = "job-distributor:0.28.0"
+
+[infra]
+  type = "docker"
+
+[[nodesets]]
+  nodes = 4
+  name = "workflow"
+  don_family = "test-don-family"
+  don_types = ["workflow"]
+  override_mode = "each"
+  http_port_range_start = 10100
+
+  supported_evm_chains = [1337]
+  env_vars = { CL_CRE_SETTINGS_DEFAULT = '{"PerWorkflow":{"CapabilityCallTimeout":"5m0s","ChainAllowed":{"Default":"false","Values":{"1337":"true","4457093679053095497":"true"}},"ExecutionTimestampsEnabled":"true","FeatureAptosWriteReportBlockTimestampActivePeriod":"[2020-01-01 00:00:00 +0000 UTC,2030-01-01 00:00:00 +0000 UTC]"}}' }
+  capabilities = ["cron", "consensus", "aptos-4", "don-time"]
+  registry_based_launch_allowlist = ["cron-trigger@1.0.0"]
+
+  [nodesets.db]
+    image = "postgres:12.0"
+    port = 13000
+
+  [[nodesets.node_specs]] # node 0 - PR
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      image = "${CRE_PR_IMAGE}"
+      pull_image = true
+      user_config_overrides = ""
+  [[nodesets.node_specs]] # node 1 - PR
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      image = "${CRE_PR_IMAGE}"
+      pull_image = true
+      user_config_overrides = ""
+  [[nodesets.node_specs]] # node 2 - develop
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      image = "${CRE_BASELINE_IMAGE}"
+      pull_image = true
+      user_config_overrides = ""
+  [[nodesets.node_specs]] # node 3 - develop
+    roles = ["plugin"]
+    [nodesets.node_specs.node]
+      image = "${CRE_BASELINE_IMAGE}"
+      pull_image = true
+      user_config_overrides = ""
+
+[[nodesets]]
+  nodes = 1
+  name = "bootstrap-gateway"
+  don_family = "test-don-family"
+  don_types = ["bootstrap", "gateway"]
+  override_mode = "each"
+  http_port_range_start = 10300
+
+  supported_evm_chains = [1337]
+
+  [nodesets.db]
+    image = "postgres:12.0"
+    port = 13200
+
+  [[nodesets.node_specs]]
+    roles = ["bootstrap", "gateway"]
+    [nodesets.node_specs.node]
+      image = "${CRE_BASELINE_IMAGE}"
+      pull_image = true
+      custom_ports = ["5002:5002", "15002:15002"]
+      user_config_overrides = ""