Skip to content

Commit 07cd4bb

Browse files
sadielbartholomewsadielbartholomew
committed
Separate run-test-suite.yml into job per OS for fail-fast control
1 parent 7c42610 commit 07cd4bb

1 file changed

Lines changed: 247 additions & 139 deletions

File tree

‎.github/workflows/run-test-suite.yml‎

Lines changed: 247 additions & 139 deletions
Original file line numberDiff line numberDiff line change
@@ -14,143 +14,251 @@ on:
1414
branches:
1515
- main
1616

17-
# Note a workflow can have 1+ jobs that can run sequentially or in parallel.
1817
jobs:
19-
# TODO: setup parallel runs (-job-2 etc.) of sub-tests for speed-up
20-
test-suite-job-0:
21-
22-
# Set-up the build matrix. We run on different distros and Python versions.
23-
strategy:
24-
matrix:
25-
# Test on macos-intel rather than macos (ARM64 architecture)
26-
os: [ubuntu-latest, macos-15-intel]
27-
# Note: keep versions quoted as strings else 3.14 taken as 3.1, etc.
28-
python-version: ["3.11", "3.12", "3.13", "3.14"]
29-
30-
# Set up the build matrix. We test on Ubuntu (representative Linux distro)
31-
# and macOS, with all of the major Python versions in support
32-
runs-on: ${{ matrix.os }}
33-
34-
# The sequence of tasks that will be executed as part of this job:
35-
steps:
36-
37-
- name: Checkout cf-python
38-
uses: actions/checkout@v7
39-
with:
40-
path: main
41-
42-
# Provide a notification message
43-
- name: Notify about setup
44-
run: echo Now setting up the environment for the cf-python test suite...
45-
46-
- name: Checkout the current cfdm main to use as the dependency
47-
uses: actions/checkout@v7
48-
with:
49-
repository: NCAS-CMS/cfdm
50-
path: cfdm
51-
52-
# Setup conda, which is the simplest way to access all dependencies,
53-
# especially as some are C-based so otherwise difficult to setup.
54-
- name: Setup Miniconda
55-
uses: conda-incubator/setup-miniconda@v4
56-
with:
57-
auto-update-conda: true
58-
miniconda-version: "latest"
59-
activate-environment: cf-latest
60-
python-version: ${{ matrix.python-version }}
61-
channels: ncas, conda-forge
62-
63-
# Ensure shell is configured with conda and pip correctly activated.
64-
- name: Check conda and pip config
65-
shell: bash -l {0}
66-
run: |
67-
echo "*Conda report:*"
68-
conda info
69-
conda list
70-
conda config --show-sources
71-
conda config --show
72-
echo "*Pip report:*"
73-
pip --version
74-
pip list
75-
76-
# Install cf-python dependencies, excluding cfunits and cfdm, pre-testing
77-
# We do so with conda (and pip) which was setup in a previous step.
78-
- name: Install dependencies excluding the NCAS CF Data Tools libraries
79-
shell: bash -l {0}
80-
run: |
81-
conda install -c ncas -c conda-forge cf-plot udunits2
82-
conda install -c conda-forge mpich esmpy
83-
conda install scipy matplotlib dask
84-
pip install pycodestyle
85-
86-
# Install cfunits and cfdm (from development main branch) separately,
87-
# since it is most robust to test a no-dependency installation of cf,
88-
# then finally install the cf-python development version.
89-
- name: Install cfunits then development versions of cfdm and cf-python
90-
shell: bash -l {0}
91-
run: |
92-
pip install cfunits
93-
cd cfdm
94-
pip install -e .
95-
cd ../main
96-
pip install --no-deps -e .
97-
98-
# Install the coverage library
99-
# We do so with conda which was setup in a previous step.
100-
- name: Install coverage
101-
shell: bash -l {0}
102-
run: |
103-
conda install coverage
104-
105-
# Provide another notification message
106-
- name: Notify about starting testing
107-
run: echo Setup complete. Now starting to run the cf-python test suite...
108-
109-
# Create netCDF files needed for testing. A separate step is required
110-
# for this so the files can be registered and recognised first; locally
111-
# they are created and used on-the-fly by 'run_tests_and_coverage'.
112-
- name: Create netCDF test files, e.g. test_file.nc
113-
shell: bash -l {0}
114-
working-directory: ${{ github.workspace }}/main/cf/test
115-
run: |
116-
# Exit code 5 means no tests were found, which is expected for
117-
# create_test_files.py. Allow that code but fail the workflow
118-
# for any other non-zero exit code, to avoid hiding other failures.
119-
python create_test_files.py || test $? -eq 5
120-
python setup_create_field.py
121-
ls -la
122-
123-
# Finally run the test suite and generate a coverage report!
124-
- name: Run test suite and generate a coverage report
125-
shell: bash -l {0}
126-
working-directory: ${{ github.workspace }}/main/cf/test
127-
run: |
128-
cd ${{ github.workspace }}/main/cf/test
129-
./run_tests_and_coverage --nohtml
130-
131-
# For one job only, generate a coverage report:
132-
- name: Upload coverage report to Codecov
133-
# Get coverage from only one job (choose with Ubuntu Python 3.14 as
134-
# representative). Note that we could use a separate workflow
135-
# to setup Codecov reports, but given the amount of steps required to
136-
# install including dependencies via conda, that a separate workflow
137-
# would have to run anyway, it is simplest to add it in this flow.
138-
# Also, it means code coverage is only generated if the test suite is
139-
# passing at least for that job, avoiding useless coverage reports.
140-
uses: codecov/codecov-action@v7
141-
# Warning: GitHub Actions expression string literals require
142-
# single (not double) quoted strings.
143-
if: |
144-
matrix.os == 'ubuntu-latest' && matrix.python-version == '3.14'
145-
with:
146-
file: |
147-
${{ github.workspace }}/main/cf/test/cf_coverage_reports/coverage.xml
148-
fail_ci_if_error: true
149-
flags: unittests
150-
name: codecov-umbrella
151-
152-
# End with a message indicating the suite has completed its run
153-
- name: Notify about a completed run
154-
run: |
155-
echo The cf-python test suite has completed and you may now
156-
echo inspect the results.
18+
# Use different jobs for a Linux OS and for a Mac OS, so we can fail all on
19+
# a given OS if one of those (per Python version) jobs fails, instead of
20+
# either having to choose between all other jobs cancelling or none of
21+
# them with one failure.
22+
23+
# Test the test suite on Linux (with Ubuntu as representative dist.)
24+
test-on-linux:
25+
runs-on: ubuntu-latest
26+
strategy:
27+
fail-fast: true
28+
matrix:
29+
python-version: ["3.11", "3.12", "3.13", "3.14"]
30+
31+
steps:
32+
33+
- name: Checkout cf-python
34+
uses: actions/checkout@v7
35+
with:
36+
path: main
37+
38+
# Provide a notification message
39+
- name: Notify about setup
40+
run: echo Now setting up the environment for the cf-python test suite...
41+
42+
- name: Checkout the current cfdm main to use as the dependency
43+
uses: actions/checkout@v7
44+
with:
45+
repository: NCAS-CMS/cfdm
46+
path: cfdm
47+
48+
# Setup conda, which is the simplest way to access all dependencies,
49+
# especially as some are C-based so otherwise difficult to setup.
50+
- name: Setup Miniconda
51+
uses: conda-incubator/setup-miniconda@v4
52+
with:
53+
auto-update-conda: true
54+
miniconda-version: "latest"
55+
activate-environment: cf-latest
56+
python-version: ${{ matrix.python-version }}
57+
channels: ncas, conda-forge
58+
59+
# Ensure shell is configured with conda and pip correctly activated.
60+
- name: Check conda and pip config
61+
shell: bash -l {0}
62+
run: |
63+
echo "*Conda report:*"
64+
conda info
65+
conda list
66+
conda config --show-sources
67+
conda config --show
68+
echo "*Pip report:*"
69+
pip --version
70+
pip list
71+
72+
# Install cf-python dependencies, excluding cfunits and cfdm, pre-testing
73+
# We do so with conda (and pip) which was setup in a previous step.
74+
- name: Install dependencies excluding the NCAS CF Data Tools libraries
75+
shell: bash -l {0}
76+
run: |
77+
conda install -c ncas -c conda-forge cf-plot udunits2
78+
conda install -c conda-forge mpich esmpy
79+
conda install scipy matplotlib dask
80+
pip install pycodestyle
81+
82+
# Install cfunits and cfdm (from development main branch) separately,
83+
# since it is most robust to test a no-dependency installation of cf,
84+
# then finally install the cf-python development version.
85+
- name: Install cfunits then development versions of cfdm and cf-python
86+
shell: bash -l {0}
87+
run: |
88+
pip install cfunits
89+
cd cfdm
90+
pip install -e .
91+
cd ../main
92+
pip install --no-deps -e .
93+
94+
# Install the coverage library
95+
# We do so with conda which was setup in a previous step.
96+
- name: Install coverage
97+
shell: bash -l {0}
98+
run: |
99+
conda install coverage
100+
101+
# Provide another notification message
102+
- name: Notify about starting testing
103+
run: echo Setup complete. Now starting to run the cf-python test suite...
104+
105+
# Create netCDF files needed for testing. A separate step is required
106+
# for this so the files can be registered and recognised first; locally
107+
# they are created and used on-the-fly by 'run_tests_and_coverage'.
108+
- name: Create netCDF test files, e.g. test_file.nc
109+
shell: bash -l {0}
110+
working-directory: ${{ github.workspace }}/main/cf/test
111+
run: |
112+
# Exit code 5 means no tests were found, which is expected for
113+
# create_test_files.py. Allow that code but fail the workflow
114+
# for any other non-zero exit code, to avoid hiding other failures.
115+
python create_test_files.py || test $? -eq 5
116+
python setup_create_field.py
117+
ls -la
118+
119+
# Finally run the test suite and generate a coverage report!
120+
- name: Run test suite and generate a coverage report
121+
shell: bash -l {0}
122+
working-directory: ${{ github.workspace }}/main/cf/test
123+
run: |
124+
./run_tests_and_coverage --nohtml
125+
126+
# For one job only, generate a coverage report:
127+
- name: Upload coverage report to Codecov
128+
# Get coverage from only one job (choose with Ubuntu Python 3.14 as
129+
# representative). Note that we could use a separate workflow
130+
# to setup Codecov reports, but given the amount of steps required to
131+
# install including dependencies via conda, that a separate workflow
132+
# would have to run anyway, it is simplest to add it in this flow.
133+
# Also, it means code coverage is only generated if the test suite is
134+
# passing at least for that job, avoiding useless coverage reports.
135+
uses: codecov/codecov-action@v7
136+
# Warning: GitHub Actions expression string literals require
137+
# single (not double) quoted strings.
138+
if: |
139+
matrix.python-version == '3.14'
140+
with:
141+
file: |
142+
${{ github.workspace }}/main/cf/test/cf_coverage_reports/coverage.xml
143+
fail_ci_if_error: true
144+
flags: unittests
145+
name: codecov-umbrella
146+
147+
# End with a message indicating the suite has completed its run
148+
- name: Notify about a completed run
149+
run: |
150+
echo The cf-python test suite has completed and you may now
151+
echo inspect the results.
152+
153+
154+
# Test the test suite on Mac OS: use on 'macos-*-intel' rather than
155+
# 'macos-*' (which specifies ARM64 architecture)
156+
test-on-mac:
157+
# Mac OS ARM64 via 'macos-latest' is also available but not compatible
158+
# with our conda installation support
159+
runs-on: macos-15-intel
160+
strategy:
161+
fail-fast: true
162+
matrix:
163+
python-version: ["3.11", "3.12", "3.13", "3.14"]
164+
165+
steps:
166+
167+
- name: Checkout cf-python
168+
uses: actions/checkout@v7
169+
with:
170+
path: main
171+
172+
# Provide a notification message
173+
- name: Notify about setup
174+
run: echo Now setting up the environment for the cf-python test suite...
175+
176+
- name: Checkout the current cfdm main to use as the dependency
177+
uses: actions/checkout@v7
178+
with:
179+
repository: NCAS-CMS/cfdm
180+
path: cfdm
181+
182+
# Setup conda, which is the simplest way to access all dependencies,
183+
# especially as some are C-based so otherwise difficult to setup.
184+
- name: Setup Miniconda
185+
uses: conda-incubator/setup-miniconda@v4
186+
with:
187+
auto-update-conda: true
188+
miniconda-version: "latest"
189+
activate-environment: cf-latest
190+
python-version: ${{ matrix.python-version }}
191+
channels: ncas, conda-forge
192+
193+
# Ensure shell is configured with conda and pip correctly activated.
194+
- name: Check conda and pip config
195+
shell: bash -l {0}
196+
run: |
197+
echo "*Conda report:*"
198+
conda info
199+
conda list
200+
conda config --show-sources
201+
conda config --show
202+
echo "*Pip report:*"
203+
pip --version
204+
pip list
205+
206+
# Install cf-python dependencies, excluding cfunits and cfdm, pre-testing
207+
# We do so with conda (and pip) which was setup in a previous step.
208+
- name: Install dependencies excluding the NCAS CF Data Tools libraries
209+
shell: bash -l {0}
210+
run: |
211+
conda install -c ncas -c conda-forge cf-plot udunits2
212+
conda install -c conda-forge mpich esmpy
213+
conda install scipy matplotlib dask
214+
pip install pycodestyle
215+
216+
# Install cfunits and cfdm (from development main branch) separately,
217+
# since it is most robust to test a no-dependency installation of cf,
218+
# then finally install the cf-python development version.
219+
- name: Install cfunits then development versions of cfdm and cf-python
220+
shell: bash -l {0}
221+
run: |
222+
pip install cfunits
223+
cd cfdm
224+
pip install -e .
225+
cd ../main
226+
pip install --no-deps -e .
227+
228+
# Install the coverage library
229+
# We do so with conda which was setup in a previous step.
230+
- name: Install coverage
231+
shell: bash -l {0}
232+
run: |
233+
conda install coverage
234+
235+
# Provide another notification message
236+
- name: Notify about starting testing
237+
run: echo Setup complete. Now starting to run the cf-python test suite...
238+
239+
# Create netCDF files needed for testing. A separate step is required
240+
# for this so the files can be registered and recognised first; locally
241+
# they are created and used on-the-fly by 'run_tests_and_coverage'.
242+
- name: Create netCDF test files, e.g. test_file.nc
243+
shell: bash -l {0}
244+
working-directory: ${{ github.workspace }}/main/cf/test
245+
run: |
246+
# Exit code 5 means no tests were found, which is expected for
247+
# create_test_files.py. Allow that code but fail the workflow
248+
# for any other non-zero exit code, to avoid hiding other failures.
249+
python create_test_files.py || test $? -eq 5
250+
python setup_create_field.py
251+
ls -la
252+
253+
# Finally run the test suite and generate a coverage report!
254+
- name: Run test suite and generate a coverage report
255+
shell: bash -l {0}
256+
working-directory: ${{ github.workspace }}/main/cf/test
257+
run: |
258+
./run_tests_and_coverage --nohtml
259+
260+
# End with a message indicating the suite has completed its run
261+
- name: Notify about a completed run
262+
run: |
263+
echo The cf-python test suite on Mac has completed and you may now
264+
echo inspect the results.

0 commit comments

Comments
 (0)