diff --git a/dev/ci/cases/gcafsv1/C384_gcafs_cycled.yaml b/dev/ci/cases/gcafsv1/C384_gcafs_cycled.yaml index 2a2677fa08d..587a40b7382 100644 --- a/dev/ci/cases/gcafsv1/C384_gcafs_cycled.yaml +++ b/dev/ci/cases/gcafsv1/C384_gcafs_cycled.yaml @@ -6,7 +6,7 @@ experiment: resdetatmos: 384 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327 + icsdir: {{ BASE_IC }}/C96C48/20250327 idate: 2021122012 edate: 2021122100 nens: 0 diff --git a/dev/ci/cases/gcafsv1/C384_gcafs_cycled_noDA.yaml b/dev/ci/cases/gcafsv1/C384_gcafs_cycled_noDA.yaml index ecb8ec678f1..58519b3e583 100644 --- a/dev/ci/cases/gcafsv1/C384_gcafs_cycled_noDA.yaml +++ b/dev/ci/cases/gcafsv1/C384_gcafs_cycled_noDA.yaml @@ -6,7 +6,7 @@ experiment: resdetatmos: 384 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327 + icsdir: {{ BASE_IC }}/C96C48/20250327 idate: 2021122012 edate: 2021122100 nens: 0 diff --git a/dev/ci/cases/gcafsv1/C384_gcafs_cycled_noDA_dev.yaml b/dev/ci/cases/gcafsv1/C384_gcafs_cycled_noDA_dev.yaml index 773258b85fb..e3e27c27003 100644 --- a/dev/ci/cases/gcafsv1/C384_gcafs_cycled_noDA_dev.yaml +++ b/dev/ci/cases/gcafsv1/C384_gcafs_cycled_noDA_dev.yaml @@ -6,7 +6,7 @@ experiment: resdetatmos: 384 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327 + icsdir: {{ BASE_IC }}/C96C48/20250327 idate: 2021122012 edate: 2021122100 nens: 0 diff --git a/dev/ci/cases/gcafsv1/C96_gcafs_cycled.yaml b/dev/ci/cases/gcafsv1/C96_gcafs_cycled.yaml index 937c1263397..9781f4af06b 100644 --- a/dev/ci/cases/gcafsv1/C96_gcafs_cycled.yaml +++ b/dev/ci/cases/gcafsv1/C96_gcafs_cycled.yaml @@ -6,7 +6,7 @@ experiment: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 + icsdir: {{ BASE_IC }}/C96C48/20250808 idate: 2021122012 edate: 2021122100 nens: 0 diff --git a/dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA.yaml b/dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA.yaml index 1e70b22bc75..10c7a64f120 100644 --- a/dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA.yaml +++ b/dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA.yaml @@ -6,7 +6,7 @@ experiment: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 + icsdir: {{ BASE_IC }}/C96C48/20250808 idate: 2021122012 edate: 2021122100 nens: 0 diff --git a/dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA_dev.yaml b/dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA_dev.yaml index 20fde3e50c1..ec147924fd2 100644 --- a/dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA_dev.yaml +++ b/dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA_dev.yaml @@ -6,7 +6,7 @@ experiment: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 + icsdir: {{ BASE_IC }}/C96C48/20250808 idate: 2021122012 edate: 2021122200 nens: 0 diff --git a/dev/ci/cases/pr/C48_gsienkf_atmDA.yaml b/dev/ci/cases/pr/C48_gsienkf_atmDA.yaml index fd1f2d12ac2..931e9ea8626 100644 --- a/dev/ci/cases/pr/C48_gsienkf_atmDA.yaml +++ b/dev/ci/cases/pr/C48_gsienkf_atmDA.yaml @@ -7,7 +7,7 @@ experiment: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 + icsdir: {{ BASE_IC }}/C96C48/20250808 idate: 2024022318 edate: 2024022406 nens: 2 diff --git a/dev/ci/cases/pr/C48_ufsenkf_atmDA.yaml b/dev/ci/cases/pr/C48_ufsenkf_atmDA.yaml index 2067ccbb956..2e81a2fd3d1 100644 --- a/dev/ci/cases/pr/C48_ufsenkf_atmDA.yaml +++ b/dev/ci/cases/pr/C48_ufsenkf_atmDA.yaml @@ -7,7 +7,7 @@ experiment: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 + icsdir: {{ BASE_IC }}/C96C48/20250808 idate: 2024022318 edate: 2024022406 nens: 2 diff --git a/dev/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml b/dev/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml index c6788b9d532..1be12e2977b 100644 --- a/dev/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml +++ b/dev/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml @@ -7,7 +7,7 @@ experiment: resdetocean: 5.0 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20250808 + icsdir: {{ BASE_IC }}/C48mx500/20250808 idate: 2021032418 edate: 2021032500 nens: 0 diff --git a/dev/ci/cases/pr/C48mx500_hybAOWCDA.yaml b/dev/ci/cases/pr/C48mx500_hybAOWCDA.yaml index d1fa21c367e..37af0db1c4d 100644 --- a/dev/ci/cases/pr/C48mx500_hybAOWCDA.yaml +++ b/dev/ci/cases/pr/C48mx500_hybAOWCDA.yaml @@ -8,7 +8,7 @@ experiment: resdetocean: 5.0 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20250808 + icsdir: {{ BASE_IC }}/C48mx500/20250808 idate: 2021032418 edate: 2021032500 nens: 2 diff --git a/dev/ci/cases/pr/C96C48_hybatmDA.yaml b/dev/ci/cases/pr/C96C48_hybatmDA.yaml index cbfc8c70763..c85e68e859a 100644 --- a/dev/ci/cases/pr/C96C48_hybatmDA.yaml +++ b/dev/ci/cases/pr/C96C48_hybatmDA.yaml @@ -8,7 +8,7 @@ experiment: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 + icsdir: {{ BASE_IC }}/C96C48/20250808 idate: 2021122018 edate: 2021122106 nens: 2 diff --git a/dev/ci/cases/pr/C96C48_hybatmsnowDA.yaml b/dev/ci/cases/pr/C96C48_hybatmsnowDA.yaml index ad136129a3d..6c1412a229b 100644 --- a/dev/ci/cases/pr/C96C48_hybatmsnowDA.yaml +++ b/dev/ci/cases/pr/C96C48_hybatmsnowDA.yaml @@ -7,7 +7,7 @@ experiment: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 + icsdir: {{ BASE_IC }}/C96C48/20250808 idate: 2021122012 edate: 2021122100 nens: 2 diff --git a/dev/ci/cases/pr/C96C48_hybatmsoilDA.yaml b/dev/ci/cases/pr/C96C48_hybatmsoilDA.yaml index 9c4c60d2fc8..11b8f6f6668 100644 --- a/dev/ci/cases/pr/C96C48_hybatmsoilDA.yaml +++ b/dev/ci/cases/pr/C96C48_hybatmsoilDA.yaml @@ -7,7 +7,7 @@ experiment: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808/ + icsdir: {{ BASE_IC }}/C96C48/20250808 idate: 2022051506 edate: 2022051518 nens: 2 diff --git a/dev/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/dev/ci/cases/pr/C96C48_ufs_hybatmDA.yaml index 41937cf88bb..d122a452038 100644 --- a/dev/ci/cases/pr/C96C48_ufs_hybatmDA.yaml +++ b/dev/ci/cases/pr/C96C48_ufs_hybatmDA.yaml @@ -7,7 +7,7 @@ experiment: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 + icsdir: {{ BASE_IC }}/C96C48/20250808 idate: 2024022318 edate: 2024022406 nens: 2 diff --git a/dev/ci/cases/pr/C96C48_ufsgsi_hybatmDA.yaml b/dev/ci/cases/pr/C96C48_ufsgsi_hybatmDA.yaml index dd81c485f8e..cbe5abba619 100644 --- a/dev/ci/cases/pr/C96C48_ufsgsi_hybatmDA.yaml +++ b/dev/ci/cases/pr/C96C48_ufsgsi_hybatmDA.yaml @@ -7,7 +7,7 @@ experiment: resensatmos: 48 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 + icsdir: {{ BASE_IC }}/C96C48/20250808 idate: 2024022318 edate: 2024022406 nens: 2 diff --git a/dev/ci/cases/pr/C96C48mx500_S2SW_cyc_gfs.yaml b/dev/ci/cases/pr/C96C48mx500_S2SW_cyc_gfs.yaml index b70f88ccb13..8c643a0a5d4 100644 --- a/dev/ci/cases/pr/C96C48mx500_S2SW_cyc_gfs.yaml +++ b/dev/ci/cases/pr/C96C48mx500_S2SW_cyc_gfs.yaml @@ -8,7 +8,7 @@ experiment: resdetocean: 5.0 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 + icsdir: {{ BASE_IC }}/C96C48/20250808 idate: 2021122012 edate: 2021122100 nens: 2 diff --git a/dev/ci/cases/pr/C96_atm3DVar.yaml b/dev/ci/cases/pr/C96_atm3DVar.yaml index ef5bebbd54d..cf18207fa88 100644 --- a/dev/ci/cases/pr/C96_atm3DVar.yaml +++ b/dev/ci/cases/pr/C96_atm3DVar.yaml @@ -6,7 +6,7 @@ experiment: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 + icsdir: {{ BASE_IC }}/C96C48/20250808 idate: 2021122018 edate: 2021122106 nens: 0 diff --git a/dev/ci/cases/pr/C96_atm3DVar_extended.yaml b/dev/ci/cases/pr/C96_atm3DVar_extended.yaml index f62dc7a49b4..48c3e558408 100644 --- a/dev/ci/cases/pr/C96_atm3DVar_extended.yaml +++ b/dev/ci/cases/pr/C96_atm3DVar_extended.yaml @@ -6,7 +6,7 @@ experiment: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 + icsdir: {{ BASE_IC }}/C96C48/20250808 idate: 2021122018 edate: 2021122118 nens: 0 diff --git a/dev/ci/cases/pr/C96_gcafs_cycled.yaml b/dev/ci/cases/pr/C96_gcafs_cycled.yaml index 674dc788251..fe78983dad2 100644 --- a/dev/ci/cases/pr/C96_gcafs_cycled.yaml +++ b/dev/ci/cases/pr/C96_gcafs_cycled.yaml @@ -6,7 +6,7 @@ experiment: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 + icsdir: {{ BASE_IC }}/C96C48/20250808 idate: 2021122012 edate: 2021122100 nens: 0 diff --git a/dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml b/dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml index 1b7a40528be..d5788e5b4c3 100644 --- a/dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml +++ b/dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml @@ -6,7 +6,7 @@ experiment: resdetatmos: 96 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808 + icsdir: {{ BASE_IC }}/C96C48/20250808 idate: 2021122012 edate: 2021122100 nens: 0 diff --git a/dev/ci/cases/pr/C96mx100_S2S.yaml b/dev/ci/cases/pr/C96mx100_S2S.yaml index 41a8242995b..96297d5a9cd 100644 --- a/dev/ci/cases/pr/C96mx100_S2S.yaml +++ b/dev/ci/cases/pr/C96mx100_S2S.yaml @@ -12,7 +12,7 @@ experiment: nens: 2 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20250808 + icsdir: {{ BASE_IC }}/C96mx100/20250808 yaml: {{ HOMEglobal }}/dev/ci/cases/yamls/sfs_defaults.yaml skip_ci_on_hosts: diff --git a/dev/ci/cases/sfsv1/C96mx025_S2S.yaml b/dev/ci/cases/sfsv1/C96mx025_S2S.yaml index 829419ed6b6..3cd39bcef44 100644 --- a/dev/ci/cases/sfsv1/C96mx025_S2S.yaml +++ b/dev/ci/cases/sfsv1/C96mx025_S2S.yaml @@ -12,7 +12,7 @@ experiment: nens: 2 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx025/20251217 + icsdir: {{ BASE_IC }}/C96mx025/20251217 yaml: {{ HOMEglobal }}/dev/ci/cases/yamls/sfs_CPC_defaults.yaml skip_ci_on_hosts: diff --git a/dev/ci/cases/sfsv1/C96mx100_S2S.yaml b/dev/ci/cases/sfsv1/C96mx100_S2S.yaml index 5f68fc1ac46..2db7be8f72a 100644 --- a/dev/ci/cases/sfsv1/C96mx100_S2S.yaml +++ b/dev/ci/cases/sfsv1/C96mx100_S2S.yaml @@ -12,7 +12,7 @@ experiment: nens: 10 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20240610 + icsdir: {{ BASE_IC }}/C96mx100/20240610 yaml: {{ HOMEglobal }}/dev/ci/cases/yamls/sfs_full.yaml workflow: diff --git a/dev/ci/cases/weekly/C384C192_hybatmda.yaml b/dev/ci/cases/weekly/C384C192_hybatmda.yaml index 2b85317114b..f1c507c3a2c 100644 --- a/dev/ci/cases/weekly/C384C192_hybatmda.yaml +++ b/dev/ci/cases/weekly/C384C192_hybatmda.yaml @@ -8,7 +8,7 @@ experiment: resensatmos: 192 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C384C192/20240610 + icsdir: {{ BASE_IC }}/C384C192/20240610 idate: 2023040118 edate: 2023040200 nens: 2 diff --git a/dev/ci/cases/weekly/C384_atm3DVar.yaml b/dev/ci/cases/weekly/C384_atm3DVar.yaml index 740e7f99825..b3d3a53d072 100644 --- a/dev/ci/cases/weekly/C384_atm3DVar.yaml +++ b/dev/ci/cases/weekly/C384_atm3DVar.yaml @@ -8,7 +8,7 @@ experiment: resensatmos: 192 comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR - icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C384C192/20240610 + icsdir: {{ BASE_IC }}/C384C192/20240610 idate: 2023040118 edate: 2023040200 nens: 0 diff --git a/dev/ci/platforms/config.gaeac6 b/dev/ci/platforms/config.gaeac6 index 5b06b46f4f9..f7468354517 100644 --- a/dev/ci/platforms/config.gaeac6 +++ b/dev/ci/platforms/config.gaeac6 @@ -10,9 +10,9 @@ # Main CI root directory - Base directory for all CI/CD operations export GFS_CI_ROOT="/gpfs/f6/drsa-precip3/proj-shared/${USER}/GFS_CI_CD" -# ICSDIR root directory - Contains initial condition data -# Used by create_experiment.py for setting up test cases -export ICSDIR_ROOT=/gpfs/f6/drsa-precip3/world-shared/role.glopara/data/ICSDIR +# BASE_IC directory - Platform home for staged initial conditions +# Used by case yaml files for setting up the IC path +export BASE_IC=/gpfs/f6/drsa-precip3/world-shared/role.glopara/data/ICSDIR ######################################################################### # Jenkins configuration settings diff --git a/dev/ci/platforms/config.hera b/dev/ci/platforms/config.hera index 6b846ca52ed..9baec74704d 100644 --- a/dev/ci/platforms/config.hera +++ b/dev/ci/platforms/config.hera @@ -12,9 +12,9 @@ # main path to establish a unique path export GFS_CI_ROOT=/scratch3/NCEPDEV/global/role.glopara/GFS_CI_CD/HERA -# ICSDIR root directory - Contains initial condition data -# Used by create_experiment.py for setting up test cases -export ICSDIR_ROOT=/scratch3/NCEPDEV/global/role.glopara/data/ICSDIR +# BASE_IC directory - Platform home for staged initial conditions +# Used by case yaml files for setting up the IC path +export BASE_IC=/scratch3/NCEPDEV/global/role.glopara/data/ICSDIR ######################################################################### diff --git a/dev/ci/platforms/config.hercules b/dev/ci/platforms/config.hercules index 0b1bb6f15e4..50ff747a992 100644 --- a/dev/ci/platforms/config.hercules +++ b/dev/ci/platforms/config.hercules @@ -10,9 +10,9 @@ # Main CI root directory export GFS_CI_ROOT=/work2/noaa/global/role-global/GFS_CI_CD/HERCULES -# ICSDIR root directory - Contains initial condition data -# Used by create_experiment.py for setting up test cases -export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR +# BASE_IC directory - Platform home for staged initial conditions +# Used by case yaml files for setting up the IC path +export BASE_IC=/work2/noaa/global/role-global/data/ICSDIR ######################################################################### # Jenkins configuration settings diff --git a/dev/ci/platforms/config.orion b/dev/ci/platforms/config.orion index 32af60cef84..724f3abbabd 100644 --- a/dev/ci/platforms/config.orion +++ b/dev/ci/platforms/config.orion @@ -10,9 +10,9 @@ # Main CI root directory export GFS_CI_ROOT=/work2/noaa/global/${USER}/GFS_CI_CD/ORION -# ICSDIR root directory - Contains initial condition data -# Used by create_experiment.py for setting up test cases -export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR +# BASE_IC directory - Platform home for staged initial conditions +# Used by case yaml files for setting up the IC path +export BASE_IC=/work2/noaa/global/role-global/data/ICSDIR ######################################################################### # Jenkins configuration settings diff --git a/dev/ci/platforms/config.ursa b/dev/ci/platforms/config.ursa index 0a9e6dfd61a..3672db4a0ec 100644 --- a/dev/ci/platforms/config.ursa +++ b/dev/ci/platforms/config.ursa @@ -12,9 +12,9 @@ # main path to establish a unique path export GFS_CI_ROOT=/scratch3/NCEPDEV/global/role.glopara/GFS_CI_CD/URSA -# ICSDIR root directory - Contains initial condition data -# Used by create_experiment.py for setting up test cases -export ICSDIR_ROOT=/scratch3/NCEPDEV/global/role.glopara/data/ICSDIR +# BASE_IC directory - Platform home for staged initial conditions +# Used by case yaml files for setting up the IC path +export BASE_IC=/scratch3/NCEPDEV/global/role.glopara/data/ICSDIR ######################################################################### diff --git a/dev/ci/platforms/config.wcoss2 b/dev/ci/platforms/config.wcoss2 index 2d461a8256a..ecbb8c5e4dc 100644 --- a/dev/ci/platforms/config.wcoss2 +++ b/dev/ci/platforms/config.wcoss2 @@ -1,7 +1,7 @@ #!/usr/bin/bash export GFS_CI_ROOT=/lfs/h2/emc/global/noscrub/globalworkflow.ci/GFS_CI_ROOT -export ICSDIR_ROOT=/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR +export BASE_IC=/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR # HPC account which overides the default account export HPC_ACCOUNT=${HPC_ACCOUNT:-GFS-DEV} diff --git a/dev/ci/scripts/unittests/test_create_experiment.py b/dev/ci/scripts/unittests/test_create_experiment.py index 6f58385b9b6..92c9fd43322 100644 --- a/dev/ci/scripts/unittests/test_create_experiment.py +++ b/dev/ci/scripts/unittests/test_create_experiment.py @@ -6,7 +6,6 @@ HOMEglobal = find_upward('.github') current_dir = os.path.dirname(os.path.abspath(__file__)) RUNDIR_FAKE = os.path.join(current_dir, 'testdata/RUNTESTS') -ICSDIR_FAKE = os.path.join(current_dir, 'testdata/ICSDIR') def test_create_experiment(): @@ -16,7 +15,6 @@ def test_create_experiment(): yaml_dir = yaml_dir = os.path.join(HOMEglobal, 'dev/ci/cases/pr') env = os.environ.copy() env['RUNTESTS'] = RUNDIR_FAKE - env['ICSDIR_ROOT'] = ICSDIR_FAKE err = 0 for case in os.listdir(yaml_dir): diff --git a/dev/ctests/CMakeLists.txt b/dev/ctests/CMakeLists.txt index 62d25f8ace3..e30f214ecf4 100644 --- a/dev/ctests/CMakeLists.txt +++ b/dev/ctests/CMakeLists.txt @@ -39,17 +39,10 @@ if (NOT DEFINED STAGED_CTESTS) return() endif() -# Set ICSDIR_ROOT -set_from_env_or_default(ICSDIR_ROOT ICSDIR_ROOT "") -if (NOT DEFINED ICSDIR_ROOT) - message(WARNING "ICSDIR_ROOT must be set. CTests will not be created.") - return() -endif() message(STATUS "gw: global-workflow baselines will be used from: '${HOMEglobal}'") message(STATUS "gw: global-workflow tests will be run at: '${RUNTESTS}'") message(STATUS "gw: global-workflow tests will use the allocation: '${HPC_ACCOUNT}'") -message(STATUS "gw: global-workflow tests will use ICSDIR_ROOT: '${ICSDIR_ROOT}'") # Prepare test scripts configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/setup.sh.in diff --git a/dev/ctests/README.md b/dev/ctests/README.md index e4654ac25fe..fb7b3de11e4 100644 --- a/dev/ctests/README.md +++ b/dev/ctests/README.md @@ -13,7 +13,6 @@ The following environment variables must be set (either in your environment or v ```bash HPC_ACCOUNT # Your HPC allocation account STAGED_CTESTS # Path to nightly baseline COMROOT -ICSDIR_ROOT # Path to initial condition files ``` These are typically defined in `$HOMEglobal/dev/ci/platforms/config.$MACHINE_ID`. @@ -31,7 +30,6 @@ cmake ../../.. # Or configure with explicit command-line options cmake -DHPC_ACCOUNT=myaccount \ -DSTAGED_CTESTS=/path/to/baselines/RUNTESTS \ - -DICSDIR_ROOT=/path/to/ics \ ../../.. ``` @@ -242,7 +240,7 @@ Platform-specific settings are in `$HOMEglobal/dev/ci/platforms/config.$MACHINE_ export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT export GITLAB_BUILDS_DIR=${GFS_CI_ROOT}/BUILDS/GITLAB export STAGED_CTESTS=${GITLAB_BUILDS_DIR}/stable/RUNTESTS -export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR +export BASE_IC=/scratch1/NCEPDEV/global/glopara/data/ICSDIR export HPC_ACCOUNT=nems ``` diff --git a/dev/ctests/scripts/setup.sh.in b/dev/ctests/scripts/setup.sh.in index ed6e7ab014c..5d2bcab73b9 100755 --- a/dev/ctests/scripts/setup.sh.in +++ b/dev/ctests/scripts/setup.sh.in @@ -8,7 +8,6 @@ YAML_FILE=${2:?"Name of the CI yaml file for the test"} # CMake to fill these variables HOMEglobal="@PROJECT_SOURCE_DIR@" RUNTESTS="@RUNTESTS@" -ICSDIR_ROOT="@ICSDIR_ROOT@" HPC_ACCOUNT="@HPC_ACCOUNT@" set +x @@ -24,7 +23,6 @@ sha=$(git rev-parse --short HEAD) pslot="${TEST_NAME}_${sha}" \ RUNTESTS="${RUNTESTS}" \ -ICSDIR_ROOT="${ICSDIR_ROOT}" \ HPC_ACCOUNT="${HPC_ACCOUNT}" \ ./create_experiment.py --yaml "${YAML_FILE}" --overwrite rc=$? diff --git a/dev/workflow/create_experiment.py b/dev/workflow/create_experiment.py index 584ddab6f27..0f9e1358de1 100755 --- a/dev/workflow/create_experiment.py +++ b/dev/workflow/create_experiment.py @@ -34,6 +34,7 @@ import setup_expt import setup_workflow +from hosts import Host _here = os.path.dirname(__file__) @@ -85,7 +86,11 @@ def input_args(): user_inputs = input_args() # Create a dictionary to pass to parse_j2yaml for parsing the yaml file + # Load host info first so host variables (e.g. BASE_IC) are available as + # template variables when the case YAML is rendered. + host = Host() data = AttrDict(HOMEglobal=_top) + data.update(host.info) data.update(os.environ) testconf = parse_j2yaml(path=user_inputs.yaml, data=data) diff --git a/dev/workflow/generate_workflows.sh b/dev/workflow/generate_workflows.sh index b2cfd404ec2..8b2d39f3837 100755 --- a/dev/workflow/generate_workflows.sh +++ b/dev/workflow/generate_workflows.sh @@ -57,6 +57,10 @@ function _usage() { \$HOMEglobal/dev/ci/platform/config.\$machine will be used. + -I "/path/to/base_ic" Override BASE_IC for all cases. + If this is not set, BASE_IC is read from the hosts YAML + (\$HOMEglobal/dev/workflow/hosts/\$machine.yaml). + -c Append the chosen set of tests to your existing crontab If this option is not chosen, the new entries that would have been written to your crontab will be printed to stdout. @@ -83,6 +87,9 @@ EOF set -eu +# --------------------------------------------------------------------------- # +# Defaults and Runtime State +# --------------------------------------------------------------------------- # # Set default options HOMEglobal="" _specified_home=false @@ -100,6 +107,8 @@ _run_all_sfs=false _run_all_gcafs=false _hpc_account="" _set_account=false +_base_ic="" +_set_base_ic=false _update_cron=false _email="" _tag="" @@ -112,71 +121,100 @@ _cwd=$(pwd) _runtests="${RUNTESTS:-${_runtests:-}}" _auto_del=false _nonflag_option_count=0 +# --------------------------------------------------------------------------- # +# Argument Parsing +# --------------------------------------------------------------------------- # + +function _set_yaml_list_from_arg() { + # Start over with an empty list and normalize names to no .yaml suffix. + declare -a _yaml_list=() + for _yaml in ${OPTARG}; do + _yaml_list+=("${_yaml//.yaml/}") + done + _specified_yaml_list=true +} -while [[ $# -gt 0 && "$1" != "--" ]]; do - while getopts ":H:bBDuy:Y:GESCA:ce:t:vVdh" option; do - case "${option}" in - H) - HOMEglobal="${OPTARG}" - _specified_home=true - if [[ ! -d "${HOMEglobal}" ]]; then - echo "Specified HOMEglobal directory (${HOMEglobal}) does not exist" - exit 1 - fi - ;; - b) _build=true ;; - B) _build=true && _compute_build=true ;; - D) _auto_del=true ;; - u) _update_submods=true ;; - y) # Start over with an empty _yaml_list - declare -a _yaml_list=() - for _yaml in ${OPTARG}; do - # Strip .yaml from the end of each and append to _yaml_list - _yaml_list+=("${_yaml//.yaml/}") - done - _specified_yaml_list=true - ;; - Y) _yaml_dir="${OPTARG}" && _specified_yaml_dir=true ;; - G) _run_all_gfs=true ;; - E) _run_all_gefs=true ;; - S) _run_all_sfs=true ;; - C) _run_all_gcafs=true ;; - c) _update_cron=true ;; - e) _email="${OPTARG}" && _set_email=true ;; - t) _tag="_${OPTARG}" ;; - v) _verbose=true ;; - V) _very_verbose=true && _verbose=true && _verbose_flag="-v" ;; - A) _set_account=true && _hpc_account="${OPTARG}" ;; - d) _debug=true && _very_verbose=true && _verbose=true && _verbose_flag="-v" && PS4='${LINENO}: ' ;; - h) _usage && exit 0 ;; - :) - echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" - _usage - exit 1 - ;; - *) - echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" - _usage +function _parse_option() { + case "${option}" in + # Core paths and build mode + H) + HOMEglobal="${OPTARG}" + _specified_home=true + if [[ ! -d "${HOMEglobal}" ]]; then + echo "Specified HOMEglobal directory (${HOMEglobal}) does not exist" exit 1 - ;; - esac - done + fi + ;; + b) _build=true ;; + B) _build=true && _compute_build=true ;; + D) _auto_del=true ;; + u) _update_submods=true ;; + + # Test/case selection + y) _set_yaml_list_from_arg ;; + Y) _yaml_dir="${OPTARG}" && _specified_yaml_dir=true ;; + G) _run_all_gfs=true ;; + E) _run_all_gefs=true ;; + S) _run_all_sfs=true ;; + C) _run_all_gcafs=true ;; + + # Workflow behavior and notifications + c) _update_cron=true ;; + e) _email="${OPTARG}" && _set_email=true ;; + t) _tag="_${OPTARG}" ;; + I) _set_base_ic=true && _base_ic="${OPTARG}" ;; + + # Logging/debug + v) _verbose=true ;; + V) _very_verbose=true && _verbose=true && _verbose_flag="-v" ;; + d) _debug=true && _very_verbose=true && _verbose=true && _verbose_flag="-v" && PS4='${LINENO}: ' ;; + + # HPC account and usage + A) _set_account=true && _hpc_account="${OPTARG}" ;; + h) _usage && exit 0 ;; + + :) + echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" + _usage + exit 1 + ;; + *) + echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" + _usage + exit 1 + ;; + esac +} - if [[ ${OPTIND:-0} -gt 0 ]]; then - shift $((OPTIND - 1)) - fi +function _parse_args() { + while [[ $# -gt 0 && "$1" != "--" ]]; do + while getopts ":H:bBDuy:Y:GESCA:I:ce:t:vVdh" option; do + _parse_option + done - while [[ $# -gt 0 && ! "$1" =~ ^- ]]; do - _runtests=${1} - ((_nonflag_option_count += 1)) - if [[ ${_nonflag_option_count} -gt 1 ]]; then - echo "Too many arguments specified." - _usage - exit 2 + if [[ ${OPTIND:-0} -gt 0 ]]; then + shift $((OPTIND - 1)) + OPTIND=1 fi - shift + + while [[ $# -gt 0 && ! "$1" =~ ^- ]]; do + _runtests=${1} + ((_nonflag_option_count += 1)) + if [[ ${_nonflag_option_count} -gt 1 ]]; then + echo "Too many arguments specified." + _usage + exit 2 + fi + shift + done done -done +} + +_parse_args "$@" + +# --------------------------------------------------------------------------- # +# Common Helpers +# --------------------------------------------------------------------------- # function send_email() { # Send an email to $_email. @@ -226,6 +264,10 @@ function delete_dir() { done } +# --------------------------------------------------------------------------- # +# Validate Required Inputs +# --------------------------------------------------------------------------- # + if [[ -z "${_runtests}" ]]; then echo "Missing run directory (RUNTESTS) argument/environment variable." sleep 2 @@ -238,6 +280,10 @@ if [[ "${_debug}" == "true" ]]; then set -x fi +# --------------------------------------------------------------------------- # +# Prepare RUNTESTS Directory +# --------------------------------------------------------------------------- # + # Create the RUNTESTS directory # Start by getting the full path _runtests="$(realpath "${_runtests}")" @@ -262,6 +308,10 @@ else fi fi +# --------------------------------------------------------------------------- # +# Resolve Initial Case Selection +# --------------------------------------------------------------------------- # + # Empty the _yaml_list array if -G, -E, -S and/or -C were selected if [[ "${_run_all_gfs}" == "true" || "${_run_all_gefs}" == "true" || @@ -293,6 +343,10 @@ if [[ "${_specified_yaml_dir}" == false ]]; then _yaml_dir="${HOMEglobal}/dev/ci/cases/pr" fi +# --------------------------------------------------------------------------- # +# Case Discovery Helper +# --------------------------------------------------------------------------- # + function select_all_yamls() { # A helper function to select all of the YAMLs for a specified system (gfs, gefs, sfs) @@ -355,6 +409,10 @@ EOM fi } +# --------------------------------------------------------------------------- # +# Expand Case List By System Flags +# --------------------------------------------------------------------------- # + # Check if running all GEFS cases if [[ "${_run_all_gefs}" == "true" ]]; then # Append -w to build_all.sh flags if -E was specified @@ -392,6 +450,10 @@ if [[ "${_run_all_gcafs}" == "true" ]]; then _yaml_list=("${_yaml_list[@]}" "${_gcafs_yaml_list[@]}") fi +# --------------------------------------------------------------------------- # +# Optional Submodule Update +# --------------------------------------------------------------------------- # + # Update submodules if requested if [[ "${_update_submods}" == "true" ]]; then printf "Updating submodules\n\n" @@ -418,6 +480,10 @@ EOM fi fi +# --------------------------------------------------------------------------- # +# Load Workflow Environment +# --------------------------------------------------------------------------- # + # Loading modules sometimes raises unassigned errors, so disable checks set +u if [[ "${_verbose}" == "true" ]]; then @@ -440,26 +506,49 @@ if [[ "${_debug}" == "true" ]]; then fi set -u machine=${MACHINE_ID} -platform_config="${HOMEglobal}/dev/ci/platforms/config.${machine}" -if [[ -f "${platform_config}" ]]; then - source "${HOMEglobal}/dev/ci/platforms/config.${machine}" -else - if [[ "${_set_account}" == "false" ]]; then - echo "ERROR Unknown HPC account! Please use the -A option to specify." - exit 11 - fi -fi # If _yaml_dir is not set, set it to $HOMEglobal/dev/ci/cases/pr if [[ -z ${_yaml_dir} ]]; then _yaml_dir="${HOMEglobal}/dev/ci/cases/pr" fi +# --------------------------------------------------------------------------- # +# Resolve HPC Account +# --------------------------------------------------------------------------- # + +# Update the account: -A flag > existing env var > platform config default +if [[ "${_set_account}" == true ]]; then + export HPC_ACCOUNT="${_hpc_account}" + if [[ "${_verbose}" == true ]]; then + printf "Setting HPC account to %s\n\n" "${HPC_ACCOUNT}" + fi +elif [[ -z "${HPC_ACCOUNT:-}" ]]; then + platform_config="${HOMEglobal}/dev/ci/platforms/config.${machine}" + if [[ -f "${platform_config}" ]]; then + _platform_account=$(sed -n "s/^export HPC_ACCOUNT=\${HPC_ACCOUNT:-\([^}]*\)}.*/\1/p" "${platform_config}") + export HPC_ACCOUNT="${_platform_account}" + if [[ "${_verbose}" == true ]]; then + printf "Setting HPC account to %s from platform config\n\n" "${HPC_ACCOUNT}" + fi + else + echo "ERROR Unknown HPC account! Please use the -A option to specify." + exit 11 + fi +fi + +# --------------------------------------------------------------------------- # +# Build and Link Workflow +# --------------------------------------------------------------------------- # + # Build the system if requested if [[ "${_build}" == "true" ]]; then printf "Building via build_all.sh %s\n\n" "${_build_flags}" # Let the output of build_all.sh go to stdout regardless of verbose options if [[ "${_compute_build}" == true ]]; then + if [[ -z "${HPC_ACCOUNT:-}" ]]; then + echo "ERROR Unknown HPC account! Please use the -A option to specify." + exit 11 + fi _compute_build_flag="-c -A ${HPC_ACCOUNT}" fi #shellcheck disable=SC2086,SC2248 @@ -482,6 +571,10 @@ if ! "${HOMEglobal}/sorc/link_workflow.sh" >&stdout; then fi rm -f stdout +# --------------------------------------------------------------------------- # +# Validate YAML Inputs For This Host +# --------------------------------------------------------------------------- # + # Configure the environment for running create_experiment.py if [[ "${_verbose}" == true ]]; then printf "Setting up the environment to run create_experiment.py\n\n" @@ -521,14 +614,22 @@ EOM done done -# Update the account if specified -if [[ "${_set_account}" == true ]]; then - export HPC_ACCOUNT=${_hpc_account} +# --------------------------------------------------------------------------- # +# Apply BASE_IC Override +# --------------------------------------------------------------------------- # + +# Override BASE_IC if specified via -I +if [[ "${_set_base_ic}" == true ]]; then + export BASE_IC="${_base_ic}" if [[ "${_verbose}" == true ]]; then - printf "Setting HPC account to %s\n\n" "${HPC_ACCOUNT}" + printf "Overriding BASE_IC to %s\n\n" "${BASE_IC}" fi fi +# --------------------------------------------------------------------------- # +# Create Experiments and Collect Schedule Entries +# --------------------------------------------------------------------------- # + # Create the experiments rm -f "tests.cron" "${_verbose_flag}" echo "Running create_experiment.py for ${#_yaml_list[@]} cases" @@ -606,6 +707,10 @@ for _case in "${_yaml_list[@]}"; do done echo +# --------------------------------------------------------------------------- # +# Configure Mail Behavior +# --------------------------------------------------------------------------- # + # Add MAILTO to tests.cron for regular crontab if [[ "${_use_scron}" == false ]]; then if [[ "${_set_email}" == "true" ]]; then @@ -620,6 +725,10 @@ if [[ "${_use_scron}" == false ]]; then fi fi +# --------------------------------------------------------------------------- # +# Install or Print Scheduler Entries +# --------------------------------------------------------------------------- # + # Update the cron if [[ "${_update_cron}" == "true" ]]; then printf "Updating the existing crontab\n\n" @@ -698,6 +807,10 @@ else fi fi +# --------------------------------------------------------------------------- # +# Cleanup and Completion +# --------------------------------------------------------------------------- # + # Cleanup if [[ "${_debug}" == "false" ]]; then rm -f final.cron existing.cron tests.cron "${_verbose_flag}" diff --git a/dev/workflow/hosts.py b/dev/workflow/hosts.py index 2a2fcf42cb6..972053d3a99 100644 --- a/dev/workflow/hosts.py +++ b/dev/workflow/hosts.py @@ -32,6 +32,7 @@ def __init__(self, host=None): self.detect() self.info = self._get_info + self._apply_env_overrides(self.info) self.scheduler = self.info['SCHEDULER'] def __str__(self) -> str: @@ -103,6 +104,19 @@ def detect(self) -> None: 'Currently supported hosts are:\n' + f'{" | ".join(Host.SUPPORTED_HOSTS)}') + def _apply_env_overrides(self, info: dict) -> None: + """Override host configuration values with environment variables. + + Parameters + ---------- + info : dict + Host configuration dictionary to update in-place. + """ + # Allow BASE_IC to be overridden via the environment + # (e.g. from the -I flag in generate_workflows.sh) + if 'BASE_IC' in os.environ: + info['BASE_IC'] = os.environ['BASE_IC'] + @property def _get_info(self) -> dict: diff --git a/ush/postamble.sh b/ush/postamble.sh index b26ead789a2..dd9ebd74cd1 100644 --- a/ush/postamble.sh +++ b/ush/postamble.sh @@ -22,9 +22,9 @@ postamble() { # rc: exit code of the script [default: $?] # + local rc="${2:-$?}" # Capture exit code FIRST, before set +x clobbers $? set +x local start_time="${1}" - local rc="${2:-$?}" # Execute any commands registered in POSTAMBLE_CMD #