Eliminate Reliance on CI Platform Settings in generate_workflows.sh and Prioritize Hosts Configuration

dev/ci/cases/gcafsv1/C384_gcafs_cycled.yaml

@@ -6,7 +6,7 @@ experiment:
   resdetatmos: 384
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327
+  icsdir: {{ BASE_IC }}/C96C48/20250327
   idate: 2021122012
   edate: 2021122100
   nens: 0

dev/ci/cases/gcafsv1/C384_gcafs_cycled_noDA.yaml

@@ -6,7 +6,7 @@ experiment:
   resdetatmos: 384
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327
+  icsdir: {{ BASE_IC }}/C96C48/20250327
   idate: 2021122012
   edate: 2021122100
   nens: 0

dev/ci/cases/gcafsv1/C384_gcafs_cycled_noDA_dev.yaml

@@ -6,7 +6,7 @@ experiment:
   resdetatmos: 384
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250327
+  icsdir: {{ BASE_IC }}/C96C48/20250327
   idate: 2021122012
   edate: 2021122100
   nens: 0

dev/ci/cases/gcafsv1/C96_gcafs_cycled.yaml

@@ -6,7 +6,7 @@ experiment:
   resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808
+  icsdir: {{ BASE_IC }}/C96C48/20250808
   idate: 2021122012
   edate: 2021122100
   nens: 0

dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA.yaml

@@ -6,7 +6,7 @@ experiment:
   resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808
+  icsdir: {{ BASE_IC }}/C96C48/20250808
   idate: 2021122012
   edate: 2021122100
   nens: 0

dev/ci/cases/gcafsv1/C96_gcafs_cycled_noDA_dev.yaml

@@ -6,7 +6,7 @@ experiment:
   resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808
+  icsdir: {{ BASE_IC }}/C96C48/20250808
   idate: 2021122012
   edate: 2021122200
   nens: 0

dev/ci/cases/pr/C48_gsienkf_atmDA.yaml

@@ -7,7 +7,7 @@ experiment:
   resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808
+  icsdir: {{ BASE_IC }}/C96C48/20250808
   idate: 2024022318
   edate: 2024022406
   nens: 2

dev/ci/cases/pr/C48_ufsenkf_atmDA.yaml

@@ -7,7 +7,7 @@ experiment:
   resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808
+  icsdir: {{ BASE_IC }}/C96C48/20250808
   idate: 2024022318
   edate: 2024022406
   nens: 2

dev/ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

@@ -7,7 +7,7 @@ experiment:
   resdetocean: 5.0
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20250808
+  icsdir: {{ BASE_IC }}/C48mx500/20250808
   idate: 2021032418
   edate: 2021032500
   nens: 0

dev/ci/cases/pr/C48mx500_hybAOWCDA.yaml

@@ -8,7 +8,7 @@ experiment:
   resdetocean: 5.0
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20250808
+  icsdir: {{ BASE_IC }}/C48mx500/20250808
   idate: 2021032418
   edate: 2021032500
   nens: 2

dev/ci/cases/pr/C96C48_hybatmDA.yaml

@@ -8,7 +8,7 @@ experiment:
   resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808
+  icsdir: {{ BASE_IC }}/C96C48/20250808
   idate: 2021122018
   edate: 2021122106
   nens: 2

dev/ci/cases/pr/C96C48_hybatmsnowDA.yaml

@@ -7,7 +7,7 @@ experiment:
   resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808
+  icsdir: {{ BASE_IC }}/C96C48/20250808
   idate: 2021122012
   edate: 2021122100
   nens: 2

dev/ci/cases/pr/C96C48_hybatmsoilDA.yaml

@@ -7,7 +7,7 @@ experiment:
   resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808/
+  icsdir: {{ BASE_IC }}/C96C48/20250808
   idate: 2022051506
   edate: 2022051518
   nens: 2

dev/ci/cases/pr/C96C48_ufs_hybatmDA.yaml

@@ -7,7 +7,7 @@ experiment:
   resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808
+  icsdir: {{ BASE_IC }}/C96C48/20250808
   idate: 2024022318
   edate: 2024022406
   nens: 2

dev/ci/cases/pr/C96C48_ufsgsi_hybatmDA.yaml

@@ -7,7 +7,7 @@ experiment:
   resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808
+  icsdir: {{ BASE_IC }}/C96C48/20250808
   idate: 2024022318
   edate: 2024022406
   nens: 2

dev/ci/cases/pr/C96C48mx500_S2SW_cyc_gfs.yaml

@@ -8,7 +8,7 @@ experiment:
   resdetocean: 5.0
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808
+  icsdir: {{ BASE_IC }}/C96C48/20250808
   idate: 2021122012
   edate: 2021122100
   nens: 2

dev/ci/cases/pr/C96_atm3DVar.yaml

@@ -6,7 +6,7 @@ experiment:
   resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808
+  icsdir: {{ BASE_IC }}/C96C48/20250808
   idate: 2021122018
   edate: 2021122106
   nens: 0

dev/ci/cases/pr/C96_atm3DVar_extended.yaml

@@ -6,7 +6,7 @@ experiment:
   resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808
+  icsdir: {{ BASE_IC }}/C96C48/20250808
   idate: 2021122018
   edate: 2021122118
   nens: 0

dev/ci/cases/pr/C96_gcafs_cycled.yaml

@@ -6,7 +6,7 @@ experiment:
   resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808
+  icsdir: {{ BASE_IC }}/C96C48/20250808
   idate: 2021122012
   edate: 2021122100
   nens: 0

dev/ci/cases/pr/C96_gcafs_cycled_noDA.yaml

@@ -6,7 +6,7 @@ experiment:
   resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20250808
+  icsdir: {{ BASE_IC }}/C96C48/20250808
   idate: 2021122012
   edate: 2021122100
   nens: 0

dev/ci/cases/pr/C96mx100_S2S.yaml

@@ -12,7 +12,7 @@ experiment:
   nens: 2
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20250808
+  icsdir: {{ BASE_IC }}/C96mx100/20250808
   yaml: {{ HOMEglobal }}/dev/ci/cases/yamls/sfs_defaults.yaml
 
 skip_ci_on_hosts:

dev/ci/cases/sfsv1/C96mx025_S2S.yaml

@@ -12,7 +12,7 @@ experiment:
   nens: 2
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx025/20251217
+  icsdir: {{ BASE_IC }}/C96mx025/20251217
   yaml: {{ HOMEglobal }}/dev/ci/cases/yamls/sfs_CPC_defaults.yaml
 
 skip_ci_on_hosts:

dev/ci/cases/sfsv1/C96mx100_S2S.yaml

@@ -12,7 +12,7 @@ experiment:
   nens: 10
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20240610
+  icsdir: {{ BASE_IC }}/C96mx100/20240610
   yaml: {{ HOMEglobal }}/dev/ci/cases/yamls/sfs_full.yaml
 
 workflow:

dev/ci/cases/weekly/C384C192_hybatmda.yaml

@@ -8,7 +8,7 @@ experiment:
   resensatmos: 192
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C384C192/20240610
+  icsdir: {{ BASE_IC }}/C384C192/20240610
   idate: 2023040118
   edate: 2023040200
   nens: 2

dev/ci/cases/weekly/C384_atm3DVar.yaml

@@ -8,7 +8,7 @@ experiment:
   resensatmos: 192
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C384C192/20240610
+  icsdir: {{ BASE_IC }}/C384C192/20240610
   idate: 2023040118
   edate: 2023040200
   nens: 0

dev/ci/platforms/config.gaeac6

@@ -10,9 +10,9 @@
 # Main CI root directory - Base directory for all CI/CD operations
 export GFS_CI_ROOT="/gpfs/f6/drsa-precip3/proj-shared/${USER}/GFS_CI_CD"
 
-# ICSDIR root directory - Contains initial condition data
-# Used by create_experiment.py for setting up test cases
-export ICSDIR_ROOT=/gpfs/f6/drsa-precip3/world-shared/role.glopara/data/ICSDIR
+# BASE_IC directory - Platform home for staged initial conditions
+# Used by case yaml files for setting up the IC path
+export BASE_IC=/gpfs/f6/drsa-precip3/world-shared/role.glopara/data/ICSDIR
 
 #########################################################################
 # Jenkins configuration settings

dev/ci/platforms/config.hera

@@ -12,9 +12,9 @@
 # main path to establish a unique path
 export GFS_CI_ROOT=/scratch3/NCEPDEV/global/role.glopara/GFS_CI_CD/HERA
 
-# ICSDIR root directory - Contains initial condition data
-# Used by create_experiment.py for setting up test cases
-export ICSDIR_ROOT=/scratch3/NCEPDEV/global/role.glopara/data/ICSDIR
+# BASE_IC directory - Platform home for staged initial conditions
+# Used by case yaml files for setting up the IC path
+export BASE_IC=/scratch3/NCEPDEV/global/role.glopara/data/ICSDIR
 
 
 #########################################################################

dev/ci/platforms/config.hercules

@@ -10,9 +10,9 @@
 # Main CI root directory
 export GFS_CI_ROOT=/work2/noaa/global/role-global/GFS_CI_CD/HERCULES
 
-# ICSDIR root directory - Contains initial condition data
-# Used by create_experiment.py for setting up test cases
-export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
+# BASE_IC directory - Platform home for staged initial conditions
+# Used by case yaml files for setting up the IC path
+export BASE_IC=/work2/noaa/global/role-global/data/ICSDIR
 
 #########################################################################
 # Jenkins configuration settings

dev/ci/platforms/config.orion

@@ -10,9 +10,9 @@
 # Main CI root directory
 export GFS_CI_ROOT=/work2/noaa/global/${USER}/GFS_CI_CD/ORION
 
-# ICSDIR root directory - Contains initial condition data
-# Used by create_experiment.py for setting up test cases
-export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
+# BASE_IC directory - Platform home for staged initial conditions
+# Used by case yaml files for setting up the IC path
+export BASE_IC=/work2/noaa/global/role-global/data/ICSDIR
 
 #########################################################################
 # Jenkins configuration settings

dev/ci/platforms/config.ursa

@@ -12,9 +12,9 @@
 # main path to establish a unique path
 export GFS_CI_ROOT=/scratch3/NCEPDEV/global/role.glopara/GFS_CI_CD/URSA
 
-# ICSDIR root directory - Contains initial condition data
-# Used by create_experiment.py for setting up test cases
-export ICSDIR_ROOT=/scratch3/NCEPDEV/global/role.glopara/data/ICSDIR
+# BASE_IC directory - Platform home for staged initial conditions
+# Used by case yaml files for setting up the IC path
+export BASE_IC=/scratch3/NCEPDEV/global/role.glopara/data/ICSDIR
 
 
 #########################################################################

dev/ci/platforms/config.wcoss2

@@ -1,7 +1,7 @@
 #!/usr/bin/bash
 
 export GFS_CI_ROOT=/lfs/h2/emc/global/noscrub/globalworkflow.ci/GFS_CI_ROOT
-export ICSDIR_ROOT=/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR
+export BASE_IC=/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR
 # HPC account which overides the default account
 export HPC_ACCOUNT=${HPC_ACCOUNT:-GFS-DEV}
 

dev/ci/scripts/unittests/test_create_experiment.py

@@ -6,7 +6,6 @@
 HOMEglobal = find_upward('.github')
 current_dir = os.path.dirname(os.path.abspath(__file__))
 RUNDIR_FAKE = os.path.join(current_dir, 'testdata/RUNTESTS')
-ICSDIR_FAKE = os.path.join(current_dir, 'testdata/ICSDIR')
 
 
 def test_create_experiment():
@@ -16,7 +15,6 @@ def test_create_experiment():
     yaml_dir = yaml_dir = os.path.join(HOMEglobal, 'dev/ci/cases/pr')
     env = os.environ.copy()
     env['RUNTESTS'] = RUNDIR_FAKE
-    env['ICSDIR_ROOT'] = ICSDIR_FAKE
 
     err = 0
     for case in os.listdir(yaml_dir):

dev/ctests/CMakeLists.txt

@@ -39,17 +39,10 @@ if (NOT DEFINED STAGED_CTESTS)
   return()
 endif()
 
-# Set ICSDIR_ROOT
-set_from_env_or_default(ICSDIR_ROOT ICSDIR_ROOT "")
-if (NOT DEFINED ICSDIR_ROOT)
-  message(WARNING "ICSDIR_ROOT must be set. CTests will not be created.")
-  return()
-endif()
 
 message(STATUS "gw: global-workflow baselines will be used from: '${HOMEglobal}'")
 message(STATUS "gw: global-workflow tests will be run at: '${RUNTESTS}'")
 message(STATUS "gw: global-workflow tests will use the allocation: '${HPC_ACCOUNT}'")
-message(STATUS "gw: global-workflow tests will use ICSDIR_ROOT: '${ICSDIR_ROOT}'")
 
 # Prepare test scripts
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/setup.sh.in

dev/ctests/README.md

@@ -13,7 +13,6 @@ The following environment variables must be set (either in your environment or v
 ```bash
 HPC_ACCOUNT       # Your HPC allocation account
 STAGED_CTESTS     # Path to nightly baseline COMROOT
-ICSDIR_ROOT       # Path to initial condition files
 ```
 
 These are typically defined in `$HOMEglobal/dev/ci/platforms/config.$MACHINE_ID`.
@@ -31,7 +30,6 @@ cmake ../../..
 # Or configure with explicit command-line options
 cmake -DHPC_ACCOUNT=myaccount \
       -DSTAGED_CTESTS=/path/to/baselines/RUNTESTS \
-      -DICSDIR_ROOT=/path/to/ics \
       ../../..
 ```
 
@@ -242,7 +240,7 @@ Platform-specific settings are in `$HOMEglobal/dev/ci/platforms/config.$MACHINE_
 export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT
 export GITLAB_BUILDS_DIR=${GFS_CI_ROOT}/BUILDS/GITLAB
 export STAGED_CTESTS=${GITLAB_BUILDS_DIR}/stable/RUNTESTS
-export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR
+export BASE_IC=/scratch1/NCEPDEV/global/glopara/data/ICSDIR
 export HPC_ACCOUNT=nems
 ```
 

dev/ctests/scripts/setup.sh.in

@@ -8,7 +8,6 @@ YAML_FILE=${2:?"Name of the CI yaml file for the test"}
 # CMake to fill these variables
 HOMEglobal="@PROJECT_SOURCE_DIR@"
 RUNTESTS="@RUNTESTS@"
-ICSDIR_ROOT="@ICSDIR_ROOT@"
 HPC_ACCOUNT="@HPC_ACCOUNT@"
 
 set +x
@@ -24,7 +23,6 @@ sha=$(git rev-parse --short HEAD)
 
 pslot="${TEST_NAME}_${sha}" \
 RUNTESTS="${RUNTESTS}" \
-ICSDIR_ROOT="${ICSDIR_ROOT}" \
 HPC_ACCOUNT="${HPC_ACCOUNT}" \
 ./create_experiment.py --yaml "${YAML_FILE}" --overwrite
 rc=$?

dev/workflow/create_experiment.py

@@ -34,6 +34,7 @@
 
 import setup_expt
 import setup_workflow
+from hosts import Host
 
 
 _here = os.path.dirname(__file__)
@@ -85,7 +86,11 @@ def input_args():
     user_inputs = input_args()
 
     # Create a dictionary to pass to parse_j2yaml for parsing the yaml file
+    # Load host info first so host variables (e.g. BASE_IC) are available as
+    # template variables when the case YAML is rendered.
+    host = Host()
     data = AttrDict(HOMEglobal=_top)
+    data.update(host.info)
     data.update(os.environ)
     testconf = parse_j2yaml(path=user_inputs.yaml, data=data)