diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 308a8585f20..9fa2ce6f2c3 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -46,7 +46,6 @@ body: - Gaea-C6 - Orion - Hercules - - Jet - Cloud validations: required: true diff --git a/dev/jobs/JGFS_ATMOS_VERIFICATION b/dev/jobs/JGFS_ATMOS_VERIFICATION index 15986bffa9d..261e180b0ae 100755 --- a/dev/jobs/JGFS_ATMOS_VERIFICATION +++ b/dev/jobs/JGFS_ATMOS_VERIFICATION @@ -37,18 +37,20 @@ for grid in '1p00'; do declare -rx "${prod_dir}"="${ROTDIR}/${RUN}.${PDY}/${cyc}/products/atmos/grib2/${grid}" done -# TODO: If none of these are on, why are we running this job? if [[ "${RUN_GRID2GRID_STEP1}" == "YES" || "${RUN_GRID2OBS_STEP1}" == "YES" || "${RUN_PRECIP_STEP1}" == "YES" ]]; then # Override the -e in VERIF_GLOBALSH's shebang and un-export SHELLOPTS # TODO: clean up the verif-global script so it does not raise false-positive errors source "${USHglobal}/unset_strict.sh" export -n SHELLOPTS - bash -x "${VERIF_GLOBALSH}" + "${VERIF_GLOBALSH}" err=$? source "${USHglobal}/set_strict.sh" if [[ ${err} -ne 0 ]]; then exit "${err}" fi +else + export err=1 + err_exit "FATAL ERROR: No verification steps are enabled. Please check your configuration." fi if [[ ${KEEPDATA:-"NO"} == "NO" ]]; then diff --git a/dev/parm/config/gcafs/config.resources b/dev/parm/config/gcafs/config.resources index 3bd2ed049c8..e19209d99fa 100644 --- a/dev/parm/config/gcafs/config.resources +++ b/dev/parm/config/gcafs/config.resources @@ -63,43 +63,6 @@ case ${machine} in max_tasks_per_node=80 mem_node_max="500GB" ;; - "JET") - case ${PARTITION_BATCH} in - "xjet") - max_tasks_per_node=24 - mem_node_max="61GB" - ;; - "vjet") - max_tasks_per_node=16 - mem_node_max="61GB" - ;; - "sjet") - max_tasks_per_node=16 - mem_node_max="29GB" - ;; - "kjet") - max_tasks_per_node=40 - mem_node_max="88GB" - ;; - *) - echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}" - exit 3 - esac - ;; - "S4") - case ${PARTITION_BATCH} in - "s4") max_tasks_per_node=32 - mem_node_max="168GB" - ;; - "ivy") - max_tasks_per_node=20 - mem_node_max="128GB" - ;; - *) - echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}" - exit 3 - esac - ;; "AWSPW") export PARTITION_BATCH="compute" npe_node_max=48 diff --git a/dev/parm/config/gfs/config.base.j2 b/dev/parm/config/gfs/config.base.j2 index 441d5f32dbd..5d57595ce5a 100644 --- a/dev/parm/config/gfs/config.base.j2 +++ b/dev/parm/config/gfs/config.base.j2 @@ -487,6 +487,12 @@ export DO_CA="YES" export DO_METP="{{ DO_METP }}" # Run METPLUS jobs - set METPLUS settings in config.metp export DO_FIT2OBS="YES" # Run fit to observations package +# METplus verification is not enabled on Hera or Orion +if [[ "${machine}" == "hera" || "${machine}" == "orion" ]] && [[ ${DO_METP} == "YES" ]]; then + echo "WARNING: METplus verification not supported on ${machine}. Setting DO_METP=NO" + export DO_METP="NO" +fi + #--online archive of netcdf files for fit2obs verification export FHMAX_FITS=132 if [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]]; then @@ -513,12 +519,6 @@ if [[ "${DO_JEDIATMVAR}" = "YES" ]]; then export DO_VMINMON="NO" # GSI minimization monitoring fi -# TODO: Enable METplus on Ursa when verif-global has been upgraded to spack-stack 1.9.x+ -# TODO: Clean this up by allowing DO_METP to be system-specific and/or allow overriding that parameter -if [[ "${machine}" == "URSA" || "${machine}" == "GAEAC6" || "${machine}" == "ORION" || "${machine}" == "HERCULES" ]]; then - export DO_METP=NO -fi - if [[ "${DOENKFONLY_ATM:-NO}" == "YES" ]] ; then export RECENTER_ENKF="NO" # Turn off recentering ensemble analysis export DO_VERFOZN="NO" # Ozone data assimilation monitoring diff --git a/dev/parm/config/gfs/config.metp b/dev/parm/config/gfs/config.metp index b15622eaf77..7e2e57b2f63 100644 --- a/dev/parm/config/gfs/config.metp +++ b/dev/parm/config/gfs/config.metp @@ -24,8 +24,8 @@ export VERIF_GLOBALSH=${HOMEverif_global}/ush/run_verif_global_in_global_workflo ## INPUT DATA SETTINGS export model=${PSLOT} export model_file_format="pgbf{lead?fmt=%2H}.${RUN}.{init?fmt=%Y%m%d%H}.grib2" -export model_hpss_dir=${ATARDIR}/.. -export model_dir=${ARCDIR}/.. +export model_hpss_dir=${ATARDIR} +export model_dir=${ARCDIR} export get_data_from_hpss="NO" export hpss_walltime="10" ## OUTPUT SETTINGS @@ -45,22 +45,19 @@ export g2g1_anom_truth_file_format="pgbanl.${RUN}.{valid?fmt=%Y%m%d%H}.grib2" export g2g1_anom_fhr_min=${FHMIN_GFS} export g2g1_anom_fhr_max=${FHMAX_GFS} export g2g1_anom_grid="G002" -export g2g1_anom_gather_by="VSDB" +export g2g1_anom_gather_by="VALID" export g2g1_pres_truth_name="self_anl" export g2g1_pres_truth_file_format="pgbanl.${RUN}.{valid?fmt=%Y%m%d%H}.grib2" export g2g1_pres_fhr_min=${FHMIN_GFS} export g2g1_pres_fhr_max=${FHMAX_GFS} export g2g1_pres_grid="G002" -export g2g1_pres_gather_by="VSDB" +export g2g1_pres_gather_by="VALID" export g2g1_sfc_truth_name="self_f00" export g2g1_sfc_truth_file_format="pgbf00.${RUN}.{valid?fmt=%Y%m%d%H}.grib2" export g2g1_sfc_fhr_min=${FHMIN_GFS} export g2g1_sfc_fhr_max=${FHMAX_GFS} export g2g1_sfc_grid="G002" -export g2g1_sfc_gather_by="VSDB" -export g2g1_mv_database_name="mv_${PSLOT}_grid2grid_metplus" -export g2g1_mv_database_group="NOAA NCEP" -export g2g1_mv_database_desc="Grid-to-grid METplus data for global workflow experiment ${PSLOT}" +export g2g1_sfc_gather_by="VALID" # GRID-TO-OBS STEP 1: gfsmetpg2o1 export g2o1_type_list="upper_air conus_sfc" export g2o1_upper_air_msg_type_list="ADPUPA" @@ -68,23 +65,20 @@ export g2o1_upper_air_vhr_list="00 06 12 18" export g2o1_upper_air_fhr_min=${FHMIN_GFS} export g2o1_upper_air_fhr_max="240" export g2o1_upper_air_grid="G003" -export g2o1_upper_air_gather_by="VSDB" +export g2o1_upper_air_gather_by="VALID" export g2o1_conus_sfc_msg_type_list="ONLYSF ADPUPA" export g2o1_conus_sfc_vhr_list="00 03 06 09 12 15 18 21" export g2o1_conus_sfc_fhr_min=${FHMIN_GFS} export g2o1_conus_sfc_fhr_max="240" export g2o1_conus_sfc_grid="G104" -export g2o1_conus_sfc_gather_by="VSDB" +export g2o1_conus_sfc_gather_by="VALID" export g2o1_polar_sfc_msg_type_list="IABP" export g2o1_polar_sfc_vhr_list="00 03 06 09 12 15 18 21" export g2o1_polar_sfc_fhr_min=${FHMIN_GFS} export g2o1_polar_sfc_fhr_max="240" export g2o1_polar_sfc_grid="G219" -export g2o1_polar_sfc_gather_by="VSDB" +export g2o1_polar_sfc_gather_by="VALID" export g2o1_prepbufr_data_run_hpss="NO" -export g2o1_mv_database_name="mv_${PSLOT}_grid2obs_metplus" -export g2o1_mv_database_group="NOAA NCEP" -export g2o1_mv_database_desc="Grid-to-obs METplus data for global workflow experiment ${PSLOT}" # PRECIP STEP 1: gfsmetppcp1 export precip1_type_list="ccpa_accum24hr" export precip1_ccpa_accum24hr_model_bucket="06" @@ -93,10 +87,7 @@ export precip1_ccpa_accum24hr_model_file_format="pgbf{lead?fmt=%2H}.${RUN}.{init export precip1_ccpa_accum24hr_fhr_min=${FHMIN_GFS} export precip1_ccpa_accum24hr_fhr_max="180" export precip1_ccpa_accum24hr_grid="G211" -export precip1_ccpa_accum24hr_gather_by="VSDB" +export precip1_ccpa_accum24hr_gather_by="VALID" export precip1_obs_data_run_hpss="NO" -export precip1_mv_database_name="mv_${PSLOT}_precip_metplus" -export precip1_mv_database_group="NOAA NCEP" -export precip1_mv_database_desc="Precip METplus data for global workflow experiment ${PSLOT}" echo "END: config.metp" diff --git a/dev/parm/config/gfs/config.resources b/dev/parm/config/gfs/config.resources index 38fec5b7dbc..6ccf9b18a2b 100644 --- a/dev/parm/config/gfs/config.resources +++ b/dev/parm/config/gfs/config.resources @@ -1110,6 +1110,7 @@ case ${step} in ntasks=1 tasks_per_node=1 memory="80G" + prepost=True ;; "echgres") diff --git a/dev/ush/gw_setup.sh b/dev/ush/gw_setup.sh index 0dcdf2a35d7..e3b4a408f01 100755 --- a/dev/ush/gw_setup.sh +++ b/dev/ush/gw_setup.sh @@ -10,7 +10,7 @@ # Determine if HOMEglobal is already set unset_homegfs=NO -if [[ -z "${HOMEglobal+x}" ]]; then +if [[ -z "${HOMEglobal:-}" ]]; then script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" HOMEglobal=$(cd "${script_dir}" && git rev-parse --show-toplevel) export HOMEglobal diff --git a/dev/ush/load_modules.sh b/dev/ush/load_modules.sh index e5d4ffd04d7..ad3b0dc905d 100755 --- a/dev/ush/load_modules.sh +++ b/dev/ush/load_modules.sh @@ -155,13 +155,34 @@ case "${MODULE_TYPE}" in export PYTHONPATH ;; - "run" | "gsi" | "verif" | "setup" | "upp") + "verif") + # EMC_verif-global modules -- use that submodule's module files + if [[ "${MACHINE_ID}" == "wcoss2" ]]; then + source "${HOMEglobal}/sorc/verif-global.fd/versions/run.ver" + fi + module use "${HOMEglobal}/sorc/verif-global.fd/modulefiles" + module load "emc_verif_global_${MACHINE_ID}" + export err=$? + if [[ ${err} -ne 0 ]]; then + echo "FATAL ERROR: Failed to load emc_verif_global_${MACHINE_ID}" + exit 1 + fi + module list + + ;; + + "run" | "gsi" | "setup" | "upp") # Test that the version file exists if [[ ! -f "${HOMEglobal}/versions/run.ver" ]]; then echo "FATAL ERROR: ${HOMEglobal}/versions/run.ver does not exist!" echo "HINT: Run link_workflow.sh first." - exit 1 + # Exit with 0 if loading setup modules (so the user's terminal doesn't close), else with 1 + if [[ "${MODULE_TYPE}" == "setup" ]]; then + exit 0 + else + exit 1 + fi fi # Load our modules: @@ -181,8 +202,8 @@ case "${MODULE_TYPE}" in mod_type="${MODULE_TYPE}" fi - # Source versions file (except for upp) - if [[ "${mod_type}" != "upp" ]]; then + # Source versions file (except for upp and verification) + if [[ "${mod_type}" != "upp" && "${mod_type}" != "verif" ]]; then source "${HOMEglobal}/versions/run.ver" fi diff --git a/docs/source/hpc.rst b/docs/source/hpc.rst index eb7d943d50d..44f58015a48 100644 --- a/docs/source/hpc.rst +++ b/docs/source/hpc.rst @@ -17,10 +17,7 @@ The following system software requirements are the minimum for any new or existi +==============+=============+=======================================+ | Bash | 4.4.20 | | +--------------+-------------+---------------------------------------+ -| Python | * 3.8.6 | * 3.10.x is not supported by METplus | -| | * 3.10.13+ | verification software | -| | * 3.11.6+ | * 3.11.6 is packaged with spack-stack | -| | | * 3.9.x is untested | +| Python | 3.11.6 | | +--------------+-------------+---------------------------------------+ | Spack-Stack | 1.6.0 | * Available everywhere but WCOSS2 | +--------------+-------------+---------------------------------------+ @@ -106,7 +103,7 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas - - - X - - + - X - X * - Hercules - 1 @@ -121,7 +118,7 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas - - - X - - + - X - X * - Gaea C6 - 1 @@ -136,10 +133,10 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas - - - X - - + - X - X * - Hera - - 1 + - 2 - X - X - X @@ -151,7 +148,7 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas - X - - X - - X + - - X * - Orion - 2 diff --git a/modulefiles/gw_run.common.lua b/modulefiles/gw_run.common.lua index 997466b5748..e8013f6922a 100644 --- a/modulefiles/gw_run.common.lua +++ b/modulefiles/gw_run.common.lua @@ -46,9 +46,6 @@ local common_modules = { "py-pandas", "py-python-dateutil", "py-xarray", - -- TODO: Reenable when MET/METplus and verif-global are at compatible versions - -- "met", - -- "metplus", } for _, name in pairs(common_modules) do diff --git a/modulefiles/gw_run.gaeac6.lua b/modulefiles/gw_run.gaeac6.lua index 82be4745e03..e5c8331a74a 100644 --- a/modulefiles/gw_run.gaeac6.lua +++ b/modulefiles/gw_run.gaeac6.lua @@ -5,7 +5,7 @@ Load environment to run GFS on Gaea C6 -- Test that HOMEglobal is set. -- If not, load_modules.sh was not sourced to load this module. local homegfssdir=os.getenv("HOMEglobal") or "None" -if (homegfssdir == "None") then +if (homegfssdir == "None" and mode() == "load") then LmodError("FATAL ERROR HOMEglobal variable is unset.\n" .. "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n") end diff --git a/modulefiles/gw_run.hera.lua b/modulefiles/gw_run.hera.lua index c151c62f5a8..f8538f8df56 100644 --- a/modulefiles/gw_run.hera.lua +++ b/modulefiles/gw_run.hera.lua @@ -5,7 +5,7 @@ Load environment to run GFS on Hera -- Test that HOMEglobal is set. -- If not, load_modules.sh was not sourced to load this module. local homegfssdir=os.getenv("HOMEglobal") or "None" -if (homegfssdir == "None") then +if (homegfssdir == "None" and mode() == "load") then LmodError("FATAL ERROR HOMEglobal variable is unset.\n" .. "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n") end diff --git a/modulefiles/gw_run.hercules.lua b/modulefiles/gw_run.hercules.lua index 2942e96822d..86b5e5bf625 100644 --- a/modulefiles/gw_run.hercules.lua +++ b/modulefiles/gw_run.hercules.lua @@ -5,7 +5,7 @@ Load environment to run GFS on Hercules -- Test that HOMEglobal is set. -- If not, load_modules.sh was not sourced to load this module. local homegfssdir=os.getenv("HOMEglobal") or "None" -if (homegfssdir == "None") then +if (homegfssdir == "None" and mode() == "load") then LmodError("FATAL ERROR HOMEglobal variable is unset.\n" .. "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n") end diff --git a/modulefiles/gw_run.noaacloud.lua b/modulefiles/gw_run.noaacloud.lua index 2508d6de0f5..4f48650f2e3 100644 --- a/modulefiles/gw_run.noaacloud.lua +++ b/modulefiles/gw_run.noaacloud.lua @@ -5,7 +5,7 @@ Load environment to run GFS on NOAA cloud -- Test that HOMEglobal is set. -- If not, load_modules.sh was not sourced to load this module. local homegfssdir=os.getenv("HOMEglobal") or "None" -if (homegfssdir == "None") then +if (homegfssdir == "None" and mode() == "load") then LmodError("FATAL ERROR HOMEglobal variable is unset.\n" .. "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n") end diff --git a/modulefiles/gw_run.orion.lua b/modulefiles/gw_run.orion.lua index 9e4a32defc2..183dbce9917 100644 --- a/modulefiles/gw_run.orion.lua +++ b/modulefiles/gw_run.orion.lua @@ -5,7 +5,7 @@ Load environment to run GFS on Orion -- Test that HOMEglobal is set. -- If not, load_modules.sh was not sourced to load this module. local homegfssdir=os.getenv("HOMEglobal") or "None" -if (homegfssdir == "None") then +if (homegfssdir == "None" and mode() == "load") then LmodError("FATAL ERROR HOMEglobal variable is unset.\n" .. "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n") end diff --git a/modulefiles/gw_run.ursa.lua b/modulefiles/gw_run.ursa.lua index 0fe3847f1a0..bbd83a92e3f 100644 --- a/modulefiles/gw_run.ursa.lua +++ b/modulefiles/gw_run.ursa.lua @@ -5,7 +5,7 @@ Load environment to run GFS on Ursa -- Test that HOMEglobal is set. -- If not, load_modules.sh was not sourced to load this module. local homegfssdir=os.getenv("HOMEglobal") or "None" -if (homegfssdir == "None") then +if (homegfssdir == "None" and mode() == "load") then LmodError("FATAL ERROR HOMEglobal variable is unset.\n" .. "Please \"source dev/ush/load_modules.sh\" rather than loading this module directly.\n") end diff --git a/modulefiles/gw_verif.hera.lua b/modulefiles/gw_verif.hera.lua deleted file mode 100644 index ae567e30e35..00000000000 --- a/modulefiles/gw_verif.hera.lua +++ /dev/null @@ -1,15 +0,0 @@ -help([[ -Load environment to run EMC_verif-global on Hera -]]) - -prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-fms-2024.01/install/modulefiles/Core") -load("stack-intel") -load("stack-python") -load("stack-intel-oneapi-mpi") -load("met/9.1.3") -load("metplus/3.1.1") -load("grib-util") -load("prod_util") -load("wgrib2") - -whatis("Description: EMC_verif-global run environment") diff --git a/modulefiles/gw_verif.wcoss2.lua b/modulefiles/gw_verif.wcoss2.lua deleted file mode 100644 index 700e2d4f8e0..00000000000 --- a/modulefiles/gw_verif.wcoss2.lua +++ /dev/null @@ -1,35 +0,0 @@ -help([[ -Load environment to run verification on WCOSS2 -]]) - -load(pathJoin("PrgEnv-intel", (os.getenv("PrgEnv_intel_ver") or "None"))) -load(pathJoin("craype", (os.getenv("craype_ver") or "None"))) -load(pathJoin("intel", (os.getenv("intel_ver") or "None"))) -load(pathJoin("cray-mpich", (os.getenv("cray_mpich_ver") or "None"))) -load(pathJoin("cray-pals", (os.getenv("cray_pals_ver") or "None"))) -load(pathJoin("cfp", (os.getenv("cfp_ver") or "None"))) -setenv("USE_CFP","YES") - --- Use Python 3.8.6 for verification -load(pathJoin("python", '3.8.6')) - -load(pathJoin("libjpeg", (os.getenv("libjpeg_ver") or "None"))) - --- MET was build with these versions of HDF5 and netCDF -load(pathJoin("hdf5", '1.10.6')) -load(pathJoin("netcdf", '4.7.4')) - --- The cray library path for C MPI libraries (needed by C-only programs using netCDF) -local cray_lib_path=os.getenv("CRAY_LD_LIBRARY_PATH") or "" -prepend_path("LD_LIBRARY_PATH", cray_lib_path) - -load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) -load(pathJoin("grib_util", (os.getenv("grib_util_ver") or "None"))) -load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) - -prepend_path("MODULEPATH", "/apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304") -setenv("HPC_OPT", "/apps/ops/para/libs") -load(pathJoin("met", (os.getenv("met_ver") or "None"))) -load(pathJoin("metplus", (os.getenv("metplus_ver") or "None"))) - -whatis("Description: Verification environment") diff --git a/sorc/verif-global.fd b/sorc/verif-global.fd index d7991eae9e7..0f7aed29d3c 160000 --- a/sorc/verif-global.fd +++ b/sorc/verif-global.fd @@ -1 +1 @@ -Subproject commit d7991eae9e73845b8feea11c9173eeec3c7e08f2 +Subproject commit 0f7aed29d3c84c19e0bddacc511926d074a2e721