Re-enable and upgrade verif-global to use spack-stack 2.0.0, metplus 6, met 12

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -46,7 +46,6 @@ body:
         - Gaea-C6
         - Orion
         - Hercules
-        - Jet
         - Cloud
     validations:
       required: true

dev/jobs/JGFS_ATMOS_VERIFICATION

@@ -34,18 +34,18 @@ for grid in '1p00'; do
     declare -rx "${prod_dir}"="${ROTDIR}/${RUN}.${PDY}/${cyc}/products/atmos/grib2/${grid}"
 done
 
-# TODO: If none of these are on, why are we running this job?
 if [[ "${RUN_GRID2GRID_STEP1}" == "YES" || "${RUN_GRID2OBS_STEP1}" == "YES" || "${RUN_PRECIP_STEP1}" == "YES" ]]; then
-    # Override the -e in VERIF_GLOBALSH's shebang and un-export SHELLOPTS
-    # TODO: clean up the verif-global script so it does not raise false-positive errors
     unset_strict
     export -n SHELLOPTS
-    bash -x "${VERIF_GLOBALSH}"
+    "${VERIF_GLOBALSH}"
     err=$?
     set_strict
     if [[ ${err} -ne 0 ]]; then
         exit "${err}"
     fi
+else
+    export err=1
+    err_exit "FATAL ERROR: No verification steps are enabled. Please check your configuration."
 fi
 
 if [[ ${KEEPDATA:-"NO"} == "NO" ]]; then

dev/parm/config/gcafs/config.resources

@@ -63,43 +63,6 @@ case ${machine} in
               max_tasks_per_node=80
               mem_node_max="500GB"
     ;;
-  "JET")
-    case ${PARTITION_BATCH} in
-      "xjet")
-              max_tasks_per_node=24
-              mem_node_max="61GB"
-        ;;
-      "vjet")
-              max_tasks_per_node=16
-              mem_node_max="61GB"
-        ;;
-      "sjet")
-              max_tasks_per_node=16
-              mem_node_max="29GB"
-        ;;
-      "kjet")
-              max_tasks_per_node=40
-              mem_node_max="88GB"
-        ;;
-      *)
-        echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}"
-        exit 3
-    esac
-    ;;
-  "S4")
-    case ${PARTITION_BATCH} in
-      "s4")   max_tasks_per_node=32
-              mem_node_max="168GB"
-        ;;
-      "ivy")
-              max_tasks_per_node=20
-              mem_node_max="128GB"
-        ;;
-      *)
-        echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}"
-        exit 3
-    esac
-    ;;
   "AWSPW")
     export PARTITION_BATCH="compute"
     npe_node_max=48

dev/parm/config/gfs/config.base.j2

@@ -485,6 +485,12 @@ export DO_CA="YES"
 export DO_METP="{{ DO_METP }}"   # Run METPLUS jobs - set METPLUS settings in config.metp
 export DO_FIT2OBS="YES"      # Run fit to observations package
 
+# METplus verification is not enabled on Hera or Orion
+if [[ "${machine}" == "hera" || "${machine}" == "orion" ]] && [[ ${DO_METP} == "YES" ]]; then
+   echo "WARNING: METplus verification not supported on ${machine}. Setting DO_METP=NO"
+   export DO_METP="NO"
+fi
+
 #--online archive of netcdf files for fit2obs verification
 export FHMAX_FITS=132
 if [[ "${FHMAX_FITS}" -gt "${FHMAX_GFS}" ]]; then
@@ -511,12 +517,6 @@ if [[ "${DO_JEDIATMVAR}" = "YES" ]]; then
    export DO_VMINMON="NO"     # GSI minimization monitoring
 fi
 
-# TODO: Enable METplus on Ursa when verif-global has been upgraded to spack-stack 1.9.x+
-# TODO: Clean this up by allowing DO_METP to be system-specific and/or allow overriding that parameter
-if [[ "${machine}" == "URSA" || "${machine}" == "GAEAC6" || "${machine}" == "ORION" || "${machine}" == "HERCULES" ]]; then
-    export DO_METP=NO
-fi
-
 if [[ "${DOENKFONLY_ATM:-NO}" == "YES" ]] ; then
     export RECENTER_ENKF="NO"   # Turn off recentering ensemble analysis
     export DO_VERFOZN="NO"      # Ozone data assimilation monitoring

dev/parm/config/gfs/config.metp

@@ -24,8 +24,8 @@ export VERIF_GLOBALSH=${HOMEverif_global}/ush/run_verif_global_in_global_workflo
 ## INPUT DATA SETTINGS
 export model=${PSLOT}
 export model_file_format="pgbf{lead?fmt=%2H}.${RUN}.{init?fmt=%Y%m%d%H}.grib2"
-export model_hpss_dir=${ATARDIR}/..
-export model_dir=${ARCDIR}/..
+export model_hpss_dir=${ATARDIR}
+export model_dir=${ARCDIR}
 export get_data_from_hpss="NO"
 export hpss_walltime="10"
 ## OUTPUT SETTINGS
@@ -45,19 +45,19 @@ export g2g1_anom_truth_file_format="pgbanl.${RUN}.{valid?fmt=%Y%m%d%H}.grib2"
 export g2g1_anom_fhr_min=${FHMIN_GFS}
 export g2g1_anom_fhr_max=${FHMAX_GFS}
 export g2g1_anom_grid="G002"
-export g2g1_anom_gather_by="VSDB"
+export g2g1_anom_gather_by="VALID"
 export g2g1_pres_truth_name="self_anl"
 export g2g1_pres_truth_file_format="pgbanl.${RUN}.{valid?fmt=%Y%m%d%H}.grib2"
 export g2g1_pres_fhr_min=${FHMIN_GFS}
 export g2g1_pres_fhr_max=${FHMAX_GFS}
 export g2g1_pres_grid="G002"
-export g2g1_pres_gather_by="VSDB"
+export g2g1_pres_gather_by="VALID"
 export g2g1_sfc_truth_name="self_f00"
 export g2g1_sfc_truth_file_format="pgbf00.${RUN}.{valid?fmt=%Y%m%d%H}.grib2"
 export g2g1_sfc_fhr_min=${FHMIN_GFS}
 export g2g1_sfc_fhr_max=${FHMAX_GFS}
 export g2g1_sfc_grid="G002"
-export g2g1_sfc_gather_by="VSDB"
+export g2g1_sfc_gather_by="VALID"
 export g2g1_mv_database_name="mv_${PSLOT}_grid2grid_metplus"
 export g2g1_mv_database_group="NOAA NCEP"
 export g2g1_mv_database_desc="Grid-to-grid METplus data for global workflow experiment ${PSLOT}"
@@ -68,19 +68,19 @@ export g2o1_upper_air_vhr_list="00 06 12 18"
 export g2o1_upper_air_fhr_min=${FHMIN_GFS}
 export g2o1_upper_air_fhr_max="240"
 export g2o1_upper_air_grid="G003"
-export g2o1_upper_air_gather_by="VSDB"
+export g2o1_upper_air_gather_by="VALID"
 export g2o1_conus_sfc_msg_type_list="ONLYSF ADPUPA"
 export g2o1_conus_sfc_vhr_list="00 03 06 09 12 15 18 21"
 export g2o1_conus_sfc_fhr_min=${FHMIN_GFS}
 export g2o1_conus_sfc_fhr_max="240"
 export g2o1_conus_sfc_grid="G104"
-export g2o1_conus_sfc_gather_by="VSDB"
+export g2o1_conus_sfc_gather_by="VALID"
 export g2o1_polar_sfc_msg_type_list="IABP"
 export g2o1_polar_sfc_vhr_list="00 03 06 09 12 15 18 21"
 export g2o1_polar_sfc_fhr_min=${FHMIN_GFS}
 export g2o1_polar_sfc_fhr_max="240"
 export g2o1_polar_sfc_grid="G219"
-export g2o1_polar_sfc_gather_by="VSDB"
+export g2o1_polar_sfc_gather_by="VALID"
 export g2o1_prepbufr_data_run_hpss="NO"
 export g2o1_mv_database_name="mv_${PSLOT}_grid2obs_metplus"
 export g2o1_mv_database_group="NOAA NCEP"
@@ -93,7 +93,7 @@ export precip1_ccpa_accum24hr_model_file_format="pgbf{lead?fmt=%2H}.${RUN}.{init
 export precip1_ccpa_accum24hr_fhr_min=${FHMIN_GFS}
 export precip1_ccpa_accum24hr_fhr_max="180"
 export precip1_ccpa_accum24hr_grid="G211"
-export precip1_ccpa_accum24hr_gather_by="VSDB"
+export precip1_ccpa_accum24hr_gather_by="VALID"
 export precip1_obs_data_run_hpss="NO"
 export precip1_mv_database_name="mv_${PSLOT}_precip_metplus"
 export precip1_mv_database_group="NOAA NCEP"

dev/parm/config/gfs/config.resources

@@ -1110,6 +1110,7 @@ case ${step} in
     ntasks=1
     tasks_per_node=1
     memory="80G"
+    prepost=True
     ;;
 
   "echgres")

dev/ush/gw_setup.sh

@@ -10,7 +10,7 @@
 
 # Determine if HOMEglobal is already set
 unset_homegfs=NO
-if [[ -z "${HOMEglobal+x}" ]]; then
+if [[ -z "${HOMEglobal:-}" ]]; then
     script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
     HOMEglobal=$(cd "${script_dir}" && git rev-parse --show-toplevel)
     export HOMEglobal

dev/ush/load_modules.sh

@@ -155,13 +155,34 @@ case "${MODULE_TYPE}" in
         export PYTHONPATH
         ;;
 
-    "run" | "gsi" | "verif" | "setup" | "upp")
+    "verif")
+        # EMC_verif-global modules -- use that submodule's module files
+        if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
+            source "${HOMEglobal}/sorc/verif-global.fd/versions/run.ver"
+        fi
+        module use "${HOMEglobal}/sorc/verif-global.fd/modulefiles"
+        module load "emc_verif_global_${MACHINE_ID}"
+        export err=$?
+        if [[ ${err} -ne 0 ]]; then
+            echo "FATAL ERROR: Failed to load emc_verif_global_${MACHINE_ID}"
+            exit 1
+        fi
+        module list
+
+        ;;
+
+    "run" | "gsi" | "setup" | "upp")
 
         # Test that the version file exists
         if [[ ! -f "${HOMEglobal}/versions/run.ver" ]]; then
             echo "FATAL ERROR: ${HOMEglobal}/versions/run.ver does not exist!"
             echo "HINT: Run link_workflow.sh first."
-            exit 1
+            # Exit with 0 if loading setup modules (so the user's terminal doesn't close), else with 1
+            if [[ "${MODULE_TYPE}" == "setup" ]]; then
+                exit 0
+            else
+                exit 1
+            fi
         fi
 
         # Load our modules:
@@ -181,8 +202,8 @@ case "${MODULE_TYPE}" in
             mod_type="${MODULE_TYPE}"
         fi
 
-        # Source versions file (except for upp)
-        if [[ "${mod_type}" != "upp" ]]; then
+        # Source versions file (except for upp and verification)
+        if [[ "${mod_type}" != "upp" && "${mod_type}" != "verif" ]]; then
             source "${HOMEglobal}/versions/run.ver"
         fi
 

docs/source/hpc.rst

@@ -17,10 +17,7 @@ The following system software requirements are the minimum for any new or existi
 +==============+=============+=======================================+
 | Bash         | 4.4.20      |                                       |
 +--------------+-------------+---------------------------------------+
-| Python       | * 3.8.6     | * 3.10.x is not supported by METplus  |
-|              | * 3.10.13+  |   verification software               |
-|              | * 3.11.6+   | * 3.11.6 is packaged with spack-stack |
-|              |             | * 3.9.x is untested                   |
+| Python       | 3.11.6      |                                       |
 +--------------+-------------+---------------------------------------+
 | Spack-Stack  | 1.6.0       | * Available everywhere but WCOSS2     |
 +--------------+-------------+---------------------------------------+
@@ -106,7 +103,7 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas
      -
      -
      - X
-     -
+     - X
      - X
    * - Hercules
      - 1
@@ -121,7 +118,7 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas
      -
      -
      - X
-     -
+     - X
      - X
    * - Gaea C6
      - 1
@@ -136,10 +133,10 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas
      -
      -
      - X
-     -
+     - X
      - X
    * - Hera
-     - 1
+     - 2
      - X
      - X
      - X
@@ -151,7 +148,7 @@ The Global Workflow provides capabilities for deterministic and ensemble forecas
      - X
      -
      - X
-     - X
+     -
      - X
    * - Orion
      - 2

modulefiles/gw_run.common.lua

@@ -46,9 +46,6 @@ local common_modules = {
   "py-pandas",
   "py-python-dateutil",
   "py-xarray",
-  -- TODO: Reenable when MET/METplus and verif-global are at compatible versions
-  -- "met",
-  -- "metplus",
 }
 
 for _, name in pairs(common_modules) do