[GFS V17] Add real-time tropical cyclone QC to v17

.gitignore

@@ -80,6 +80,7 @@ parm/post/post_tag_gfs128
 parm/post/gfs
 parm/post/gefs
 parm/post/sfs
+parm/post/gcafs
 parm/post/ocean.csv
 parm/post/ocean_gefs.csv
 parm/post/ocean_gfs.csv
@@ -96,9 +97,11 @@ parm/ufs/MOM6_data_table.IN
 parm/ufs/ice_in.IN
 parm/ufs/ufs.configure.*.IN
 parm/ufs/post_itag_gfs
+parm/ufs/post_itag_gcafs
 parm/ufs/ww3_shel.nml.IN
 parm/ufs/global_control.nml.IN
 parm/regrid_sfc
+parm/chem/nexus
 
 # Ignore sorc and logs folders from externals
 #--------------------------------------------
@@ -173,6 +176,7 @@ ush/global_cycle_driver.sh
 ush/gsi_satbias2ioda_all.sh
 ush/jediinc2fv3.py
 ush/imsfv3_scf2ioda.py
+ush/ghcn_snod2ioda.py
 ush/bufr_snocvr_snomad.py
 ush/atparse.bash
 ush/bufr2ioda_insitu*
@@ -183,6 +187,7 @@ ush/python/soca
 ush/python/gen_bufr2ioda_json.py
 ush/python/gen_bufr2ioda_yaml.py
 ush/python/run_bufr2ioda.py
+ush/python/wxflow
 ush/spoc
 # ush log file
 ush/fetch-fix-data.log

dev/ci/cases/gfsv17/retrov17_realtime.yaml

@@ -13,7 +13,7 @@ experiment:
   expdir: /lfs/h2/emc/gfstemp/emc.global/expdir
   idate: 2025111306
   edate: 2027011600
-  icsdir: /lfs/h2/emc/gfstemp/emc.global/comroot/rt17_upd03_realtime 
+  icsdir: /lfs/h2/emc/gfstemp/emc.global/comroot/rt17_upd03_realtime
   yaml: {{ HOMEgfs }}/dev/ci/cases/gfsv17/s2sw_realtime.yaml
 
 workflow:

dev/ci/cases/gfsv17/s2sw_realtime.yaml

@@ -24,10 +24,11 @@ base:
   DO_VERFRAD: "YES"
   EUPD_CYC: "gdas"
   INTERVAL_GFS: 6
-  SDATE_GFS: 2025111312
+  SDATE_GFS: 2026031912
   FHMAX_GFS: 384
   FHMAX_HF_GFS: 0
   CASE_HIST: "C768"
+  NMEM_ENS_GFS_OFFSET: 0
 
 prepoceanobs:
   use_exp_obs: "YES"
@@ -36,6 +37,10 @@ prepoceanobs:
   # realtime
   dmpdir_exp: /lfs/h2/emc/global/noscrub/emc.global/ocean
 
+prep:
+  # This can only be enabled in realtime
+  PROCESS_TROPCY: "NO"
+
 marinebmat:
   SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca
   SOCA_ANL_GEOM: {{ HOMEgfs }}/fix/gdas/soca/720x540x75/soca

dev/jobs/JGLOBAL_ATMOS_TROPCY_QC_RELOC

@@ -20,7 +20,7 @@ export TANK_TROPCY=${TANK_TROPCY:-${DCOMROOT}} # path to tropical cyclone record
 ##############################################
 # Define COM directories
 ##############################################
-declare_from_tmpl COMOUT_OBS:COM_OBS_TMPL
+YMD=${PDY} HH=${cyc} declare_from_tmpl COMOUT_OBS:COM_OBS_TMPL
 mkdir -p "${COMOUT_OBS}"
 
 export CRES=${CASE_HIST:1}

dev/parm/config/gfs/config.base.j2

@@ -49,6 +49,8 @@ export FIXgdas="${FIXgfs}/gdas"
 # GLOBAL static environment parameters
 export PACKAGEROOT="{{ PACKAGEROOT }}"    # TODO: set via prod_envir in Ops
 export COMROOT="{{ COMROOT }}"    # TODO: set via prod_envir in Ops
+# Available for WCOSS2/realtime only
+export DCOMROOT="/lfs/h1/ops/prod/dcom"    # TODO: set via prod_envir in ops
 export COMINsyn="{{ COMINsyn }}"
 export DMPDIR="{{ DMPDIR }}"
 export IODADIR="{{ IODADIR }}"

dev/parm/config/gfs/config.prep.j2

@@ -11,7 +11,7 @@ source "${EXPDIR}/config.resources" prep
 export cdate10=${PDY}${cyc}
 
 # Relocation and syndata QC
-export PROCESS_TROPCY=${PROCESS_TROPCY:-NO}
+export PROCESS_TROPCY='{{PROCESS_TROPCY}}'
 export TROPCYQCRELOSH="${SCRgfs}/exglobal_atmos_tropcy_qc_reloc.sh"
 export COMINsyn=${COMINsyn:-$(compath.py "${envir}/com/gfs/${gfs_ver}")/syndat}
 

dev/parm/config/gfs/config.resources

@@ -1378,7 +1378,7 @@ case ${step} in
     elif [[ "${RUN}" = *gfs ]]; then
       ntasks=28
       tasks_per_node=28
-      memory="6GB"
+      memory="8GB"
     fi
     ;;
 

dev/parm/config/gfs/yaml/defaults.yaml

@@ -76,6 +76,9 @@ marineanlletkf:
   SOCA_ANL_GEOM: "${FIXgfs}/gdas/soca/72x35x25/soca"
   SOCA_OBS_LIST: "${PARMgfs}/gdas/marine/obs/obs_list.yaml.j2"
 
+prep:
+  PROCESS_TROPCY: "NO"
+
 prepoceanobs:
   use_exp_obs: "YES"
   dmpdir_exp: "${BASE_DATA}/experimental_obs"

dev/scripts/exgfs_wave_prdgen_gridded.sh

@@ -85,7 +85,7 @@ while [[ "${fhcnt}" -le "${FHMAX_WAV}" ]]; do
 
         iparam=1
         while [[ ${iparam} -le ${nparam} ]]; do
-            nip=${arrpar[${iparam} - 1]}
+            nip=${arrpar[iparam-1]}
             prepar=${nip::-1} # Part prefix (assumes 1 digit index)
             paridx="${nip:0-1}"
             npart=0

dev/scripts/exglobal_atmos_tropcy_qc_reloc.sh

@@ -55,7 +55,7 @@ if [[ "${PROCESS_TROPCY}" = 'YES' ]]; then
     ls -ltr ./*syndata*
     cd "${ARCHSYND}" || exit 1
     pwd
-    ls -ltr
+    ls -ltr ./*syndata* || true
     cat syndat_dateck
     cd "${HOMENHC}" || exit 1
     pwd

ush/syndat_getjtbul.sh

@@ -108,7 +108,7 @@ if [[ -s prep_step ]]; then
     source prep_step
     set -u
 else
-    [[ -f errfile ]] && rm errfile
+    rm -f errfile
     # shellcheck disable=SC2046,SC2312
     unset FORT00 $(env | grep "^FORT[0-9]\{1,\}=" | awk -F= '{print $1}')
 fi

ush/syndat_qctropcy.sh

@@ -73,24 +73,12 @@ files_override=${files_override:-""}
 
 cd "${DATA}" || exit 2
 
-msg="Tropical Cyclone tcvitals QC processing has begun"
-set +x
-echo
-echo "${msg}"
-echo
-set_trace
+echo "Tropical Cyclone tcvitals QC processing has begun"
 
 if [[ "$#" -ne '1' ]]; then
-    msg="**NON-FATAL ERROR PROGRAM  SYNDAT_QCTROPCY  run date not in \
+    echo "**NON-FATAL ERROR PROGRAM  SYNDAT_QCTROPCY  run date not in \
 positional parameter 1"
-    set +x
-    echo
-    echo "${msg}"
-    echo
-    msg="**NO TROPICAL CYCLONE tcvitals processed --> non-fatal"
-    echo
-    echo "${msg}"
-    echo
+    echo "**NO TROPICAL CYCLONE tcvitals processed --> non-fatal"
 
     # Copy null files into "${COMOUT_OBS}/${RUN}.${cycle}.syndata.tcvitals.$tmmark" and
     #  "${COMOUT_OBS}/${RUN}.${cycle}.jtwc-fnoc.tcvitals.$tmmark" so later ftp attempts will find and
@@ -110,11 +98,7 @@ fi
 
 run_date=$1
 
-set +x
-echo
 echo "Run date is ${run_date}"
-echo
-set_trace
 
 year=${run_date:0:4}
 
@@ -134,12 +118,8 @@ touch dateck
 dateck_size=$(find ./ -name dateck -printf "%s")
 
 if [[ ${dateck_size} -lt 10 ]]; then
-    msg="WARNING: Archive run date check file not available or shorter than expected.\
-  Using dummy date 1900010100 to allow code to continue"
     echo 1900010100 > dateck
-    set +x
-    echo -e "\n${msg}\n"
-    set_trace
+    echo "WARNING: Archive run date check file not available or shorter than expected. Using dummy date 1900010100 to allow code to continue"
 fi
 
 #  Generate the correct RUNID and FILES value based on $NET, $RUN and $cyc
@@ -162,14 +142,11 @@ if [[ -n "${files_override}" ]]; then # for testing, typically want FILES=F
     files_override=${files_override//./}
     files_override=${files_override:0:1}
     if [[ "${files_override}" == 'T' || "${files_override}" == 'F' ]]; then
-        msg="WARNING: Variable files setting will be overriden from ${files} to ${files_override}. Override expected if testing."
+        echo "WARNING: Variable files setting will be overriden from ${files} to ${files_override}. Override expected if testing."
         files=${files_override}
     else
-        msg="WARNING: Invalid attempt to override files setting. Will stay with default for this job"
+        echo "WARNING: Invalid attempt to override files setting. Will stay with default for this job"
     fi
-    set +x
-    echo -e "\n${msg}\n"
-    set_trace
 fi
 
 echo " &INPUT  RUNID = '${net}_${tmmark}_${cyc}', FILES = ${files} " > vitchk.inp
@@ -233,7 +210,7 @@ if [[ -s prep_step ]]; then
     source prep_step
     set -u
 else
-    [[ -f errfile ]] && rm errfile
+    rm -f errfile
     # shellcheck disable=SC2046,SC2312
     unset FORT00 $(env | grep "^FORT[0-9]\{1,\}=" | awk -F= '{print $1}')
 fi
@@ -243,24 +220,10 @@ export FORT11=slmask.126
 export FORT12=run_date.dat
 "${EXECgfs}/${pgm}"
 errqct=$?
-set +x
-echo
 echo "The foreground exit status for SYNDAT_QCTROPCY is ${errqct}"
-echo
-set_trace
 if [[ "${errqct}" -gt '0' ]]; then
-    msg="**NON-FATAL ERROR PROGRAM  SYNDAT_QCTROPCY  RETURN CODE ${errqct}"
-    set +x
-    echo
-    echo "${msg}"
-    echo
-    set_trace
-    msg="**NO TROPICAL CYCLONE tcvitals processed --> non-fatal"
-    set +x
-    echo
-    echo "${msg}"
-    echo
-    set_trace
+    echo "**NON-FATAL ERROR PROGRAM  SYNDAT_QCTROPCY  RETURN CODE ${errqct}"
+    echo "**NO TROPICAL CYCLONE tcvitals processed --> non-fatal"
 
     # In the event of a ERROR in PROGRAM SYNDAT_QCTROPCY, copy null files into
     #  "${COMOUT_OBS}/${RUN}.${cycle}.syndata.tcvitals.$tmmark" and "${COMOUT_OBS}/${RUN}.${cycle}.jtwc-fnoc.tcvitals.$tmmark"
@@ -277,13 +240,11 @@ if [[ "${errqct}" -gt '0' ]]; then
 
     exit
 fi
-set +x
-echo
-echo "----------------------------------------------------------"
-echo "**********  COMPLETED PROGRAM syndat_qctropcy   **********"
-echo "----------------------------------------------------------"
-echo
-set_trace
+cat << EOF
+----------------------------------------------------------
+**********  COMPLETED PROGRAM syndat_qctropcy   **********
+----------------------------------------------------------
+EOF
 
 if [[ "${copy_back}" == 'YES' ]]; then
     cat lthistry >> "${ARCHSYND}/syndat_lthistry.${year}"
@@ -312,29 +273,16 @@ if [[ "${errdiff}" -ne 0 ]]; then
         err=$?
 
         if [[ "${err}" -ne 0 ]]; then
-            msg="###ERROR: Previous NHC Synthetic Data Record File \
-${HOMENHC}/tcvitals not updated by syndat_qctropcy"
+            echo "###ERROR: Previous NHC Synthetic Data Record File ${HOMENHC}/tcvitals not updated by syndat_qctropcy"
         else
-            msg="Previous NHC Synthetic Data Record File \
-${HOMENHC}/tcvitals successfully updated by syndat_qctropcy"
+            echo "Previous NHC Synthetic Data Record File ${HOMENHC}/tcvitals successfully updated by syndat_qctropcy"
         fi
 
-        set +x
-        echo
-        echo "${msg}"
-        echo
-        set_trace
     fi
 
 else
 
-    msg="Previous NHC Synthetic Data Record File ${HOMENHC}/tcvitals \
-not changed by syndat_qctropcy"
-    set +x
-    echo
-    echo "${msg}"
-    echo
-    set_trace
+    echo "Previous NHC Synthetic Data Record File ${HOMENHC}/tcvitals not changed by syndat_qctropcy"
 
 fi