Create operational scripts for jaigfs_forecast

ecf/jaigfs_cleanup.ecf

@@ -0,0 +1,34 @@
+#PBS -N aigfs_cleanup_%CYC%
+#PBS -j oe
+#PBS -S /bin/bash
+#PBS -q %QUEUE%
+#PBS -l walltime=00:05:00
+#PBS -A %PROJ%-%PROJENVIR%
+#PBS -l place=shared,select=1:ncpus=1:mem=2GB
+#PBS -l debug=true
+
+export model=aigfs
+
+%include <head.h>
+%include <envir-p1.h>
+
+export cyc=%CYC%
+
+module list
+
+${HOMEaigfs}/jobs/JAIGFS_CLEANUP
+
+if [[ $? -ne 0 ]]; then
+    ecflow_client --msg="***JOB ${ECF_NAME} ERROR RUNNING J-SCRIPT"
+    ecflow_client --abort
+    exit
+fi
+
+%include <tail.h>
+%manual
+###############################################################################
+#
+#  PURPOSE: Clean up old data from shared working directory
+#
+###############################################################################
+%end

ecf/jaigfs_forecast.ecf

@@ -0,0 +1,40 @@
+#PBS -N aigfs_forecast_%CYC%
+#PBS -j oe
+#PBS -S /bin/bash
+#PBS -q %QUEUE%
+#PBS -l walltime=01:30:00
+#PBS -A %PROJ%-%PROJENVIR%
+#PBS -l place=vscatter:shared,select=1:ncpus=128:mem=150GB
+#PBS -l debug=true
+
+export model=aigfs
+
+%include <head.h>
+%include <envir-p1.h>
+
+export cyc=%CYC%
+
+module load intel/${intel_ver:?}
+module load wgrib2/${wgrib2_ver:?}
+module load libjpeg-turbo/${libjpeg_turbo_ver:?}
+
+source ${HOMEaigfs}/venv/bin/activate
+
+module list
+
+${HOMEaigfs}/jobs/JAIGFS_FORECAST
+
+if [[ $? -ne 0 ]]; then
+    ecflow_client --msg="*** JOB ${ECF_NAME} ERROR RUNNING J-SCRIPT"
+    ecflow_client --abort
+    exit
+fi
+
+%include <tail.h>
+%manual
+###############################################################################
+#
+#  PURPOSE: Run a machine learning forecast with GFS inputs
+#
+###############################################################################
+%end

ecf/post/create_links.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+target=jaigfs_post.ecf
+for i in $(seq -w 0 6 384); do
+  ln -s $target jaigfs_post_f${i}.ecf
+done
+

ecf/post/jaigfs_post.ecf

@@ -0,0 +1,39 @@
+#PBS -N aigfs_post_%CYC%_f%FHR%
+#PBS -j oe
+#PBS -S /bin/bash
+#PBS -q %QUEUE%
+#PBS -l walltime=00:05:00
+#PBS -A %PROJ%-%PROJENVIR%
+#PBS -l place=shared,select=1:ncpus=1:mem=4GB
+#PBS -l debug=true
+
+export model=aigfs
+
+%include <head.h>
+%include <envir-p1.h>
+
+export cyc=%CYC%
+export fh=%FHR%
+
+module load intel/${intel_ver:?}
+module load wgrib2/${wgrib2_ver:?}
+#module load libjpeg-turbo/${libjpeg_turbo_ver:?}
+
+module list
+
+${HOMEaigfs}/jobs/JAIGFS_POST
+
+if [[ $? -ne 0 ]]; then
+    ecflow_client --msg="***JOB ${ECF_NAME} ERROR RUNNING J-SCRIPT"
+    ecflow_client --abort
+    exit
+fi
+
+%include <tail.h>
+%manual
+###############################################################################
+#
+#  PURPOSE: Copy forecast files from tmp directory to COM, create GRIB2 idx files
+#
+###############################################################################
+%end

jobs/JAIGFS_CLEANUP

@@ -0,0 +1,36 @@
+#!/bin/bash
+set -x
+
+# Prepend time to output
+export PS4=' $SECONDS + '
+
+# Create temporary working directory
+export DATA=${DATA:-${DATAROOT:?}/${jobid:?}}
+mkdir -p $DATA
+cd $DATA
+
+# Create directories to store gfs inputs and final output
+mkdir -p data output
+
+export cycle=${cycle:-t${cyc}z}
+setpdy.sh
+. ${DATA}/PDY
+echo $PDY
+
+# Variables used in com directory organization
+export NET=${NET:-aigfs}
+export RUN=${RUN:-aigfs}
+
+# loop over members and remove shared working directories
+export DATAshared=${DATAROOT:?}/${RUN}_forecast_${cyc}_${aigfs_ver%.*}_${envir}
+# if directory exists, remove it
+[ -d "$DATAshared" ] && rm -rf "$DATAshared"
+
+# Post successful completion message
+msg="JOB $job HAS COMPLETED NORMALLY."
+postmsg $jlogfile "$msg"
+
+# Remove temporary working directory
+if [ "${KEEPDATA^^}" != YES ]; then
+  rm -rf $DATA
+fi

jobs/JAIGFS_FORECAST

@@ -0,0 +1,57 @@
+#!/bin/bash
+set -x
+
+# Prepend time to output
+export PS4=' $SECONDS + '
+
+# Variables used in com directory organization
+export NET=${NET:-aigfs}
+export RUN=${RUN:-aigfs}
+
+# Create temporary working directory
+export DATAshared=${DATAROOT:?}/${RUN}_forecast_${cyc}_${aigfs_ver%.*}_${envir}
+export DATA=${DATAshared}/${jobid:?}
+mkdir -p $DATA
+cd $DATA
+
+export cycle=${cycle:-t${cyc}z}
+setpdy.sh
+. ${DATA}/PDY
+echo $PDY
+
+export curr_PDYHH=${PDY}${cyc}
+# Set up parent directory and all sub-directories for current model
+export FIXaigfs=${HOMEaigfs}/fix
+export USHaigfs=${HOMEaigfs}/ush
+export SETEVENTSH=${USHaigfs}/set_event.sh
+
+export MAILTO=${MAILTO:-'nco.spa@noaa.gov'}
+
+# Locations of incoming data
+export COMIN=${COMIN:-$(compath.py ${envir}/$NET/$aigfs_ver)}/$RUN.$PDY/$cyc/model/atmos/init
+
+# Locations of outgoing data
+export COMOUT=${COMOUT:-$(compath.py -o $NET/$aigfs_ver/$RUN.$PDY)}/${cyc}/model/atmos/grib2
+
+# Create output directories
+mkdir -p $COMOUT
+
+# Execute ex-script
+$HOMEaigfs/scripts/exaigfs_forecast.sh
+export err=$?; err_chk
+
+# Post successful completion message
+msg="JOB $job HAS COMPLETED NORMALLY."
+postmsg $jlogfile "$msg"
+
+# Print exec output
+if [ -e $pgmout ]; then
+  cat $pgmout
+fi
+
+# Remove temporary working directory
+if [ "${KEEPDATA^^}" != YES ]; then
+  rm -rf $DATA
+fi
+
+date

jobs/JAIGFS_POST

@@ -0,0 +1,48 @@
+#!/bin/bash
+set -x
+
+# Prepend time to output
+export PS4=' $SECONDS + '
+
+# Variables used in com directory organization
+export NET=${NET:-aigfs}
+export RUN=${RUN:-aigfs}
+
+# Create temporary working directory
+export DATAshared=${DATAROOT:?}/${RUN}_forecast_${cyc}_${aigfs_ver%.*}_${envir}
+export DATA=${DATAshared}/${jobid:?}
+mkdir -p $DATA
+cd $DATA
+
+export cycle=${cycle:-t${cyc}z}
+setpdy.sh
+. ${DATA}/PDY
+echo $PDY
+
+export MAILTO=${MAILTO:-'nco.spa@noaa.gov'}
+
+# Locations of outgoing data
+export COMOUT=${COMOUT:-$(compath.py -o $NET/$aigfs_ver/$RUN.$PDY)}/${cyc}/model/atmos/grib2
+
+# Create output directories
+mkdir -p $COMOUT
+
+# Execute ex-script
+$HOMEaigfs/scripts/exaigfs_post.sh
+export err=$?; err_chk
+
+# Post successful completion message
+msg="JOB $job HAS COMPLETED NORMALLY."
+postmsg $jlogfile "$msg"
+
+# Print exec output
+if [ -e $pgmout ]; then
+  cat $pgmout
+fi
+
+# Remove temporary working directory
+if [ "${KEEPDATA^^}" != YES ]; then
+  rm -rf $DATA
+fi
+
+date

parm/transfer_aigfs.list

@@ -0,0 +1,50 @@
+# This file specifies the directories to be transfered and, optionally, the files within
+# those directories to include or exclude.  If one directory is specified per line, it
+# will be used as both the source and destination.  If two directories are specified per
+# line, separated by one or more spaces, the first will be used as the source and the
+# second the destination.  Directories that begin with "com/" will be resolved using
+# the compath.py utility.  Rules may be placed below each directory or directory pair
+# and must begin with one of the following characters:
+#  -  exclude, specifies an exclude pattern
+#  +  include, specifies an include pattern
+#  .  merge, specifies a merge-file to read for more rules
+#  :  dir-merge, specifies a per-directory merge-file
+#  H  hide, specifies a pattern for hiding files from the transfer
+#  S  show, files that match the pattern are not hidden
+#  P  protect, specifies a pattern for protecting files from deletion
+#  R  risk, files that match the pattern are not protected
+#  !  clear, clears the current include/exclude list (takes no arg)
+#  B  bytes, relative size of the path in relation to the other paths in the list
+#  D  delete, delete extraneous files from destination directories (takes no arg)
+#  E  encrypt, enables data encryption [two cores should be allocated] (takes no arg)
+#  T  two-way syncronization will update both sides with latest changes (takes no arg)
+#  Z  compress data as it is sent, accepts optional compression level argument (1-9)
+# Rules higher in the list take precedence over lower ones.  By default, all files in a
+# directory are included, so if no exclude patterns match that file, it will be
+# transferred.
+
+_COMROOT_/aigfs/_SHORTVER_/aigfs._PDY_/
++ /??/
++ /??/model/
++ /??/model/atmos/
++ /??/model/atmos/init/
++ /??/model/atmos/init/aigfs.t??z.ic.nc
++ /??/model/atmos/grib2/
++ /??/model/atmos/grib2/aigfs.t??z.pres.f???.grib2
++ /??/model/atmos/grib2/aigfs.t??z.pres.f???.grib2.idx
++ /??/model/atmos/grib2/aigfs.t??z.sfc.f???.grib2
++ /??/model/atmos/grib2/aigfs.t??z.sfc.f???.grib2.idx
+- *
+
+_COMROOT_/aigfs/_SHORTVER_/aigfs._PDYm1_/
++ /??/
++ /??/model/
++ /??/model/atmos/
++ /??/model/atmos/init/
++ /??/model/atmos/init/aigfs.t??z.ic.nc
++ /??/model/atmos/grib2/
++ /??/model/atmos/grib2/aigfs.t??z.pres.f???.grib2
++ /??/model/atmos/grib2/aigfs.t??z.pres.f???.grib2.idx
++ /??/model/atmos/grib2/aigfs.t??z.sfc.f???.grib2
++ /??/model/atmos/grib2/aigfs.t??z.sfc.f???.grib2.idx
+- *

scripts/exaigfs_forecast.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -x
+
+echo "Starting $0"
+
+mkdir -p ${DATA}/data
+mkdir -p ${DATAshared}/output
+
+# copy data in
+cpreq $COMIN/aigfs.${cycle}.ic.nc $DATA/data
+ls -l $DATA/data
+
+# run graphcast forecast
+$USHaigfs/run_graphcast.py \
+	-n aigfs \
+	-i ${DATA}/data/aigfs.${cycle}.ic.nc \
+	-o ${DATAshared}/output \
+	-w $FIXaigfs \
+	-l 64
+export err=$?; err_chk
+
+echo "$0 completed normally"

scripts/exaigfs_post.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -x
+
+echo "Starting $0"
+
+for prod in pres sfc; do
+  file=aigfs.${cycle}.${prod}.f${fh}.grib2
+
+  echo "[$(date)]: Opening GRIB2 file"
+  wgrib2 -s ${DATAshared}/output/${file} > ${DATAshared}/output/${file}.idx
+  export err=$?; err_chk
+  echo "[$(date)]: Index file created successfully"
+
+  cpfs ${DATAshared}/output/${file} ${COMOUT}
+  cpfs ${DATAshared}/output/${file}.idx ${COMOUT}
+
+  if [ "$SENDDBN" = "YES" ]; then 
+    $DBNROOT/bin/dbn_alert MODEL AIGFS_GB2 $job $COMOUT/${file}
+    $DBNROOT/bin/dbn_alert MODEL AIGFS_GB2_IDX $job $COMOUT/${file}.idx
+  fi
+done
+
+echo "$0 completed normally"

ush/set_event.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -x
+
+# =========================================================
+# set_event.sh
+# ------------
+# Set an event to release an ecFlow task corresponding to a 
+# forecast lead time.
+#
+# Parameters
+# ----------
+# fhour : str
+#   Zero-padded forecast hour lead time as HHH 
+# =========================================================
+
+fhour=$1
+
+ecflow_client --force=set ${ECF_NAME}:release_f${fhour}