diff --git a/.github/workflows/check-cr-approved.yaml b/.github/workflows/check-cr-approved.yaml new file mode 100644 index 00000000..ee897d05 --- /dev/null +++ b/.github/workflows/check-cr-approved.yaml @@ -0,0 +1,11 @@ +name: Check CR approved + +on: + pull_request_review: + types: [submitted, edited, dismissed] + workflow_dispatch: + +jobs: + check_cr_approved: + if: ${{ github.event.pull_request.number }} + uses: MetOffice/growss/.github/workflows/check-cr-approved.yaml@main diff --git a/.github/workflows/track-review-project.yaml b/.github/workflows/track-review-project.yaml new file mode 100644 index 00000000..639477cd --- /dev/null +++ b/.github/workflows/track-review-project.yaml @@ -0,0 +1,17 @@ +name: Track Review Project + +on: + workflow_run: + workflows: [Trigger Review Project] + types: + - completed + +permissions: + actions: read + contents: read + pull-requests: write + +jobs: + track_review_project: + uses: MetOffice/growss/.github/workflows/track-review-project.yaml@main + secrets: inherit diff --git a/.github/workflows/trigger-project-workflow.yaml b/.github/workflows/trigger-project-workflow.yaml new file mode 100644 index 00000000..ccb7a55b --- /dev/null +++ b/.github/workflows/trigger-project-workflow.yaml @@ -0,0 +1,17 @@ +name: Trigger Review Project + +on: + pull_request_target: + types: ["opened", "synchronize", "reopened", "edited", "review_requested", "review_request_removed", "closed"] + pull_request_review: + pull_request_review_comment: + +permissions: + actions: read + contents: read + pull-requests: write + +jobs: + trigger_project_workflow: + uses: MetOffice/growss/.github/workflows/trigger-project-workflow.yaml@main + secrets: inherit diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index c34eb45d..8f347ab8 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1,5 +1,8 @@ # Contributors -| GitHub user | Real Name | Affiliation | Date | -| ----------- | --------- | ----------- | ---- | -| james-bruten-mo | James Bruten | Met Office | 2025-12-09 | +| GitHub user | Real Name | Affiliation | Date | +|-----------------|-----------------|-------------|------------| +| james-bruten-mo | James Bruten | Met Office | 2025-12-09 | +| Pierre-siddall | Pierre Siddall | Met Office | 2025-01-19 | +| ericaneininger | Erica Neininger | Met Office | 2026-02-04 | +| dcalve | Daley Calvert | Met Office | 2026-02-04 | \ No newline at end of file diff --git a/Coupled_Drivers/run_model b/Coupled_Drivers/run_model index 23504bf6..4e4b690c 100755 --- a/Coupled_Drivers/run_model +++ b/Coupled_Drivers/run_model @@ -39,7 +39,13 @@ L_MCT_VALIDATE=${L_MCT_VALIDATE:=false} # Copy scripts to work directory so they can be found. We use the dependency # checker script so we only copy the files we need. -FILES_TO_INSTALL=$($DRIVER_EXTRACT_DIR/extract/drivers/Coupled_Drivers/driver_dependencies.py --extract-directory $DRIVER_EXTRACT_DIR/extract/drivers/Coupled_Drivers) +if [[ $DRIVER_EXTRACT_DIR == *"fcm_make"* ]] ; then + # fcm_make infrastructure + extract_path=extract/drivers +else + extract_path= +fi +FILES_TO_INSTALL=$($DRIVER_EXTRACT_DIR/$extract_path/Coupled_Drivers/driver_dependencies.py --extract-directory $DRIVER_EXTRACT_DIR/$extract_path/Coupled_Drivers) for file in $FILES_TO_INSTALL; do # handle any packages package_name=$(dirname "${file}") @@ -47,7 +53,7 @@ for file in $FILES_TO_INSTALL; do mkdir $package_name fi # as package_name will default to . then this will handle all cases - cp $DRIVER_EXTRACT_DIR/extract/drivers/Coupled_Drivers/$file $package_name + cp $DRIVER_EXTRACT_DIR/$extract_path/Coupled_Drivers/$file $package_name done # If the run uses LFRic, and environment variable NAMCOUPLE_STATIC is .false. @@ -56,8 +62,8 @@ done # compatibility NAMCOUPLE_STATIC=${NAMCOUPLE_STATIC:=True} if [[ "$models" == *lfric* && "$NAMCOUPLE_STATIC" == .false. ]]; then - cp $DRIVER_EXTRACT_DIR/extract/drivers/Utilities/NGMS_utils/ngms_namcouple_gen/*py ./ - cp $DRIVER_EXTRACT_DIR/extract/drivers/Utilities/NGMS_utils/ngms_suite_lib/*py ./ + cp $DRIVER_EXTRACT_DIR/$extract_path/Utilities/NGMS_utils/ngms_namcouple_gen/*py ./ + cp $DRIVER_EXTRACT_DIR/$extract_path/Utilities/NGMS_utils/ngms_suite_lib/*py ./ fi EXTRA_LINK_DRIVERS_INFO=$( echo ${EXTRA_LINK_DRIVERS_INFO:-} | tr '[:upper:]' '[:lower:]') @@ -80,7 +86,7 @@ echo '[DRIVER_TEST_SCRIPT] Drivers successfully linked' # had been completed if [ "$L_MCT_VALIDATE" = True ]; then echo "Running MCT validate" - cp $DRIVER_EXTRACT_DIR/extract/drivers/Coupled_Drivers/driver_utilities/mct_validate/mct_validate.py ./ + cp $DRIVER_EXTRACT_DIR/$extract_path/Coupled_Drivers/driver_utilities/mct_validate/mct_validate.py ./ ./mct_validate.py fi diff --git a/Postprocessing/build/build_pp.sh b/Postprocessing/build/build_pp.sh new file mode 100755 index 00000000..051ec854 --- /dev/null +++ b/Postprocessing/build/build_pp.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# (C) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +# ----------------------------------------------------------------------------- +# NAME +# build_pp +# +# DESCRIPTION +# Move PostProc application code to executable location within a rose suite +# +# ----------------------------------------------------------------------------- + +# Fail if environment variables are unset +set -eu + +# Required environment +PP_SOURCE_DIR=${PP_SOURCE_DIR:=$CYLC_WORKFLOW_RUN_DIR/share/source/moci_postproc} +PP_TARGET_DIR=${PP_TARGET_DIR:-$CYLC_WORKFLOW_SHARE_DIR/bin} + +MOCILIB=${MOCILIB:=true} +MOCILIB_PATH=${MOCILIB_PATH:=$CYLC_WORKFLOW_RUN_DIR/share/moci/mocilib} + +PP_COMPONENTS="atmos nemocice unicicles archive_verify" +PP_TESTS=${PP_TESTS:=false} + +echo [INFO] Building PostProc application from $PP_SOURCE_DIR +echo [INFO] -- Build location: $PP_TARGET_DIR + +if [[ ! -d $PP_SOURCE_DIR ]] ; then + echo [ERROR] $PP_SOURCE_DIR does not exist >&2 + exit 1 +fi +if [[ ! -d $PP_TARGET_DIR ]] ; then + echo [INFO] Creating build directory... + mkdir -p $PP_TARGET_DIR +fi + +# Copy mocilib +if [[ "$MOCILIB" != true ]] ; then + echo [INFO] MOCIlib library not installed +elif [[ -d "$MOCILIB_PATH" ]] ; then + cp -r $MOCILIB $PP_TARGET_DIR +else + echo [ERROR] Failed to find the required \"mocilib\" library >&2 + exit 1 +fi + +# Copy NEMO tools +if [[ -d "$PP_SOURCE_DIR/../nemotools" ]] ; then + cp $PP_SOURCE_DIR/../nemotools/* $PP_TARGET_DIR +else + echo [INFO] NEMO iceberg rebuilding tools not available +fi + +# Copy main_pp executable +mainscr=$PP_SOURCE_DIR/Postprocessing/main_pp.py +if [[ -f "$mainscr" ]] ; then + cp $mainscr $PP_TARGET_DIR +else + echo [ERROR] Source for PostProc executable $mainscr does not exist >&2 + exit 1 +fi + +# Copy component directory contents +src_dirs="common platforms $PP_COMPONENTS" +if [[ "$PP_TESTS" = true ]] ; then + src_dirs="$src_dirs unittests" +fi +for directory in $src_dirs; do + if [[ -d $PP_SOURCE_DIR/Postprocessing/$directory ]] ; then + cp $PP_SOURCE_DIR/Postprocessing/$directory/* $PP_TARGET_DIR + else + echo [ERROR] Source for PostProc component $directory does not exist >&2 + exit 1 + fi +done diff --git a/Utilities/mean_nemo/compiler.xc40 b/Utilities/mean_nemo/compiler.xc40 index 6a67dc8c..ed80d48d 100755 --- a/Utilities/mean_nemo/compiler.xc40 +++ b/Utilities/mean_nemo/compiler.xc40 @@ -1,7 +1,41 @@ #!/bin/bash +#PBS -N compile_mean_nemo +#PBS -o compile_mean_nemo.o +#PBS -e compile_mean_nemo.e +#PBS -q shared +#PBS -l ncpus=1 +#PBS -l walltime=00:01:00 +#PBS -l mem=500mb -rm -f mean_nemo.exe +set -eu -module unload cray-netcdf -module load cray-netcdf -ftn -O0 mean_nemo.f90 -o mean_nemo.exe +NAME=mean_nemo + +MODE=prod +#MODE=dev +#MODE=debug + +case ${MODE} in + prod) + opts="-O2 -Ovector1 -hfp0 -hflex_mp=intolerant" + ;; + dev) + opts="-O0" + ;; + debug) + opts="-O0 -Ovector0 -hflex_mp=intolerant -e CID -Ktrap=fp -g" + ;; + *) + echo "Compilation mode \"${MODE}\" not supported" + exit 1 + ;; +esac + +cd ${PBS_O_WORKDIR} + +[ -f ${NAME}.exe ] && rm -f ${NAME}.exe + +module unload cray-hdf5 cray-netcdf 2>/dev/null +module load cray-hdf5 cray-netcdf + +ftn ${opts} mean_nemo.f90 -o ${NAME}.exe diff --git a/Utilities/mean_nemo/compiler.xc40_login b/Utilities/mean_nemo/compiler.xc40_login index 343759fb..2e211f0f 100755 --- a/Utilities/mean_nemo/compiler.xc40_login +++ b/Utilities/mean_nemo/compiler.xc40_login @@ -1,8 +1,32 @@ #!/bin/bash -rm -f mean_nemo.exe +set -eu -module swap craype-haswell craype-ivybridge -module unload cray-netcdf -module load cray-netcdf -ftn -O0 mean_nemo.f90 -o mean_nemo_login.exe +NAME=mean_nemo_login + +MODE=prod +#MODE=dev +#MODE=debug + +case ${MODE} in + prod) + opts="-O2 -Ovector1 -hfp0 -hflex_mp=intolerant" + ;; + dev) + opts="-O0" + ;; + debug) + opts="-O0 -Ovector0 -hflex_mp=intolerant -e CID -Ktrap=fp -g" + ;; + *) + echo "Mode \"${MODE}\" not supported" + exit 1 + ;; +esac + +[ -f ${NAME}.exe ] && rm -f ${NAME}.exe + +module unload cray-hdf5 cray-netcdf 2>/dev/null +module load cray-hdf5 cray-netcdf + +ftn ${opts} mean_nemo.f90 -o ${NAME}.exe diff --git a/Utilities/mean_nemo/mean_nemo.f90 b/Utilities/mean_nemo/mean_nemo.f90 index baaa757f..8fadb201 100644 --- a/Utilities/mean_nemo/mean_nemo.f90 +++ b/Utilities/mean_nemo/mean_nemo.f90 @@ -1,3 +1,9 @@ +! ----------------------------------------------------------------------------- +! (C) Crown copyright Met Office. All rights reserved. +! The file LICENCE, distributed with this code, contains details of the terms +! under which the code may be used. +! ----------------------------------------------------------------------------- + PROGRAM mean_nemo !----------------------------------------------------------------------------- @@ -13,6 +19,8 @@ PROGRAM mean_nemo ! Dave Storkey Feb 2016 - Add support for thickness weighted time-mean variables ! Daley Calvert Oct 2024 - Use a more robust approach to missing data ! and resolve issues with thickness-weighting + ! Daley Calvert Feb 2026 - Add support for time-varying masking + ! - Add support for 5D data !----------------------------------------------------------------------------- USE netcdf @@ -26,59 +34,66 @@ PROGRAM mean_nemo INTEGER,PARAMETER :: sp=SELECTED_REAL_KIND(6,37) INTEGER,PARAMETER :: dp=SELECTED_REAL_KIND(12,307) - LOGICAL, PARAMETER :: l_verbose = .true. - + LOGICAL, PARAMETER :: l_verbose = .true. + LOGICAL, PARAMETER :: l_timing = .false. + CHARACTER(LEN=nf90_max_name) :: outfile, attname, dimname, varname, time, date, zone, timestamp CHARACTER(LEN=nf90_max_name), ALLOCATABLE :: filenames(:), indimnames(:) CHARACTER(LEN=256) :: standard_name,cell_methods - LOGICAL :: l_thckwgt, l_doavg, l_ismasked + LOGICAL :: l_thckwgt, l_doavg, l_ismasked, l_inputdata_ismasked, l_cellthick_ismasked INTEGER :: nargs, ifile , iargc, no_fill INTEGER :: ncid, outid, iostat, idim, istop, itime - INTEGER :: natts, attid, xtype, varid, icellthick_type - ! ntimes is total number of time points to average over + INTEGER :: natts, attid, xtype, varid + ! ntimes is total number of time points to average over (for unmasked data) ! ntimes_local is number of time points in each file - INTEGER :: jv_loop, jv, jv_thickness, ndims, nvars, dimlen, dimids(4), ntimes, ntimes_local + INTEGER :: jv, jv_thickness, ndims, nvars, dimlen, dimids(5), ntimes, ntimes_local INTEGER :: dimid, unlimitedDimId, unlimitedDimId_local, varunlimitedDimId INTEGER :: chunksize = 32000000 INTEGER, ALLOCATABLE :: outdimids(:), outdimlens(:), inncids(:) INTEGER, ALLOCATABLE :: indimlens(:), start(:) ! Scalars - INTEGER(i1) :: ntimes_i1 - INTEGER(i2) :: ntimes_i2 - INTEGER(i4) :: ntimes_i4, inputdata_fill_value_i4 - REAL(sp) :: ntimes_sp, inputdata_fill_value_sp - REAL(dp) :: ntimes_dp, inputdata_fill_value_dp - - ! Logical data masks - LOGICAL, ALLOCATABLE, DIMENSION(:,:,: ) :: l_mask_3d + INTEGER(i4) :: inputdata_fill_value_i4, outputdata_fill_value_i4 + REAL(sp) :: inputdata_fill_value_sp, outputdata_fill_value_sp, cellthick_fill_value_sp + REAL(dp) :: inputdata_fill_value_dp, outputdata_fill_value_dp, cellthick_fill_value_dp + + ! Logical data masks (for 4d data only, which may be thickness-weighted and/or masked) LOGICAL, ALLOCATABLE, DIMENSION(:,:,:,:) :: l_mask_4d + ! Time counters (for masked data) + INTEGER(i2), ALLOCATABLE, DIMENSION(:,:,:) :: ntimes_3d + INTEGER(i2), ALLOCATABLE, DIMENSION(:,:,:,:) :: ntimes_4d + INTEGER(i2), ALLOCATABLE, DIMENSION(:,:,:,:,:) :: ntimes_5d + !Int 1 versions of the local data arrays INTEGER(i1), ALLOCATABLE, SAVE, DIMENSION(:) :: inputdata_1d_i1 INTEGER(i1), ALLOCATABLE, SAVE, DIMENSION(:,:) :: inputdata_2d_i1 INTEGER(i1), ALLOCATABLE, SAVE, DIMENSION(:,:,:) :: inputdata_3d_i1 INTEGER(i1), ALLOCATABLE, SAVE, DIMENSION(:,:,:,:) :: inputdata_4d_i1 + INTEGER(i1), ALLOCATABLE, SAVE, DIMENSION(:,:,:,:,:) :: inputdata_5d_i1 !Int 2 versions of the local data arrays INTEGER(i2), ALLOCATABLE, SAVE, DIMENSION(:) :: inputdata_1d_i2 INTEGER(i2), ALLOCATABLE, SAVE, DIMENSION(:,:) :: inputdata_2d_i2 INTEGER(i2), ALLOCATABLE, SAVE, DIMENSION(:,:,:) :: inputdata_3d_i2 INTEGER(i2), ALLOCATABLE, SAVE, DIMENSION(:,:,:,:) :: inputdata_4d_i2 + INTEGER(i2), ALLOCATABLE, SAVE, DIMENSION(:,:,:,:,:) :: inputdata_5d_i2 !Int 4 versions of the local data arrays INTEGER(i4), ALLOCATABLE, SAVE, DIMENSION(:) :: inputdata_1d_i4 INTEGER(i4), ALLOCATABLE, SAVE, DIMENSION(:,:) :: inputdata_2d_i4 INTEGER(i4), ALLOCATABLE, SAVE, DIMENSION(:,:,:) :: inputdata_3d_i4 INTEGER(i4), ALLOCATABLE, SAVE, DIMENSION(:,:,:,:) :: inputdata_4d_i4 + INTEGER(i4), ALLOCATABLE, SAVE, DIMENSION(:,:,:,:,:) :: inputdata_5d_i4 !Real 4 versions of the local data arrays REAL(sp), ALLOCATABLE, SAVE, DIMENSION(:) :: inputdata_1d_sp REAL(sp), ALLOCATABLE, SAVE, DIMENSION(:,:) :: inputdata_2d_sp REAL(sp), ALLOCATABLE, SAVE, DIMENSION(:,:,:) :: inputdata_3d_sp REAL(sp), ALLOCATABLE, SAVE, DIMENSION(:,:,:,:) :: inputdata_4d_sp + REAL(sp), ALLOCATABLE, SAVE, DIMENSION(:,:,:,:,:) :: inputdata_5d_sp REAL(sp), ALLOCATABLE, SAVE, DIMENSION(:,:,:,:) :: cellthick_4d_sp !Real 8 versions of the local data arrays @@ -86,6 +101,7 @@ PROGRAM mean_nemo REAL(dp), ALLOCATABLE, SAVE, DIMENSION(:,:) :: inputdata_2d_dp REAL(dp), ALLOCATABLE, SAVE, DIMENSION(:,:,:) :: inputdata_3d_dp REAL(dp), ALLOCATABLE, SAVE, DIMENSION(:,:,:,:) :: inputdata_4d_dp + REAL(dp), ALLOCATABLE, SAVE, DIMENSION(:,:,:,:,:) :: inputdata_5d_dp REAL(dp), ALLOCATABLE, SAVE, DIMENSION(:,:,:,:) :: cellthick_4d_dp !Int 1 versions of the global data arrays @@ -94,6 +110,7 @@ PROGRAM mean_nemo INTEGER(i1), ALLOCATABLE, DIMENSION(:,:) :: meandata_2d_i1 INTEGER(i1), ALLOCATABLE, DIMENSION(:,:,:) :: meandata_3d_i1 INTEGER(i1), ALLOCATABLE, DIMENSION(:,:,:,:) :: meandata_4d_i1 + INTEGER(i1), ALLOCATABLE, DIMENSION(:,:,:,:,:) :: meandata_5d_i1 !Int 2 versions of the global data arrays INTEGER(i2) :: meandata_0d_i2 @@ -101,6 +118,7 @@ PROGRAM mean_nemo INTEGER(i2), ALLOCATABLE, DIMENSION(:,:) :: meandata_2d_i2 INTEGER(i2), ALLOCATABLE, DIMENSION(:,:,:) :: meandata_3d_i2 INTEGER(i2), ALLOCATABLE, DIMENSION(:,:,:,:) :: meandata_4d_i2 + INTEGER(i2), ALLOCATABLE, DIMENSION(:,:,:,:,:) :: meandata_5d_i2 !Int 4 versions of the global data arrays INTEGER(i4) :: meandata_0d_i4 @@ -108,13 +126,15 @@ PROGRAM mean_nemo INTEGER(i4), ALLOCATABLE, DIMENSION(:,:) :: meandata_2d_i4 INTEGER(i4), ALLOCATABLE, DIMENSION(:,:,:) :: meandata_3d_i4 INTEGER(i4), ALLOCATABLE, DIMENSION(:,:,:,:) :: meandata_4d_i4 - + INTEGER(i4), ALLOCATABLE, DIMENSION(:,:,:,:,:) :: meandata_5d_i4 + !Real 4 versions of the global data arrays REAL(sp) :: meandata_0d_sp REAL(sp), ALLOCATABLE, DIMENSION(:) :: meandata_1d_sp REAL(sp), ALLOCATABLE, DIMENSION(:,:) :: meandata_2d_sp REAL(sp), ALLOCATABLE, DIMENSION(:,:,:) :: meandata_3d_sp REAL(sp), ALLOCATABLE, DIMENSION(:,:,:,:) :: meandata_4d_sp + REAL(sp), ALLOCATABLE, DIMENSION(:,:,:,:,:) :: meandata_5d_sp REAL(sp), ALLOCATABLE, DIMENSION(:,:,:,:) :: meancellthick_4d_sp !Real 8 versions of the global data arrays @@ -123,11 +143,17 @@ PROGRAM mean_nemo REAL(dp), ALLOCATABLE, DIMENSION(:,:) :: meandata_2d_dp REAL(dp), ALLOCATABLE, DIMENSION(:,:,:) :: meandata_3d_dp REAL(dp), ALLOCATABLE, DIMENSION(:,:,:,:) :: meandata_4d_dp + REAL(dp), ALLOCATABLE, DIMENSION(:,:,:,:,:) :: meandata_5d_dp REAL(dp), ALLOCATABLE, DIMENSION(:,:,:,:) :: meancellthick_4d_dp + ! Timing-related scalars + INTEGER(i8) :: t_section, t_total, t_variable + REAL(dp) :: secondclock + ! Initialise timing + CALL timing_init(secondclock) - !End of definitions + !End of definitions !-------------------------------------------------------------------------------- !1. Read in the arguments (input and output filenames) @@ -152,10 +178,14 @@ PROGRAM mean_nemo !--------------------------------------------------------------------------- !2. Read in the global dimensions from the first input file and set up the output file - + + CALL timing_start(t_total) ! Total time taken + CALL timing_start(t_section) + iostat = nf90_open( TRIM(filenames(1)), nf90_share, ncid ) IF( iostat /= nf90_noerr ) THEN - WRITE(6,*) TRIM(nf90_strerror(iostat)) + WRITE(6,*)'E R R O R opening input file '//TRIM(filenames(1))//':' + WRITE(6,*) ' '//TRIM(nf90_strerror(iostat)) STOP 11 ENDIF iostat = nf90_inquire( ncid, ndims, nvars, natts ) @@ -198,7 +228,39 @@ PROGRAM mean_nemo zone iostat = nf90_put_att( outid, nf90_global, "TimeStamp", timestamp) IF (l_verbose) WRITE(6,*)'Writing new TimeStamp attribute' - + + jv_thickness = -1 + l_cellthick_ismasked = .FALSE. + + ! Find out if there is a cell thickness variable in this set of files in case we need to do thickness weighting + DO jv = 1, nvars + iostat = nf90_get_att(ncid, jv, "standard_name", standard_name) + + IF( (iostat == nf90_noerr) .AND. (TRIM(standard_name) == "cell_thickness") ) THEN + jv_thickness = jv + iostat = nf90_inquire_variable( ncid, jv, xtype=xtype ) + + ! Save cell thickness fill value if masked + SELECT CASE( xtype ) + CASE( NF90_FLOAT ) + iostat = nf90_get_att(ncid, jv, "_FillValue", cellthick_fill_value_sp ) + IF( iostat == nf90_noerr ) cellthick_fill_value_dp = REAL(cellthick_fill_value_sp, dp) + CASE( NF90_DOUBLE ) + iostat = nf90_get_att(ncid, jv, "_FillValue", cellthick_fill_value_dp ) + IF( iostat == nf90_noerr ) cellthick_fill_value_sp = REAL(cellthick_fill_value_dp, sp) + CASE DEFAULT + WRITE(6,*) "ERROR : Cell thickness variable must be single or double precision float" + STOP 13 + END SELECT + + ! Is the cell thickness data masked? + l_cellthick_ismasked = (iostat == nf90_noerr) + + ! Exit loop if the cell thickness has been found + EXIT + ENDIF + END DO + !2.2.3 Copy the variable definitions and attributes into the output file. DO jv = 1, nvars iostat = nf90_inquire_variable( ncid, jv, varname, xtype, ndims, dimids, natts) @@ -206,6 +268,7 @@ PROGRAM mean_nemo DO idim = 1, ndims outdimids(idim) = dimids(idim) END DO + iostat = nf90_def_var( outid, varname, xtype, outdimids, varid ) DEALLOCATE(outdimids) IF (l_verbose) WRITE(6,*)'Defining variable '//TRIM(varname)//'...' @@ -215,6 +278,23 @@ PROGRAM mean_nemo iostat = nf90_copy_att( ncid, varid, attname, outid, varid ) END DO ENDIF + + ! For thickness weighting, output data will use the fill value of the cell thickness if the input data is not masked + IF( jv /= jv_thickness ) THEN + iostat = nf90_get_att(ncid, jv, "cell_methods", cell_methods) + l_thckwgt = ( (iostat == nf90_noerr) .AND. (TRIM(cell_methods) == "time: mean (thickness weighted)") ) + iostat = nf90_inquire_attribute(ncid, jv, "_FillValue") + l_inputdata_ismasked = (iostat == nf90_noerr .AND. ndims >= 3) + + IF( l_thckwgt .AND. l_cellthick_ismasked .AND. (.NOT. l_inputdata_ismasked) ) THEN + SELECT CASE( xtype ) + CASE( NF90_FLOAT ) + iostat = nf90_put_att( outid, varid, '_FillValue', cellthick_fill_value_sp ) + CASE( NF90_DOUBLE ) + iostat = nf90_put_att( outid, varid, '_FillValue', cellthick_fill_value_dp ) + END SELECT + ENDIF + ENDIF END DO !2.3 End definitions in output file and copy 1st file ncid to the inncids array @@ -222,7 +302,8 @@ PROGRAM mean_nemo iostat = nf90_enddef( outid ) inncids(1) = ncid IF (l_verbose) WRITE(6,*)'Finished defining output file.' - + CALL timing_stop(t_section, 'define output file') ; CALL timing_start(t_section) + !--------------------------------------------------------------------------- !3. Read in data from each file for each variable @@ -232,48 +313,19 @@ PROGRAM mean_nemo DO ifile = 2, nargs-1 iostat = nf90_open( TRIM(filenames(ifile)), nf90_share, ncid, chunksize=chunksize) IF( iostat /= nf90_noerr ) THEN - WRITE(6,*) TRIM(nf90_strerror(iostat)) - WRITE(6,*)'E R R O R opening input file '//TRIM(filenames(ifile)) + WRITE(6,*)'E R R O R opening input file '//TRIM(filenames(ifile))//':' + WRITE(6,*) ' '//TRIM(nf90_strerror(iostat)) STOP 12 ELSE inncids(ifile) = ncid ENDIF END DO IF (l_verbose) WRITE(6,*)'All input files open.' + CALL timing_stop(t_section, 'open input files') ; CALL timing_start(t_section) - !Find out if there is a cell thickness variable in this set of files - !in case we need to do thickness weighting. - jv_thickness = -1 - DO jv = 1, nvars - iostat = nf90_inquire_variable( ncid, jv, varname, xtype, ndims, dimids, natts) - iostat = nf90_get_att(ncid, jv, "standard_name", standard_name) - IF( iostat == nf90_noerr .AND. TRIM(standard_name) == "cell_thickness" ) THEN - jv_thickness = jv - EXIT - ENDIF - ENDDO - !Loop over all variables in first input file - DO jv_loop = 0, nvars - - !Need to make sure that we mean up any cell thickness variable first so we can subsequently - !use the time-mean cell thickness in the meaning of thickness-weighted variables. - IF( jv_loop == 0 ) THEN - IF( jv_thickness /= -1 ) THEN - jv = jv_thickness - ELSE - CYCLE - ENDIF - ELSE - IF( jv_loop == jv_thickness ) THEN - CYCLE - ELSE - jv = jv_loop - ENDIF - ENDIF - - !Initialise ntimes (number of records to be averaged) - ntimes = 0 + DO jv = 1, nvars + CALL timing_start(t_variable) ! Total time taken for variable !3.2 Inquire variable to find out name and how many dimensions it has @@ -290,20 +342,42 @@ PROGRAM mean_nemo ! Does averaging need to account for thickness-weighting and masked data? l_thckwgt = .FALSE. + l_inputdata_ismasked = .FALSE. l_ismasked = .FALSE. IF( l_doavg ) THEN + ! Thickness-weighting- if an unsupported data type/shape has this attribute, raise an error below l_thckwgt = ( iostat == nf90_noerr .AND. TRIM(cell_methods) == "time: mean (thickness weighted)" ) - SELECT CASE( xtype ) - CASE( NF90_INT ) - iostat = nf90_get_att(ncid, jv, "_FillValue", inputdata_fill_value_i4 ) - CASE( NF90_FLOAT ) - iostat = nf90_get_att(ncid, jv, "_FillValue", inputdata_fill_value_sp ) - CASE( NF90_DOUBLE ) - iostat = nf90_get_att(ncid, jv, "_FillValue", inputdata_fill_value_dp ) - CASE DEFAULT - iostat = nf90_noerr + 1 - END SELECT - l_ismasked = (iostat == nf90_noerr .AND. ndims >= 3) + + ! Masked data- get the fill value of the input data and/or cell thickness, and set the fill value of the + ! output file (that of the input data is prioritised). Unsupported data types/shapes proceed silently using + ! the unmasked averaging algorithm. + IF( ndims >= 3 ) THEN + SELECT CASE( xtype ) + CASE( NF90_INT ) + iostat = nf90_get_att(ncid, jv, "_FillValue", inputdata_fill_value_i4 ) + IF( iostat == nf90_noerr ) outputdata_fill_value_i4 = inputdata_fill_value_i4 + CASE( NF90_FLOAT ) + iostat = nf90_get_att(ncid, jv, "_FillValue", inputdata_fill_value_sp ) + IF( iostat == nf90_noerr ) THEN + outputdata_fill_value_sp = inputdata_fill_value_sp + ELSE IF( l_thckwgt .AND. l_cellthick_ismasked ) THEN + outputdata_fill_value_sp = cellthick_fill_value_sp + ENDIF + CASE( NF90_DOUBLE ) + iostat = nf90_get_att(ncid, jv, "_FillValue", inputdata_fill_value_dp ) + IF( iostat == nf90_noerr ) THEN + outputdata_fill_value_dp = inputdata_fill_value_dp + ELSE IF( l_thckwgt .AND. l_cellthick_ismasked ) THEN + outputdata_fill_value_dp = cellthick_fill_value_dp + ENDIF + CASE DEFAULT + iostat = nf90_noerr + 1 + END SELECT + ! Is the input data masked? + l_inputdata_ismasked = iostat == nf90_noerr + ! We need to perform masked averaging if the input data and/or cell thickness data is masked + l_ismasked = l_inputdata_ismasked .OR. (l_thckwgt .AND. l_cellthick_ismasked) + ENDIF ENDIF IF( l_thckwgt .AND. jv_thickness == -1 ) THEN @@ -312,11 +386,10 @@ PROGRAM mean_nemo WRITE(6,*) " If you want to go ahead anyway (not recommended) remove cell_methods attribute from variable." STOP 13 ENDIF - IF( l_thckwgt .AND. ( ( xtype /= NF90_FLOAT .AND. xtype /= NF90_DOUBLE ) .OR. ndims /= 4 ) ) THEN + IF( l_thckwgt .AND. .NOT. ( ( xtype == NF90_FLOAT .OR. xtype == NF90_DOUBLE ) .AND. ndims == 4 ) ) THEN WRITE(6,*) "ERROR : Thickness-weighted time-mean variable "//TRIM(varname)//" found in file "//TRIM(filenames(ifile)) - WRITE(6,*) " This utility currently only takes account of thickness weighting for 4D FLOATS or 4D DOUBLES." - WRITE(6,*) " "//TRIM(varname)//" either has a different number of dimensions or is not a FLOAT or a DOUBLE." - WRITE(6,*) " If you want to mean the variable without thickness weighting remove cell_methods attribute from variable." + WRITE(6,*) " Thickness-weighted averaging is only supported for 4D single- or double-precision "//& + &"floating point variables. Remove the cell_methods attribute to perform averaging without thickness-weighting." STOP 13 ENDIF @@ -329,10 +402,13 @@ PROGRAM mean_nemo IF( l_thckwgt ) WRITE(6,*)'Applying thickness-weighting.' ENDIF - ! Allocate global variables ahead of looping over input files + ! Allocate and initialise arrays used in the calculation of the average IF( ndims == 1 ) THEN + ! Denominator- # of records, always a scalar (no support for masked averages) + ntimes = 0 + ! Numerator- sum over time of the data SELECT CASE( xtype ) CASE( NF90_BYTE ) ALLOCATE(meandata_1d_i1(outdimlens(dimids(1)))) @@ -355,7 +431,10 @@ PROGRAM mean_nemo END SELECT ELSEIF( ndims == 2 ) THEN + ! Denominator- # of records, always a scalar (no support for masked averages) + ntimes = 0 + ! Numerator- sum over time of the data SELECT CASE( xtype ) CASE( NF90_BYTE ) ALLOCATE(meandata_2d_i1(outdimlens(dimids(1)),outdimlens(dimids(2)))) @@ -378,7 +457,16 @@ PROGRAM mean_nemo END SELECT ELSEIF( ndims == 3 ) THEN + ! Denominator- # of records, either a scalar (normal average) or array (masked average) + IF( l_ismasked ) THEN + ALLOCATE( ntimes_3d( outdimlens(dimids(1)), outdimlens(dimids(2)), & + & outdimlens(dimids(3)) ) ) + ntimes_3d(:,:,:) = 0 + ELSE + ntimes = 0 + ENDIF + ! Numerator- sum over time of the data SELECT CASE( xtype ) CASE( NF90_BYTE ) ALLOCATE(meandata_3d_i1(outdimlens(dimids(1)),outdimlens(dimids(2)), & @@ -404,11 +492,31 @@ PROGRAM mean_nemo WRITE(6,*)'Unknown nf90 type: ', xtype STOP 14 END SELECT - IF( l_ismasked ) ALLOCATE(l_mask_3d(outdimlens(dimids(1)), outdimlens(dimids(2)), & - & outdimlens(dimids(3)))) ELSEIF( ndims == 4 ) THEN + ! Denominator- either the sum over time of the cell thickness (thickness-weighted average), + ! or # of records which is either a scalar (normal average) or array (masked average) + IF( l_thckwgt ) THEN + SELECT CASE( xtype ) + CASE( NF90_FLOAT ) + ALLOCATE(meancellthick_4d_sp(outdimlens(dimids(1)),outdimlens(dimids(2)), & + & outdimlens(dimids(3)),outdimlens(dimids(4)))) + meancellthick_4d_sp(:,:,:,:)=0.0 + CASE( NF90_DOUBLE ) + ALLOCATE(meancellthick_4d_dp(outdimlens(dimids(1)),outdimlens(dimids(2)), & + & outdimlens(dimids(3)),outdimlens(dimids(4)))) + meancellthick_4d_dp(:,:,:,:)=0.0 + END SELECT + ELSE IF( l_ismasked ) THEN + ALLOCATE( ntimes_4d( outdimlens(dimids(1)), outdimlens(dimids(2)), & + & outdimlens(dimids(3)), outdimlens(dimids(4)) ) ) + ntimes_4d(:,:,:,:) = 0 + ELSE + ntimes = 0 + ENDIF + + ! Numerator- sum over time of the data (potentially thickness-weighted) SELECT CASE( xtype ) CASE( NF90_BYTE ) ALLOCATE(meandata_4d_i1(outdimlens(dimids(1)),outdimlens(dimids(2)), & @@ -426,39 +534,73 @@ PROGRAM mean_nemo ALLOCATE(meandata_4d_sp(outdimlens(dimids(1)),outdimlens(dimids(2)), & & outdimlens(dimids(3)),outdimlens(dimids(4)))) meandata_4d_sp(:,:,:,:)=0.0 - IF( jv == jv_thickness ) THEN - ALLOCATE(meancellthick_4d_sp(outdimlens(dimids(1)),outdimlens(dimids(2)), & - & outdimlens(dimids(3)),outdimlens(dimids(4)))) - meancellthick_4d_sp(:,:,:,:)=0.0 - icellthick_type = NF90_FLOAT - ENDIF CASE( NF90_DOUBLE ) ALLOCATE(meandata_4d_dp(outdimlens(dimids(1)),outdimlens(dimids(2)), & & outdimlens(dimids(3)),outdimlens(dimids(4)))) meandata_4d_dp(:,:,:,:)=0.0 - IF( jv == jv_thickness ) THEN - ALLOCATE(meancellthick_4d_dp(outdimlens(dimids(1)),outdimlens(dimids(2)), & - & outdimlens(dimids(3)),outdimlens(dimids(4)))) - meancellthick_4d_dp(:,:,:,:)=0.0 - icellthick_type = NF90_DOUBLE - ENDIF CASE DEFAULT WRITE(6,*)'Unknown nf90 type: ', xtype STOP 14 END SELECT - IF( l_ismasked ) ALLOCATE(l_mask_4d(outdimlens(dimids(1)), outdimlens(dimids(2)), & - & outdimlens(dimids(3)), outdimlens(dimids(4)))) + + ELSEIF( ndims == 5 ) THEN + + ! Denominator- # of records, either a scalar (normal average) or array (masked average) + IF( l_ismasked ) THEN + ALLOCATE( ntimes_5d( outdimlens(dimids(1)),outdimlens(dimids(2)), & + & outdimlens(dimids(3)),outdimlens(dimids(4)), & + & outdimlens(dimids(5)) )) + ntimes_5d(:,:,:,:,:) = 0 + ELSE + ntimes = 0 + ENDIF + + ! Numerator- sum over time of the data + SELECT CASE( xtype ) + CASE( NF90_BYTE ) + ALLOCATE(meandata_5d_i1(outdimlens(dimids(1)),outdimlens(dimids(2)), & + & outdimlens(dimids(3)),outdimlens(dimids(4)), & + & outdimlens(dimids(5)))) + meandata_5d_i1(:,:,:,:,:)=0 + CASE( NF90_SHORT ) + ALLOCATE(meandata_5d_i2(outdimlens(dimids(1)),outdimlens(dimids(2)), & + & outdimlens(dimids(3)),outdimlens(dimids(4)), & + & outdimlens(dimids(5)))) + meandata_5d_i2(:,:,:,:,:)=0 + CASE( NF90_INT ) + ALLOCATE(meandata_5d_i4(outdimlens(dimids(1)),outdimlens(dimids(2)), & + & outdimlens(dimids(3)),outdimlens(dimids(4)), & + & outdimlens(dimids(5)))) + meandata_5d_i4(:,:,:,:,:)=0 + CASE( NF90_FLOAT ) + ALLOCATE(meandata_5d_sp(outdimlens(dimids(1)),outdimlens(dimids(2)), & + & outdimlens(dimids(3)),outdimlens(dimids(4)), & + & outdimlens(dimids(5)))) + meandata_5d_sp(:,:,:,:,:)=0.0 + CASE( NF90_DOUBLE ) + ALLOCATE(meandata_5d_dp(outdimlens(dimids(1)),outdimlens(dimids(2)), & + & outdimlens(dimids(3)),outdimlens(dimids(4)), & + & outdimlens(dimids(5)))) + meandata_5d_dp(:,:,:,:,:)=0.0 + CASE DEFAULT + WRITE(6,*)'Unknown nf90 type: ', xtype + STOP 14 + END SELECT + ELSE WRITE(6,*)'E R R O R: ' - WRITE(6,*)'The netcdf variable has more than 4 dimensions which is not taken into account' + WRITE(6,*)'The netcdf variable has more than 5 dimensions which is not taken into account' STOP 15 ENDIF - + + CALL timing_stop(t_section, 'query input files and allocate arrays') + istop = 0 ! If this variable is a function of the unlimited dimension then ! Average over unlimited dimension IF( l_doavg ) THEN + CALL timing_start(t_section) DO ifile = 1, nargs-1 !Loop through input files @@ -478,43 +620,39 @@ PROGRAM mean_nemo indimlens(idim)=dimlen ENDIF END DO - ntimes = ntimes + ntimes_local - - DO itime = 1, ntimes_local !Loop through records in file + + ! Loop through records in file, accumulate the numerator & denominator of the average + DO itime = 1, ntimes_local !start is the offset variable used in call to nf90_get_var start(varunlimitedDimId)=itime IF( ndims == 1 ) THEN - + ntimes = ntimes + 1 + SELECT CASE( xtype ) CASE( NF90_BYTE ) ALLOCATE(inputdata_1d_i1(outdimlens(dimids(1)))) - inputdata_1d_i1(:)=0 iostat = nf90_get_var( ncid, jv, inputdata_1d_i1, start, indimlens) meandata_1d_i1(:)=meandata_1d_i1(:)+inputdata_1d_i1(:) DEALLOCATE(inputdata_1d_i1) CASE( NF90_SHORT ) ALLOCATE(inputdata_1d_i2(outdimlens(dimids(1)))) - inputdata_1d_i2(:)=0 iostat = nf90_get_var( ncid, jv, inputdata_1d_i2, start, indimlens) meandata_1d_i2(:)=meandata_1d_i2(:)+inputdata_1d_i2(:) DEALLOCATE(inputdata_1d_i2) CASE( NF90_INT ) ALLOCATE(inputdata_1d_i4(outdimlens(dimids(1)))) - inputdata_1d_i4(:)=0 iostat = nf90_get_var( ncid, jv, inputdata_1d_i4, start, indimlens) meandata_1d_i4(:)=meandata_1d_i4(:)+inputdata_1d_i4(:) DEALLOCATE(inputdata_1d_i4) CASE( NF90_FLOAT ) ALLOCATE(inputdata_1d_sp(outdimlens(dimids(1)))) - inputdata_1d_sp(:)=0.0 iostat = nf90_get_var( ncid, jv, inputdata_1d_sp, start, indimlens) meandata_1d_sp(:)=meandata_1d_sp(:)+inputdata_1d_sp(:) DEALLOCATE(inputdata_1d_sp) CASE( NF90_DOUBLE ) ALLOCATE(inputdata_1d_dp(outdimlens(dimids(1)))) - inputdata_1d_dp(:)=0.0 iostat = nf90_get_var( ncid, jv, inputdata_1d_dp, start, indimlens) meandata_1d_dp(:)=meandata_1d_dp(:)+inputdata_1d_dp(:) DEALLOCATE(inputdata_1d_dp) @@ -524,35 +662,31 @@ PROGRAM mean_nemo END SELECT ELSEIF( ndims == 2 ) THEN + ntimes = ntimes + 1 SELECT CASE( xtype ) CASE( NF90_BYTE ) ALLOCATE(inputdata_2d_i1(outdimlens(dimids(1)),outdimlens(dimids(2)))) - inputdata_2d_i1(:,:)=0 iostat = nf90_get_var( ncid, jv, inputdata_2d_i1, start, indimlens) meandata_2d_i1(:,:)=meandata_2d_i1(:,:)+inputdata_2d_i1(:,:) DEALLOCATE(inputdata_2d_i1) CASE( NF90_SHORT ) ALLOCATE(inputdata_2d_i2(outdimlens(dimids(1)),outdimlens(dimids(2)))) - inputdata_2d_i2(:,:)=0 iostat = nf90_get_var( ncid, jv, inputdata_2d_i2, start, indimlens ) meandata_2d_i2(:,:)=meandata_2d_i2(:,:)+inputdata_2d_i2(:,:) DEALLOCATE(inputdata_2d_i2) CASE( NF90_INT ) ALLOCATE(inputdata_2d_i4(outdimlens(dimids(1)),outdimlens(dimids(2)))) - inputdata_2d_i4(:,:)=0 iostat = nf90_get_var( ncid, jv, inputdata_2d_i4, start, indimlens ) meandata_2d_i4(:,:)=meandata_2d_i4(:,:)+inputdata_2d_i4(:,:) DEALLOCATE(inputdata_2d_i4) CASE( NF90_FLOAT ) ALLOCATE(inputdata_2d_sp(outdimlens(dimids(1)),outdimlens(dimids(2)))) - inputdata_2d_sp(:,:)=0.0 iostat = nf90_get_var( ncid, jv, inputdata_2d_sp, start, indimlens ) meandata_2d_sp(:,:)=meandata_2d_sp(:,:)+inputdata_2d_sp(:,:) DEALLOCATE(inputdata_2d_sp) CASE( NF90_DOUBLE ) ALLOCATE(inputdata_2d_dp(outdimlens(dimids(1)),outdimlens(dimids(2)))) - inputdata_2d_dp(:,:)=0.0 iostat = nf90_get_var( ncid, jv, inputdata_2d_dp, start, indimlens ) meandata_2d_dp(:,:)=meandata_2d_dp(:,:)+inputdata_2d_dp(:,:) DEALLOCATE(inputdata_2d_dp) @@ -567,49 +701,69 @@ PROGRAM mean_nemo CASE( NF90_BYTE ) ALLOCATE(inputdata_3d_i1(outdimlens(dimids(1)),outdimlens(dimids(2)), & & outdimlens(dimids(3)))) - inputdata_3d_i1(:,:,:)=0 iostat = nf90_get_var( ncid, jv, inputdata_3d_i1, start, indimlens ) meandata_3d_i1(:,:,:)=meandata_3d_i1(:,:,:)+inputdata_3d_i1(:,:,:) DEALLOCATE(inputdata_3d_i1) + + ntimes = ntimes + 1 CASE( NF90_SHORT ) ALLOCATE(inputdata_3d_i2(outdimlens(dimids(1)),outdimlens(dimids(2)), & & outdimlens(dimids(3)))) - inputdata_3d_i2(:,:,:)=0 iostat = nf90_get_var( ncid, jv, inputdata_3d_i2, start, indimlens ) meandata_3d_i2(:,:,:)=meandata_3d_i2(:,:,:)+inputdata_3d_i2(:,:,:) DEALLOCATE(inputdata_3d_i2) + + ntimes = ntimes + 1 CASE( NF90_INT ) ALLOCATE(inputdata_3d_i4(outdimlens(dimids(1)),outdimlens(dimids(2)), & & outdimlens(dimids(3)))) - inputdata_3d_i4(:,:,:)=0 iostat = nf90_get_var( ncid, jv, inputdata_3d_i4, start, indimlens ) - ! If the variable is masked, get the mask for the first time and file (assume constant in time) - IF( l_ismasked .AND. ifile == 1 .AND. itime == 1 ) THEN - l_mask_3d(:,:,:) = (inputdata_3d_i4(:,:,:) == inputdata_fill_value_i4) + + ! Do not include masked data in the average + IF( l_ismasked ) THEN + WHERE( inputdata_3d_i4(:,:,:) /= inputdata_fill_value_i4 ) + meandata_3d_i4(:,:,:) = meandata_3d_i4(:,:,:) + inputdata_3d_i4(:,:,:) + ntimes_3d(:,:,:) = ntimes_3d(:,:,:) + 1 + ENDWHERE + ELSE + meandata_3d_i4(:,:,:) = meandata_3d_i4(:,:,:) + inputdata_3d_i4(:,:,:) + ntimes = ntimes + 1 ENDIF - meandata_3d_i4(:,:,:)=meandata_3d_i4(:,:,:)+inputdata_3d_i4(:,:,:) + DEALLOCATE(inputdata_3d_i4) CASE( NF90_FLOAT ) ALLOCATE(inputdata_3d_sp(outdimlens(dimids(1)),outdimlens(dimids(2)), & & outdimlens(dimids(3)))) - inputdata_3d_sp(:,:,:)=0.0 iostat = nf90_get_var( ncid, jv, inputdata_3d_sp, start, indimlens ) - ! If the variable is masked, get the mask for the first time and file (assume constant in time) - IF( l_ismasked .AND. ifile == 1 .AND. itime == 1 ) THEN - l_mask_3d(:,:,:) = (inputdata_3d_sp(:,:,:) == inputdata_fill_value_sp) + + ! Do not include masked data in the average + IF( l_ismasked ) THEN + WHERE( inputdata_3d_sp(:,:,:) /= inputdata_fill_value_sp ) + meandata_3d_sp(:,:,:) = meandata_3d_sp(:,:,:) + inputdata_3d_sp(:,:,:) + ntimes_3d(:,:,:) = ntimes_3d(:,:,:) + 1 + ENDWHERE + ELSE + meandata_3d_sp(:,:,:) = meandata_3d_sp(:,:,:) + inputdata_3d_sp(:,:,:) + ntimes = ntimes + 1 ENDIF - meandata_3d_sp(:,:,:)=meandata_3d_sp(:,:,:)+inputdata_3d_sp(:,:,:) + DEALLOCATE(inputdata_3d_sp) CASE( NF90_DOUBLE ) ALLOCATE(inputdata_3d_dp(outdimlens(dimids(1)),outdimlens(dimids(2)), & & outdimlens(dimids(3)))) - inputdata_3d_dp(:,:,:)=0.0 iostat = nf90_get_var( ncid, jv, inputdata_3d_dp, start, indimlens ) - ! If the variable is masked, get the mask for the first time and file (assume constant in time) - IF( l_ismasked .AND. ifile == 1 .AND. itime == 1 ) THEN - l_mask_3d(:,:,:) = (inputdata_3d_dp(:,:,:) == inputdata_fill_value_dp) + + ! Do not include masked data in the average + IF( l_ismasked ) THEN + WHERE( inputdata_3d_dp(:,:,:) /= inputdata_fill_value_dp ) + meandata_3d_dp(:,:,:) = meandata_3d_dp(:,:,:) + inputdata_3d_dp(:,:,:) + ntimes_3d(:,:,:) = ntimes_3d(:,:,:) + 1 + ENDWHERE + ELSE + meandata_3d_dp(:,:,:) = meandata_3d_dp(:,:,:) + inputdata_3d_dp(:,:,:) + ntimes = ntimes + 1 ENDIF - meandata_3d_dp(:,:,:)=meandata_3d_dp(:,:,:)+inputdata_3d_dp(:,:,:) + DEALLOCATE(inputdata_3d_dp) CASE DEFAULT WRITE(6,*)'Unknown nf90 type: ', xtype @@ -622,88 +776,232 @@ PROGRAM mean_nemo CASE( NF90_BYTE ) ALLOCATE(inputdata_4d_i1(outdimlens(dimids(1)),outdimlens(dimids(2)), & & outdimlens(dimids(3)),outdimlens(dimids(4)))) - inputdata_4d_i1(:,:,:,:)=0 iostat = nf90_get_var( ncid, jv, inputdata_4d_i1, start, indimlens ) meandata_4d_i1(:,:,:,:)=meandata_4d_i1(:,:,:,:)+inputdata_4d_i1(:,:,:,:) DEALLOCATE(inputdata_4d_i1) + + ntimes = ntimes + 1 CASE( NF90_SHORT ) ALLOCATE(inputdata_4d_i2(outdimlens(dimids(1)),outdimlens(dimids(2)), & & outdimlens(dimids(3)),outdimlens(dimids(4)))) - inputdata_4d_i2(:,:,:,:)=0 iostat = nf90_get_var( ncid, jv, inputdata_4d_i2, start, indimlens ) meandata_4d_i2(:,:,:,:)=meandata_4d_i2(:,:,:,:)+inputdata_4d_i2(:,:,:,:) DEALLOCATE(inputdata_4d_i2) + + ntimes = ntimes + 1 CASE( NF90_INT ) ALLOCATE(inputdata_4d_i4(outdimlens(dimids(1)),outdimlens(dimids(2)), & & outdimlens(dimids(3)),outdimlens(dimids(4)))) - inputdata_4d_i4(:,:,:,:)=0 iostat = nf90_get_var( ncid, jv, inputdata_4d_i4, start, indimlens ) - ! If the variable is masked, get the mask for the first time and file (assume constant in time) - IF( l_ismasked .AND. ifile == 1 .AND. itime == 1 ) THEN - l_mask_4d(:,:,:,:) = (inputdata_4d_i4(:,:,:,:) == inputdata_fill_value_i4) + + ! Do not include masked data in the average + IF( l_ismasked ) THEN + WHERE( inputdata_4d_i4(:,:,:,:) /= inputdata_fill_value_i4 ) + meandata_4d_i4(:,:,:,:) = meandata_4d_i4(:,:,:,:) + inputdata_4d_i4(:,:,:,:) + ntimes_4d(:,:,:,:) = ntimes_4d(:,:,:,:) + 1 + ENDWHERE + ELSE + meandata_4d_i4(:,:,:,:) = meandata_4d_i4(:,:,:,:) + inputdata_4d_i4(:,:,:,:) + ntimes = ntimes + 1 ENDIF - meandata_4d_i4(:,:,:,:)=meandata_4d_i4(:,:,:,:)+inputdata_4d_i4(:,:,:,:) + DEALLOCATE(inputdata_4d_i4) CASE( NF90_FLOAT ) ALLOCATE(inputdata_4d_sp(outdimlens(dimids(1)),outdimlens(dimids(2)), & & outdimlens(dimids(3)),outdimlens(dimids(4)))) - inputdata_4d_sp(:,:,:,:)=0.0 iostat = nf90_get_var( ncid, jv, inputdata_4d_sp, start, indimlens ) - ! If the variable is masked, get the mask for the first time and file (assume constant in time) - IF( l_ismasked .AND. ifile == 1 .AND. itime == 1 ) THEN - l_mask_4d(:,:,:,:) = (inputdata_4d_sp(:,:,:,:) == inputdata_fill_value_sp) - ENDIF - ! Thickness-weighting + + ! Thickness-weighting- get cell thickness IF( l_thckwgt ) THEN ALLOCATE(cellthick_4d_sp(outdimlens(dimids(1)),outdimlens(dimids(2)), & & outdimlens(dimids(3)),outdimlens(dimids(4)))) - cellthick_4d_sp(:,:,:,:)=0.0 iostat = nf90_get_var( ncid, jv_thickness, cellthick_4d_sp, start, indimlens ) - ! Avoid floating point errors - IF( l_ismasked ) THEN ; WHERE( l_mask_4d ) cellthick_4d_sp(:,:,:,:) = 1. ; ENDIF - meandata_4d_sp(:,:,:,:)=meandata_4d_sp(:,:,:,:)+inputdata_4d_sp(:,:,:,:)*cellthick_4d_sp(:,:,:,:) - DEALLOCATE(cellthick_4d_sp) + ENDIF + + ! Do not include masked data in the average + IF( l_ismasked ) THEN + ! Use the union of the input data and cell thickness masks + ALLOCATE(l_mask_4d(outdimlens(dimids(1)),outdimlens(dimids(2)), & + & outdimlens(dimids(3)),outdimlens(dimids(4)))) + IF( l_inputdata_ismasked .AND. (l_thckwgt .AND. l_cellthick_ismasked) ) THEN + l_mask_4d(:,:,:,:) = (inputdata_4d_sp(:,:,:,:) /= inputdata_fill_value_sp) .AND. & + & (cellthick_4d_sp(:,:,:,:) /= cellthick_fill_value_sp) + ELSE IF( l_inputdata_ismasked ) THEN + l_mask_4d(:,:,:,:) = (inputdata_4d_sp(:,:,:,:) /= inputdata_fill_value_sp) + ELSE + l_mask_4d(:,:,:,:) = (cellthick_4d_sp(:,:,:,:) /= cellthick_fill_value_sp) + ENDIF + + IF( l_thckwgt ) THEN + WHERE( l_mask_4d ) + meandata_4d_sp(:,:,:,:) = meandata_4d_sp(:,:,:,:) + inputdata_4d_sp(:,:,:,:) * cellthick_4d_sp(:,:,:,:) + meancellthick_4d_sp(:,:,:,:) = meancellthick_4d_sp(:,:,:,:) + cellthick_4d_sp(:,:,:,:) + ENDWHERE + ELSE + WHERE( l_mask_4d ) + meandata_4d_sp(:,:,:,:) = meandata_4d_sp(:,:,:,:) + inputdata_4d_sp(:,:,:,:) + ntimes_4d(:,:,:,:) = ntimes_4d(:,:,:,:) + 1 + ENDWHERE + ENDIF + + DEALLOCATE(l_mask_4d) ELSE - meandata_4d_sp(:,:,:,:)=meandata_4d_sp(:,:,:,:)+inputdata_4d_sp(:,:,:,:) + IF( l_thckwgt ) THEN + meandata_4d_sp(:,:,:,:) = meandata_4d_sp(:,:,:,:) + inputdata_4d_sp(:,:,:,:) * cellthick_4d_sp(:,:,:,:) + meancellthick_4d_sp(:,:,:,:) = meancellthick_4d_sp(:,:,:,:) + cellthick_4d_sp(:,:,:,:) + ELSE + meandata_4d_sp(:,:,:,:) = meandata_4d_sp(:,:,:,:) + inputdata_4d_sp(:,:,:,:) + ntimes = ntimes + 1 + ENDIF ENDIF + + IF( l_thckwgt ) DEALLOCATE(cellthick_4d_sp) DEALLOCATE(inputdata_4d_sp) CASE( NF90_DOUBLE ) ALLOCATE(inputdata_4d_dp(outdimlens(dimids(1)),outdimlens(dimids(2)), & & outdimlens(dimids(3)),outdimlens(dimids(4)))) - inputdata_4d_dp(:,:,:,:)=0.0 iostat = nf90_get_var( ncid, jv, inputdata_4d_dp, start, indimlens ) - ! If the variable is masked, get the mask for the first time and file (assume constant in time) - IF( l_ismasked .AND. ifile == 1 .AND. itime == 1 ) THEN - l_mask_4d(:,:,:,:) = (inputdata_4d_dp(:,:,:,:) == inputdata_fill_value_dp) - ENDIF - ! Thickness-weighting + + ! Thickness-weighting- get cell thickness IF( l_thckwgt ) THEN ALLOCATE(cellthick_4d_dp(outdimlens(dimids(1)),outdimlens(dimids(2)), & & outdimlens(dimids(3)),outdimlens(dimids(4)))) - cellthick_4d_dp(:,:,:,:)=0.0 iostat = nf90_get_var( ncid, jv_thickness, cellthick_4d_dp, start, indimlens ) - ! Avoid floating point errors - IF( l_ismasked ) THEN ; WHERE( l_mask_4d ) cellthick_4d_dp(:,:,:,:) = 1. ; ENDIF - meandata_4d_dp(:,:,:,:)=meandata_4d_dp(:,:,:,:)+inputdata_4d_dp(:,:,:,:)*cellthick_4d_dp(:,:,:,:) - DEALLOCATE(cellthick_4d_dp) + ENDIF + + ! Do not include masked data in the average + IF( l_ismasked ) THEN + ! Use the union of the input data and cell thickness masks + ALLOCATE(l_mask_4d(outdimlens(dimids(1)),outdimlens(dimids(2)), & + & outdimlens(dimids(3)),outdimlens(dimids(4)))) + IF( l_inputdata_ismasked .AND. (l_thckwgt .AND. l_cellthick_ismasked) ) THEN + l_mask_4d(:,:,:,:) = (inputdata_4d_dp(:,:,:,:) /= inputdata_fill_value_dp) .AND. & + & (cellthick_4d_dp(:,:,:,:) /= cellthick_fill_value_dp) + ELSE IF( l_inputdata_ismasked ) THEN + l_mask_4d(:,:,:,:) = (inputdata_4d_dp(:,:,:,:) /= inputdata_fill_value_dp) + ELSE + l_mask_4d(:,:,:,:) = (cellthick_4d_dp(:,:,:,:) /= cellthick_fill_value_dp) + ENDIF + + IF( l_thckwgt ) THEN + WHERE( l_mask_4d ) + meandata_4d_dp(:,:,:,:) = meandata_4d_dp(:,:,:,:) + inputdata_4d_dp(:,:,:,:) * cellthick_4d_dp(:,:,:,:) + meancellthick_4d_dp(:,:,:,:) = meancellthick_4d_dp(:,:,:,:) + cellthick_4d_dp(:,:,:,:) + ENDWHERE + ELSE + WHERE( l_mask_4d ) + meandata_4d_dp(:,:,:,:) = meandata_4d_dp(:,:,:,:) + inputdata_4d_dp(:,:,:,:) + ntimes_4d(:,:,:,:) = ntimes_4d(:,:,:,:) + 1 + ENDWHERE + ENDIF + + DEALLOCATE(l_mask_4d) ELSE - meandata_4d_dp(:,:,:,:)=meandata_4d_dp(:,:,:,:)+inputdata_4d_dp(:,:,:,:) + IF( l_thckwgt ) THEN + meandata_4d_dp(:,:,:,:) = meandata_4d_dp(:,:,:,:) + inputdata_4d_dp(:,:,:,:) * cellthick_4d_dp(:,:,:,:) + meancellthick_4d_dp(:,:,:,:) = meancellthick_4d_dp(:,:,:,:) + cellthick_4d_dp(:,:,:,:) + ELSE + meandata_4d_dp(:,:,:,:) = meandata_4d_dp(:,:,:,:) + inputdata_4d_dp(:,:,:,:) + ntimes = ntimes + 1 + ENDIF ENDIF + + IF( l_thckwgt ) DEALLOCATE(cellthick_4d_dp) DEALLOCATE(inputdata_4d_dp) CASE DEFAULT WRITE(6,*)'Unknown nf90 type: ', xtype STOP 14 END SELECT + ELSEIF( ndims == 5 ) THEN + + SELECT CASE( xtype ) + CASE( NF90_BYTE ) + ALLOCATE(inputdata_5d_i1(outdimlens(dimids(1)),outdimlens(dimids(2)), & + & outdimlens(dimids(3)),outdimlens(dimids(4)), & + & outdimlens(dimids(5)))) + iostat = nf90_get_var( ncid, jv, inputdata_5d_i1, start, indimlens ) + meandata_5d_i1(:,:,:,:,:)=meandata_5d_i1(:,:,:,:,:)+inputdata_5d_i1(:,:,:,:,:) + DEALLOCATE(inputdata_5d_i1) + + ntimes = ntimes + 1 + CASE( NF90_SHORT ) + ALLOCATE(inputdata_5d_i2(outdimlens(dimids(1)),outdimlens(dimids(2)), & + & outdimlens(dimids(3)),outdimlens(dimids(4)), & + & outdimlens(dimids(5)))) + iostat = nf90_get_var( ncid, jv, inputdata_5d_i2, start, indimlens ) + meandata_5d_i2(:,:,:,:,:)=meandata_5d_i2(:,:,:,:,:)+inputdata_5d_i2(:,:,:,:,:) + DEALLOCATE(inputdata_5d_i2) + + ntimes = ntimes + 1 + CASE( NF90_INT ) + ALLOCATE(inputdata_5d_i4(outdimlens(dimids(1)),outdimlens(dimids(2)), & + & outdimlens(dimids(3)),outdimlens(dimids(4)), & + & outdimlens(dimids(5)))) + iostat = nf90_get_var( ncid, jv, inputdata_5d_i4, start, indimlens ) + + ! Do not include masked data in the average + IF( l_ismasked ) THEN + WHERE( inputdata_5d_i4(:,:,:,:,:) /= inputdata_fill_value_i4 ) + meandata_5d_i4(:,:,:,:,:) = meandata_5d_i4(:,:,:,:,:) + inputdata_5d_i4(:,:,:,:,:) + ntimes_5d(:,:,:,:,:) = ntimes_5d(:,:,:,:,:) + 1 + ENDWHERE + ELSE + meandata_5d_i4(:,:,:,:,:) = meandata_5d_i4(:,:,:,:,:) + inputdata_5d_i4(:,:,:,:,:) + ntimes = ntimes + 1 + ENDIF + + DEALLOCATE(inputdata_5d_i4) + CASE( NF90_FLOAT ) + ALLOCATE(inputdata_5d_sp(outdimlens(dimids(1)),outdimlens(dimids(2)), & + & outdimlens(dimids(3)),outdimlens(dimids(4)), & + & outdimlens(dimids(5)))) + iostat = nf90_get_var( ncid, jv, inputdata_5d_sp, start, indimlens ) + + ! Do not include masked data in the average + IF( l_ismasked ) THEN + WHERE( inputdata_5d_sp(:,:,:,:,:) /= inputdata_fill_value_sp ) + meandata_5d_sp(:,:,:,:,:) = meandata_5d_sp(:,:,:,:,:) + inputdata_5d_sp(:,:,:,:,:) + ntimes_5d(:,:,:,:,:) = ntimes_5d(:,:,:,:,:) + 1 + ENDWHERE + ELSE + meandata_5d_sp(:,:,:,:,:) = meandata_5d_sp(:,:,:,:,:) + inputdata_5d_sp(:,:,:,:,:) + ntimes = ntimes + 1 + ENDIF + + DEALLOCATE(inputdata_5d_sp) + CASE( NF90_DOUBLE ) + ALLOCATE(inputdata_5d_dp(outdimlens(dimids(1)),outdimlens(dimids(2)), & + & outdimlens(dimids(3)),outdimlens(dimids(4)), & + & outdimlens(dimids(5)))) + iostat = nf90_get_var( ncid, jv, inputdata_5d_dp, start, indimlens ) + + ! Do not include masked data in the average + IF( l_ismasked ) THEN + WHERE( inputdata_5d_dp(:,:,:,:,:) /= inputdata_fill_value_dp ) + meandata_5d_dp(:,:,:,:,:) = meandata_5d_dp(:,:,:,:,:) + inputdata_5d_dp(:,:,:,:,:) + ntimes_5d(:,:,:,:,:) = ntimes_5d(:,:,:,:,:) + 1 + ENDWHERE + ELSE + meandata_5d_dp(:,:,:,:,:) = meandata_5d_dp(:,:,:,:,:) + inputdata_5d_dp(:,:,:,:,:) + ntimes = ntimes + 1 + ENDIF + + DEALLOCATE(inputdata_5d_dp) + CASE DEFAULT + WRITE(6,*)'Unknown nf90 type: ', xtype + STOP 14 + END SELECT + ELSE WRITE(6,*)'E R R O R: ' - WRITE(6,*)'The netcdf variable has more than 4 dimensions which is not taken into account' + WRITE(6,*)'The netcdf variable has more than 5 dimensions which is not taken into account' STOP 15 ENDIF !End of if statement over number of dimensions IF( iostat /= nf90_noerr ) THEN - WRITE(6,*) TRIM(nf90_strerror(iostat)) - WRITE(6,*) 'E R R O R reading variable '//TRIM(varname)//' from file '//TRIM(filenames(ifile)) + WRITE(6,*) 'E R R O R reading variable '//TRIM(varname)//' from file '//TRIM(filenames(ifile))//':' + WRITE(6,*) ' '//TRIM(nf90_strerror(iostat)) istop = 1 ENDIF @@ -715,27 +1013,23 @@ PROGRAM mean_nemo END DO !loop over files - !Divide by number of records to get mean - ! cast ntimes to appropriate types to allow for the divisions - ntimes_i1 = INT(ntimes) - ntimes_i2 = INT(ntimes) - ntimes_i4 = INT(ntimes) - ntimes_sp = REAL(ntimes) - ntimes_dp = REAL(ntimes) - + CALL timing_stop(t_section, 'variable '//TRIM(varname)//'- calculate sums for mean') ; CALL timing_start(t_section) + + ! Divide numerator (sum of data, possibly thickness-weighted) by denominator (sum of cell thickness + ! or number of time records) to get the mean, setting masked results to a fill value IF( ndims == 1 ) THEN SELECT CASE( xtype ) CASE( NF90_BYTE ) - meandata_1d_i1(:)=meandata_1d_i1(:)/(ntimes_i1) + meandata_1d_i1(:)=meandata_1d_i1(:)/ntimes CASE( NF90_SHORT ) - meandata_1d_i2(:)=meandata_1d_i2(:)/(ntimes_i2) + meandata_1d_i2(:)=meandata_1d_i2(:)/ntimes CASE( NF90_INT ) - meandata_1d_i4(:)=meandata_1d_i4(:)/(ntimes_i4) + meandata_1d_i4(:)=meandata_1d_i4(:)/ntimes CASE( NF90_FLOAT ) - meandata_1d_sp(:)=meandata_1d_sp(:)/(ntimes_sp) + meandata_1d_sp(:)=meandata_1d_sp(:)/ntimes CASE( NF90_DOUBLE ) - meandata_1d_dp(:)=meandata_1d_dp(:)/(ntimes_dp) + meandata_1d_dp(:)=meandata_1d_dp(:)/ntimes CASE DEFAULT WRITE(6,*)'Unknown nf90 type: ', xtype STOP 14 @@ -745,15 +1039,15 @@ PROGRAM mean_nemo SELECT CASE( xtype ) CASE( NF90_BYTE ) - meandata_2d_i1(:,:)=meandata_2d_i1(:,:)/(ntimes_i1) + meandata_2d_i1(:,:)=meandata_2d_i1(:,:)/ntimes CASE( NF90_SHORT ) - meandata_2d_i2(:,:)=meandata_2d_i2(:,:)/(ntimes_i2) + meandata_2d_i2(:,:)=meandata_2d_i2(:,:)/ntimes CASE( NF90_INT ) - meandata_2d_i4(:,:)=meandata_2d_i4(:,:)/(ntimes_i4) + meandata_2d_i4(:,:)=meandata_2d_i4(:,:)/ntimes CASE( NF90_FLOAT ) - meandata_2d_sp(:,:)=meandata_2d_sp(:,:)/(ntimes_sp) + meandata_2d_sp(:,:)=meandata_2d_sp(:,:)/ntimes CASE( NF90_DOUBLE ) - meandata_2d_dp(:,:)=meandata_2d_dp(:,:)/(ntimes_dp) + meandata_2d_dp(:,:)=meandata_2d_dp(:,:)/ntimes CASE DEFAULT WRITE(6,*)'Unknown nf90 type: ', xtype STOP 14 @@ -763,87 +1057,166 @@ PROGRAM mean_nemo SELECT CASE( xtype ) CASE( NF90_BYTE ) - meandata_3d_i1(:,:,:)=meandata_3d_i1(:,:,:)/(ntimes_i1) + meandata_3d_i1(:,:,:)=meandata_3d_i1(:,:,:)/ntimes CASE( NF90_SHORT ) - meandata_3d_i2(:,:,:)=meandata_3d_i2(:,:,:)/(ntimes_i2) + meandata_3d_i2(:,:,:)=meandata_3d_i2(:,:,:)/ntimes CASE( NF90_INT ) - meandata_3d_i4(:,:,:)=meandata_3d_i4(:,:,:)/(ntimes_i4) - ! Restore masked points IF( l_ismasked ) THEN - WHERE( l_mask_3d ) meandata_3d_i4(:,:,:) = inputdata_fill_value_i4 + WHERE( ntimes_3d(:,:,:) == 0 ) + meandata_3d_i4(:,:,:) = outputdata_fill_value_i4 + ELSEWHERE + meandata_3d_i4(:,:,:) = meandata_3d_i4(:,:,:) / ntimes_3d(:,:,:) + ENDWHERE + ELSE + meandata_3d_i4(:,:,:) = meandata_3d_i4(:,:,:) / ntimes ENDIF CASE( NF90_FLOAT ) - meandata_3d_sp(:,:,:)=meandata_3d_sp(:,:,:)/(ntimes_sp) - ! Restore masked points IF( l_ismasked ) THEN - WHERE( l_mask_3d ) meandata_3d_sp(:,:,:) = inputdata_fill_value_sp + WHERE( ntimes_3d(:,:,:) == 0 ) + meandata_3d_sp(:,:,:) = outputdata_fill_value_sp + ELSEWHERE + meandata_3d_sp(:,:,:) = meandata_3d_sp(:,:,:) / ntimes_3d(:,:,:) + ENDWHERE + ELSE + meandata_3d_sp(:,:,:) = meandata_3d_sp(:,:,:) / ntimes ENDIF CASE( NF90_DOUBLE ) - meandata_3d_dp(:,:,:)=meandata_3d_dp(:,:,:)/(ntimes_dp) - ! Restore masked points IF( l_ismasked ) THEN - WHERE( l_mask_3d ) meandata_3d_dp(:,:,:) = inputdata_fill_value_dp + WHERE( ntimes_3d(:,:,:) == 0 ) + meandata_3d_dp(:,:,:) = outputdata_fill_value_dp + ELSEWHERE + meandata_3d_dp(:,:,:) = meandata_3d_dp(:,:,:) / ntimes_3d(:,:,:) + ENDWHERE + ELSE + meandata_3d_dp(:,:,:) = meandata_3d_dp(:,:,:) / ntimes ENDIF CASE DEFAULT WRITE(6,*)'Unknown nf90 type: ', xtype STOP 14 END SELECT + IF( l_ismasked ) DEALLOCATE(ntimes_3d) ELSEIF( ndims == 4 ) THEN SELECT CASE( xtype ) CASE( NF90_BYTE ) - meandata_4d_i1(:,:,:,:)=meandata_4d_i1(:,:,:,:)/(ntimes_i1) + meandata_4d_i1(:,:,:,:)=meandata_4d_i1(:,:,:,:)/ntimes CASE( NF90_SHORT ) - meandata_4d_i2(:,:,:,:)=meandata_4d_i2(:,:,:,:)/(ntimes_i2) + meandata_4d_i2(:,:,:,:)=meandata_4d_i2(:,:,:,:)/ntimes CASE( NF90_INT ) - meandata_4d_i4(:,:,:,:)=meandata_4d_i4(:,:,:,:)/(ntimes_i4) - ! Restore masked points IF( l_ismasked ) THEN - WHERE( l_mask_4d ) meandata_4d_i4(:,:,:,:) = inputdata_fill_value_i4 + WHERE( ntimes_4d(:,:,:,:) == 0 ) + meandata_4d_i4(:,:,:,:) = outputdata_fill_value_i4 + ELSEWHERE + meandata_4d_i4(:,:,:,:) = meandata_4d_i4(:,:,:,:) / ntimes_4d(:,:,:,:) + ENDWHERE + ELSE + meandata_4d_i4(:,:,:,:) = meandata_4d_i4(:,:,:,:) / ntimes ENDIF CASE( NF90_FLOAT ) - IF(l_thckwgt) THEN - IF( icellthick_type == NF90_DOUBLE ) THEN - meandata_4d_sp(:,:,:,:)=meandata_4d_sp(:,:,:,:)/( REAL(meancellthick_4d_dp, sp) * ntimes_sp ) + IF( l_ismasked ) THEN + IF( l_thckwgt ) THEN + WHERE( meancellthick_4d_sp(:,:,:,:) == 0._sp ) + meandata_4d_sp(:,:,:,:) = outputdata_fill_value_sp + ELSEWHERE + meandata_4d_sp(:,:,:,:) = meandata_4d_sp(:,:,:,:) / meancellthick_4d_sp(:,:,:,:) + ENDWHERE ELSE - meandata_4d_sp(:,:,:,:)=meandata_4d_sp(:,:,:,:)/(meancellthick_4d_sp(:,:,:,:) * ntimes_sp) + WHERE( ntimes_4d(:,:,:,:) == 0 ) + meandata_4d_sp(:,:,:,:) = outputdata_fill_value_sp + ELSEWHERE + meandata_4d_sp(:,:,:,:) = meandata_4d_sp(:,:,:,:) / ntimes_4d(:,:,:,:) + ENDWHERE ENDIF ELSE - meandata_4d_sp(:,:,:,:)=meandata_4d_sp(:,:,:,:)/(ntimes_sp) - ENDIF - ! Restore masked points - IF( l_ismasked ) THEN - WHERE( l_mask_4d ) meandata_4d_sp(:,:,:,:) = inputdata_fill_value_sp + IF( l_thckwgt ) THEN + meandata_4d_sp(:,:,:,:) = meandata_4d_sp(:,:,:,:) / meancellthick_4d_sp(:,:,:,:) + ELSE + meandata_4d_sp(:,:,:,:) = meandata_4d_sp(:,:,:,:) / ntimes + ENDIF ENDIF - ! If this is the cell thickness, save to normalise thickness-weighted time-means - IF( jv == jv_thickness ) meancellthick_4d_sp = meandata_4d_sp + IF( l_thckwgt ) DEALLOCATE(meancellthick_4d_sp) CASE( NF90_DOUBLE ) - IF(l_thckwgt) THEN - IF( icellthick_type == NF90_FLOAT ) THEN - meandata_4d_dp(:,:,:,:)=meandata_4d_dp(:,:,:,:)/( REAL(meancellthick_4d_sp, dp) * ntimes_dp ) + IF( l_ismasked ) THEN + IF( l_thckwgt ) THEN + WHERE( meancellthick_4d_dp(:,:,:,:) == 0._dp ) + meandata_4d_dp(:,:,:,:) = outputdata_fill_value_dp + ELSEWHERE + meandata_4d_dp(:,:,:,:) = meandata_4d_dp(:,:,:,:) / meancellthick_4d_dp(:,:,:,:) + ENDWHERE ELSE - meandata_4d_dp(:,:,:,:)=meandata_4d_dp(:,:,:,:)/(meancellthick_4d_dp(:,:,:,:) * ntimes_dp) + WHERE( ntimes_4d(:,:,:,:) == 0 ) + meandata_4d_dp(:,:,:,:) = outputdata_fill_value_dp + ELSEWHERE + meandata_4d_dp(:,:,:,:) = meandata_4d_dp(:,:,:,:) / ntimes_4d(:,:,:,:) + ENDWHERE ENDIF ELSE - meandata_4d_dp(:,:,:,:)=meandata_4d_dp(:,:,:,:)/(ntimes_dp) + IF( l_thckwgt ) THEN + meandata_4d_dp(:,:,:,:) = meandata_4d_dp(:,:,:,:) / meancellthick_4d_dp(:,:,:,:) + ELSE + meandata_4d_dp(:,:,:,:) = meandata_4d_dp(:,:,:,:) / ntimes + ENDIF ENDIF - ! Restore masked points + IF( l_thckwgt ) DEALLOCATE(meancellthick_4d_dp) + CASE DEFAULT + WRITE(6,*)'Unknown nf90 type: ', xtype + STOP 14 + END SELECT + IF( l_ismasked .AND. .NOT. l_thckwgt ) DEALLOCATE(ntimes_4d) + + ELSEIF( ndims == 5 ) THEN + + SELECT CASE( xtype ) + CASE( NF90_BYTE ) + meandata_5d_i1(:,:,:,:,:)=meandata_5d_i1(:,:,:,:,:)/ntimes + CASE( NF90_SHORT ) + meandata_5d_i2(:,:,:,:,:)=meandata_5d_i2(:,:,:,:,:)/ntimes + CASE( NF90_INT ) IF( l_ismasked ) THEN - WHERE( l_mask_4d ) meandata_4d_dp(:,:,:,:) = inputdata_fill_value_dp + WHERE( ntimes_5d(:,:,:,:,:) == 0 ) + meandata_5d_i4(:,:,:,:,:) = outputdata_fill_value_i4 + ELSEWHERE + meandata_5d_i4(:,:,:,:,:) = meandata_5d_i4(:,:,:,:,:) / ntimes_5d(:,:,:,:,:) + ENDWHERE + ELSE + meandata_5d_i4(:,:,:,:,:) = meandata_5d_i4(:,:,:,:,:) / ntimes + ENDIF + CASE( NF90_FLOAT ) + IF( l_ismasked ) THEN + WHERE( ntimes_5d(:,:,:,:,:) == 0 ) + meandata_5d_sp(:,:,:,:,:) = outputdata_fill_value_sp + ELSEWHERE + meandata_5d_sp(:,:,:,:,:) = meandata_5d_sp(:,:,:,:,:) / ntimes_5d(:,:,:,:,:) + ENDWHERE + ELSE + meandata_5d_sp(:,:,:,:,:) = meandata_5d_sp(:,:,:,:,:) / ntimes + ENDIF + CASE( NF90_DOUBLE ) + IF( l_ismasked ) THEN + WHERE( ntimes_5d(:,:,:,:,:) == 0 ) + meandata_5d_dp(:,:,:,:,:) = outputdata_fill_value_dp + ELSEWHERE + meandata_5d_dp(:,:,:,:,:) = meandata_5d_dp(:,:,:,:,:) / ntimes_5d(:,:,:,:,:) + ENDWHERE + ELSE + meandata_5d_dp(:,:,:,:,:) = meandata_5d_dp(:,:,:,:,:) / ntimes ENDIF - ! If this is the cell thickness, save to normalise thickness-weighted time-means - IF( jv == jv_thickness ) meancellthick_4d_dp = meandata_4d_dp CASE DEFAULT WRITE(6,*)'Unknown nf90 type: ', xtype STOP 14 END SELECT + IF( l_ismasked ) DEALLOCATE(ntimes_5d) + ENDIF + CALL timing_stop(t_section, 'variable '//TRIM(varname)//'- calculate mean') + ELSE ! Else if the variable does not contain the unlimited dimension just read ! in from first file to be copied to outfile as it should be the same in all ! files (e.g. coordinates) + CALL timing_start(t_section) ncid = inncids(1) iostat = nf90_inquire_variable( ncid, jv, varname, xtype, ndims, dimids, natts) @@ -920,18 +1293,39 @@ PROGRAM mean_nemo STOP 14 END SELECT + ELSEIF( ndims == 5 ) THEN + + SELECT CASE( xtype ) + CASE( NF90_BYTE ) + iostat = nf90_get_var( ncid, jv, meandata_5d_i1 ) + CASE( NF90_SHORT ) + iostat = nf90_get_var( ncid, jv, meandata_5d_i2 ) + CASE( NF90_INT ) + iostat = nf90_get_var( ncid, jv, meandata_5d_i4 ) + CASE( NF90_FLOAT ) + iostat = nf90_get_var( ncid, jv, meandata_5d_sp ) + CASE( NF90_DOUBLE ) + iostat = nf90_get_var( ncid, jv, meandata_5d_dp ) + CASE DEFAULT + WRITE(6,*)'Unknown nf90 type: ', xtype + STOP 14 + END SELECT + ENDIF !End of ndims if statements IF( iostat /= nf90_noerr ) THEN - WRITE(6,*) TRIM(nf90_strerror(iostat)) - WRITE(6,*) 'E R R O R reading variable '//TRIM(varname)//' from file '//TRIM(filenames(ifile)) + WRITE(6,*) 'E R R O R reading variable '//TRIM(varname)//' from file '//TRIM(filenames(ifile))//':' + WRITE(6,*) ' '//TRIM(nf90_strerror(iostat)) STOP 16 ENDIF + CALL timing_stop(t_section, 'variable '//TRIM(varname)//'- copy data without time coordinate') + ENDIF !End of check for unlimited dimension !--------------------------------------------------------------------------- !4. Write data to output file and close files + CALL timing_start(t_section) IF (l_verbose) WRITE(6,*)'Writing variable '//TRIM(varname)//'...' @@ -957,7 +1351,7 @@ PROGRAM mean_nemo DEALLOCATE(meandata_1d_dp) END SELECT - ELSEIF( ndims == 2 ) THEN + ELSEIF( ndims == 2 ) THEN SELECT CASE( xtype ) CASE( NF90_BYTE ) @@ -979,7 +1373,7 @@ PROGRAM mean_nemo WRITE(6,*)'Unknown nf90 type: ', xtype STOP 14 END SELECT - + ELSEIF( ndims == 3 ) THEN SELECT CASE( xtype ) @@ -1001,8 +1395,7 @@ PROGRAM mean_nemo CASE DEFAULT WRITE(6,*)'Unknown nf90 type: ', xtype STOP 14 - END SELECT - IF( l_ismasked ) DEALLOCATE( l_mask_3d ) + END SELECT ELSEIF( ndims == 4 ) THEN @@ -1025,22 +1418,46 @@ PROGRAM mean_nemo CASE DEFAULT WRITE(6,*)'Unknown nf90 type: ', xtype STOP 14 - END SELECT - IF( l_ismasked ) DEALLOCATE( l_mask_4d ) + END SELECT + + ELSEIF( ndims == 5 ) THEN + + SELECT CASE( xtype ) + CASE( NF90_BYTE ) + iostat = nf90_put_var( outid, jv, meandata_5d_i1 ) + DEALLOCATE(meandata_5d_i1) + CASE( NF90_SHORT ) + iostat = nf90_put_var( outid, jv, meandata_5d_i2 ) + DEALLOCATE(meandata_5d_i2) + CASE( NF90_INT ) + iostat = nf90_put_var( outid, jv, meandata_5d_i4 ) + DEALLOCATE(meandata_5d_i4) + CASE( NF90_FLOAT ) + iostat = nf90_put_var( outid, jv, meandata_5d_sp ) + DEALLOCATE(meandata_5d_sp) + CASE( NF90_DOUBLE ) + iostat = nf90_put_var( outid, jv, meandata_5d_dp ) + DEALLOCATE(meandata_5d_dp) + CASE DEFAULT + WRITE(6,*)'Unknown nf90 type: ', xtype + STOP 14 + END SELECT ENDIF - IF( iostat /= 0 ) THEN - WRITE(6,*) 'E R R O R writing variable '//TRIM(varname) + IF( iostat /= nf90_noerr ) THEN + WRITE(6,*) 'E R R O R writing variable '//TRIM(varname)//':' + WRITE(6,*) ' '//TRIM(nf90_strerror(iostat)) STOP 17 ENDIF - - END DO !loop over variables - IF( allocated(meancellthick_4d_sp) ) DEALLOCATE(meancellthick_4d_sp) - IF( allocated(meancellthick_4d_dp) ) DEALLOCATE(meancellthick_4d_dp) + CALL timing_stop(t_section, 'variable '//TRIM(varname)//'- write to file') + CALL timing_stop(t_variable, 'variable '//TRIM(varname)//'- total time') ! Total time taken for variable + + END DO !loop over variables !4.1 Close all input files + CALL timing_start(t_section) IF (l_verbose) WRITE(6,*)'Closing input files...' DO ifile = 1, nargs-1 @@ -1048,7 +1465,8 @@ PROGRAM mean_nemo iostat = nf90_close( ncid ) IF( iostat /= nf90_noerr ) THEN WRITE(6,*) TRIM(nf90_strerror(iostat)) - WRITE(6,*)'E R R O R closing input file '//TRIM(filenames(ifile)) + WRITE(6,*)'E R R O R closing input file '//TRIM(filenames(ifile))//':' + WRITE(6,*) ' '//TRIM(nf90_strerror(iostat)) STOP 18 ENDIF END DO @@ -1059,8 +1477,49 @@ PROGRAM mean_nemo iostat = nf90_close( outid ) IF( iostat /= nf90_noerr ) THEN WRITE(6,*) TRIM(nf90_strerror(iostat)) - WRITE(6,*)'E R R O R closing output file' + WRITE(6,*)'E R R O R closing output file:' + WRITE(6,*) ' '//TRIM(nf90_strerror(iostat)) STOP 19 ENDIF - + + CALL timing_stop(t_section, 'close files') + CALL timing_stop(t_total, 'TOTAL') ! Total time taken + + CONTAINS + + SUBROUTINE timing_init( secondclock ) + ! Timing initialisation- get the processor clock count rate + REAL(dp), INTENT(out) :: secondclock + INTEGER(i8) :: count_rate + + IF( .NOT. l_timing ) RETURN + + CALL SYSTEM_CLOCK(COUNT_RATE=count_rate) + secondclock = 1._dp / REAL(count_rate, dp) + END SUBROUTINE + + SUBROUTINE timing_start( count ) + ! Start a timing section + INTEGER(i8), INTENT(out) :: count + + IF( .NOT. l_timing ) RETURN + + CALL SYSTEM_CLOCK(COUNT=count) + END SUBROUTINE + + SUBROUTINE timing_stop( count, sect ) + ! Stop a timing section and report the time + INTEGER(i8), INTENT(in) :: count ! Counter returned by timing_start + CHARACTER(len=*), INTENT(in) :: sect + INTEGER(i8) :: count2 + REAL(dp) :: elapsed + CHARACTER(len=128) :: clfmt = "(a,f0.6,'s')" + + IF( .NOT. l_timing ) RETURN + + CALL SYSTEM_CLOCK(COUNT=count2) + elapsed = REAL(count2-count, dp) * secondclock + WRITE(6,clfmt) 'TIMING ('//TRIM(sect)//'): ', elapsed + END SUBROUTINE + END PROGRAM mean_nemo diff --git a/Utilities/mean_nemo/simple_test_data.py b/Utilities/mean_nemo/simple_test_data.py new file mode 100755 index 00000000..1fff1dce --- /dev/null +++ b/Utilities/mean_nemo/simple_test_data.py @@ -0,0 +1,386 @@ +#! /usr/bin/env python3 + +# ----------------------------------------------------------------------------- +# (C) Crown copyright Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +# ----------------------------------------------------------------------------- + +import os +import xarray as xr +import numpy as np + +""" +Script to generate a simple dataset for testing mean_nemo + +Two types of files will be generated- input data (intended as input to mean_nemo) and expected results +(the results expected from running mean_nemo with the input data). The following `nccmp` command is recommended +for comparing the latter file with the output from mean_nemo: + + `nccmp -F -f -c 1 -d -m ` + +The expected results are obtained by using xarray/numpy averaging methods. + +Both types of file will contain several variables that test different components of the mean_nemo code, named: + + {data_type}_{data_shape}_{weighting}_{time_coordinate}_{masking} + +where: + + * `data_type`- the data precision, using their numpy alias names ("byte", "short", "intc", "single", "double") + * `data_shape`- the shape of the data ("1d", "2d", "3d", "4d") + * `weighting`- the weighting method used in the average: + * "unweighted"- no weighting is used + * "weighted"- the average is weighted by an unmasked cell thickness + * "weighted-masked"- the average is weighted by a masked cell thickness + * `time_coordinate`- whether the data has a time coordinate ("time") or not ("notime") + * `masking`- whether the data is masked ("masked") or not ("unmasked") +""" + +# User input +# ============================================================================================================================ + +# Paths for files generated by this script +dir_out = '' # Output directory +file_out = f'{dir_out}/input.nc' # Input data (unmasked cell thickness) +file_out_cellthk_masked = f'{dir_out}/input_cellthk_masked.nc' # Input data (masked cell thickness) +file_expected = f'{dir_out}/expected.nc' # Expected results (unmasked cell thickness) +file_expected_cellthk_masked = f'{dir_out}/expected_cellthk_masked.nc' # Expected results (masked cell thickness) + +# Properties of the test data +data_dims = ('t', 'b', 'z', 'y', 'x') # Order of dimensions in the file +data_shape = {'t': 6, 'x': 10, 'y': 10, 'z': 10, 'b': 5} # Shape of the FULL test data (sliced by data_isel entries + # to generate the different data shapes) +data_time_split = [slice(0, 2), slice(2, 4), slice(4, 6)] # How data should be split in time between files +data_masked_isel = [ # Points to be masked in the FULL test data + {'x': [2, 4], 'y': [2, 4, 6]}, + {'z': [7, 8, 9]}, + {'t': [0], 'x': [0], 'y': [0], 'z': [0]}, + {'t': [2], 'x': [1], 'y': [0], 'z': [0]}, + {'t': [4], 'x': [1], 'y': [1], 'z': [0]}, + {'t': [2, 4], 'b': [3], 'x': [2]}, + ] +cellthk_masked_isel = [ # Points to be masked in the FULL cell thickness data + {'x': [2], 'y': [4]}, + {'t': [0], 'x': [0], 'y': [1, 3, 5]}, + {'t': [3], 'x': [1], 'y': [1, 3, 5]}, + {'t': [5], 'x': [2], 'y': [1, 3, 5]}, + ] +data_isel = { # Data shapes to be generated and the coordinates + '1d': {'b': 0, 'z': 0, 'y': 0, 'x': 0}, # used to generate them with respect to the FULL data + '2d': {'b': 0, 'z': 0, 'y': 0}, + '3d': {'b': 0, 'z': 0}, + '4d': {'b': 0}, + '5d': None, + } +data_fillvals = { # Data types to be generated and the fill values + 'byte': -1, # to be used for masked data + 'short': -1, + 'intc': -1, + 'single': 1e20, + 'double': 1e20, + } + +# ============================================================================================================================ + +# Numpy data type objects +data_dtypes = {i: getattr(np, i) for i in data_fillvals} + +# Dimensions +dict_da_dims = {k: xr.DataArray(range(v), dims=k) for k, v in data_shape.items()} + +# Full test data (double precision)- linear increase over each dimension +data_full = np.mgrid[*[slice(None, data_shape[i]) for i in data_dims]].sum(axis=0).astype('double') +da_full = xr.DataArray(data_full, + coords=[dict_da_dims[i] for i in data_dims]) + +# Mask for full test data (True where valid) +da_isvalid_full = xr.DataArray(True, coords=[dict_da_dims[i] for i in data_dims]) +for isel in data_masked_isel: + da_isvalid_full[isel] = False + +# Full cell thickness data (single precision)- linear increase with time +da_cellthk_full = xr.DataArray(0, + coords=[dict_da_dims[i] for i in ('t', 'z', 'y', 'x')], + attrs={'standard_name': 'cell_thickness'}, + name='cellthk').astype('single') +da_cellthk_full += da_cellthk_full.t + 1 + +# Mask for full cell thickness data (True where valid) +da_cellthk_isvalid_full = xr.DataArray(True, coords=[dict_da_dims[i] for i in ('t', 'z', 'y', 'x')]) +for isel in cellthk_masked_isel: + da_cellthk_isvalid_full[isel] = False + +# DataArray lists to be combined into Datasets later +da_list = [] +da_list_cellthk_masked = [] +da_expected_list = [] +da_expected_list_cellthk_masked = [] + + +def input_data(da: xr.DataArray, da_isvalid: xr.DataArray | None, + dtype: np.dtype, slices: dict | None, fillval: np.typing.ArrayLike, + drop_time_coord: bool = False, is_weighted: bool = False, masked_in_file: bool = False + ) -> (xr.DataArray, xr.DataArray | None): + """ + Generate input data. + + Parameters + ---------- + da : xr.DataArray + Full input data to use as the basis + da_isvalid : xr.DataArray | None + Mask to apply to `da` (`False` where masked) + dtype : np.dtype + Numpy dtype to convert `da` to + slices : dict | None + Indexing slices to apply to `da` + fillval : np.typing.ArrayLike + Value to set when applying `da_isvalid` mask + drop_time_coord : bool + Whether the time coordinate should be dropped from `da` + is_weighted : bool + Whether a `cell_methods` attribute will be applied to `da` to indicate it is weighted by the cell thickness + masked_in_file : bool + Whether a `_FillValue` attribute will be applied to `da` + + Returns + ------- + da : xr.DataArray + Input data + xr.DataArray | None + `masked_in_file == True:` + Input data mask (`True` where valid) corresponding to `da` + `masked_in_file == False`: + None + """ + # Slice full data and mask + if slices is not None: + da = da.isel(**slices, drop=True) + da_isvalid = da_isvalid.isel(**slices, drop=True) + + # Convert to given data type + da = da.astype(dtype) + + # Set masked data to given fill value (np.nan not representable for non-floats) + da = da.where(da_isvalid, fillval) + + # No time coordinate- use first time and drop coordinate + if drop_time_coord: + da = da.isel(t=0, drop=True) + + # Thickness-weighting- assign cell_methods recognised by mean_nemo + if is_weighted: + da = da.assign_attrs(cell_methods='time: mean (thickness weighted)') + + # Set _FillValue if data is to be read in as masked by mean_nemo + da.encoding = {'_FillValue': fillval if masked_in_file else None} + + return da, (da_isvalid if masked_in_file else None) + + +def cellthk_data(da: xr.DataArray, slices: dict | None = None, da_isvalid: xr.DataArray | None = None + ) -> xr.DataArray: + """ + Generate cell thickness data. + + Parameters + ---------- + da : xr.DataArray + Full cell thickness data to use as the basis + slices : dict | None + Indexing slices to apply to `da` + da_isvalid : xr.DataArray | None + Mask to apply to `da` (`False` where masked) + + Returns + ------- + da: xr.DataArray + Cell thickness data + """ + # If data will be masked, set masked points to np.nan and set a _FillValue for these points to be written to file + if da_isvalid is not None: + da = da.where(da_isvalid) + da.encoding = {'_FillValue': np.single(2e20)} + else: + da.encoding = {'_FillValue': None} + + # Slice full data + if slices is not None: + da = da.isel(**slices, drop=True, missing_dims='ignore') + + return da + + +def calc_mean(da: xr.DataArray, da_isvalid: xr.DataArray | None = None, da_cellthk: xr.DataArray | None = None + ) -> xr.DataArray: + """ + Calculate the time-average (i.e. the expected result) using xarray/numpy routines. + + Parameters + ---------- + da : xr.DataArray + Data to be averaged + da_isvalid : xr.DataArray | None + Mask to apply to `da` for the average (`False` where masked) + da_cellthk : xr.DataArray | None + Weighting to apply to `da` for the average (`np.nan` where masked) + + Returns + ------- + da: xr.DataArray + Time average of `da` + """ + dtype = da.dtype + fillval = da.encoding['_FillValue'] + attrs = da.attrs + + # Time coordinate to be added to average data + t_mean = da.t.mean().astype(np.intc).expand_dims('t') + + # Thickness-weighting + if da_cellthk is not None: + + # When the input data has no fill value, use the cell thickness fill value instead + if fillval is None: + fillval = da_cellthk.encoding['_FillValue'] + + # Set masked cell thickness data to 0 (exclude from weighted average- np.nan not allowed) + da_cellthk = da_cellthk.fillna(0) + + # Masked averaging + if da_isvalid is not None: + + # We can use the normal masking-aware aggregators for floats, which support np.nan + if dtype in (np.single, np.double): + if da_cellthk is not None: + da = da.where(da_isvalid.values).weighted(da_cellthk).mean('t') + else: + da = da.where(da_isvalid.values).mean('t') + + # Otherwise, we have to pass a `where` argument since np.nan isn't correctly represented for non-floats + else: + da = da.mean('t', where=da_isvalid.values, skipna=False) + + # The mean results in zeroes where there is no data over time, so we must restore the fill values + da = da.where(da_isvalid.any('t'), fillval) + + # Normal averaging + else: + if da_cellthk is not None: + + # The weighted mean will contain np.nan where there is no data, but this isn't supported for non-floats. + # Stop so we can see what needs to be done. + if dtype not in (np.single, np.double): + raise TypeError(f'Weighted unmasked average not supported for non-floats') + + da = da.weighted(da_cellthk).mean('t') + else: + da = da.mean('t') + + # Recast the result back to the original precision + da = da.astype(dtype) + + # Restore _FillValue and other attributes (averaging removes them) + da.encoding = {'_FillValue': fillval} + da = da.assign_attrs(**attrs) + + # Assign a time coordinate (dropped by averaging)- use an unweighted average since this is what mean_nemo does + da = da.expand_dims('t').assign_coords(t=t_mean) + + return da + + +# Generate data for each of the: +# Data types +for dtype_name in data_dtypes: + # Data shapes + for shape_name in data_isel: + # Data to be used in thickness-weighted (where cell thickness is masked or unmasked) or normal averages + for thk_name in ('unweighted', 'weighted', 'weighted-masked'): + # Data with/without a time coordinate + for time_name in ('time', 'notime'): + # Data that will be read in by mean_nemo as masked or unmasked + for mask_name in ('masked', 'unmasked'): + + dtype = data_dtypes[dtype_name] + fillval = dtype(data_fillvals[dtype_name]) + slices = data_isel[shape_name] + + # Skip thickness-weighted cases not supported by mean_nemo + if (thk_name.startswith('weighted') and + not (time_name == 'time' and shape_name in ('4d',) and dtype in (np.single, np.double))): + continue + # Skip scalars, which aren't supported by mean_nemo + if (time_name == 'notime') and (shape_name == '1d'): + continue + # Skip masked cases not supported by mean_nemo + if (mask_name == 'masked') and not (shape_name in ('3d', '4d', '5d') and + dtype in (np.intc, np.single, np.double)): + continue + + # Get input data and its mask + da_input, da_isvalid = input_data(da_full, da_isvalid_full, dtype, slices, fillval, + drop_time_coord=(time_name == 'notime'), + is_weighted=(thk_name.startswith('weighted')), + masked_in_file=(mask_name == 'masked'), + ) + + # Variable name + da_input.name = f'{dtype_name}_{shape_name}_{thk_name}_{time_name}_{mask_name}' + + # Get the cell thickness if required for thickness-weighting + if thk_name.startswith('weighted'): + da_cellthk = cellthk_data(da_cellthk_full, slices=slices, + da_isvalid=(da_cellthk_isvalid_full if thk_name == 'weighted-masked' else None), + ) + else: + da_cellthk = None + + # Calculate the expected result + if time_name == 'time': + da_expected = calc_mean(da_input, da_isvalid=da_isvalid, da_cellthk=da_cellthk) + else: + da_expected = da_input + + # We can only have one cell thickness variable per file (mean_nemo limitation)- + # put the cell thickness and the variables weighted by it in separate files + if thk_name == 'weighted-masked': + da_list_cellthk_masked.append(da_input) + da_expected_list_cellthk_masked.append(da_expected) + else: + da_list.append(da_input) + da_expected_list.append(da_expected) + +# Add unmasked cell thickness to data +da_cellthk = cellthk_data(da_cellthk_full) +da_expected = calc_mean(da_cellthk) +da_list.append(da_cellthk) +da_expected_list.append(da_expected) + +# Add masked cell thickness to data +da_cellthk = cellthk_data(da_cellthk_full, da_isvalid=da_cellthk_isvalid_full) +da_expected = calc_mean(da_cellthk) +da_list_cellthk_masked.append(da_cellthk) +da_expected_list_cellthk_masked.append(da_expected) + +# Combine DataArrays to Datasets +ds_out = xr.merge(da_list) +ds_out_cellthk_masked = xr.merge(da_list_cellthk_masked) +ds_expected = xr.merge(da_expected_list) +ds_expected_cellthk_masked = xr.merge(da_expected_list_cellthk_masked) + +# Create output directory if needed +if not os.path.isdir(dir_out): + os.makedirs(dir_out) + +# Write input data to files- potentially split over multiple files (file_0.nc, file_1.nc, etc) +for i_file, time_slice in enumerate(data_time_split): + fname = f'{file_out.replace('.nc', '')}_{i_file}.nc' + ds_out.isel(t=time_slice).to_netcdf(fname, mode='w', format='NETCDF4_CLASSIC', unlimited_dims=['t']) + fname = f'{file_out_cellthk_masked.replace('.nc', '')}_{i_file}.nc' + ds_out_cellthk_masked.isel(t=time_slice).to_netcdf(fname, mode='w', format='NETCDF4_CLASSIC', unlimited_dims=['t']) + +# Write expected results to files- currently NC3 with 64-bit offset +ds_expected.to_netcdf(file_expected, mode='w', format='NETCDF3_64BIT', unlimited_dims=['t']) +ds_expected_cellthk_masked.to_netcdf(file_expected_cellthk_masked, mode='w', format='NETCDF3_64BIT', unlimited_dims=['t']) diff --git a/dependencies.yaml b/dependencies.yaml new file mode 100644 index 00000000..dfc7b058 --- /dev/null +++ b/dependencies.yaml @@ -0,0 +1,26 @@ +############################################################################## +# (c) Crown copyright 2026 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + +# This file contains a list of all linked external repositories containing code +# and data required to build and run the moci rose-stem +# +# Each section is the name of the repository +# source: Is either the url to the repository (the upstream or +# a fork) or a path to a local clone with format `host:/path/to/clone` +# ref: Is a valid git reference. This can either be a commit hash, a tag or a +# branch name +# +# The entry for this repository will have blank values for both source and ref +# by default, which will cause the test suite to use the source code from the +# local clone. These can be filled to set a different source if desired. + +moci: + source: + ref: + +SimSys_Scripts: + source: git@github.com:MetOffice/SimSys_Scripts.git + ref: main diff --git a/rose-stem/.cylcignore b/rose-stem/.cylcignore new file mode 100644 index 00000000..aa634ee8 --- /dev/null +++ b/rose-stem/.cylcignore @@ -0,0 +1,4 @@ +bin/suite_report_git.py +bin/suite_data.py +bin/git_bdiff.py +bin/get_git_sources.py diff --git a/rose-stem/app/build_pp/rose-app.conf b/rose-stem/app/build_pp/rose-app.conf new file mode 100644 index 00000000..e62a0435 --- /dev/null +++ b/rose-stem/app/build_pp/rose-app.conf @@ -0,0 +1,21 @@ +[command] +default=$SOURCE_DIRECTORY/moci/Postprocessing/build/build_pp.sh + +[env] +# MOCIlib doesn't exist yet! Default is true. +MOCILIB=false +NEMO_TOOLS=fcm:nemo.xm/utils/tools_r4.0-HEAD +NEMO_TOOLS_REV=@16016 +!PP_COMPONENTS=atmos nemocice unicicles archive_verify +# Default name for extracted repository source = "moci_postproc" +PP_SOURCE_DIR=$SOURCE_DIRECTORY/moci +!PP_TARGET_DIR=$CYLC_WORKFLOW_SHARE_DIR/bin +# Unit tests not copied by default +PP_TESTS=true + +# Icebergs not currently used in rose-stem - nemotools not required +[file:$SOURCE_DIRECTORY/nemotools/icb_combrest.py] +source=$NEMO_TOOLS/REBUILD_NEMO/icb_combrest.py$NEMO_TOOLS_REV + +[!file:$SOURCE_DIRECTORY/nemotools/icb_pp.py] +source=$NEMO_TOOLS/MISCELLANEOUS/icb_pp.py$NEMO_TOOLS_REV diff --git a/rose-stem/app/extract_source/opt/rose-app-coupled.conf b/rose-stem/app/extract_source/opt/rose-app-coupled.conf new file mode 100644 index 00000000..972c8333 --- /dev/null +++ b/rose-stem/app/extract_source/opt/rose-app-coupled.conf @@ -0,0 +1,21 @@ +[command] +default=echo "[INFO] Extracting UM and related repositories" + +[file:$SOURCE_DIRECTORY/casim] +source=git:${GIT_LOC}MetOffice/casim.git::./::2025.12.1 + +[file:$SOURCE_DIRECTORY/jules] +source=git:${GIT_LOC}/MetOffice/jules.git::./::2025.12.1 + +[file:$SOURCE_DIRECTORY/shumlib] +source=git:${GIT_LOC}MetOffice/shumlib.git::./::2025.10.1 + +[file:$SOURCE_DIRECTORY/socrates] +source=git:${GIT_LOC}MetOffice/socrates.git::./::2025.12.1 + +[file:$SOURCE_DIRECTORY/ukca] +source=git:${GIT_LOC}MetOffice/ukca.git::./::2025.12.1 + +[file:$SOURCE_DIRECTORY/um] +# UM repository is private - Use local mirror +source=git:${GIT_LOC}MetOffice/um.git::./::vn14.0 diff --git a/rose-stem/app/extract_source/rose-app.conf b/rose-stem/app/extract_source/rose-app.conf new file mode 100644 index 00000000..90364b5c --- /dev/null +++ b/rose-stem/app/extract_source/rose-app.conf @@ -0,0 +1,8 @@ +[command] +default=python rose_stem_extract_source.py + +[file:get_git_sources.py] +source=git:https://github.com/MetOffice/SimSys_Scripts.git::github_scripts/get_git_sources.py::main + +[file:rose_stem_extract_source.py] +source=git:https://github.com/MetOffice/SimSys_Scripts.git::github_scripts/rose_stem_extract_source.py::main diff --git a/rose-stem/app/fcm_make_um/file/fcm-make_main.cfg b/rose-stem/app/fcm_make_um/file/fcm-make_main.cfg index 411ff93d..6d4afeb3 100644 --- a/rose-stem/app/fcm_make_um/file/fcm-make_main.cfg +++ b/rose-stem/app/fcm_make_um/file/fcm-make_main.cfg @@ -3,11 +3,12 @@ use = $prebuild $flags_coupling = -I$prism_path/build/lib/psmile.MPI1 $ldflags_coupling = -L$prism_path/lib -lpsmile.MPI1 -lmct -lmpeu -lscrip -include = $config_root_path/fcm-make/$platform_config_dir/um-$config_type-$optimisation_level.cfg$config_revision - -extract.location{diff}[um] = $um_sources -extract.location{diff}[ukca] = $ukca_sources -extract.location{diff}[shumlib] = $shumlib_sources -extract.location{diff}[casim] = $casim_sources -extract.location{diff}[jules] = $jules_sources -extract.location{diff}[socrates] = $socrates_sources +include = $config_root_path/fcm-make/$platform_config_dir/um-$config_type-$optimisation_level.cfg + +extract.location{diff}[um] = $source_directory_um +extract.location{diff}[casim] = $source_directory_casim +extract.location{diff}[jules] = $source_directory_jules +extract.location{diff}[socrates] = $source_directory_socrates +extract.location{diff}[shumlib] = $source_directory_shumlib +extract.location{diff}[ukca] = $source_directory_ukca + diff --git a/rose-stem/app/fcm_make_um/file/fcm-make_slash.cfg b/rose-stem/app/fcm_make_um/file/fcm-make_slash.cfg index 8ba44f2e..f4a1e00e 100644 --- a/rose-stem/app/fcm_make_um/file/fcm-make_slash.cfg +++ b/rose-stem/app/fcm_make_um/file/fcm-make_slash.cfg @@ -5,9 +5,9 @@ $ldflags_coupling = -L\$prism_path/lib -lpsmile.MPI1 -lmct -lmpeu -lscrip include = $config_root_path/fcm-make/$platform_config_dir/um-$config_type-$optimisation_level.cfg$config_revision -extract.location{diff}[um] = $um_sources -extract.location{diff}[ukca] = $ukca_sources -extract.location{diff}[shumlib] = $shumlib_sources -extract.location{diff}[casim] = $casim_sources -extract.location{diff}[jules] = $jules_sources -extract.location{diff}[socrates] = $socrates_sources +extract.location{diff}[um] = $source_directory_um +extract.location{diff}[casim] = $source_directory_casim +extract.location{diff}[jules] = $source_directory_jules +extract.location{diff}[socrates] = $source_directory_socrates +extract.location{diff}[shumlib] = $source_directory_shumlib +extract.location{diff}[ukca] = $source_directory_ukca diff --git a/rose-stem/app/fcm_make_um/rose-app.conf b/rose-stem/app/fcm_make_um/rose-app.conf index ac243a45..1ad1af23 100644 --- a/rose-stem/app/fcm_make_um/rose-app.conf +++ b/rose-stem/app/fcm_make_um/rose-app.conf @@ -3,8 +3,6 @@ meta=um-fcm-make/vn13.1 [env] COUPLER=oasis3_mct DR_HOOK=false -casim_rev=um13.1 -casim_sources= compile_atmos=preprocess-atmos build-atmos !!compile_createbc=preprocess-createbc build-createbc !!compile_crmstyle_coarse_grid=preprocess-crmstyle_coarse_grid build-crmstyle_coarse_grid @@ -13,16 +11,14 @@ compile_recon=preprocess-recon build-recon !!compile_scm=preprocess-scm build-scm !!compile_sstpert_lib=preprocess-sstpert_lib build-sstpert_lib !!compile_wafccb_lib=preprocess-wafccb_lib build-wafccb_lib -config_revision=@vn13.1 -config_root_path=fcm:um.xm_tr +config_revision= +config_root_path=$SOURCE_DIRECTORY/um config_type=atmos eccodes=false extract=extract fcflags_overrides= gcom_root_path=/common/internal/umdir/lib/cce-15.0.0/gcom/gcom-7.9.2 gwd_ussp_precision=double -jules_rev=um13.1 -jules_sources= land_surface_model=jules ldflags_overrides_prefix= ldflags_overrides_suffix= @@ -39,17 +35,12 @@ platform_config_dir=$CONFIG portio_version=2A prebuild= !!recon_mpi=parallel -shumlib_rev=um13.1 -shumlib_sources= -socrates_rev=um13.1 -socrates_sources= -stash_version=1A -thread_utils=false -timer_version=3A -ukca_rev=um13.1 -ukca_sources= -um_rev=vn13.1 -um_sources= +source_directory_casim=$SOURCE_DIRECTORY/casim +source_directory_jules=$SOURCE_DIRECTORY/jules +source_directory_shumlib=$SOURCE_DIRECTORY/shumlib +source_directory_socrates=$SOURCE_DIRECTORY/socrates +source_directory_ukca=$SOURCE_DIRECTORY/ukca +source_directory_um=$SOURCE_DIRECTORY/um [file:fcm-make.cfg] -source=$ROSE_SUITE_DIR/app/fcm_make_um/file/fcm-make_${FCM_MAKE_FILE}.cfg +source=$CYLC_WORKFLOW_RUN_DIR/app/fcm_make_um/file/fcm-make_${FCM_MAKE_FILE}.cfg diff --git a/rose-stem/app/test_plot_cpmip_summary/rose-app.conf b/rose-stem/app/test_plot_cpmip_summary/rose-app.conf index 378f5825..6181864e 100644 --- a/rose-stem/app/test_plot_cpmip_summary/rose-app.conf +++ b/rose-stem/app/test_plot_cpmip_summary/rose-app.conf @@ -1,4 +1,4 @@ meta=coupled_drivers/plot_loadbalance/drivers_2.0.1 [command] -default=run_python_env.sh $CYLC_SUITE_SHARE_DIR/fcm_make_drivers/extract/drivers/Coupled_Drivers/driver_utilities/plot_cpmip_summary/plot_cpmip_summary.py --input-file cpmip.output --filename plot_out.png +default=run_python_env.sh $SOURCE_DIRECTORY/moci/Coupled_Drivers/driver_utilities/plot_cpmip_summary/plot_cpmip_summary.py --input-file cpmip.output --filename plot_out.png diff --git a/rose-stem/app/um/rose-app.conf b/rose-stem/app/um/rose-app.conf index 42b6ed3c..82473dfc 100644 --- a/rose-stem/app/um/rose-app.conf +++ b/rose-stem/app/um/rose-app.conf @@ -1,4 +1,4 @@ -meta=um-atmos/vn13.1 +meta=um-atmos/vn14.0 [command] default=um-atmos @@ -25,7 +25,7 @@ RECON_KEEP_MPP_STDOUT=true RECON_STDOUT_FILE=pe_output/${RUNID}.fort6.pe SPECTRAL_FILE_DIR=$UMDIR/vn${UMVN}/ctldata/spectral/ga7_1 UM_THREAD_LEVEL=MULTIPLE -VN=13.1 +VN=14.0 [file:$DATAM] mode=mkdir @@ -36,7 +36,7 @@ mode=mkdir source= [file:ATMOSCNTL] -source=namelist:configid namelist:nlstcgen namelist:nlst_mpp namelist:run_track namelist:run_calc_pmsl namelist:lbc_options namelist:run_nudging namelist:run_sl namelist:run_diffusion namelist:run_cosp namelist:radfcdia namelist:r2swclnl namelist:r2lwclnl namelist:clmchfcg namelist:acp namelist:acdiag namelist:jules_nvegparm namelist:jules_pftparm namelist:jules_triffid namelist:jules_elevate (namelist:jules_deposition_species(:)) (namelist:jules_deposition_species_specific) namelist:iau_nl namelist:tuning_segments (namelist:nlstcall_pp(:)) (namelist:nlstcall_nc(:)) namelist:nlstcall_nc_options +source=namelist:configid namelist:nlstcgen namelist:nlst_mpp namelist:run_track namelist:run_calc_pmsl namelist:lbc_options namelist:run_nudging namelist:run_sl namelist:run_diffusion namelist:radfcdia namelist:r2swclnl namelist:r2lwclnl namelist:clmchfcg namelist:acp namelist:acdiag namelist:jules_nvegparm namelist:jules_pftparm (namelist:jules_red) (namelist:jules_triffid) namelist:jules_elevate (namelist:jules_deposition_species(:)) (namelist:jules_deposition_species_specific) namelist:iau_nl namelist:tuning_segments (namelist:nlstcall_pp(:)) (namelist:nlstcall_nc(:)) namelist:nlstcall_nc_options [!!file:HYBRID_JNR2SNR] source=namelist:hybrid_cpl namelist:hybrid_jnr @@ -60,7 +60,7 @@ source=namelist:oasis_send_nml source=namelist:recon_technical namelist:recon_science namelist:recon_vertical namelist:horizont namelist:headers (namelist:trans(:)) (namelist:recon_idealised) (namelist:ideal_free_tracer(:)) [file:SHARED] -source=(namelist:nlcfiles) namelist:nlstcall namelist:ancilcta namelist:temp_fixes namelist:jules_temp_fixes namelist:carbon_options namelist:coupling_control namelist:model_domain namelist:planet_constants namelist:run_dust namelist:run_glomap_aeroclim namelist:run_ukca namelist:run_gwd namelist:run_vera namelist:run_murk namelist:run_convection (namelist:run_comorph) namelist:run_bl namelist:run_precip namelist:run_radiation namelist:run_cloud namelist:run_aerosol namelist:lam_config namelist:run_ozone namelist:run_free_tracers namelist:run_eng_corr namelist:gen_phys_inputs namelist:run_dyn namelist:run_dyntest namelist:easyaerosol namelist:jules_model_environment namelist:jules_surface_types namelist:jules_surface namelist:jules_radiation namelist:jules_hydrology namelist:jules_rivers (namelist:jules_overbank) namelist:jules_water_resources namelist:jules_sea_seaice namelist:jules_soil namelist:jules_vegetation namelist:jules_irrig namelist:jules_soil_biogeochem namelist:jules_snow (namelist:jules_urban) namelist:jules_deposition namelist:run_stochastic namelist:run_electric namelist:ancilcta (namelist:items(:)) namelist:exceptions +source=(namelist:nlcfiles) namelist:nlstcall namelist:ancilcta namelist:temp_fixes namelist:jules_temp_fixes namelist:carbon_options namelist:coupling_control namelist:model_domain namelist:planet_constants namelist:run_dust namelist:run_glomap_aeroclim namelist:run_ukca namelist:run_gwd namelist:run_vera namelist:run_murk namelist:run_convection (namelist:run_comorph) namelist:run_bl namelist:run_precip namelist:run_radiation namelist:run_cloud namelist:run_aerosol namelist:lam_config namelist:run_ozone namelist:run_free_tracers namelist:run_eng_corr namelist:gen_phys_inputs namelist:run_dyn namelist:run_dyntest namelist:run_cosp namelist:easyaerosol namelist:jules_model_environment namelist:jules_surface_types namelist:jules_surface namelist:jules_radiation namelist:jules_hydrology namelist:jules_rivers (namelist:jules_overbank) namelist:jules_water_resources namelist:jules_sea_seaice namelist:jules_soil namelist:jules_vegetation namelist:jules_irrig namelist:jules_soil_biogeochem namelist:jules_snow (namelist:jules_urban) namelist:jules_deposition namelist:run_stochastic namelist:run_electric namelist:ancilcta (namelist:items(:)) namelist:exceptions [file:SIZES] source=namelist:nlsizes @@ -1040,6 +1040,7 @@ update_anc=.false. !!user_prog_rconst=0.0 [namelist:jules_deposition] +!!dep_h2_soil_scheme=1 !!dry_dep_model=1 !!dzl_const=-1073741824 l_deposition=.false. @@ -1078,6 +1079,8 @@ l_elev_absolute_height=9*.false. surf_hgt_io=9*0.00 [namelist:jules_hydrology] +!!b_pdm=1.0 +!!dz_pdm=1.0 l_hydrology=.true. l_limit_gsoil=.false. l_pdm=.false. @@ -1088,6 +1091,9 @@ l_wetland_unfrozen=.false. nfita=30 !!s_pdm=0.0 !!slope_pdm_max=6.0 +ti_max=10.0 +ti_wetl=1.5 +zw_max=6.0 [namelist:jules_irrig] !!frac_irrig_all_tiles=.false. @@ -1122,13 +1128,11 @@ z0hm_nvg_io=1.00000e-7,2.50000e-1,2.00000e-1,2.00000e-1 !!coef_b=0.0 !!ent_ratio=0.0 !!exp_c=0.0 -!!l_riv_hypsometry=.false. -!!l_riv_overbank=.false. +!!overbank_model=1 !!riv_a=0.0 !!riv_b=0.0 !!riv_c=0.0 !!riv_f=0.0 -!!use_rosgen=.false. [namelist:jules_pftparm] a_wl_io=0.65,0.65,0.005,0.005,0.10 @@ -1175,14 +1179,22 @@ eta_sl_io=5*0.01 f0_io=0.875,0.875,0.900,0.800,0.900 fd_io=0.015,0.015,0.015,0.025,0.015 !!fef_bc_io=0.56 0.56 0.56 0.47 0.56 +!!fef_c2h4_io=1.11E+00,1.54E+00,8.30E-01,1.99E+00,8.30E-01 +!!fef_c2h6_io=8.80E-01,9.70E-01,4.20E-01,1.01E+00,4.20E-01 +!!fef_c3h8_io=5.30E-01,2.90E-01,1.30E-01,3.12E-01,1.30E-01 !!fef_ch4_io=6.8 4.8 4.8 2.4 4.8 !!fef_co2_io=1631 1576 1576 1654 1576 !!fef_co_io=100 106 106 64 106 +!!fef_dms_io=2.00E-03,2.00E-03,8.00E-03,1.92E-02,8.00E-03 +!!fef_hcho_io=2.40E+00,1.75E+00,1.23E+00,2.95E+00,1.23E+00 +!!fef_mecho_io=2.26E+00,8.10E-01,8.40E-01,2.02E+00,8.40E-01 +!!fef_nh3_io=1.33E+00,2.50E+00,8.90E-01,2.14E+00,8.90E-01 !!fef_nox_io=2.55 3.24 3.24 2.49 3.24 !!fef_oc_io=4.3 9.1 9.1 3.2 9.1 !!fef_so2_io=0.40 0.40 0.40 0.48 0.40 !!fire_mort_io=5*1.0 !!fl_o3_ct_io=5*-1073741824 +!!fsmc_mod_io=5*0 fsmc_of_io=5*0.00 fsmc_p0_io=5*0.0 !!g1_stomata_io=5*2.0 @@ -1190,7 +1202,7 @@ g_leaf_0_io=5*0.25 glmin_io=5*1.0e-6 !!gpp_st_io=1.29E-07,2.58E-08,2.07E-07,3.42E-07,1.68E-007 gsoil_f_io=5*1.0 -hw_sw_io=5*0.5 +!!hw_sw_io=5*0.5 !!ief_io=25.0,8.00,16.00,24.00,20.00 infil_f_io=4.00000,4.00000,2.00000,2.00000,2.00000 !!jv25_ratio_io=5*1.97 @@ -1199,15 +1211,15 @@ kn_io=5*0.78 knl_io=5*0.20 kpar_io=5*0.50 lai_alb_lim_io=5*0.005 -lma_io=0.0824,0.2263,0.0498,0.1370,0.0695 +!!lma_io=0.0824,0.2263,0.0498,0.1370,0.0695 !!mef_io=0.9,1.8,0.6,0.9,0.57 neff_io=0.8e-3,0.8e-3,0.8e-3,0.4e-3,0.8e-3 nl0_io=0.040,0.030,0.060,0.030,0.030 -nmass_io=0.0210,0.0115,0.0219,0.0131,0.0219 -nr_io=0.01726,0.00784,0.0162,0.0084,0.01726 +!!nmass_io=0.0210,0.0115,0.0219,0.0131,0.0219 +!!nr_io=0.01726,0.00784,0.0162,0.0084,0.01726 nr_nl_io=5*1.00 ns_nl_io=0.10,0.10,1.00,1.00,0.10 -nsw_io=0.0072,0.0083,0.01604,0.0202,0.0072 +!!nsw_io=0.0072,0.0083,0.01604,0.0202,0.0072 omega_io=0.15,0.15,0.15,0.17,0.15 omegal_io=0.10,0.10,0.10,0.12,0.10 omegau_io=0.23,0.23,0.35,0.35,0.35 @@ -1221,12 +1233,18 @@ q10_leaf_io=5*2.00 r_grow_io=5*0.25 rootd_ft_io=3.00000,1.00000,5.00000e-1,5.00000e-1,5.00000e-1 sigl_io=0.0375,0.1000,0.0250,0.0500,0.0500 +!!sox_a_io=5*0.0 +!!sox_p50_io=5*0.0 +!!sox_rp_min_io=5*0.0 +!!sug_g0_io=5*1.0 +!!sug_grec_io=5*1.0 +!!sug_yg_io=5*1.0 !!tef_io=1.2,2.4,0.8,1.2,0.8 tleaf_of_io=273.15,243.15,258.15,258.15,243.15 tlow_io=0.0,-5.0,0.0,13.0,0.0 tupp_io=36.0,31.0,36.0,45.0,36.0 -vint_io=5.73,6.32,6.42,0.00,14.71 -vsl_io=29.81,18.15,40.96,10.24,23.15 +!!vint_io=5.73,6.32,6.42,0.00,14.71 +!!vsl_io=29.81,18.15,40.96,10.24,23.15 z0hm_classic_pft_io=5*1.00000e-1 z0hm_pft_io=1.00,1.000,1.00000e-2,1.00000e-2,1.00000e-2 z0v_io=1.1,1.1,0.22,0.22,1.0 @@ -1251,6 +1269,21 @@ l_spec_sea_alb=.true. !!swdn_frac_albsoil=0.5 wght_alb=0.0,0.5,0.0,0.5 +[!!namelist:jules_red] +!!alpha_recrt=5*0.0 +!!crwn_area0=5*0.0 +!!dom_order=5*0 +!!height0=5*0.0 +!!lai_bal0=5*0.0 +!!mass0=5*0.0 +!!massi=5*0.0 +!!mclass=5*0 +!!mort_base=5*0.0 +!!phi_a=5*0.50 +!!phi_g=5*0.75 +!!phi_h=5*0.25 +!!phi_l=5*0.25 + [namelist:jules_rivers] !!a_thresh=1 !!cbland=0.1 @@ -1259,13 +1292,19 @@ wght_alb=0.0,0.5,0.0,0.5 !!criver=0.62 i_river_vn=1 l_inland=.true. +!!l_riv_overbank=.false. l_rivers=.true. +lake_water_conserve_method=1 nstep_rivers=$NSTEP_RIVERS !!retl=0.0 !!retr=0.005 rivers_meander=1.400000 rivers_speed=0.400000 !!runoff_factor=1.0 +trip_globe_shape=2 + +[!!namelist:jules_rivers_props] +!!rivers_regrid=.false. [namelist:jules_sea_seaice] a_chrn_coare=0.0016 @@ -1388,6 +1427,7 @@ l_soil_sat_down=.true. !!l_tile_soil=.false. l_vg_soil=.true. !!ns_deep=100 +sm_levels=4 soilhc_method=2 zsmc=1.0 zst=1.0 @@ -1406,7 +1446,7 @@ const_ch4_resps=4.36e-3 !!frz_ch4=0.5 !!k2_ch4=0.01 kaps=0.5e-8 -!!kaps_roth=3.22e-7,9.65e-9,2.12e-8,6.43e-10 +!!kaps_4pool=3.22e-7,9.65e-9,2.12e-8,6.43e-10 !!kd_ch4=0.0003 l_ch4_interactive=.false. l_ch4_microbe=.false. @@ -1430,9 +1470,12 @@ t0_ch4=273.15 !!tau_ch4=6.5 !!tau_lit=5.0 !!tau_resp=2.0 +!!z_burn_max=0.2 [namelist:jules_surface] all_tiles=1 +!!anthrop_heat_mean=20.0 +!!anthrop_heat_option=0 beta_cnv_bl=0.04 cor_mo_iter=4 fd_hill_option=2 @@ -1483,13 +1526,14 @@ l_fix_albsnow_ts=.true. !!l_fix_improve_drydep=.true. l_fix_lake_ice_temperatures=.false. l_fix_moruses_roof_rad_coupling=.true. +l_fix_neg_snow=.false. l_fix_osa_chloro=.true. l_fix_snow_frac=.false. !!l_fix_ukca_h2dd_x=.true. l_fix_ustar_dust=.true. l_fix_wind_snow=.true. -[namelist:jules_triffid] +[!!namelist:jules_triffid] !!ag_expand_io=5*0 alloc_fast_io=0.6,0.6,1.0,1.0,0.8 alloc_med_io=0.3,0.4,0.0,0.0,0.2 @@ -1549,10 +1593,12 @@ l_limit_canhc=.true. !!l_o3_damage=.false. l_phenol=.false. !!l_prescsow=.false. +!!l_red=.false. !!l_rsl_scalar=.false. l_scale_resp_pm=.false. l_spec_veg_z0=.true. l_stem_resp_fix=.false. +!!l_sugar=.false. l_trait_phys=.false. !!l_trif_biocrop=.false. !!l_trif_crop=.false. @@ -1588,10 +1634,12 @@ stomata_model=1 !!l_water_transfers=.false. !!nr_gwater_model=0 !!nstep_water_res=1 +!!partition_method=2 !!priority='irr' !!rf_domestic=0.1 !!rf_industry=0.1 !!rf_livestock=0.1 +!!sfc_water_factor=2.0 [namelist:lam_config] !!delta_lat=0 @@ -1608,6 +1656,7 @@ stomata_model=1 !!rimweightsa=0 [!!namelist:lustre_control] +default_ost_count=0 default_stripe_count=0 default_stripe_offset=-1 default_stripe_pattern=0 @@ -1616,6 +1665,7 @@ default_stripe_size=0 number_custom_files=0 [!!namelist:lustre_control_custom_files] +!!ost_count=0 !!stripe_count=0 !!stripe_offset=-1 !!stripe_pattern=0 @@ -1851,6 +1901,7 @@ dump_packim=3 dumpfreqim=1 !!dumptimesim=40*0 !!end_of_run_dump=.false. +!!greg_dump_ref=0 !!greg_monthly_dump=.false. i_dump_output=2 l_meaning_sequence=.false. @@ -1863,9 +1914,12 @@ l_meaning_sequence=.false. !!ppselectim=1,1,1,1 !!ppxm=5 !!psum_filename_base='$DATAM/${RUNID}a_s' +!!psum_header_time=0 secs_per_periodim=86400 steps_per_periodim=$ATMOS_TIMESTEPS_PER_DAY +[!!namelist:oasis_rivers] + [!!namelist:oasis_send_nml] !!hybrid_rmp_mapping='BILINEAR' !!hybrid_weight=500 @@ -1885,6 +1939,8 @@ l_planet_g=.false. !!l_planet_intrinsic_flux=.false. l_planet_orbit=.false. l_set_planet_rotation=.false. +l_time_mean_solar_irradiance=.true. +!!l_ve_at_epoch=.false. !!lapse=0.0 !!omega=0.0 !!planet_a=0.0 @@ -2261,17 +2317,22 @@ a_ent_shr_nml=1.6 alpha_cd_in=2.0,1.5 !!bdy_tke=1 bl_res_inv=1 +blend_height_opt=0 blending_option=0 +c_gust=4.0 calc_prob_of_vis=0.400 !!cbl_mix_fac_nml=0.0 cbl_op=2 dec_thres_cloud=0.1 +dzrad_disc_opt=1 entr_smooth_dec=1 flux_bc_opt=0 flux_grad=0 fric_heating=1 !!fric_number=0 +!!h_blend_fix=0.0 i_bl_vn=3 +i_impsolve_loc=2 i_interp_local=1 idyndiag=3 keep_ri_fa=1 @@ -2280,6 +2341,7 @@ kprof_cu=2 !!l_adv_turb_field=.true. l_bl_mix_qcf=.false. l_conv_tke=.true. +l_converge_ga=.false. !!l_local_above_tkelvs=.false. !!l_my_condense=.false. !!l_my_ini_zero=.false. @@ -2291,6 +2353,7 @@ l_noice_in_turb=.false. l_reset_dec_thres=.true. l_reset_neg_q=.true. !!l_shcu_buoy=.false. +l_use_sml_dsc_fixes=.false. l_use_surf_in_ri=.false. l_use_var_fixes=.false. lambda_min_nml=40.0 @@ -2299,11 +2362,13 @@ local_fa=1 !!my_lowest_pd_surf=0 !!my_z_limit_elb=0 near_neut_z_on_l=1.6 +ng_stress=2 +num_sweeps_bflux=3 pstb=2.000 puns=1.0 -relax_sc_over_cu=0 ritrans=0.1 sbl_op=6 +sc_diag_opt=0 sg_orog_mixing=0 !!shallow_cu_maxtop=5000.0 !!shcu_levels=0 @@ -2324,17 +2389,22 @@ npmsl_height=500.00 [namelist:run_cloud] allicetdegc=-20.00 cff_spread_rate=1.00e-5 +cloud_pc2_tol=0.005 +cloud_pc2_tol_2=0.001 dbsdtbs_turb_0=1.50e-4 +!!ent_coef_bm=0.2 ez_max_bm=400.0 falliceshear_method=2 forced_cu=2 forced_cu_fac=0.5 +i_bm_ez_opt=1 i_cld_area=0 i_cld_vn=2 !!i_eacf=0 i_pc2_checks_cld_frac_method=2 i_pc2_conv_coupling=3 i_pc2_erosion_method=3 +i_pc2_erosion_numerics=1 i_pc2_homog_g_method=1 i_pc2_init_logic=1 i_pc2_init_method=1 @@ -2342,7 +2412,8 @@ i_rhcpt=2 ice_fraction_method=1 ice_width=0.020 l_add_cca_to_mcica=.true. -l_bm_ez_subcrit_only=.false. +l_bm_sigma_s_grad=.false. +l_bm_tweaks=.false. l_ceil_cld_filter=.false. l_cloud_call_b4_conv=.false. l_ensure_min_in_cloud_qcf=.false. @@ -2350,14 +2421,16 @@ l_micro_eros=.false. l_od_cld_filter=.true. l_pc2_check_init=.true. l_pc2_homog_conv_pressure=.false. -l_pc2_implicit_erosion=.false. !!l_pc2_lbc=.false. l_pc2_sl_advection=.false. l_sharpen_cbh_diags=.false. l_subgrid_qv=.true. +max_sigmas=3.0 +min_sigx_ft=0.0 rhcrit=0.920,0.918,0.916,0.912,0.908,0.903,79*0.900 starticetkelvin=263.15 tau_thresh=0.01 +turb_var_fac_bm=1.0 [!!namelist:run_comorph] ass_min_radius=500.0 @@ -2368,9 +2441,11 @@ col_eff_coef=1.0 core_ent_fac=1.0 drag_coef_cond=0.5 drag_coef_par=0.5 +dx_ref=50000.0 ent_coef=0.2 hetnuc_temp=263.0 l_core_ent_cmr=.true. +l_resdep_precipramp=.false. n_dndraft_types=1 overlap_power=0.5 par_gen_core_fac=3.0 @@ -2402,6 +2477,7 @@ anvil_factor=1.0000 !!betts_rh=0.7 !!betts_tau=7200.0 bl_cnv_mix=1 +c_mass_sh=0.03 cape_bottom=5 cape_min=0.5 cape_timescale=3600 @@ -2439,6 +2515,7 @@ eff_dcfl=1.0 !!ent_dp_power=1.00 ent_fac_dp=1.0 ent_fac_md=0.90 +ent_fac_sh=1.0 !!ent_md_power=0 ent_opt_dp=7 ent_opt_md=0 @@ -2460,7 +2537,9 @@ l_anvil=.true. !!l_ccp_blv=.false. !!l_ccp_parcel_dp=.false. !!l_ccp_parcel_md=.false. +!!l_ccp_seabreeze=.false. !!l_ccp_trig=.false. +!!l_ccp_wind=.false. l_ccrad=.true. l_cloud_deep=.true. l_cmt_heating=.true. @@ -2470,6 +2549,7 @@ l_conv_prog_dtheta=.true. l_conv_prog_flx=.false. l_conv_prog_precip=.true. l_cv_conserve_check=.false. +l_cvdiag_ctop_qmax=.false. l_eman_dd=.false. l_fcape=.false. !!l_fix_udfactor=.false. @@ -2498,6 +2578,7 @@ mid_cnv_pmin=10000.00 midtrig_opt=0 mparwtr=1.0000e-3 n_conv_calls=2 +orig_mdet_fac=1.0 !!phi_ccp=1.0e-3 plume_water_load=0 pr_melt_frz_opt=2 @@ -2626,15 +2707,18 @@ qlimit=1.000e-8 w_print_limit=0 [namelist:run_dust] +dust_bl_mixfac=1.0 dust_veg_emiss=1 horiz_d=2.25 i_dust=1 +l_dust_clay_as_max=.false. l_dust_div1_lbc=.false. l_dust_div2_lbc=.false. l_dust_div3_lbc=.false. l_dust_div4_lbc=.false. l_dust_div5_lbc=.false. l_dust_div6_lbc=.false. +l_dust_emp_sc=.false. l_fix_size_dist=.false. l_twobin_dust=.false. pwsdiag_sfc_em=0.0 @@ -2738,6 +2822,7 @@ l_pv_tracer=.false. l_theta_tracer=.false. l_wtrac=.false. !!n_noniso_wtrac=0 +!!n_noniso_wtrac_jls=0 !!n_norm_wtrac=1 !!qlimit_h218o_wtrac=-1 !!qlimit_hdo_wtrac=-1 @@ -2747,17 +2832,21 @@ l_wtrac=.false. !!gclmacsw='unset' !!gclmanlw='unset' !!gclmansw='unset' +!!gclmcnlw='unset' +!!gclmcnsw='unset' !!gclmcrlw='unset' !!gclmcrsw='unset' !!gclmprec='unset' +!!gclmsulw='unset' +!!gclmsusw='unset' !!i_glomap_clim_activation_scheme=0 !!i_glomap_clim_nwbins=20 !!i_glomap_clim_setup=2 +!!i_glomap_clim_tune_bc=0 !!l_glomap_clim_aie1=.false. !!l_glomap_clim_aie2=.false. !!l_glomap_clim_radaer=.false. !!l_glomap_clim_radaer_sustrat=.false. -!!l_glomap_clim_tune_bc=.false. l_glomap_mode_clim=.false. [namelist:run_gwd] @@ -2783,6 +2872,8 @@ var=0.18 wavelstar=4300.00 [namelist:run_murk] +!!krain_murk=0.0 +!!ksnow_murk=0.0 l_murk=.false. !!l_murk_advect=.true. !!l_murk_lbc=.false. @@ -2798,15 +2889,20 @@ l_nudging=.false. !!ndg_dampwidth=-1.0 !!ndg_datapath='' !!ndg_filename_template=' ',' ' +!!ndg_freq=720.0 !!ndg_hours_perdata= !!ndg_lat_max=99.0 !!ndg_lat_min=-99.0 +!!ndg_length_scale=0.3 !!ndg_lev_bottom= !!ndg_lev_top=0 !!ndg_lon_max=361.0 !!ndg_lon_min=-1.0 +!!ndg_method=0 !!ndg_on_lev_bottom=0 !!ndg_on_lev_top=0 +!!ndg_ramp_end=0 +!!ndg_ramp_start=0 !!ndg_relax_tvalue=0 !!ndg_relax_uvalue=0 !!ndg_relax_vvalue=0 @@ -2833,9 +2929,12 @@ di_input=4.16e-01 dic_input=1.0 !!fcrit=1.0 graupel_option=0 +heavy_rain_evap_fac=0.0 i_mcr_iter=2 +!!i_update_precfrac=1 !!l_autoconv_murk=.false. l_casim=.false. +!!l_casim_warmstart=.false. l_diff_icevt=.true. l_droplet_tpr=.false. l_fsd_generator=.true. @@ -2846,10 +2945,12 @@ l_mcr_precfrac=.false. !!l_mcr_qgraup_lbc=.false. l_mcr_qrain=.true. !!l_mcr_qrain_lbc=.false. +!!l_micro_in_rim=.false. l_mphys_nonshallow=.false. l_orograin=.false. !!l_orograin_block=.false. !!l_orogrime=.false. +l_proc_fluxes=.false. l_progn_tnuc=.false. l_psd=.true. l_rain=.true. @@ -2862,7 +2963,9 @@ l_subgrid_qcl_mp=.true. !!l_use_seasalt_autoconv=.false. !!l_use_sulphate_autoconv=.true. l_warm_new=.true. +mp_czero=10.0 mp_dz_scal=1.0 +mp_tau_lim=1200.0 !!ndrop_surf=20.0e6 !!niters_mp=0 !!nscalesf=1.0 @@ -2898,6 +3001,9 @@ h_swbands=6 hcfc22mmr=0 hfc125mmr=0.0 hfc134ammr=9.0535e-10 +!!i_calc_orbprm_year=-32768 +!!i_cloud_entrapment=2 +!!i_cloud_entrapment_2=2 i_cloud_representation=2 !!i_cloud_representation_2=2 i_fsd=2 @@ -2915,6 +3021,7 @@ i_sw_radstep_perday_prog=24 inhom_cloud_lw=0 inhom_cloud_sw=0 l_bs1999_abundances=.false. +l_calc_orbprm_year=.false. l_consistent_cdnc=.true. l_extra_top=.false. l_fsd_eff_res=.true. @@ -2983,6 +3090,12 @@ tr_priestley_opt=2 !!a_ent_1_rp=0,0,0 !!a_ent_shr_rp=5.0 !!a_ent_shr_rp_max=8.7 +!!alnir_rp=0.45,0.35,0.58,0.58,0.58 +!!alnir_rp_max=0.45,0.35,0.58,0.58,0.58 +!!alnir_rp_min=0.45,0.35,0.58,0.58,0.58 +!!alpar_rp=0.10,0.07,0.10,0.10,0.10 +!!alpar_rp_max=0.10,0.07,0.10,0.10,0.10 +!!alpar_rp_min=0.10,0.07,0.10,0.10,0.10 !!alphac=2.00e-3 br=0.027 !!cbl_mix_fac_rp=0.0,0.0,0.0 @@ -2992,15 +3105,15 @@ cdispfac=1.00 conv_std=1.73 !!cs_rp=0.2,0.2,0.2 !!decorr_ts_pert_theta=600.0 -!!dz0v_dh_rp=5.00000e-2,5.00000e-2,1.00000e-1,1.00000e-1,1.00000e-1 -!!dz0v_dh_rp_max=5.00000e-2,5.00000e-2,1.00000e-1,1.00000e-1,1.00000e-1 -!!dz0v_dh_rp_min=5.00000e-2,5.00000e-2,1.00000e-1,1.00000e-1,1.00000e-1 +!!fxd_cld_num_rp=150.0e+06,150.0e+06,150.0e+06 !!g0_rp=0,0,0 !!g1_rp=0,0,0 gwd_std=1.5 i_pert_theta=0 !!i_pert_theta_type=0 !!i_rp_scheme=0 +!!ice_fspd_rp=6000000.0,6000000.0,6000000.0 +!!l_only_pert_near_edge=.false. !!l_pert_all_points=.false. !!l_pert_shape=.false. l_rp2=.false. @@ -3023,15 +3136,26 @@ l_spt_qcons=.true. l_spt_rad=.true. l_spt_rain=.true. l_x_eq_sin_x=.false. +!!lai_mult_rp=5*1.0 !!lai_mult_rp_max=5*1.5 !!lai_mult_rp_min=5*0.5 !!lambda_min_rp=0,0,0 !!m_ci_rp=0,0,0 !!mag_pert_theta=0.5 +!!mp_czero_rp=10.0,10.0,10.0 +!!mpof_rp=0.5,0.5,0.5 !!ndrop_surf_rp=2.0e+07,7.5e+07,10.0e+07 +!!npts_pert_from_edge=0 !!npts_pert_theta=8 nsmooth=3 nsmooth_spt=3 +!!omega_rp=0.15,0.15,0.15,0.17,0.15 +!!omega_rp_max=0.15,0.15,0.15,0.17,0.15 +!!omega_rp_min=0.15,0.15,0.15,0.17,0.15 +!!omnir_rp=0.70,0.45,0.83,0.83,0.83 +!!omnir_rp_max=0.70,0.45,0.83,0.83,0.83 +!!omnir_rp_min=0.70,0.45,0.83,0.83,0.83 +!!orog_drag_param_rp=0.15,0.15,0.15 !!par_mezcla_rp=0,0,0 psif_orog_thres=0.5 rad_std=1.73 @@ -3050,6 +3174,7 @@ skeb2_botlev=2 skeb2_cdisp=4 skeb2_sdisp=7 skeb2_toplev=50 +!!snow_fspd_rp=12.0,12.0,12.0 spt_bot_tap_lev=9 spt_botlev=15 spt_top_tap_lev=45 @@ -3061,9 +3186,15 @@ tau_skeb=2.00e+4 tau_spt=2.00e+4 tot_backscat=1.00e-4 !!x1r_rp=0.07,0.22,0.52 +!!z0_soil_rp=1.00000e-3,1.00000e-3,1.00000e-3 +!!z0_urban_mult_rp=1.0,1.0,1.0 !!z0hm_pft_rp=1.65000,1.65000,1.00000e-1,1.00000e-1,1.00000e-1 !!z0hm_pft_rp_max=1.65000,1.65000,1.00000e-1,1.00000e-1,1.00000e-1 !!z0hm_pft_rp_min=1.65000,1.65000,1.00000e-1,1.00000e-1,1.00000e-1 +!!z0hm_soil_rp=2.00000e-1,2.00000e-1,2.00000e-1 +!!z0v_rp=1.1,1.1,0.22,0.22,1.0 +!!z0v_rp_max=1.1,1.1,0.22,0.22,1.0 +!!z0v_rp_min=1.1,1.1,0.22,0.22,1.0 !!zhloc_depth_fac_rp=0.5,0.5,0.5 !!zmax_pert_theta=400.0 !!zmin_pert_theta=0.0 @@ -3077,19 +3208,28 @@ ntop_tc=63 sm=0.1 [namelist:run_ukca] +!!acc_cor_scav_scaling=1.0 +!!anth_so2_ems_scaling=1.0 +!!bc_refrac_im_scaling=1.0 biom_aer_ems_scaling=2.0 chem_timestep=3600 !!dir_strat_aer='' +!!dry_depvel_acc_scaling=1.0 +!!dry_depvel_so2_scaling=1.0 dts0=300 !!fastjx_mode=1 !!fastjx_numwl=8 !!fastjx_prescutoff=0 !!file_strat_aer='' +!!file_volc_so2='$UMDIR/vn$VN/ctldata/UKCA/strat/SO2_expvolc_1979-2021.csv' +!!hno3_uptake_coeff=0.193 i_ageair_reset_method=1 +!!i_chem_timestep_halvings=0 !!i_inferno_emi=20 !!i_mode_bln_param_method=0 i_mode_nzts=15 i_mode_setup=2 +i_primss_method=2 i_ukca_activation_scheme=1 i_ukca_chem=14 !!i_ukca_chem_version=111 @@ -3102,29 +3242,35 @@ i_ukca_nwbins=20 !!i_ukca_photol=2 !!i_ukca_quasinewton_end=3 !!i_ukca_quasinewton_start=2 +!!i_ukca_radaer_prescribe_ssa=0 !!i_ukca_sad_months=0 !!i_ukca_sad_start_year=0 !!i_ukca_scenario=0 !!i_ukca_solcyc=0 !!i_ukca_solcyc_start_year=1882 !!i_ukca_topboundary=1 +i_ukca_tune_bc=0 !!jvscat_file='' !!jvsolar_file=FJX_solcyc_May17.dat !!jvspec_dir='' !!jvspec_file='' +l_aero_rainout=.true. l_bcoc_bf=.true. l_bcoc_bm=.true. l_bcoc_ff=.true. -!!l_dust_slinn_impc_scav=.false. +!!l_dust_mp_ageing=.false. +!!l_dust_mp_slinn_impc_scav=.false. !!l_environ_jo2=.false. !!l_environ_jo2b=.false. l_mode_bhn_on=.true. l_mode_bln_on=.false. +!!l_no3_prod_in_aero_step=.false. l_ukca=.true. l_ukca_ageair=.true. l_ukca_aie1=.true. l_ukca_aie2=.true. !!l_ukca_asad_columns=.false. +!!l_ukca_asad_full=.false. l_ukca_chem_aero=.true. !!l_ukca_classic_hetchem=.false. !!l_ukca_coarse_no3_prod=.false. @@ -3145,6 +3291,8 @@ l_ukca_intph=.false. !!l_ukca_limit_nat=.false. !!l_ukca_linox_scaling=.false. l_ukca_mode=.true. +!!l_ukca_mp_fibre=.false. +!!l_ukca_mp_fragment=.false. !!l_ukca_prescribech4=.false. l_ukca_prim_moc=.true. l_ukca_primbcoc=.true. @@ -3167,15 +3315,17 @@ l_ukca_radaer_sustrat=.true. !!l_ukca_sa_clim=.false. l_ukca_scale_biom_aer_ems=.true. l_ukca_scale_marine_pom_ems=.false. +l_ukca_scale_ppe=.false. l_ukca_scale_sea_salt_ems=.false. l_ukca_scale_seadms_ems=.false. -l_ukca_scale_soa_yield=.true. +!!l_ukca_scale_soa_yield_isop=.false. +l_ukca_scale_soa_yield_mt=.true. !!l_ukca_set_trace_gases=.false. l_ukca_sfix=.false. l_ukca_so2ems_expvolc=.false. +!!l_ukca_so2ems_plumeria=.false. l_ukca_src_in_conservation=.true. !!l_ukca_trophet=.false. -l_ukca_tune_bc=.false. !!l_ukca_use_background_aerosol=.false. !!lightnox_scale_fac=1.0 !!marine_pom_ems_scaling=1.0 @@ -3185,6 +3335,7 @@ mode_activation_dryr=37.5 mode_aitsol_cvscav=0.5 mode_incld_so2_rfrac=0.25 mode_parfrac=2.5 +!!nerupt=178 !!nit=0 !!nrsteps=45 !!ph_fit_coeff_a=0.142 @@ -3193,7 +3344,9 @@ mode_parfrac=2.5 !!phot2d_dir='' !!sea_salt_ems_scaling=1.0 !!seadms_ems_scaling=1.0 -soa_yield_scaling=2.0 +!!sigma_updraught_scaling=1.0 +!!soa_yield_scaling_isop=1.0 +soa_yield_scaling_mt=2.0 !!tc_lbc_ukca=150*0 !!ukca_ccl4mmr=0 !!ukca_cfc115mmr=0 @@ -3230,15 +3383,24 @@ ukca_offline_files='$UM_NETCDF_UKCAOXID_O3_DIR/$UM_NETCDF_UKCAOXID_O3_FILE', ='$UM_NETCDF_UKCAOXID_NO3_DIR/$UM_NETCDF_UKCAOXID_NO3_FILE', ='$UM_NETCDF_UKCAOXID_H2O2_DIR/$UM_NETCDF_UKCAOXID_H2O2_FILE', ='$UM_NETCDF_UKCAOXID_HO2_DIR/$UM_NETCDF_UKCAOXID_HO2_FILE' +!!ukca_radaer_dir='' +!!ukca_radaer_lwabs_file='' +!!ukca_radaer_lwext_file='' +!!ukca_radaer_swabs_file='' +!!ukca_radaer_swext_file='' !!ukca_rcpdir='' !!ukca_rcpfile='' ukcaaclw='$UMDIR/vn$VN/ctldata/UKCA/radaer/ga7_1/nml_ac_lw' ukcaacsw='$UMDIR/vn$VN/ctldata/UKCA/radaer/ga7_1/nml_ac_sw' ukcaanlw='$UMDIR/vn$VN/ctldata/UKCA/radaer/ga7_1/nml_an_lw' ukcaansw='$UMDIR/vn$VN/ctldata/UKCA/radaer/ga7_1/nml_an_sw' +ukcacnlw='$UMDIR/vn$VN/ctldata/UKCA/radaer/ga7_1/nml_cn_lw' +ukcacnsw='$UMDIR/vn$VN/ctldata/UKCA/radaer/ga7_1/nml_cn_sw' ukcacrlw='$UMDIR/vn$VN/ctldata/UKCA/radaer/ga7_1/nml_cr_lw' ukcacrsw='$UMDIR/vn$VN/ctldata/UKCA/radaer/ga7_1/nml_cr_sw' ukcaprec='$SPECTRAL_FILE_DIR/RADAER_pcalc.ukca' +ukcasulw='$UMDIR/vn$VN/ctldata/UKCA/radaer/ga7_1/nml_su_lw' +ukcasusw='$UMDIR/vn$VN/ctldata/UKCA/radaer/ga7_1/nml_su_sw' [namelist:run_vera] !!a0=1.2e-9 @@ -3321,9 +3483,10 @@ l_fix_ukca_activate_vert_rep=.false. l_fix_ukca_cloud_frac=.false. l_fix_ukca_h2dd_x=.true. !!l_fix_ukca_h2so4_ystore=.true. +l_fix_ukca_hygroscopicities=.false. l_fix_ukca_impscav=.true. -!!l_fix_ukca_meoh_emiss_input=.true. l_fix_ukca_offox_h2o_fac=.false. +l_fix_ukca_water_content=.false. l_fix_zh=.true. l_improve_aero_drydep=.false. !!l_improve_cv_cons=.false. diff --git a/rose-stem/flow.cylc b/rose-stem/flow.cylc index ea4d408c..ced3c51a 100644 --- a/rose-stem/flow.cylc +++ b/rose-stem/flow.cylc @@ -1,11 +1,35 @@ #!jinja2 +{% from "cylc.flow" import LOG %} +{% from "utils" import get_site %} +{% from "read_sources" import read_sources %} + +{% if SITE is not defined %} + {% set SITE = get_site() %} +{% endif %} +{% do LOG.info("Site: " + SITE) %} + +{% set SOURCE_DIRECTORY = ROSE_ORIG_HOST~":"~CYLC_WORKFLOW_SRC_DIR~"/.." %} +{# Include log message for suite_report #} +{% do LOG.info("MOCI SOURCE CLONE="~SOURCE_DIRECTORY) %} + +{# Read the sources of dependencies from the dependencies.yaml file #} +{% set dependencies = read_sources(SOURCE_DIRECTORY, "moci", false) %} + +{# Read in groups to run #} +{% if group is defined %} + {% set RUN_NAMES = group %} +{% elif g is defined %} + {% set RUN_NAMES = g %} +{% else %} + {% set RUN_NAMES = ["default"] %} +{% endif %} +{% do LOG.info("Running groups: "~RUN_NAMES|join(' + ')) %} + {% set TASK_RUN = "rose task-run --verbose -O '(" + SITE + ")'"%} {% set PP_CYCLES = range(110,113)|list + range(201,213)|list + [301,]|list %} {% set PP_UNICICLES_CYCLES = [112,203,206,209,212,]|list %} -{# Test to determine when to use single fcm_make tasks #} -{% set MULTISTEP_FCM = ['archer'] %} - {% macro format_wallclock(time_list) -%} PT{{'%02d' % (time_list[0])}}H{{'%02d' % (time_list[1])}}M{{'%02d' % (time_list[2])}}S {%- endmacro %} @@ -22,6 +46,7 @@ {# groups contains group_option-trigger_list key-value pairs. -#} {# If a group option is set, each group or task in the trigger list will be set. -#} {%- set groups = { + "tests" : [], "ALL" : [], "all" : ["ALL"] } @@ -33,6 +58,8 @@ [scheduler] UTC mode = True [[events]] + shutdown handlers = "suite_report_git.py -S $CYLC_WORKFLOW_RUN_DIR" + stall handlers = "suite_report_git.py -S $CYLC_WORKFLOW_RUN_DIR" stall timeout handlers = "echo %worklfow%s %event%s" stall timeout = PT6H abort on stall timeout = true @@ -69,11 +96,53 @@ submission timeout = PT12H # 12 hours execution timeout = PT3H # 3 hours [[[environment]]] - SOURCE_MOCI_BASE = {{ HOST_SOURCE_MOCI_BASE }} - + SOURCE_DIRECTORY = $CYLC_WORKFLOW_SHARE_DIR/source + SOURCE_MOCI_BASE = $SOURCE_DIRECTORY/moci + [[default_task]] script = "echo [ERROR] Please select a valid group within the test suite to test the component that has been modified >&2; exit 1" + [[SOURCE_EXTRACTION]] + post-script = """ +cp $SOURCE_DIRECTORY/moci/dependencies.yaml $CYLC_WORKFLOW_RUN_DIR +cp $SOURCE_DIRECTORY/SimSys_Scripts/github_scripts/suite_report_git.py $CYLC_WORKFLOW_RUN_DIR/bin +cp $SOURCE_DIRECTORY/SimSys_Scripts/github_scripts/suite_data.py $CYLC_WORKFLOW_RUN_DIR/bin +cp $SOURCE_DIRECTORY/SimSys_Scripts/github_scripts/git_bdiff.py $CYLC_WORKFLOW_RUN_DIR/bin +cp $SOURCE_DIRECTORY/SimSys_Scripts/github_scripts/get_git_sources.py $CYLC_WORKFLOW_RUN_DIR/bin +""" + [[[environment]]] + ROSE_TASK_APP = extract_source + DEPENDENCIES = {{dependencies}} + USE_MIRRORS = {{USE_MIRRORS}} + USE_TOKENS = false + USE_HEADS = false + {% if USE_MIRRORS %} + {% if GIT_MIRROR_LOC is defined and GIT_MIRROR_LOC %} + GIT_MIRROR_LOC = {{GIT_MIRROR_LOC}} + {% else %} + {{ raise( + "Trying to run with mirrors without setting a mirror location. " + "Ensure 'GIT_MIRROR_LOC' set." + ) }} + {% endif %} + {% endif %} + + [[extract_source]] + inherit = SOURCE_EXTRACTION, EXTRACT_RESOURCE + + [[mirror_source]] + inherit = SOURCE_EXTRACTION, EXTRACT_RESOURCE + script = """ +RELATIVE_SOURCE_DIRECTORY=`echo $SOURCE_DIRECTORY | sed 's|$HOME/||'` +ssh $HOSTNAME mkdir -p $RELATIVE_SOURCE_DIRECTORY +rsync -avz --exclude='.*' $SOURCE_DIRECTORY/* $HOSTNAME:$RELATIVE_SOURCE_DIRECTORY/ +sleep 5 + """ + [[[environment]]] + HOSTNAME = $(rose host-select {{COMPUTE_HOST}}) + + [[UNITTESTS]] + [[HOUSEKEEPING]] {% if HOUSEKEEPING == true %} script = "rose task-run --verbose" diff --git a/rose-stem/graph-drivers.cylc b/rose-stem/graph-drivers.cylc index 845819c7..3159de33 100644 --- a/rose-stem/graph-drivers.cylc +++ b/rose-stem/graph-drivers.cylc @@ -1,17 +1,17 @@ {# Setup graph required to run postproc tasks #} {% set SPACER = " " %} {%- do name_graphs.update({ - "test_drivers_unit" : SPACER + "fcm_make_drivers => drivers_unittests & mct_validate_unittests" + (" => drivers_local_housekeeping" if HOUSEKEEPING else ""), - "test_drivers_misc" : SPACER + "fcm_make_drivers => drivers_check_python_tabs" + (" => drivers_local_housekeeping" if HOUSEKEEPING else ""), + "test_drivers_unit" : SPACER + "extract_source => drivers_unittests & mct_validate_unittests" + (" => drivers_local_housekeeping" if HOUSEKEEPING else ""), + "test_drivers_misc" : SPACER + "extract_source => drivers_check_python_tabs" + (" => drivers_local_housekeeping" if HOUSEKEEPING else ""), "test_drivers_run" : """ - fcm_make_drivers => fcm_make2_drivers => coupled - fcm_make_um => recon => coupled - fcm_make_ocean => fcm_make2_ocean => recon + extract_source & extract_coupled_source => mirror_source + mirror_source => fcm_make_um => recon => coupled + mirror_source => fcm_make_ocean => fcm_make2_ocean => recon install_ancil => recon recon => coupled => check_coupled" + (" => drivers_hpc_housekeeping" if HOUSEKEEPING else "")+" """, - "test_drivers_verify_metrics" : SPACER + "test_verify_metrics" + (" => drivers_local_housekeeping" if HOUSEKEEPING else ""), - "test_drivers_plot_cpmip" : SPACER + "fcm_make_drivers => test_plot_cpmip_summary" + (" => drivers_local_housekeeping" if HOUSEKEEPING else ""), + "test_drivers_verify_metrics" : SPACER + "extract_source => test_verify_metrics" + (" => drivers_local_housekeeping" if HOUSEKEEPING else ""), + "test_drivers_plot_cpmip" : SPACER + "extract_source => test_plot_cpmip_summary" + (" => drivers_local_housekeeping" if HOUSEKEEPING else ""), }) %} {# Update groups with drivers tasks. -#} @@ -29,3 +29,4 @@ {%- endif %} {%- do groups["ALL"].append( "drivers" ) %} +{%- do groups["tests"].append( "drivers_non_run" ) %} diff --git a/rose-stem/graph-postproc.cylc b/rose-stem/graph-postproc.cylc index 0b908111..b5b298e7 100644 --- a/rose-stem/graph-postproc.cylc +++ b/rose-stem/graph-postproc.cylc @@ -1,16 +1,11 @@ {# Setup graph required to run postproc tasks #} -{% if SITE in MULTISTEP_FCM %} - {% set FCMPP_GRAPH = 'fcm_make_pp => fcm_make2_pp ' %} -{% else %} - {% set FCMPP_GRAPH = 'fcm_make_pp' %} -{% endif %} - +{% set BUILD_GRAPH = "extract_source => " + ("mirror_source => " if REMOTE_PP else "") + "build_pp" %} {# Setup Multi-date tasks #} {% set SPACER = " " %} {% set FAM_SUCCEED = ":succeed-all" if SPLIT_PP else "" %} -{% set COUPLED_GRAPH = [ SPACER + "pp_clearlog => " + FCMPP_GRAPH + " => coupled_dummy_198" + PP_CYCLES[0]|string + "01" + " & pp_unicicles_startup" ] %} +{% set COUPLED_GRAPH = [ SPACER + "pp_clearlog => " + BUILD_GRAPH + " => coupled_dummy_198" + PP_CYCLES[0]|string + "01" + " & pp_unicicles_startup" ] %} {% set PP_GRAPH = [] %} {% set VERIFY_GRAPH = [] %} {% for i in range(PP_CYCLES|length - 1) %} @@ -42,7 +37,7 @@ {%- do name_graphs.update({ "postproc_tests" : " test_mule:fail? => remove_mule_from_postproc - test_mule? | remove_mule_from_postproc => " + FCMPP_GRAPH + " => pp_unittests", + test_mule? | remove_mule_from_postproc => " + BUILD_GRAPH + " => pp_unittests", "coupled_run" : COUPLED_GRAPH|join("\n"), "postproc_run" : PP_GRAPH|join("\n"), "postproc_transfer": PPTRANSFER_GRAPH|join("\n"), @@ -60,3 +55,4 @@ {%- do groups['ALL'].append( "postproc" ) %} +{%- do groups['tests'].append( "pp_unittests" ) %} diff --git a/rose-stem/lib/python/read_sources.py b/rose-stem/lib/python/read_sources.py new file mode 100644 index 00000000..f7dabe0b --- /dev/null +++ b/rose-stem/lib/python/read_sources.py @@ -0,0 +1,66 @@ +############################################################################## +# (c) Crown copyright 2026 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + +import yaml +import tempfile +import os +from subprocess import run +from shutil import rmtree + +def get_dependencies_file(wc_loc): + """ + Copy the dependencies file to a temporary directory on the local machine that can be + read. + """ + + tempdir = tempfile.mkdtemp() + + try: + host, path = wc_loc.split(":", 1) + path = os.path.join(path, "dependencies.yaml") + copy_command = f"scp -o StrictHostKeyChecking=no {host}:" + except ValueError: + path = os.path.join(wc_loc, "dependencies.yaml") + copy_command = "cp " + copy_command += f"{path} {tempdir}" + + result = run( + copy_command.split(), capture_output=True, text=True, timeout=120 + ) + + # Raise an error if the returncode is positive + if result.returncode: + raise RuntimeError( + f"An error occured while running the command '{copy_command}' " + "in order to read the dependencies file. The error message is:\n\n" + f"'{result.stderr}'" + ) + + return tempdir + +def read_sources(clone_source, repo, use_heads): + """ + Load the dependencies.yaml file as a dictionary + """ + + dependencies_file = get_dependencies_file(clone_source) + + with open(os.path.join(dependencies_file, "dependencies.yaml")) as stream: + dependencies = yaml.safe_load(stream) + + if not dependencies[repo]["source"]: + dependencies[repo]["source"] = clone_source + + # Populate parent, assume MetOffice is owner if not set + for dependency, values in dependencies.items(): + if "parent" not in values: + dependencies[dependency]["parent"] = f"MetOffice/{dependency}.git" + if use_heads: + dependencies[dependency]["ref"] = "" + + rmtree(dependencies_file) + + return dependencies diff --git a/rose-stem/lib/python/utils.py b/rose-stem/lib/python/utils.py new file mode 100644 index 00000000..2f8c3d68 --- /dev/null +++ b/rose-stem/lib/python/utils.py @@ -0,0 +1,20 @@ +############################################################################## +# (c) Crown copyright 2025 Met Office. All rights reserved. +# The file LICENCE, distributed with this code, contains details of the terms +# under which the code may be used. +############################################################################## + +from subprocess import Popen, PIPE + + +def get_site(): + proc = Popen( + ["rose", "config", "rose-stem", "automatic-options"], stdout=PIPE, text=True + ) + out, _ = proc.communicate() + if proc.returncode or "SITE" not in out: + raise Exception('Could not determine the rose-stem "SITE"') + # At some sites there may be many variables that are returned by rose config rose-stem + # Try to just grab the thing after SITE= and then ignore anything afterwards + site = out.split("SITE=")[1].split(' ')[0].strip() + return site diff --git a/rose-stem/meta/rose-meta.conf b/rose-stem/meta/rose-meta.conf index 2641e2d1..44ea379f 100644 --- a/rose-stem/meta/rose-meta.conf +++ b/rose-stem/meta/rose-meta.conf @@ -50,6 +50,18 @@ sort-key=drivers_a2 title=Wallclock time type=integer +[template variables=COMPUTE_HOST] +compulsory=true +description=Select which EX host to submit to: + ='exab' = EXA/EXB + ='xccd' = EXC/EXCD +help=Select the login node used for the suite when run on the EX machine. +ns=Test Options/Drivers +sort-key=drivers_a3 +title=HPC host +values='exab', + ='excd' + [template variables=EXTRACT_HOST] compulsory=true description=Select the host for source code extraction tasks @@ -65,17 +77,23 @@ ns=General Options sort-key=general014 type=character -[template variables=HOST_HPC] +[template variables=GIT_LOC] compulsory=true -description=Select which EX host to submit to: - ='exab' = EXA/EXB - ='xccd' = EXC/EXCD -help=Select the login node used for the suite when run on the EX machine. -ns=Test Options/Drivers -sort-key=drivers_a3 -title=HPC host -values='exab', - ='excd' +description=Git repository location +help=Location of the Git repositories. + = All repositories are assumed to be owned by MetOffice. +ns=General Options +sort-key=general_git2 +type=character + +[template variables=GIT_MIRROR_LOC] +compulsory=true +description=Local Mirrors repository location +help=Location of the Git repositories. + = All repositories are assumed to be owned by MetOffice. +ns=General Options +sort-key=general_git3 +type=character [template variables=HOUSEKEEPING] compulsory=true @@ -136,6 +154,14 @@ sort-key=drivers_atmos_2 title=Reconfiguration: Processes North_South type=integer +[template variables=REMOTE_PP] +compulsory=true +description=Run PostProc app on remote machine +help=PostProc application will be run with a local extract mirrored to and run on a remote machine +ns=Test Options/Postproc +sort-key=pp0 +type=boolean + [template variables=SPLIT_PP] compulsory=true description=If true, split the postproc app into multiple tasks by model @@ -172,6 +198,17 @@ title=Use Linux Server trigger=template variables=EXTRACT_HOST: false; type=boolean +[template variables=USE_MIRRORS] +compulsory=true +description=Use the local repository mirror(s) +help=Use the local repository mirrors. + = All repositories are assumed to be owned by MetOffice. +ns=General Options +sort-key=general_git1 +trigger=template variables=GIT_MIRROR_LOC: true; + =template variables=GIT_LOC: false +type=boolean + [template variables=VERBOSE] compulsory=true description=Turns on verbose output mode diff --git a/rose-stem/opt/rose-suite-archer2.conf b/rose-stem/opt/rose-suite-archer2.conf index 33629e42..859d954d 100644 --- a/rose-stem/opt/rose-suite-archer2.conf +++ b/rose-stem/opt/rose-suite-archer2.conf @@ -1,7 +1,9 @@ [template variables] COMPUTE_HOST='archer2' !!PROJECT-NAME='climate' +REMOTE_PP=true SPECIFY_PROJECT=false SYSTEM_NAME='archer2' TASKS_PER_NODE=128 +USE_MIRRORS=false USE_RAMDISK=false diff --git a/rose-stem/rose-suite.conf b/rose-stem/rose-suite.conf index f5ee299b..f166a789 100644 --- a/rose-stem/rose-suite.conf +++ b/rose-stem/rose-suite.conf @@ -1,5 +1,3 @@ -ROSE_STEM_VERSION=1 - [template variables] ATM_PROCX=8 ATM_PROCY=8 @@ -7,19 +5,22 @@ BASIS=1978,9,1,0,0,0 CICE_COL=1440 CICE_ROW=1205 CLOCK=0,25,0 +COMPUTE_HOST='exab' !!EXTRACT_HOST='$ROSE_ORIG_HOST' FCM_VERSION='' -HOST_HPC='exab' +GIT_LOC='git@github.com:' +!!GIT_MIRROR_LOC='localmirrors:' HOUSEKEEPING=true IOS_NPROC=0 NEMO_IPROC=15 NEMO_JPROC=17 POSTPROC_HOST='$ROSE_ORIG_HOST' -#PYTHON3=true RCF_PROCX=4 RCF_PROCY=8 +REMOTE_PP=false SPLIT_PP=true SYSTEM_NAME='meto_azspice' USE_LINUX_SERVER=true +USE_MIRRORS=false VERBOSE=true XIOS_NPROC=6 diff --git a/rose-stem/site/meto_azspice.cylc b/rose-stem/site/meto_azspice.cylc index a2d17de2..1e0481e3 100644 --- a/rose-stem/site/meto_azspice.cylc +++ b/rose-stem/site/meto_azspice.cylc @@ -26,18 +26,22 @@ echo *** Linux Python Environment End *** [[EXTRACT_RESOURCE]] inherit = LINUX + [[DRIVERS_BUILD_TASKS]] + [[POSTPROC_SCRIPTING_RESOURCE]] inherit = LINUX, LINUX_PYTHON_ENV - execution time limit = PT10M + execution time limit = PT10M +{% if not REMOTE_PP %} [[POSTPROC_RESOURCE]] inherit = POSTPROC_SCRIPTING_RESOURCE [[[environment]]] - STASHMASTER = /data/users/moci/KGO/pp_test_execs/STASHmaster_vn10.2 -{% if USE_LINUX_SERVER %} + STASHMASTER = /data/users/moci/KGO/pp_test_execs/STASHmaster_vn10.2 + {% if USE_LINUX_SERVER %} [[[directives]]] --mem=10G -{% endif %} + {% endif %} +{% endif %} [[DRIVERS_TESTS_RESOURCE]] inherit = LINUX, LINUX_PYTHON_ENV diff --git a/rose-stem/site/meto_ex1a.cylc b/rose-stem/site/meto_ex1a.cylc index ee6b52a8..f79a8e60 100644 --- a/rose-stem/site/meto_ex1a.cylc +++ b/rose-stem/site/meto_ex1a.cylc @@ -1,28 +1,28 @@ {% set MODULE_CMD = 'module use /data/users/moci/modules/modules; module load GC4-PrgEnv/2024-01-cpe23.05-cce15.0.0/5083; module load scitools;' %} [[HPC_RESOURCE]] - env-script = "eval $(rose task-env)" + env-script = "eval $(rose task-env)" pre-script = "{{MODULE_CMD}} module list" - platform = {{ HOST_HPC }} + platform = {{ COMPUTE_HOST }} [[[environment]]] - PLATFORM = ex1a - CONFIG = meto-ex1a-cce - UMDIR = /common/internal/umdir - UM_NSTALL_DIR = /common/internal/umdir + PLATFORM = ex1a + CONFIG = meto-ex1a-cce + UMDIR = /common/internal/umdir + UM_NSTALL_DIR = /common/internal/umdir OCEANDIR = /common/internal/ocean - INPUT_FILES = /data/users/moci/INPUT_FILES - CMIP6_ANCILS = $INPUT_FILES/GC4_INPUTS_20220804/cmip6 + INPUT_FILES = /data/users/moci/INPUT_FILES + CMIP6_ANCILS = $INPUT_FILES/GC4_INPUTS_20220804/cmip6 [[[directives]]] -W umask=0022 [[EXTRACT_RESOURCE]] - [[[environment]]] - CONFIG = meto-ex-cce - OPTIM_LEVEL = safe + [[[environment]]] + CONFIG = meto-ex-cce + OPTIM_LEVEL = safe [[BUILD_RESOURCE]] inherit = HPC_RESOURCE - execution time limit = PT1H30M + execution time limit = PT1H30M [[[environment]]] ARCHIVE_FCM_MAKE = true ROSE_TASK_N_JOBS = 6 @@ -35,11 +35,11 @@ [[SCRIPTS_INSTALL_HPC_RESOURCE]] inherit = None, HPC_RESOURCE - execution time limit = PT10M + execution time limit = PT10M [[[environment]]] ROSE_TASK_N_JOBS = 1 [[[directives]]] - -l ncpus = 1 + -l ncpus = 1 -q = shared [[DRIVERS_SCRIPTING_RESOURCE]] @@ -53,7 +53,7 @@ [[INSTALL_ANCIL_RESOURCE]] inherit = None, SCRIPTS_INSTALL_HPC_RESOURCE - execution time limit = PT20M + execution time limit = PT20M [[ATMOS_RESOURCE]] [[[environment]]] @@ -74,7 +74,7 @@ {% set NEMO_SELECT = '{{OCEAN_NODES}}:ncpus=256:mpiprocs={{OCEAN_PPN}}' %} {% set XIOS_SELECT = '{{XIOS_NODES}}:ncpus=256:mpiprocs={{XIOS_NPROC}}' %} {% if IOS_NPROC > 0 %} - UKMO_THREAD_MULTI = true + UKMO_THREAD_MULTI = true {% endif %} ROSE_LAUNCHER_PREOPTS_NEMO = -n {{NEMO_NPROC}} -d 1 ROSE_LAUNCHER_PREOPTS_XIOS = -n {{XIOS_NPROC}} -d 1 @@ -87,14 +87,14 @@ {% if IOS_NPROC is defined and IOS_NPROC > 0 %} UKMO_THREAD_MULTI=true {% endif %} - PPN = {{PPN}} + PPN = {{PPN}} L_MCT_VALIDATE = true CPMIP_ANALYSIS = true - IO_COST = true + IO_COST = true DATA_INTENSITY = true - COKMPLEXITY = false - TOTAL_POWER_CONSUMPTION_NEEDED = 4.8 - NODES_IN_HPC_NEEDED = 12932 + COMPLEXITY = false + TOTAL_POWER_CONSUMPTION_NEEDED = 4.8 + NODES_IN_HPC_NEEDED = 12932 [[[directives]]] -l select = '{{ATMOS_NODES}}:ncpus=256:mpiprocs={{ATMOS_MPIPROC_PN}}+{{OCEAN_NODES}}:ncpus=256:mpiprocs={{OCEAN_PPN}}+{{XIOS_NODES}}:ncpus=256:mpiprocs={{XIOS_NPROC}}' @@ -107,3 +107,14 @@ OMP_NUM_THREADS = 1 RECON_LAUNCHER = mpiexec -n {{RCF_PROCX * RCF_PROCY}} ROSE_LAUNCHER_PREOPTS = -n {{RCF_PROCX * RCF_PROCY}} + +{% if REMOTE_PP %} + [[POSTPROC_SCRIPTING_RESOURCE]] + inherit = None, SCRIPTS_INSTALL_HPC_RESOURCE + pre-script = 'module load scitools/default-current ; module load um_tools ; module load nco' + + [[POSTPROC_RESOURCE]] + inherit = POSTPROC_SCRIPTING_RESOURCE + [[[environment]]] + STASHMASTER = /data/users/moci/KGO/pp_test_execs/STASHmaster_vn10.2 +{% endif %} diff --git a/rose-stem/tasks-drivers.cylc b/rose-stem/tasks-drivers.cylc index 6822a00c..ff9b5f4b 100644 --- a/rose-stem/tasks-drivers.cylc +++ b/rose-stem/tasks-drivers.cylc @@ -2,32 +2,35 @@ [[DRIVERS_BUILD_TASKS]] [[DRIVERS_RUN_TASKS]] + [[[environment]]] + DRIVER_EXTRACT_DIR = $SOURCE_DIRECTORY/moci [[DRIVERS_TESTS]] - [[fcm_make_drivers]] + [[extract_coupled_source]] inherit = DRIVERS_BUILD_TASKS, EXTRACT_RESOURCE - - [[fcm_make2_drivers]] - inherit = DRIVERS_BUILD_TASKS, BUILD_RESOURCE - + [[[environment]]] + ROSE_TASK_APP = extract_source + ROSE_APP_OPT_CONF_KEYS = coupled + GIT_LOC = {{ GIT_MIRROR_LOC if (USE_MIRRORS and GIT_MIRROR_LOC is defined) else GIT_LOC }} + [[drivers_unittests]] - inherit = DRIVERS_TESTS, DRIVERS_TESTS_RESOURCE - env-script = "eval $(rose task-env)" - script = run_python_env.sh $CYLC_WORKFLOW_SHARE_DIR/fcm_make_drivers/extract/drivers/Coupled_Drivers/unittests/test.py + inherit = DRIVERS_TESTS, UNITTESTS, DRIVERS_TESTS_RESOURCE + env-script = "eval $(rose task-env)" + script = run_python_env.sh $SOURCE_DIRECTORY/moci/Coupled_Drivers/unittests/test.py [[drivers_check_python_tabs]] inherit = DRIVERS_TESTS, DRIVERS_TESTS_RESOURCE - script = test_tab.sh $CYLC_WORKFLOW_SHARE_DIR/fcm_make_drivers/extract/drivers/Coupled_Drivers + script = test_tab.sh $SOURCE_DIRECTORY/moci/Coupled_Drivers [[mct_validate_unittests]] - inherit = DRIVERS_TESTS, DRIVERS_TESTS_RESOURCE + inherit = DRIVERS_TESTS, UNITTESTS, DRIVERS_TESTS_RESOURCE env-script = "eval $(rose task-env)" - script = run_python_env.sh $CYLC_WORKFLOW_SHARE_DIR/fcm_make_drivers/extract/drivers/Coupled_Drivers/driver_utilities/mct_validate/unittests/test.py + script = run_python_env.sh $SOURCE_DIRECTORY/moci/Coupled_Drivers/driver_utilities/mct_validate/unittests/test.py [[test_verify_metrics]] inherit = DRIVERS_TESTS, DRIVERS_TESTS_RESOURCE - env-script = "eval $(rose task-env)" + env-script = "eval $(rose task-env)" script = 'rose task-run --verbose' [[test_plot_cpmip_summary]] @@ -39,12 +42,12 @@ [[drivers_local_housekeeping]] inherit=HOUSEKEEPING, HOUSEKEEP_RESOURCE [[[environment]]] - RUNDIR="$CYLC_WORKFLOW_WORK_DIR/1/fcm_make_drivers $CYLC_WORKFLOW_WORK_DIR/1/fcm_make_ocean $CYLC_WORKFLOW_WORK_DIR/1/test_plot_cpmip_summary $CYLC_WORKFLOW_WORK_DIR/1/test_verify_metrics $CYLC_WORKFLOW_SHARE_DIR/fcm_make_drivers $CYLC_WORKFLOW_SHARE_DIR/fcm_make_ocean $CYLC_WORKFLOW_WORK_DIR/1/drivers_local_housekeeping" + RUNDIR="$CYLC_WORKFLOW_WORK_DIR/1/fcm_make_ocean $CYLC_WORKFLOW_WORK_DIR/1/test_plot_cpmip_summary $CYLC_WORKFLOW_WORK_DIR/1/test_verify_metrics $CYLC_WORKFLOW_SHARE_DIR/fcm_make_ocean $CYLC_WORKFLOW_WORK_DIR/1/drivers_local_housekeeping" {% if "test_drivers_run" in groups["drivers"] %} {# The following is to test the drivers in a model run #} - {% set NON_RIGOROUS_PATH='--path= --path=$CYLC_WORKFLOW_SHARE_DIR/fcm_make_ocean/build-ocean/bin/ --path=$CYLC_WORKFLOW_SHARE_DIR/fcm_make_um/build-*/bin/ --path=$CYLC_WORKFLOW_SHARE_DIR/fcm_make_drivers/extract/drivers/Coupled_Drivers/' %} + {% set NON_RIGOROUS_PATH='--path= --path=$CYLC_WORKFLOW_SHARE_DIR/fcm_make_ocean/build-ocean/bin/ --path=$CYLC_WORKFLOW_SHARE_DIR/fcm_make_um/build-*/bin/ --path=$SOURCE_DIRECTORY/moci/Coupled_Drivers/' %} [[fcm_make_um]] inherit = DRIVERS_BUILD_TASKS, BUILD_RESOURCE @@ -57,7 +60,7 @@ [[fcm_make_ocean]] inherit = DRIVERS_BUILD_TASKS, EXTRACT_RESOURCE [[[environment]]] - CICE_ROW = {{CICE_ROW}} + CICE_ROW = {{CICE_ROW}} CICE_COL = {{CICE_COL}} [[fcm_make2_ocean]] @@ -65,7 +68,7 @@ script = """ rose task-run --verbose --define=fast-dest-root-orig=$TMPDIR --define=fast-dest-root-cont=$TMPDIR --define='args=--archive' """ - execution time limit = PT1H50M + execution time limit = PT1H50M [[[environment]]] FCM_MAKE_FILE = main @@ -82,14 +85,14 @@ ATMOS_SECONDS_PER_TIMESTEP = 1200 NSTEP_RIVERS = 3 COUPLING_HOURS = 1 - UMVN = 13.1 + UMVN = 14.0 [[OCEAN_ENV]] [[[environment]]] OCEAN_EXEC = $CYLC_WORKFLOW_SHARE_DIR/fcm_make_ocean/build-ocean/bin/nemo-cice.exe NEMO_IPROC = {{NEMO_IPROC}} NEMO_JPROC = {{NEMO_JPROC}} - NEMO_NPROC = {{NEMO_IPROC*NEMO_JPROC}} + NEMO_NPROC = {{NEMO_IPROC*NEMO_JPROC}} NEMO_VERSION = 306 NEMO_RST = $CYLC_WORKFLOW_SHARE_DIR/data/drivers/History_Data/NEMOhist OCEAN_SEAICE_TIMESTEPS_PER_DAY = 72 @@ -141,6 +144,6 @@ [[drivers_hpc_housekeeping]] inherit=HOUSEKEEPING, DRIVERS_SCRIPTING_RESOURCE [[[environment]]] - RUNDIR="$CYLC_WORKFLOW_SHARE_DIR/fcm_make_drivers $CYLC_WORKFLOW_SHARE_DIR/fcm_make_ocean $CYLC_WORKFLOW_SHARE_DIR/fcm_make_um $CYLC_WORKFLOW_SHARE_DIR/data/drivers $CYLC_WORKFLOW_WORK_DIR/1/coupled $CYLC_WORKFLOW_WORK_DIR/1/fcm_make2_drivers $CYLC_WORKFLOW_WORK_DIR/1/fcm_make2_ocean $CYLC_WORKFLOW_WORK_DIR/1/fcm_make_um $CYLC_WORKFLOW_WORK_DIR/1/install_ancil $CYLC_WORKFLOW_WORK_DIR/1/recon $CYLC_WORKFLOW_WORK_DIR/1/check_coupled $CYLC_WORKFLOW_WORK_DIR/1/drivers_hpc_housekeeping" + RUNDIR="$CYLC_WORKFLOW_SHARE_DIR/fcm_make_ocean $CYLC_WORKFLOW_SHARE_DIR/fcm_make_um $CYLC_WORKFLOW_SHARE_DIR/data/drivers $CYLC_WORKFLOW_WORK_DIR/1/coupled $CYLC_WORKFLOW_WORK_DIR/1/fcm_make2_ocean $CYLC_WORKFLOW_WORK_DIR/1/fcm_make_um $CYLC_WORKFLOW_WORK_DIR/1/install_ancil $CYLC_WORKFLOW_WORK_DIR/1/recon $CYLC_WORKFLOW_WORK_DIR/1/check_coupled $CYLC_WORKFLOW_WORK_DIR/1/drivers_hpc_housekeeping" -{% endif %} \ No newline at end of file +{% endif %} diff --git a/rose-stem/tasks-postproc.cylc b/rose-stem/tasks-postproc.cylc index ba7d3633..cf3e1f0e 100644 --- a/rose-stem/tasks-postproc.cylc +++ b/rose-stem/tasks-postproc.cylc @@ -2,13 +2,8 @@ [[PP_TESTS]] - [[fcm_make_pp]] - inherit = PP_BUILD_TASKS, EXTRACT_RESOURCE - -{% if SITE in MULTISTEP_FCM %} - [[fcm_make2_pp]] - inherit = PP_BUILD_TASKS, BUILD_RESOURCE -{% endif %} + [[build_pp]] + inherit = PP_BUILD_TASKS, {{ "BUILD_RESOURCE" if REMOTE_PP else "EXTRACT_RESOURCE" }} [[POSTPROC]] inherit = POSTPROC_RESOURCE @@ -186,13 +181,13 @@ echo Command: {{TASK_RUN}} --command-key=pp_unicicles inherit = PP_TESTS, POSTPROC_SCRIPTING_RESOURCE [[pp_unittests]] - inherit = PP_TESTS, POSTPROC_SCRIPTING_RESOURCE - env-script = "eval $(rose task-env)" + inherit = PP_TESTS, UNITTESTS, POSTPROC_SCRIPTING_RESOURCE + env-script = "eval $(rose task-env) ; PATH=$CYLC_WORKFLOW_RUN_DIR/app/postproc/bin:$PATH" # Run each of the main groups (common, atmos, nemocice) first - helps to spot any leaking tests. # Finally run 'all' - sweeps up any tests not covered in the main groups script = run_python_env.sh test_postproc -g common -g atmos -g nemocice -g platforms -g unicicles -g all [[[environment]]] - STASHMASTER = + STASHMASTER = [[test_mule]] inherit = PP_TESTS, POSTPROC_SCRIPTING_RESOURCE @@ -210,4 +205,4 @@ echo Command: {{TASK_RUN}} --command-key=pp_unicicles [[housekeeping_pp]] inherit = HOUSEKEEP_RESOURCE, HOUSEKEEPING [[[environment]]] - RUNDIR="$CYLC_WORKFLOW_SHARE_DIR/data/postproc $CYLC_WORKFLOW_WORK_DIR/1/fcm_mak*_pp $CYLC_WORKFLOW_WORK_DIR/1/p*_* $CYLC_WORKFLOW_WORK_DIR/1/housekeeping_pp $CYLC_WORKFLOW_WORK_DIR/1/coupled_dummy_*" + RUNDIR="$CYLC_WORKFLOW_SHARE_DIR/data/postproc $CYLC_WORKFLOW_WORK_DIR/1/build_pp $CYLC_WORKFLOW_WORK_DIR/1/p*_* $CYLC_WORKFLOW_WORK_DIR/1/housekeeping_pp $CYLC_WORKFLOW_WORK_DIR/1/coupled_dummy_*"