From f497a21b25df7a8c6a408a5a90ee371c46adbbf9 Mon Sep 17 00:00:00 2001 From: rem1776 Date: Tue, 3 Feb 2026 16:09:20 -0500 Subject: [PATCH 01/12] update fms2_io module description --- fms2_io/fms2_io.F90 | 94 ++++++++++++++++++++++++++++++++------------- 1 file changed, 68 insertions(+), 26 deletions(-) diff --git a/fms2_io/fms2_io.F90 b/fms2_io/fms2_io.F90 index cbbcbafea4..29c2879854 100644 --- a/fms2_io/fms2_io.F90 +++ b/fms2_io/fms2_io.F90 @@ -17,9 +17,40 @@ !*********************************************************************** !> @defgroup fms2_io_mod fms2_io_mod !> @ingroup fms2_io -!> @brief An updated library for parallel IO to replace @ref mpp_io_mod. This module contains -!! the public API for fms2 I/O interfaces and routines defined throughout this directory. -!! A conversion guide for replacing mpp/fms_io code with this module is available below. +!> @brief This module aims support I/O operations for 3 types of netcdf files within the FMS framework: +!! +!! 1) Generic netcdf files via the netcdf_io_mod module. +!! 2) Netcdf files with structured grid domains via the fms_netcdf_domain_io_mod module. +!! 3) Netcdf files with unstructured grid domains via the fms_netcdf_unstructured_domain_io_mod module. +!! +!! To accomplish this with a unified interface, this module re-exports key types and procedures from the +!! aforementioned helper modules. This allows users to interact with different types of netcdf files +!! via a consistent set of interface calls (ie. open_file, close_file, write_data, read_data, etc). +!! +!! Each file type mentioned above has its own specific derived type to represent the file's metadata and data structure: +!! - FmsNetcdfFile_t for generic netcdf files. +!! - FmsNetcdfDomainFile_t for structured domain netcdf files. Inherits FmsNetcdfFile_t. +!! - FmsNetcdfUnstructuredDomainFile_t for unstructured domain netcdf files. Also inherits FmsNetcdfFile_t. +!! These derived types are typically refered to as the "fileobj" across this module's interfaces. +!! +!! The domain file objects require a derived type (domain2d or domainug) to represent a decomposed domain for its data +!! across available MPI ranks, as defined in mpp_domains_mod. +!! See mpp_domains_mod documentation for more information on creating a domain decomposition using FMS. +!! +!! Besides standard open/close/read/write operations, this module also provides interfaces for writing and reading +!! restart files via the blackboxio module. Restart files can be created and read for both structured and unstructured +!! domain netcdf files and utilize the same unified interface calls (write_restart, read_restart, etc), but contain +!! specific metadata and grid information to facilitate model restarts. +!! +!! Additional fine tuning of the netcdf output files can be accomplished via the fms2_io_nml namelist, which defines +!! several parameters used by the netCDF library such as default netcdf file format, chunk size, deflate level, etc. +!! For more information on these parameters, see the NetCDF documentation: +!! https://docs.unidata.ucar.edu/netcdf-c/current/file_format_specifications.html +!! +!! Besides NetCDF io, this module also provides an interface for common utility routines used in FMS I/O operations, +!! such as checking for file existence, reading ascii files, parsing mask tables, and generating instance filenames. +!! +!! If converting legacy code from fms_io/mpp_io to fms2_io, please refer to the migration guide at fms2_io/readme.md. !> @addtogroup fms2_io_mod !> @{ @@ -34,15 +65,16 @@ module fms2_io_mod implicit none private +!> NetCDF constant (enum) for unlimited dimension identification public :: unlimited -public :: FmsNetcdfFile_t -public :: FmsNetcdfDomainFile_t -public :: FmsNetcdfUnstructuredDomainFile_t -public :: open_file -public :: open_virtual_file -public :: close_file + +!> File object types are defined in the helper modules (netcdf_io_mod,fms_netcdf_domain_io_mod,fms_netcdf_unstructured_domain_io_mod) +!! but are made public here for user access. +public :: FmsNetcdfFile_t, FmsNetcdfDomainFile_t, FmsNetcdfUnstructuredDomainFile_t + +!> interfaces that are defined below but utilize helper module routines. +public :: open_file, open_virtual_file, close_file public :: register_axis -public :: register_unlimited_compressed_axis public :: register_field public :: register_restart_field public :: write_data @@ -51,6 +83,9 @@ module fms2_io_mod public :: write_new_restart public :: read_restart public :: read_new_restart + +!> Routines/Interfaces from netcdf_io_mod to make public +public :: register_unlimited_compressed_axis public :: global_att_exists public :: variable_att_exists public :: register_global_attribute @@ -68,27 +103,33 @@ module fms2_io_mod public :: get_variable_num_dimensions public :: get_variable_dimension_names public :: get_variable_size -public :: get_compute_domain_dimension_indices -public :: get_global_io_domain_indices -public :: Valid_t -public :: get_valid -public :: is_valid -public :: get_unlimited_dimension_name -public :: get_variable_unlimited_dimension_index -public :: file_exists +public :: flush_file +public :: write_restart_bc +public :: read_restart_bc public :: compressed_start_and_count public :: get_variable_sense public :: get_variable_missing public :: get_variable_units public :: get_time_calendar -public :: open_check public :: is_registered_to_restart public :: check_if_open public :: set_fileobj_time_name +!! TODO: i think we can make this one internal to netcdf_io_mod +public :: Valid_t +public :: get_valid +public :: is_valid + +!> Routines/Interfaces from fms_netcdf_domain_io_mod to make public +public :: get_compute_domain_dimension_indices +public :: get_global_io_domain_indices +public :: get_unlimited_dimension_name +public :: get_variable_unlimited_dimension_index + +!> Routines/Interfaces from fms_io_utils_mod to make public +public :: file_exists +public :: open_check public :: is_dimension_registered public :: fms2_io_init -public :: write_restart_bc -public :: read_restart_bc public :: get_mosaic_tile_grid public :: ascii_read public :: get_mosaic_tile_file @@ -97,17 +138,18 @@ module fms2_io_mod public :: set_filename_appendix public :: get_instance_filename public :: nullify_filename_appendix -public :: flush_file !> @} -!> @brief Opens a given netcdf or domain file. +!> @brief Opens a netcdf dataset on disk and initializes the file object. !! !>
Example usage: !! !! io_success = open_file(fileobj, "filename", "write") !! -!! Opens a netcdf file of type @ref fmsnetcdffile_t at the given file path string. -!! File mode is set to one of "read"/"write"/"overwrite"/"append" +!! Opens a netcdf file for a standard data, domain decomposed data, or unstructured domain decomposed data based off +!! the fileobj used, and also intializes the fileobj for subsequent IO operations. +!! +!! File mode is set to one of "read"/"write"/"overwrite"/"append": !! !! io_success = open_file(fileobj, "filename", "overwrite", domain) !! @@ -125,7 +167,7 @@ module fms2_io_mod end interface open_file -!> @brief Creates a diskless netcdf or domain file +!> @brief Creates a diskless netcdf or domain file. File is created in memory only. !! !> @return true if successful, false otherwise !! From 2c797938b3d28f775e51135336d23ef3e9ac1810 Mon Sep 17 00:00:00 2001 From: rem1776 Date: Tue, 24 Feb 2026 13:16:30 -0500 Subject: [PATCH 02/12] add some expanded descriptions and mask_table description --- fms2_io/fms2_io.F90 | 29 ++++++---- fms2_io/fms_io_utils.F90 | 58 +++++++++++++++++++ fms2_io/fms_netcdf_unstructured_domain_io.F90 | 2 +- fms2_io/netcdf_io.F90 | 9 ++- 4 files changed, 84 insertions(+), 14 deletions(-) diff --git a/fms2_io/fms2_io.F90 b/fms2_io/fms2_io.F90 index 29c2879854..588f022845 100644 --- a/fms2_io/fms2_io.F90 +++ b/fms2_io/fms2_io.F90 @@ -17,7 +17,7 @@ !*********************************************************************** !> @defgroup fms2_io_mod fms2_io_mod !> @ingroup fms2_io -!> @brief This module aims support I/O operations for 3 types of netcdf files within the FMS framework: +!> @brief This module aims support I/O operations for 3 types of netcdf files within the FMS framework: !! !! 1) Generic netcdf files via the netcdf_io_mod module. !! 2) Netcdf files with structured grid domains via the fms_netcdf_domain_io_mod module. @@ -28,10 +28,19 @@ !! via a consistent set of interface calls (ie. open_file, close_file, write_data, read_data, etc). !! !! Each file type mentioned above has its own specific derived type to represent the file's metadata and data structure: -!! - FmsNetcdfFile_t for generic netcdf files. -!! - FmsNetcdfDomainFile_t for structured domain netcdf files. Inherits FmsNetcdfFile_t. -!! - FmsNetcdfUnstructuredDomainFile_t for unstructured domain netcdf files. Also inherits FmsNetcdfFile_t. -!! These derived types are typically refered to as the "fileobj" across this module's interfaces. +!! +!! - FmsNetcdfFile_t: This type provides a thin wrapper over the netCDF4 library, but allows the user to assign a +!! “pelist” to the file. If a pelist is assigned, only the first rank on the list directly interacts with the +!! netCDF library, and performs broadcasts to relay the information read to the rest of the ranks on the list. +!! When writing netcdf files, only the first rank in the pelist will perform the writes. +!! - FmsNetcdfDomainFile_t: This type does everything that the FmsNetcdfFile_t type does and it adds support for +!! “domain-decomposed” reads/writes. Here, "domain decomposed" refers to data that is on a user-defined +!! mpp_domain and is decomposed in two dimensions, in which each MPI rank has its own section of the global data. +!! This requires a domain to be associated with the fileobj. +!! - FmsNetcdfUnstructuredDomainFile_t: This type does everything that the FmsNetcdfFile_t type does and it adds +!! support for “domain-decomposed” reads/writes on a user defined mpp_domains unstructured grid. This requires +!! a unstructured domain to be associated with the fileobj. These derived types are typically refered to as the +!! "fileobj" across this module's interfaces. !! !! The domain file objects require a derived type (domain2d or domainug) to represent a decomposed domain for its data !! across available MPI ranks, as defined in mpp_domains_mod. @@ -48,7 +57,7 @@ !! https://docs.unidata.ucar.edu/netcdf-c/current/file_format_specifications.html !! !! Besides NetCDF io, this module also provides an interface for common utility routines used in FMS I/O operations, -!! such as checking for file existence, reading ascii files, parsing mask tables, and generating instance filenames. +!! such as checking for file existence, reading ascii files, parsing mask tables, and generating instance filenames. !! !! If converting legacy code from fms_io/mpp_io to fms2_io, please refer to the migration guide at fms2_io/readme.md. @@ -72,7 +81,7 @@ module fms2_io_mod !! but are made public here for user access. public :: FmsNetcdfFile_t, FmsNetcdfDomainFile_t, FmsNetcdfUnstructuredDomainFile_t -!> interfaces that are defined below but utilize helper module routines. +!> interfaces that are defined below but utilize helper module routines. public :: open_file, open_virtual_file, close_file public :: register_axis public :: register_field @@ -114,12 +123,12 @@ module fms2_io_mod public :: is_registered_to_restart public :: check_if_open public :: set_fileobj_time_name -!! TODO: i think we can make this one internal to netcdf_io_mod +!! TODO: i think we can make this one internal to netcdf_io_mod public :: Valid_t public :: get_valid public :: is_valid -!> Routines/Interfaces from fms_netcdf_domain_io_mod to make public +!> Routines/Interfaces from fms_netcdf_domain_io_mod to make public public :: get_compute_domain_dimension_indices public :: get_global_io_domain_indices public :: get_unlimited_dimension_name @@ -210,7 +219,7 @@ module fms2_io_mod module procedure close_unstructured_domain_file end interface close_file -!> @brief Add a dimension to a given file +!> @brief Adds a dimension to a given netcdf file object. !! !>
Example usage: !! diff --git a/fms2_io/fms_io_utils.F90 b/fms2_io/fms_io_utils.F90 index 83a05be992..0f591e7986 100644 --- a/fms2_io/fms_io_utils.F90 +++ b/fms2_io/fms_io_utils.F90 @@ -72,18 +72,44 @@ module fms_io_utils_mod type(char_linked_list), pointer :: head => null() endtype char_linked_list +!> Reads in a ASCII file from a given path and populates a mask that corresponds to a domain decomposition. +!! This mask differs from others across FMS in that it is for a specific rank and its associated data region, +!! instead of masking out specific indices from a corresponding array. +!! +!! Mask format is as follows: +!! +!! +!! +!! +!! +!! +!! Example: +!! +!! 2 +!! 4,4 +!! 1,1 +!! 4,4 +!! +!! For this mask file, 2 rank's would be masked: the top right corner (1,1) and the bottom left corner (4,4) +!! +!! This interface includes support for both 2D and 3D mask tables. +!! !> @ingroup fms_io_utils_mod interface parse_mask_table module procedure parse_mask_table_2d module procedure parse_mask_table_3d end interface parse_mask_table +!> Gets the output filename for a mosaic domain that contains multiple tiles. +!! Each file will include its tile number when output. !> @ingroup fms_io_utils_mod interface get_mosaic_tile_file module procedure get_mosaic_tile_file_sg module procedure get_mosaic_tile_file_ug end interface get_mosaic_tile_file +!> Interface to allocate data real, integer, or character buffers for use in read/write routines. +!! Not meant to be used externally. !> @ingroup fms_io_utils_mod interface allocate_array module procedure allocate_array_i4_kind_1d @@ -115,6 +141,16 @@ module fms_io_utils_mod end interface allocate_array +!> Utility routine that copies a smaller array into a larger one using the starting indices and sizes provided. +!! Example usage: +!! +!! call put_array_section( section, array, start, sizes ) +!! +!! Is equivalent to +!! +!! array (start(1):start(1)+sizes(1), start(2):start(2)+sizes(2), ... ) = section(:,:,...) +!! +!! This interface supports integers and reals for both 4 and 8 kind, and up to 5 dimensions. !> @ingroup fms_io_utils_mod interface put_array_section module procedure put_array_section_i4_kind_1d @@ -140,6 +176,17 @@ module fms_io_utils_mod end interface put_array_section +!> Utility routine that copies a subset of data from a larger array into a smaller one using the +!! starting indices and sizes provided. +!! Example usage: +!! +!! call get_array_section( section, array, start, sizes ) +!! +!! Is equivalent to +!! +!! section(:,:,...) = array (start(1):start(1)+sizes(1), start(2):start(2)+sizes(2), ... ) +!! +!! This interface supports integers and reals for both 4 and 8 kind, and up to 5 dimensions. !> @ingroup fms_io_utils_mod interface get_array_section module procedure get_array_section_i4_kind_1d @@ -165,6 +212,17 @@ module fms_io_utils_mod end interface get_array_section +!> Sets a string describing the datatype corresponding to the sdata argument +!! Example: +!! +!! call get_data_type_string(sdata, string) +!! +!! String values and associated data types are: +!! - "int" integers(kind=4) +!! - "i8_kind" integers(kind=8) +!! - "float" real(kind=4) +!! - "double" real(kind=8) +!! - "char" character(len=*) !> @ingroup fms_io_utils_mod interface get_data_type_string module procedure get_data_type_string_0d diff --git a/fms2_io/fms_netcdf_unstructured_domain_io.F90 b/fms2_io/fms_netcdf_unstructured_domain_io.F90 index 4f53257952..7a61466f1e 100644 --- a/fms2_io/fms_netcdf_unstructured_domain_io.F90 +++ b/fms2_io/fms_netcdf_unstructured_domain_io.F90 @@ -17,7 +17,7 @@ !*********************************************************************** !> @defgroup fms_netcdf_unstructured_domain_io_mod fms_netcdf_unstructured_domain_io_mod !> @ingroup fms2_io -!> @brief Handles netcdf I/O for unstructured domains +!> @brief Handles netcdf I/O for domains using a unstructured !! !> Mainly routines for use via interfaces in @ref fms2_io_mod diff --git a/fms2_io/netcdf_io.F90 b/fms2_io/netcdf_io.F90 index be65205eec..39989f10db 100644 --- a/fms2_io/netcdf_io.F90 +++ b/fms2_io/netcdf_io.F90 @@ -17,10 +17,13 @@ !*********************************************************************** !> @defgroup netcdf_io_mod netcdf_io_mod !> @ingroup fms2_io -!> @brief Creates a basic netcdf type and routines to extend for other uses +!> @brief This module defines a derived type, FmsNetcdfFile_t, and routines +!! to handle calls to the netcdf library in order to read and write netcdf files within a MPI fortran program. +!! +!! This module is specifically for netcdf input/output when not using a domain decomposition to split up data between +!! processors (see mpp_domains module). The FmsNetcdfFile_t type acts a base class that is extended by both +!! FmsNetcdfDomainFile_t and FmsNetcdfUnstructuredDomainFile_t. !! -!> Handles type definitions and interfaces for netcdf I/O. - !> @addtogroup netcdf_io_mod !> @{ module netcdf_io_mod From 7b79684b5a35b1e9aba1295e771739922a397532 Mon Sep 17 00:00:00 2001 From: rem1776 Date: Fri, 27 Feb 2026 17:00:33 -0500 Subject: [PATCH 03/12] add some more information for the types, domains, and helper modules --- fms2_io/fms2_io.F90 | 10 ++++----- fms2_io/fms_io_utils.F90 | 18 +++++++++------ fms2_io/fms_netcdf_domain_io.F90 | 22 ++++++++++++++++--- fms2_io/fms_netcdf_unstructured_domain_io.F90 | 16 ++++++++++---- fms2_io/netcdf_io.F90 | 7 +++++- 5 files changed, 53 insertions(+), 20 deletions(-) diff --git a/fms2_io/fms2_io.F90 b/fms2_io/fms2_io.F90 index 588f022845..570daae5c6 100644 --- a/fms2_io/fms2_io.F90 +++ b/fms2_io/fms2_io.F90 @@ -23,9 +23,9 @@ !! 2) Netcdf files with structured grid domains via the fms_netcdf_domain_io_mod module. !! 3) Netcdf files with unstructured grid domains via the fms_netcdf_unstructured_domain_io_mod module. !! -!! To accomplish this with a unified interface, this module re-exports key types and procedures from the -!! aforementioned helper modules. This allows users to interact with different types of netcdf files -!! via a consistent set of interface calls (ie. open_file, close_file, write_data, read_data, etc). +!! This module defines public facing interfaces so that each netcdf file type above can be interacted with +!! via a consistent set of interface calls. This allows the same interfaces to be used for operations (open_file, write_file, etc.) +!! regardless of the specific type of netcdf file/fileobj. !! !! Each file type mentioned above has its own specific derived type to represent the file's metadata and data structure: !! @@ -77,8 +77,8 @@ module fms2_io_mod !> NetCDF constant (enum) for unlimited dimension identification public :: unlimited -!> File object types are defined in the helper modules (netcdf_io_mod,fms_netcdf_domain_io_mod,fms_netcdf_unstructured_domain_io_mod) -!! but are made public here for user access. +!> File object types are defined in the helper modules (netcdf_io_mod,fms_netcdf_domain_io_mod, +!! fms_netcdf_unstructured_domain_io_mod) but are made public here for user access. public :: FmsNetcdfFile_t, FmsNetcdfDomainFile_t, FmsNetcdfUnstructuredDomainFile_t !> interfaces that are defined below but utilize helper module routines. diff --git a/fms2_io/fms_io_utils.F90 b/fms2_io/fms_io_utils.F90 index 0f591e7986..bd962d268e 100644 --- a/fms2_io/fms_io_utils.F90 +++ b/fms2_io/fms_io_utils.F90 @@ -72,11 +72,13 @@ module fms_io_utils_mod type(char_linked_list), pointer :: head => null() endtype char_linked_list -!> Reads in a ASCII file from a given path and populates a mask that corresponds to a domain decomposition. -!! This mask differs from others across FMS in that it is for a specific rank and its associated data region, -!! instead of masking out specific indices from a corresponding array. +!> Reads in a ASCII file from a given path and populates a maskmap that corresponds to a domain decomposition. !! -!! Mask format is as follows: +!! The logical array created will be the same shape as the domain layout, with each index representing a specific pe within the +! decomposition. This array is intended to be used as input for the mpp_define_domain routines maskmap argument, +!! to essentially exclude certain pe's from a domain decomposition. +!! +!! Mask table format is as follows: !! !! !! @@ -90,7 +92,7 @@ module fms_io_utils_mod !! 1,1 !! 4,4 !! -!! For this mask file, 2 rank's would be masked: the top right corner (1,1) and the bottom left corner (4,4) +!! For this mask table, 2 rank's would be masked: the top right corner (1,1) and the bottom left corner (4,4) !! !! This interface includes support for both 2D and 3D mask tables. !! @@ -100,8 +102,10 @@ module fms_io_utils_mod module procedure parse_mask_table_3d end interface parse_mask_table -!> Gets the output filename for a mosaic domain that contains multiple tiles. -!! Each file will include its tile number when output. +!> Gets the file name to be used when utilizing a mosaic that contains multiple tiles. +!! This is currenly used in the diag_manager and data_override in so that any output files +!! will include the tile number of the writer. Uses the format "filename.tileN.nc" and requires +!! the filename to already include the .nc suffix. !> @ingroup fms_io_utils_mod interface get_mosaic_tile_file module procedure get_mosaic_tile_file_sg diff --git a/fms2_io/fms_netcdf_domain_io.F90 b/fms2_io/fms_netcdf_domain_io.F90 index 3e49752810..666bebe7c5 100644 --- a/fms2_io/fms_netcdf_domain_io.F90 +++ b/fms2_io/fms_netcdf_domain_io.F90 @@ -17,8 +17,13 @@ !*********************************************************************** !> @defgroup fms_netcdf_domain_io_mod fms_netcdf_domain_io_mod !> @ingroup fms2_io -!> @brief Domain-specific I/O wrappers. - +!> @brief This module defines a derived type, FmsNetcdfDomainFile_t, and routines +!! to handle calls to the netcdf library when using domain decomposition for a standard +!! rectangular grid. See mpp_domains_mod for more information on domain decomposition. +!! +!! This module is not intended to be used externally, fms2_io_mod is intended to publicize the routines +!! and types defined here to provide a single set of interfaces to be used acases. +!! !> @addtogroup fms_netcdf_domain_io_mod !> @{ module fms_netcdf_domain_io_mod @@ -52,7 +57,18 @@ module fms_netcdf_domain_io_mod endtype DomainDimension_t -!> @brief netcdf domain file type. +!> @brief Type to represent a netCDF file when using a domain decomposition on a +!! standard rectangular grid. Used to do distributed I/O across ranks, +!! as determined by the io_layout. The io_layout is 2 integers set via mpp_set_io_domain +!! and determines how many PEs will be performing IO operations within a given +!! domain decompositon. The total number of writing PEs is equivalent to the +!! product of the io_layout. +!! +!! For example, if domain's layout was (4,4) so 16 PEs total, +!! then a io_layout of (2,2) would have 4 of the PEs performing I/O operations. +!! When doing a read, each IO PE will receive a data portion from 3 of the non-IO PEs and then write the aggregate. +!! When doing a write, each IO PE will read the data and then send a data portion to 3 of the non-IO PEs. +!! !> @ingroup fms_netcdf_domain_io_mod type, extends(FmsNetcdfFile_t), public :: FmsNetcdfDomainFile_t type(domain2d) :: domain !< Two-dimensional domain. diff --git a/fms2_io/fms_netcdf_unstructured_domain_io.F90 b/fms2_io/fms_netcdf_unstructured_domain_io.F90 index 7a61466f1e..fee8e32ec9 100644 --- a/fms2_io/fms_netcdf_unstructured_domain_io.F90 +++ b/fms2_io/fms_netcdf_unstructured_domain_io.F90 @@ -17,10 +17,16 @@ !*********************************************************************** !> @defgroup fms_netcdf_unstructured_domain_io_mod fms_netcdf_unstructured_domain_io_mod !> @ingroup fms2_io -!> @brief Handles netcdf I/O for domains using a unstructured +!> @brief This module defines a derived type, FmsNetcdfUnstructuredDomainFile_t, and routines to handle +!! io operations when using a domain decomposition on a unstructured grid. +!! +!! An unstructured grid has custom axes defined to behave differently than standard cartesian grids. These +!! grids are domain decomposed just like other grids, so global data is split amongst PEs. +!! For more information, see the domainUG type defined in mpp_domains_mod. +!! +!! This module is not intended to be used externally, fms2_io_mod is intended to publicize the routines +!! and types defined here to provide a single set of interfaces to be used across file types. !! -!> Mainly routines for use via interfaces in @ref fms2_io_mod - module fms_netcdf_unstructured_domain_io_mod use netcdf use mpp_domains_mod @@ -30,7 +36,9 @@ module fms_netcdf_unstructured_domain_io_mod implicit none private -!> @brief netcdf unstructured domain file type. +!> @brief Type to represent a netCDF file when using a domain decompostion +!! on a unstructured grid. +!! !> @ingroup fms_netcdf_unstructured_domain_io_mod type, public, extends(FmsNetcdfFile_t) :: FmsNetcdfUnstructuredDomainFile_t type(domainug) :: domain !< Unstructured domain. diff --git a/fms2_io/netcdf_io.F90 b/fms2_io/netcdf_io.F90 index 39989f10db..5a770d14c1 100644 --- a/fms2_io/netcdf_io.F90 +++ b/fms2_io/netcdf_io.F90 @@ -24,6 +24,9 @@ !! processors (see mpp_domains module). The FmsNetcdfFile_t type acts a base class that is extended by both !! FmsNetcdfDomainFile_t and FmsNetcdfUnstructuredDomainFile_t. !! +!! This module is not intended to be used externally, fms2_io_mod is intended to publicize the routines +!! and types defined here to provide a single set of interfaces between the three different file types. +!! !> @addtogroup netcdf_io_mod !> @{ module netcdf_io_mod @@ -137,7 +140,9 @@ module netcdf_io_mod procedure :: init endtype fmsOffloadingIn_type -!> @brief Netcdf file type. +!> @brief Type to represent a netCDF file. Can be used with multiple cores +!! but only the root pe will perform any I/O operations, before sending the +!! data to the other pes. !> @ingroup netcdf_io_mod type, public :: FmsNetcdfFile_t character(len=FMS_PATH_LEN) :: path !< File path. From 088e46f6e8177e0bb82dd40f42112b67ab11c9a4 Mon Sep 17 00:00:00 2001 From: rem1776 Date: Tue, 3 Mar 2026 16:14:28 -0500 Subject: [PATCH 04/12] address review comments and clean up for linter --- fms2_io/fms2_io.F90 | 98 +++++++++++++++++++++------------------- fms2_io/fms_io_utils.F90 | 6 +-- 2 files changed, 55 insertions(+), 49 deletions(-) diff --git a/fms2_io/fms2_io.F90 b/fms2_io/fms2_io.F90 index 570daae5c6..5adf8cef46 100644 --- a/fms2_io/fms2_io.F90 +++ b/fms2_io/fms2_io.F90 @@ -17,48 +17,54 @@ !*********************************************************************** !> @defgroup fms2_io_mod fms2_io_mod !> @ingroup fms2_io -!> @brief This module aims support I/O operations for 3 types of netcdf files within the FMS framework: -!! -!! 1) Generic netcdf files via the netcdf_io_mod module. -!! 2) Netcdf files with structured grid domains via the fms_netcdf_domain_io_mod module. -!! 3) Netcdf files with unstructured grid domains via the fms_netcdf_unstructured_domain_io_mod module. -!! -!! This module defines public facing interfaces so that each netcdf file type above can be interacted with -!! via a consistent set of interface calls. This allows the same interfaces to be used for operations (open_file, write_file, etc.) -!! regardless of the specific type of netcdf file/fileobj. -!! -!! Each file type mentioned above has its own specific derived type to represent the file's metadata and data structure: -!! -!! - FmsNetcdfFile_t: This type provides a thin wrapper over the netCDF4 library, but allows the user to assign a -!! “pelist” to the file. If a pelist is assigned, only the first rank on the list directly interacts with the -!! netCDF library, and performs broadcasts to relay the information read to the rest of the ranks on the list. -!! When writing netcdf files, only the first rank in the pelist will perform the writes. -!! - FmsNetcdfDomainFile_t: This type does everything that the FmsNetcdfFile_t type does and it adds support for -!! “domain-decomposed” reads/writes. Here, "domain decomposed" refers to data that is on a user-defined -!! mpp_domain and is decomposed in two dimensions, in which each MPI rank has its own section of the global data. -!! This requires a domain to be associated with the fileobj. -!! - FmsNetcdfUnstructuredDomainFile_t: This type does everything that the FmsNetcdfFile_t type does and it adds -!! support for “domain-decomposed” reads/writes on a user defined mpp_domains unstructured grid. This requires -!! a unstructured domain to be associated with the fileobj. These derived types are typically refered to as the -!! "fileobj" across this module's interfaces. -!! -!! The domain file objects require a derived type (domain2d or domainug) to represent a decomposed domain for its data -!! across available MPI ranks, as defined in mpp_domains_mod. +!> @brief This module aims support netCDF I/O operations for 3 use-cases within the FMS framework: +!! +!! 1) Netcdf operations done on a single core via the netcdf_io_mod module. +!! 2) Netcdf operations done with multiple cores on a structured grid via the fms_netcdf_domain_io_mod module. +!! 3) Netcdf operations done with multiple cores on a unstructured grid via the fms_netcdf_unstructured_domain_io_mod +!! module. +!! +!! fms2_io_mod is the top level module that provides open, close, read, and write interfaces to the NetCDF package. +!! This module defines public "aliases"(interfaces) to select procedures in fms_netcdf_domain_io_mod for reading/writing +!! structured grid domains; fms_netcdf_unstructured_domain_io_mod for reading/writing data on unstructured grid domains; +!! netcdf_io_mod for reading/writing netcdf files on a single core. Subroutines and functions in the latter +!! mentioned modules (fms_netcdf_domain_io_mod, fms_netcdf_unstructured_domain_io_mod, and netcdf_io_mod) are intended +!! for internally use only. We highly recommended to only call public interfaces defined in this module. +!! +!! Before any fms2_io_mod I/O operations, a file derived type must be declared first. +!! Three file derived types are currently available and are described below. Any instances +!! of these three file types are referred to as "fileobj" in this module. +!! +!! - FmsNetcdfFile_t: provides limited number of wrapper procedures to the netCDF4 library. If the +!! user provides a pelist to procedures compatible with this type, only the root rank of the pelist +!! performs I/O operations by calling the NetCDF library: the root rank either boradcasts the read-in +!! data to the remaining ranks in the pelist, or gathers data from the remaining ranks in the pelist before writing. +!! +!! - FmsNetcdfDomainFile_t: extends upon FmsNetcdfFile_t and adds supports for "domain-decomposed" reads and writes. +!! Here, "domain decomposed" refers to data that is partitioned into subdomains of the decomposed global domain, +!! and each MPI rank holds its portion of the global data. The users must provide a domain of type Domain2D from +!! mpp_domains_mod when initializing this file object. +!! +!! - FmsNetcdfUnstructuredDomainFile_t: extends upon FmsNetcdfUnstructuredDomainFile_t and adds support for +!! “domain-decomposed” reads/writes for data decomposed on subdomains of unstructured grids +!! The users must provide a domain of type DomainUG from mpp_domains_mod when initializing this file object. +!! !! See mpp_domains_mod documentation for more information on creating a domain decomposition using FMS. !! -!! Besides standard open/close/read/write operations, this module also provides interfaces for writing and reading -!! restart files via the blackboxio module. Restart files can be created and read for both structured and unstructured -!! domain netcdf files and utilize the same unified interface calls (write_restart, read_restart, etc), but contain -!! specific metadata and grid information to facilitate model restarts. +!! When doing IO on multiple cores, either via the FmsNetcdfDomainFile_t or the FmsNetcdfUnstructuredDomainFile_t, +!! the number of cores performing IO operations is set by the io_layout. The io_layout is 2 integers set via +!! mpp_set_io_domain and determines how to divide up the existing domain decomposition layout for IO operations. +!! See FmsNetcdfDomainFile_t for more information on how the io_layout works. !! -!! Additional fine tuning of the netcdf output files can be accomplished via the fms2_io_nml namelist, which defines -!! several parameters used by the netCDF library such as default netcdf file format, chunk size, deflate level, etc. -!! For more information on these parameters, see the NetCDF documentation: -!! https://docs.unidata.ucar.edu/netcdf-c/current/file_format_specifications.html +!! Besides standard open/close/read/write operations, this module also provides interfaces for writing and reading +!! "diskless" netcdf files via the blackboxio module. !! -!! Besides NetCDF io, this module also provides an interface for common utility routines used in FMS I/O operations, -!! such as checking for file existence, reading ascii files, parsing mask tables, and generating instance filenames. +!! Users can specify I/O specifications with the fms2_io_nml namelist which allows users to specify the following +!! NetCDF library parameters: Netcdf file format, chunk_size, deflate_level, shuffle. Note, fms2_io accepts the +!! Netcdf file format namelist values:"64bit", "class", and "netcdf4". For more information on optimizing NetCDF writing +!! operation, see https://docs.unidata.ucar.edu/netcdf-c/current/file_format_specifications.html !! +!! @note The legacy IO modules, fms_io_mod and mpp_io_mod, are !! If converting legacy code from fms_io/mpp_io to fms2_io, please refer to the migration guide at fms2_io/readme.md. !> @addtogroup fms2_io_mod @@ -81,7 +87,7 @@ module fms2_io_mod !! fms_netcdf_unstructured_domain_io_mod) but are made public here for user access. public :: FmsNetcdfFile_t, FmsNetcdfDomainFile_t, FmsNetcdfUnstructuredDomainFile_t -!> interfaces that are defined below but utilize helper module routines. +!> Interfaces defined below to make public public :: open_file, open_virtual_file, close_file public :: register_axis public :: register_field @@ -93,7 +99,7 @@ module fms2_io_mod public :: read_restart public :: read_new_restart -!> Routines/Interfaces from netcdf_io_mod to make public +!> Routines/functions from netcdf_io_mod to make public public :: register_unlimited_compressed_axis public :: global_att_exists public :: variable_att_exists @@ -123,18 +129,17 @@ module fms2_io_mod public :: is_registered_to_restart public :: check_if_open public :: set_fileobj_time_name -!! TODO: i think we can make this one internal to netcdf_io_mod public :: Valid_t public :: get_valid public :: is_valid -!> Routines/Interfaces from fms_netcdf_domain_io_mod to make public +!> Routines/functions from fms_netcdf_domain_io_mod to make public public :: get_compute_domain_dimension_indices public :: get_global_io_domain_indices public :: get_unlimited_dimension_name public :: get_variable_unlimited_dimension_index -!> Routines/Interfaces from fms_io_utils_mod to make public +!> Routines/functions from fms_io_utils_mod to make public public :: file_exists public :: open_check public :: is_dimension_registered @@ -149,7 +154,7 @@ module fms2_io_mod public :: nullify_filename_appendix !> @} -!> @brief Opens a netcdf dataset on disk and initializes the file object. +!> @brief Opens a netcdf dataset on disk. Initializes the file object. !! !>
Example usage: !! @@ -176,7 +181,8 @@ module fms2_io_mod end interface open_file -!> @brief Creates a diskless netcdf or domain file. File is created in memory only. +!> @brief Creates a diskless netcdf or domain file. File is created in memory only via the netcdf library's +!! NC_DISKLESS creation mode option. Data will be lost upon file closing. !! !> @return true if successful, false otherwise !! @@ -219,7 +225,7 @@ module fms2_io_mod module procedure close_unstructured_domain_file end interface close_file -!> @brief Adds a dimension to a given netcdf file object. +!> @brief Adds a dimension/axis to a given netcdf file object. !! !>
Example usage: !! @@ -244,7 +250,7 @@ module fms2_io_mod module procedure register_unstructured_dimension end interface register_axis -!> @brief Defines a new field within the given file +!> @brief Defines a new field/variable within the given file !>
Example usage: !! !! call register_field(fileobj, "lon", "double", (/"lon"/) ) diff --git a/fms2_io/fms_io_utils.F90 b/fms2_io/fms_io_utils.F90 index bd962d268e..9b6e4f9773 100644 --- a/fms2_io/fms_io_utils.F90 +++ b/fms2_io/fms_io_utils.F90 @@ -74,9 +74,9 @@ module fms_io_utils_mod !> Reads in a ASCII file from a given path and populates a maskmap that corresponds to a domain decomposition. !! -!! The logical array created will be the same shape as the domain layout, with each index representing a specific pe within the -! decomposition. This array is intended to be used as input for the mpp_define_domain routines maskmap argument, -!! to essentially exclude certain pe's from a domain decomposition. +!! The logical array created will be the same shape as the domain layout, with each index representing a specific pe +!! within the decomposition. This array is intended to be used as input for the mpp_define_domain routines maskmap +!! argument, to essentially exclude certain pe's from a domain decomposition. !! !! Mask table format is as follows: !! From 5760e8019863226ac34e38168d091e1bbb6a5dfa Mon Sep 17 00:00:00 2001 From: rem1776 Date: Thu, 5 Mar 2026 13:04:30 -0500 Subject: [PATCH 05/12] move around module description, add note for collective io in open_file --- fms2_io/fms2_io.F90 | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/fms2_io/fms2_io.F90 b/fms2_io/fms2_io.F90 index 5adf8cef46..d4cf1c5db3 100644 --- a/fms2_io/fms2_io.F90 +++ b/fms2_io/fms2_io.F90 @@ -17,12 +17,7 @@ !*********************************************************************** !> @defgroup fms2_io_mod fms2_io_mod !> @ingroup fms2_io -!> @brief This module aims support netCDF I/O operations for 3 use-cases within the FMS framework: -!! -!! 1) Netcdf operations done on a single core via the netcdf_io_mod module. -!! 2) Netcdf operations done with multiple cores on a structured grid via the fms_netcdf_domain_io_mod module. -!! 3) Netcdf operations done with multiple cores on a unstructured grid via the fms_netcdf_unstructured_domain_io_mod -!! module. +!> @brief This module aims support netCDF I/O operations. !! !! fms2_io_mod is the top level module that provides open, close, read, and write interfaces to the NetCDF package. !! This module defines public "aliases"(interfaces) to select procedures in fms_netcdf_domain_io_mod for reading/writing @@ -45,12 +40,18 @@ !! and each MPI rank holds its portion of the global data. The users must provide a domain of type Domain2D from !! mpp_domains_mod when initializing this file object. !! -!! - FmsNetcdfUnstructuredDomainFile_t: extends upon FmsNetcdfUnstructuredDomainFile_t and adds support for +!! - FmsNetcdfUnstructuredDomainFile_t: also extends upon FmsNetcdfFile_t and adds support for !! “domain-decomposed” reads/writes for data decomposed on subdomains of unstructured grids !! The users must provide a domain of type DomainUG from mpp_domains_mod when initializing this file object. !! !! See mpp_domains_mod documentation for more information on creating a domain decomposition using FMS. !! +!! These types correspond to 3 use-cases within the FMS framework. +!! +!! 1) Netcdf operations for standard data (FmsNetcdfFile_t). +!! 2) Netcdf operations for domain decomposed data on a structured grid(FmsNetcdfDomainFile_t). +!! 3) Netcdf operations for domain decomposed data on a unstructured grid(FmsNetcdfUnstructuredDomainFile_t). +!! !! When doing IO on multiple cores, either via the FmsNetcdfDomainFile_t or the FmsNetcdfUnstructuredDomainFile_t, !! the number of cores performing IO operations is set by the io_layout. The io_layout is 2 integers set via !! mpp_set_io_domain and determines how to divide up the existing domain decomposition layout for IO operations. @@ -169,6 +170,17 @@ module fms2_io_mod !! !! Opens a domain netcdf file of type @ref fmsnetcdfdomainfile_t or !! @ref fmsnetcdfunstructureddomainfile_t at the given file path name and 2D or unstructured domain. +!! +!! Netcdf's collective IO functionality can be enabled when opening a file in order to perform collective read and +!! writes. This will use netcdf libraries capabilities for parallel file access, allowing all processors +!! to perform data reads and writes, but requires hdf5 and netcdf to be built with MPI support. +!! Example usage: +!! +!! io_success = open_file(fileobj, "test_collective_io.nc", "read", domain, nc_format=nc_format, & +!! use_netcdf_mpi=.true., use_collective=.true.) +!! +!! See fms2_io/readme.md for more information. +!! !! @note For individual documentation on the listed routines, please see the appropriate helper module. !! For netcdf files with a structured domain: @ref fms_netcdf_domain_io_mod. !! For netcdf files with an unstructured domain: @ref fms_netcdf_unstructured_domain_io_mod. From 7783e2f5d93f0f3a3b52d0a4b39f592e2446db91 Mon Sep 17 00:00:00 2001 From: rem1776 Date: Thu, 5 Mar 2026 13:05:40 -0500 Subject: [PATCH 06/12] whitespace --- fms2_io/netcdf_io.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fms2_io/netcdf_io.F90 b/fms2_io/netcdf_io.F90 index 5a770d14c1..35c161f088 100644 --- a/fms2_io/netcdf_io.F90 +++ b/fms2_io/netcdf_io.F90 @@ -142,7 +142,7 @@ module netcdf_io_mod !> @brief Type to represent a netCDF file. Can be used with multiple cores !! but only the root pe will perform any I/O operations, before sending the -!! data to the other pes. +!! data to the other pes. !> @ingroup netcdf_io_mod type, public :: FmsNetcdfFile_t character(len=FMS_PATH_LEN) :: path !< File path. From 1a092068d650c3969771f72264574c0cbe01dbec Mon Sep 17 00:00:00 2001 From: Ryan Mulhall <35538242+rem1776@users.noreply.github.com> Date: Tue, 24 Mar 2026 16:35:10 -0400 Subject: [PATCH 07/12] Fix grammar and clarity in fms2_io comments --- fms2_io/fms2_io.F90 | 106 +++++++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 50 deletions(-) diff --git a/fms2_io/fms2_io.F90 b/fms2_io/fms2_io.F90 index d4cf1c5db3..323b9e46b0 100644 --- a/fms2_io/fms2_io.F90 +++ b/fms2_io/fms2_io.F90 @@ -17,16 +17,17 @@ !*********************************************************************** !> @defgroup fms2_io_mod fms2_io_mod !> @ingroup fms2_io -!> @brief This module aims support netCDF I/O operations. +!> @brief This module supports netCDF I/O operations. !! !! fms2_io_mod is the top level module that provides open, close, read, and write interfaces to the NetCDF package. !! This module defines public "aliases"(interfaces) to select procedures in fms_netcdf_domain_io_mod for reading/writing -!! structured grid domains; fms_netcdf_unstructured_domain_io_mod for reading/writing data on unstructured grid domains; -!! netcdf_io_mod for reading/writing netcdf files on a single core. Subroutines and functions in the latter -!! mentioned modules (fms_netcdf_domain_io_mod, fms_netcdf_unstructured_domain_io_mod, and netcdf_io_mod) are intended -!! for internally use only. We highly recommended to only call public interfaces defined in this module. +!! data on structured grid domains; fms_netcdf_unstructured_domain_io_mod for reading/writing data on unstructured grid domains; +!! netcdf_io_mod for reading/writing data that is not parallelized with domain decomposition. Subroutines and +!! functions in fms_netcdf_domain_io_mod, fms_netcdf_unstructured_domain_io_mod, and +!! netcdf_io_mod are intended for internally use only. We highly recommended to only call public interfaces defined +!! in this module. !! -!! Before any fms2_io_mod I/O operations, a file derived type must be declared first. +!! Before any fms2_io_mod I/O operations, a file derived type must be declared. !! Three file derived types are currently available and are described below. Any instances !! of these three file types are referred to as "fileobj" in this module. !! @@ -34,11 +35,13 @@ !! user provides a pelist to procedures compatible with this type, only the root rank of the pelist !! performs I/O operations by calling the NetCDF library: the root rank either boradcasts the read-in !! data to the remaining ranks in the pelist, or gathers data from the remaining ranks in the pelist before writing. +!! If a pelist is not provided, all ranks calling the procedure will perform the IO operation. !! !! - FmsNetcdfDomainFile_t: extends upon FmsNetcdfFile_t and adds supports for "domain-decomposed" reads and writes. !! Here, "domain decomposed" refers to data that is partitioned into subdomains of the decomposed global domain, !! and each MPI rank holds its portion of the global data. The users must provide a domain of type Domain2D from -!! mpp_domains_mod when initializing this file object. +!! mpp_domains_mod when initializing this file object. To specify pe's for performing the IO operations, see +!! mpp_set_io_layout !! !! - FmsNetcdfUnstructuredDomainFile_t: also extends upon FmsNetcdfFile_t and adds support for !! “domain-decomposed” reads/writes for data decomposed on subdomains of unstructured grids @@ -46,26 +49,20 @@ !! !! See mpp_domains_mod documentation for more information on creating a domain decomposition using FMS. !! -!! These types correspond to 3 use-cases within the FMS framework. -!! -!! 1) Netcdf operations for standard data (FmsNetcdfFile_t). -!! 2) Netcdf operations for domain decomposed data on a structured grid(FmsNetcdfDomainFile_t). -!! 3) Netcdf operations for domain decomposed data on a unstructured grid(FmsNetcdfUnstructuredDomainFile_t). -!! -!! When doing IO on multiple cores, either via the FmsNetcdfDomainFile_t or the FmsNetcdfUnstructuredDomainFile_t, -!! the number of cores performing IO operations is set by the io_layout. The io_layout is 2 integers set via -!! mpp_set_io_domain and determines how to divide up the existing domain decomposition layout for IO operations. -!! See FmsNetcdfDomainFile_t for more information on how the io_layout works. +!! When using the FmsNetcdfDomainFile_t or the FmsNetcdfUnstructuredDomainFile_t types, +!! the io_layout controls how the IO operations are parallelized. +!! See mpp_domains_mod for more information (TODO!). !! !! Besides standard open/close/read/write operations, this module also provides interfaces for writing and reading -!! "diskless" netcdf files via the blackboxio module. +!! "diskless" netcdf files via the blackboxio module. This module is used mainly for testing purposes, and should not be used +!! in production. !! -!! Users can specify I/O specifications with the fms2_io_nml namelist which allows users to specify the following +!! Users can specify additional I/O parameters with the fms2_io_nml namelist which allows users to specify the following !! NetCDF library parameters: Netcdf file format, chunk_size, deflate_level, shuffle. Note, fms2_io accepts the !! Netcdf file format namelist values:"64bit", "class", and "netcdf4". For more information on optimizing NetCDF writing !! operation, see https://docs.unidata.ucar.edu/netcdf-c/current/file_format_specifications.html !! -!! @note The legacy IO modules, fms_io_mod and mpp_io_mod, are +!! @note The legacy IO modules, fms_io_mod and mpp_io_mod, are no longer available. !! If converting legacy code from fms_io/mpp_io to fms2_io, please refer to the migration guide at fms2_io/readme.md. !> @addtogroup fms2_io_mod @@ -155,29 +152,30 @@ module fms2_io_mod public :: nullify_filename_appendix !> @} -!> @brief Opens a netcdf dataset on disk. Initializes the file object. +!> @brief Opens a NetCDF dataset on disk and initializes the file object. !! -!>
Example usage: +!> Opens a netcdf file for standard data, domain decomposed data, or unstructured domain decomposed data +!! and also intializes the fileobj for subsequent IO operations. !! -!! io_success = open_file(fileobj, "filename", "write") +!!
Example usage for opening a file with standard non-domain decomposed data: !! -!! Opens a netcdf file for a standard data, domain decomposed data, or unstructured domain decomposed data based off -!! the fileobj used, and also intializes the fileobj for subsequent IO operations. +!! io_success = open_file(fileobj, "filename", "write") +!! +!! File mode can be "read"/"write"/"overwrite"/"append" !! -!! File mode is set to one of "read"/"write"/"overwrite"/"append": +!!
Example usage for opening a file with domain decomposed data: !! -!! io_success = open_file(fileobj, "filename", "overwrite", domain) +!! io_success = open_file(fileobj, "filename", "write", domain) !! -!! Opens a domain netcdf file of type @ref fmsnetcdfdomainfile_t or -!! @ref fmsnetcdfunstructureddomainfile_t at the given file path name and 2D or unstructured domain. +!! Where fileobj is of type @ref fmsnetcdfdomainfile_t or @ref fmsnetcdfunstructureddomainfile_t !! !! Netcdf's collective IO functionality can be enabled when opening a file in order to perform collective read and !! writes. This will use netcdf libraries capabilities for parallel file access, allowing all processors -!! to perform data reads and writes, but requires hdf5 and netcdf to be built with MPI support. +!! to perform data reads and writes. To use this option, hdf5 and netcdf must be built with MPI support. !! Example usage: !! !! io_success = open_file(fileobj, "test_collective_io.nc", "read", domain, nc_format=nc_format, & -!! use_netcdf_mpi=.true., use_collective=.true.) +!! use_collective=.true.) !! !! See fms2_io/readme.md for more information. !! @@ -262,15 +260,18 @@ module fms2_io_mod module procedure register_unstructured_dimension end interface register_axis -!> @brief Defines a new field/variable within the given file +!> @brief Defines a new field/variable within the given file. After a variable is registered, +!! users can write data to the file via write_data. !>
Example usage: !! !! call register_field(fileobj, "lon", "double", (/"lon"/) ) !! -!! Adds a double variable named "lon" to the given file, corresponding to the -!! list of dimension names (which must be previously defined in the fileobj). -!! The size of dimension name list provided is the amount of ranks for the created -!! field, scalar if list not provided. +!! Adds a floating point double precision (kind=8) variable named "lon" to the given file, corresponding to the +!! list of dimension names (which must be previously registered in the fileobj with register_axis). +!! The dimension name list corresponds to the dimensions of the field. +!!
Example: +!! +!! call register_field(fileobj, "variable_2d", "double", (/"lon", "lat"/)) !! !! @note For individual documentation on the listed routines, please see the appropriate helper module. !! For netcdf files with a structured domain: @ref fms_netcdf_domain_io_mod. @@ -283,13 +284,15 @@ module fms2_io_mod module procedure register_unstructured_domain_variable end interface register_field -!> @brief Similar to @ref register_field, but occupies the field with data for restarts +!> @brief Registers a new restart field. !>
Example usage: !! -!! call register_restart_field(fileobj, "temperature", data_ptr, (/"lon", "time"/) ) +!! call register_restart_field(fileobj, "temperature", data, (/"lon", "time"/) ) !! -!! Creates a restart variable and sets it to the values from data_ptr, corresponding to -!! the list of dimension names. Rank of data_ptr must equal the amount of corresponding dimensions. +!! Creates a restart variable and stores a pointer to the data. +!! +!! This differs from the register_field interface in that registered restart fields are stored +!! in the fileobj and can be easily read/written via a call to the read_restart/write_restart interfaces. !! !! @note For individual documentation on the listed routines, please see the appropriate helper module. !! For netcdf files with a structured domain: @ref fms_netcdf_domain_io_mod. @@ -319,7 +322,7 @@ module fms2_io_mod module procedure register_restart_region_3d end interface register_restart_field -!> @brief Write data to a defined field within a file +!> @brief Write data to a registered field within a file !>
Example usage: !! !! call write_data(fileobj, "lon", data) @@ -378,13 +381,11 @@ module fms2_io_mod module procedure unstructured_domain_read_5d end interface read_data -!> @brief Writes all restart fields registered within a given restart file +!> @brief Writes previously registered restart fields to the given restart file !>
Example usage: !! !! call write_restart(fileobj) !! -!! Writes previously registered restart fields to the given restart file -!! !! @note For individual documentation on the listed routines, please see the appropriate helper module. !! For netcdf files with a structured domain: @ref fms_netcdf_domain_io_mod. !! For netcdf files with an unstructured domain: @ref fms_netcdf_unstructured_domain_io_mod. @@ -402,7 +403,9 @@ module fms2_io_mod !! call write_new_restart(fileobj, timestamp="tstring", filename="new_restartfilename") !! !! Creates a new restart file, with the provided timestamp and filename, out of the registered -!! restart fields in the given restart file. +!! restart fields in the given restart fileobj. +!! +!! @note This interface is only intended for use with diskless netcdf files. !! !! @note For individual documentation on the listed routines, please see the appropriate helper module: @ref blackboxio !> @ingroup fms2_io_mod @@ -432,6 +435,9 @@ module fms2_io_mod !! call read_new_restart(fileobj, unlimted_dimension_level) !! !! call read_new_restart(fileobj, unlimited_dimension_level, directory, timestamp, filename) +!! +!! @note This interface is only intended for use with diskless netcdf files. +!! !! @note For individual documentation on the listed routines, please see the appropriate helper module: @ref blackboxio !> @ingroup fms2_io_mod interface read_new_restart @@ -442,14 +448,14 @@ module fms2_io_mod !> @addtogroup fms2_io_mod !> @{ -logical, private :: fms2_io_is_initialized = .false. !< True after fms2_io_init is run +logical, private :: fms2_io_is_initialized = .false. !< True after calling fms2_io_init ! Namelist variables integer :: ncchksz = 64*1024 !< User defined chunksize (in bytes) argument in netcdf file !! creation calls. Replaces setting the NC_CHKSZ environment variable. character (len = 10) :: netcdf_default_format = "64bit" !< User defined netcdf file format, acceptable values - !! are: "64bit", "classic", "netcdf4". This can be overwritten if you specify - !! "nc_format" in the open_file call -integer :: header_buffer_val = 16384 !< Use defined netCDF header buffer size(in bytes) used in + !! are: "64bit", "classic", "netcdf4". This can be overwritten for a given file + !! by specifying "nc_format" in the open_file call. +integer :: header_buffer_val = 16384 !< User defined netCDF header buffer size(in bytes) used in !! NF__ENDDEF integer :: deflate_level = default_deflate_level !< Netcdf deflate level to use in nf90_def_var !! (integer between 1 to 9) @@ -459,7 +465,7 @@ module fms2_io_mod contains -!> @brief Reads the fms2_io_nml +!> @brief Reads the fms2_io_nml. Needs to be called prior to any usage of fms2_io_mod. subroutine fms2_io_init () integer :: mystat From 3f1e504ad1bc3a569ecf213a98e24cea7c3658ae Mon Sep 17 00:00:00 2001 From: Ryan Mulhall <35538242+rem1776@users.noreply.github.com> Date: Thu, 26 Mar 2026 12:19:16 -0400 Subject: [PATCH 08/12] Refine comments in fms_io_utils.F90 for mask table --- fms2_io/fms_io_utils.F90 | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/fms2_io/fms_io_utils.F90 b/fms2_io/fms_io_utils.F90 index 9b6e4f9773..ff96b45ae2 100644 --- a/fms2_io/fms_io_utils.F90 +++ b/fms2_io/fms_io_utils.F90 @@ -72,9 +72,10 @@ module fms_io_utils_mod type(char_linked_list), pointer :: head => null() endtype char_linked_list -!> Reads in a ASCII file from a given path and populates a maskmap that corresponds to a domain decomposition. +!> Reads in the mask_table file in the ASCII format from a given path and populates a maskmap array that used for +!! domain decomposition. !! -!! The logical array created will be the same shape as the domain layout, with each index representing a specific pe +!! The maskmap array is a logical array with the same shape as the domain layout, with each index representing a specific pe !! within the decomposition. This array is intended to be used as input for the mpp_define_domain routines maskmap !! argument, to essentially exclude certain pe's from a domain decomposition. !! @@ -85,14 +86,20 @@ module fms_io_utils_mod !! !! !! -!! Example: +!! For example, for a domain layout of (2,2), +!! ----------------- +!! | (1,2) | (2,2) | +!! | ------------- | +!! | (1,1) | (2,1) | +!! ----------------- +!! +!! The below mask table masks out 2 pes: the top left corner (1,1) and the bottom right corner (4,4). +!! To clarify, this means 2 pes would be required, rather than 4 if the domain decomposition was not masked. !! !! 2 -!! 4,4 +!! 2,2 !! 1,1 -!! 4,4 -!! -!! For this mask table, 2 rank's would be masked: the top right corner (1,1) and the bottom left corner (4,4) +!! 2,2 !! !! This interface includes support for both 2D and 3D mask tables. !! @@ -102,7 +109,7 @@ module fms_io_utils_mod module procedure parse_mask_table_3d end interface parse_mask_table -!> Gets the file name to be used when utilizing a mosaic that contains multiple tiles. +!> Constructs the file name to be used when utilizing a multi-tile mosaic. !! This is currenly used in the diag_manager and data_override in so that any output files !! will include the tile number of the writer. Uses the format "filename.tileN.nc" and requires !! the filename to already include the .nc suffix. From bde19f8679fc78404b7f4b46dffdb61f8a0fac0f Mon Sep 17 00:00:00 2001 From: Ryan Mulhall <35538242+rem1776@users.noreply.github.com> Date: Thu, 26 Mar 2026 14:10:24 -0400 Subject: [PATCH 09/12] Refine documentation for fms_netcdf_domain_io_mod --- fms2_io/fms_netcdf_domain_io.F90 | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/fms2_io/fms_netcdf_domain_io.F90 b/fms2_io/fms_netcdf_domain_io.F90 index 666bebe7c5..3cef01777a 100644 --- a/fms2_io/fms_netcdf_domain_io.F90 +++ b/fms2_io/fms_netcdf_domain_io.F90 @@ -17,12 +17,12 @@ !*********************************************************************** !> @defgroup fms_netcdf_domain_io_mod fms_netcdf_domain_io_mod !> @ingroup fms2_io -!> @brief This module defines a derived type, FmsNetcdfDomainFile_t, and routines -!! to handle calls to the netcdf library when using domain decomposition for a standard -!! rectangular grid. See mpp_domains_mod for more information on domain decomposition. +!> @brief This module defines the derived type, FmsNetcdfDomainFile_t, and routines +!! to handle calls to the netcdf library for data on a domain decomposed standard rectangular grid. +!! See mpp_domains_mod for more information on domain decomposition. !! -!! This module is not intended to be used externally, fms2_io_mod is intended to publicize the routines -!! and types defined here to provide a single set of interfaces to be used acases. +!! This module is not intended to be used externally. Please use the public interfaces in fms2_io_mod +!! for IO operations. !! !> @addtogroup fms_netcdf_domain_io_mod !> @{ @@ -57,16 +57,15 @@ module fms_netcdf_domain_io_mod endtype DomainDimension_t -!> @brief Type to represent a netCDF file when using a domain decomposition on a -!! standard rectangular grid. Used to do distributed I/O across ranks, -!! as determined by the io_layout. The io_layout is 2 integers set via mpp_set_io_domain +!> @brief Type to represent a netCDF file when on a domain decomposed standard rectangular +!! grid. Used to do distributed I/O across ranks, +!! as determined by the io_layout. The io_layout is a 1D array (nx_pe,ny_pe) of size 2 set via mpp_set_io_domain !! and determines how many PEs will be performing IO operations within a given -!! domain decompositon. The total number of writing PEs is equivalent to the -!! product of the io_layout. +!! domain decompositon. The total number of writing PEs is nx_pe * ny_pe. !! !! For example, if domain's layout was (4,4) so 16 PEs total, -!! then a io_layout of (2,2) would have 4 of the PEs performing I/O operations. -!! When doing a read, each IO PE will receive a data portion from 3 of the non-IO PEs and then write the aggregate. +!! then a io_layout of (2,2) would have 4 PEs performing I/O operations. +!! When doing a read, each IO PE will receive a portion of data from 3 of the non-IO PEs and then write the aggregate. !! When doing a write, each IO PE will read the data and then send a data portion to 3 of the non-IO PEs. !! !> @ingroup fms_netcdf_domain_io_mod From 5588785d3b7b70889e86356d78baa418df352f42 Mon Sep 17 00:00:00 2001 From: Ryan Mulhall <35538242+rem1776@users.noreply.github.com> Date: Thu, 26 Mar 2026 14:16:37 -0400 Subject: [PATCH 10/12] Revise documentation for fms_netcdf_unstructured_domain_io_mod Updated module documentation to clarify usage and purpose. --- fms2_io/fms_netcdf_unstructured_domain_io.F90 | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/fms2_io/fms_netcdf_unstructured_domain_io.F90 b/fms2_io/fms_netcdf_unstructured_domain_io.F90 index fee8e32ec9..b5bc3101ed 100644 --- a/fms2_io/fms_netcdf_unstructured_domain_io.F90 +++ b/fms2_io/fms_netcdf_unstructured_domain_io.F90 @@ -17,15 +17,12 @@ !*********************************************************************** !> @defgroup fms_netcdf_unstructured_domain_io_mod fms_netcdf_unstructured_domain_io_mod !> @ingroup fms2_io -!> @brief This module defines a derived type, FmsNetcdfUnstructuredDomainFile_t, and routines to handle -!! io operations when using a domain decomposition on a unstructured grid. +!> @brief This module defines the derived type, FmsNetcdfUnstructuredDomainFile_t, and routines +!! to handle calls to the netcdf library for data on a domain decomposed unstructured grid. +!! See mpp_domains_mod for more information on domain decomposition. !! -!! An unstructured grid has custom axes defined to behave differently than standard cartesian grids. These -!! grids are domain decomposed just like other grids, so global data is split amongst PEs. -!! For more information, see the domainUG type defined in mpp_domains_mod. -!! -!! This module is not intended to be used externally, fms2_io_mod is intended to publicize the routines -!! and types defined here to provide a single set of interfaces to be used across file types. +!! This module is not intended to be used externally. Please use the public interfaces in fms2_io_mod +!! for IO operations. !! module fms_netcdf_unstructured_domain_io_mod use netcdf @@ -36,9 +33,16 @@ module fms_netcdf_unstructured_domain_io_mod implicit none private -!> @brief Type to represent a netCDF file when using a domain decompostion -!! on a unstructured grid. +!> @brief Type to represent a netCDF file when on a domain decomposed unstructured grid. +!! Used to do distributed I/O across ranks, as determined by the io_layout. The io_layout +!! is a 1D array (nx_pe,ny_pe) of size 2 set via mpp_set_io_domain +!! and determines how many PEs will be performing IO operations within a given +!! domain decompositon. The total number of writing PEs is nx_pe * ny_pe. !! +!! For example, if domain's layout was (4,4) so 16 PEs total, +!! then a io_layout of (2,2) would have 4 PEs performing I/O operations. +!! When doing a read, each IO PE will receive a portion of data from 3 of the non-IO PEs and then write the aggregate. +!! When doing a write, each IO PE will read the data and then send a data portion to 3 of the non-IO PEs. !> @ingroup fms_netcdf_unstructured_domain_io_mod type, public, extends(FmsNetcdfFile_t) :: FmsNetcdfUnstructuredDomainFile_t type(domainug) :: domain !< Unstructured domain. From 3215bb0272bc67ef191098ca5c3b4186350f87a5 Mon Sep 17 00:00:00 2001 From: Ryan Mulhall <35538242+rem1776@users.noreply.github.com> Date: Thu, 26 Mar 2026 14:23:31 -0400 Subject: [PATCH 11/12] Refine comments in netcdf_io.F90 Updated comments for clarity and consistency in netcdf_io_mod. --- fms2_io/netcdf_io.F90 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/fms2_io/netcdf_io.F90 b/fms2_io/netcdf_io.F90 index 35c161f088..42cb037cb9 100644 --- a/fms2_io/netcdf_io.F90 +++ b/fms2_io/netcdf_io.F90 @@ -17,15 +17,15 @@ !*********************************************************************** !> @defgroup netcdf_io_mod netcdf_io_mod !> @ingroup fms2_io -!> @brief This module defines a derived type, FmsNetcdfFile_t, and routines -!! to handle calls to the netcdf library in order to read and write netcdf files within a MPI fortran program. +!> @brief This module defines the derived type, FmsNetcdfFile_t, and routines +!! to handle calls to the netcdf library in order to read and write netcdf files. !! -!! This module is specifically for netcdf input/output when not using a domain decomposition to split up data between -!! processors (see mpp_domains module). The FmsNetcdfFile_t type acts a base class that is extended by both +!! This module is specifically for netcdf input/output when domain decomposition is not involved +!! The FmsNetcdfFile_t type is the base class that is extended by both !! FmsNetcdfDomainFile_t and FmsNetcdfUnstructuredDomainFile_t. !! -!! This module is not intended to be used externally, fms2_io_mod is intended to publicize the routines -!! and types defined here to provide a single set of interfaces between the three different file types. +!! This module is not intended to be used externally. Please use the public interfaces in fms2_io_mod +!! for IO operations. !! !> @addtogroup netcdf_io_mod !> @{ @@ -141,7 +141,7 @@ module netcdf_io_mod endtype fmsOffloadingIn_type !> @brief Type to represent a netCDF file. Can be used with multiple cores -!! but only the root pe will perform any I/O operations, before sending the +!! but only the root pe will perform any I/O operations before sending the !! data to the other pes. !> @ingroup netcdf_io_mod type, public :: FmsNetcdfFile_t From cc02d32097df9a8380c3fe3fad770ba2461f213a Mon Sep 17 00:00:00 2001 From: rem1776 Date: Fri, 27 Mar 2026 13:02:42 -0400 Subject: [PATCH 12/12] line length fixes --- fms2_io/fms2_io.F90 | 13 +++++++------ fms2_io/fms_io_utils.F90 | 6 +++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/fms2_io/fms2_io.F90 b/fms2_io/fms2_io.F90 index 323b9e46b0..4fbff6a376 100644 --- a/fms2_io/fms2_io.F90 +++ b/fms2_io/fms2_io.F90 @@ -21,9 +21,10 @@ !! !! fms2_io_mod is the top level module that provides open, close, read, and write interfaces to the NetCDF package. !! This module defines public "aliases"(interfaces) to select procedures in fms_netcdf_domain_io_mod for reading/writing -!! data on structured grid domains; fms_netcdf_unstructured_domain_io_mod for reading/writing data on unstructured grid domains; -!! netcdf_io_mod for reading/writing data that is not parallelized with domain decomposition. Subroutines and -!! functions in fms_netcdf_domain_io_mod, fms_netcdf_unstructured_domain_io_mod, and +!! data on structured grid domains; fms_netcdf_unstructured_domain_io_mod for reading/writing data on unstructured +!! grid domains; netcdf_io_mod for reading/writing data that is not parallelized with domain decomposition. +!! +!! Subroutines and functions in fms_netcdf_domain_io_mod, fms_netcdf_unstructured_domain_io_mod, and !! netcdf_io_mod are intended for internally use only. We highly recommended to only call public interfaces defined !! in this module. !! @@ -54,8 +55,8 @@ !! See mpp_domains_mod for more information (TODO!). !! !! Besides standard open/close/read/write operations, this module also provides interfaces for writing and reading -!! "diskless" netcdf files via the blackboxio module. This module is used mainly for testing purposes, and should not be used -!! in production. +!! "diskless" netcdf files via the blackboxio module. This module is used mainly for testing purposes, and should not +!! be used in production. !! !! Users can specify additional I/O parameters with the fms2_io_nml namelist which allows users to specify the following !! NetCDF library parameters: Netcdf file format, chunk_size, deflate_level, shuffle. Note, fms2_io accepts the @@ -160,7 +161,7 @@ module fms2_io_mod !!
Example usage for opening a file with standard non-domain decomposed data: !! !! io_success = open_file(fileobj, "filename", "write") -!! +!! !! File mode can be "read"/"write"/"overwrite"/"append" !! !!
Example usage for opening a file with domain decomposed data: diff --git a/fms2_io/fms_io_utils.F90 b/fms2_io/fms_io_utils.F90 index ff96b45ae2..9375a3b1c7 100644 --- a/fms2_io/fms_io_utils.F90 +++ b/fms2_io/fms_io_utils.F90 @@ -75,9 +75,9 @@ module fms_io_utils_mod !> Reads in the mask_table file in the ASCII format from a given path and populates a maskmap array that used for !! domain decomposition. !! -!! The maskmap array is a logical array with the same shape as the domain layout, with each index representing a specific pe -!! within the decomposition. This array is intended to be used as input for the mpp_define_domain routines maskmap -!! argument, to essentially exclude certain pe's from a domain decomposition. +!! The maskmap array is a logical array with the same shape as the domain layout, with each index representing a +!! specific pe within the decomposition. This array is intended to be used as input for the mpp_define_domain +!! routines maskmap argument, to essentially exclude certain pe's from a domain decomposition. !! !! Mask table format is as follows: !!