update fms2_io docs

fms2_io/fms2_io.F90

@@ -17,9 +17,49 @@
 !***********************************************************************
 !> @defgroup fms2_io_mod fms2_io_mod
 !> @ingroup fms2_io
-!> @brief An updated library for parallel IO to replace @ref mpp_io_mod. This module contains
-!! the public API for fms2 I/O interfaces and routines defined throughout this directory.
-!! A conversion guide for replacing mpp/fms_io code with this module is available below.
+!> @brief This module aims support I/O operations for 3 types of netcdf files within the FMS framework:
+!!
+!! 1) Generic netcdf files via the netcdf_io_mod module.
+!! 2) Netcdf files with structured grid domains via the fms_netcdf_domain_io_mod module.
+!! 3) Netcdf files with unstructured grid domains via the fms_netcdf_unstructured_domain_io_mod module.
+!!
+!! This module defines public facing interfaces so that each netcdf file type above can be interacted with
+!! via a consistent set of interface calls. This allows the same interfaces to be used for operations (open_file, write_file, etc.)
+!! regardless of the specific type of netcdf file/fileobj.
+!!
+!! Each file type mentioned above has its own specific derived type to represent the file's metadata and data structure:
+!!
+!! - FmsNetcdfFile_t: This type provides a thin wrapper over the netCDF4 library, but allows the user to assign a
+!!   “pelist” to the file. If a pelist is assigned, only the first rank on the list directly interacts with the
+!!   netCDF library, and performs broadcasts to relay the information read to the rest of the ranks on the list.
+!!   When writing netcdf files, only the first rank in the pelist will perform the writes.
+!! - FmsNetcdfDomainFile_t: This type does everything that the FmsNetcdfFile_t type does and it adds support for
+!!   “domain-decomposed” reads/writes. Here, "domain decomposed" refers to data that is on a user-defined
+!!   mpp_domain and is decomposed in two dimensions, in which each MPI rank has its own section of the global data.
+!!   This requires a domain to be associated with the fileobj.
+!! - FmsNetcdfUnstructuredDomainFile_t: This type does everything that the FmsNetcdfFile_t type does and it adds
+!!   support for “domain-decomposed” reads/writes on a user defined mpp_domains unstructured grid. This requires
+!!   a unstructured domain to be associated with the fileobj. These derived types are typically refered to as the
+!!   "fileobj" across this module's interfaces.
+!!
+!! The domain file objects require a derived type (domain2d or domainug) to represent a decomposed domain for its data
+!! across available MPI ranks, as defined in mpp_domains_mod.
+!! See mpp_domains_mod documentation for more information on creating a domain decomposition using FMS.
+!!
+!! Besides standard open/close/read/write operations, this module also provides interfaces for writing and reading
+!! restart files via the blackboxio module. Restart files can be created and read for both structured and unstructured
+!! domain netcdf files and utilize the same unified interface calls (write_restart, read_restart, etc), but contain
+!! specific metadata and grid information to facilitate model restarts.
+!!
+!! Additional fine tuning of the netcdf output files can be accomplished via the fms2_io_nml namelist, which defines
+!! several parameters used by the netCDF library such as default netcdf file format, chunk size, deflate level, etc.
+!! For more information on these parameters, see the NetCDF documentation:
+!! https://docs.unidata.ucar.edu/netcdf-c/current/file_format_specifications.html
+!!
+!! Besides NetCDF io, this module also provides an interface for common utility routines used in FMS I/O operations,
+!! such as checking for file existence, reading ascii files, parsing mask tables, and generating instance filenames.
+!!
+!! If converting legacy code from fms_io/mpp_io to fms2_io, please refer to the migration guide at fms2_io/readme.md.
 
 !> @addtogroup fms2_io_mod
 !> @{
@@ -34,15 +74,16 @@ module fms2_io_mod
 implicit none
 private
 
+!> NetCDF constant (enum) for unlimited dimension identification
 public :: unlimited
-public :: FmsNetcdfFile_t
-public :: FmsNetcdfDomainFile_t
-public :: FmsNetcdfUnstructuredDomainFile_t
-public :: open_file
-public :: open_virtual_file
-public :: close_file
+
+!> File object types are defined in the helper modules (netcdf_io_mod,fms_netcdf_domain_io_mod,
+!! fms_netcdf_unstructured_domain_io_mod) but are made public here for user access.
+public :: FmsNetcdfFile_t, FmsNetcdfDomainFile_t, FmsNetcdfUnstructuredDomainFile_t
+
+!> interfaces that are defined below but utilize helper module routines.
+public :: open_file, open_virtual_file, close_file
 public :: register_axis
-public :: register_unlimited_compressed_axis
 public :: register_field
 public :: register_restart_field
 public :: write_data
@@ -51,6 +92,9 @@ module fms2_io_mod
 public :: write_new_restart
 public :: read_restart
 public :: read_new_restart
+
+!> Routines/Interfaces from netcdf_io_mod to make public
+public :: register_unlimited_compressed_axis
 public :: global_att_exists
 public :: variable_att_exists
 public :: register_global_attribute
@@ -68,27 +112,33 @@ module fms2_io_mod
 public :: get_variable_num_dimensions
 public :: get_variable_dimension_names
 public :: get_variable_size
-public :: get_compute_domain_dimension_indices
-public :: get_global_io_domain_indices
-public :: Valid_t
-public :: get_valid
-public :: is_valid
-public :: get_unlimited_dimension_name
-public :: get_variable_unlimited_dimension_index
-public :: file_exists
+public :: flush_file
+public :: write_restart_bc
+public :: read_restart_bc
 public :: compressed_start_and_count
 public :: get_variable_sense
 public :: get_variable_missing
 public :: get_variable_units
 public :: get_time_calendar
-public :: open_check
 public :: is_registered_to_restart
 public :: check_if_open
 public :: set_fileobj_time_name
+!! TODO: i think we can make this one internal to netcdf_io_mod
+public :: Valid_t
+public :: get_valid
+public :: is_valid
+
+!> Routines/Interfaces from fms_netcdf_domain_io_mod to make public
+public :: get_compute_domain_dimension_indices
+public :: get_global_io_domain_indices
+public :: get_unlimited_dimension_name
+public :: get_variable_unlimited_dimension_index
+
+!> Routines/Interfaces from fms_io_utils_mod to make public
+public :: file_exists
+public :: open_check
 public :: is_dimension_registered
 public :: fms2_io_init
-public :: write_restart_bc
-public :: read_restart_bc
 public :: get_mosaic_tile_grid
 public :: ascii_read
 public :: get_mosaic_tile_file
@@ -97,17 +147,18 @@ module fms2_io_mod
 public :: set_filename_appendix
 public :: get_instance_filename
 public :: nullify_filename_appendix
-public :: flush_file
 !> @}
 
-!> @brief Opens a given netcdf or domain file.
+!> @brief Opens a netcdf dataset on disk and initializes the file object.
 !!
 !> <br>Example usage:
 !!
 !!              io_success = open_file(fileobj, "filename", "write")
 !!
-!! Opens a netcdf file of type @ref fmsnetcdffile_t at the given file path string.
-!! File mode is set to one of "read"/"write"/"overwrite"/"append"
+!! Opens a netcdf file for a standard data, domain decomposed data, or unstructured domain decomposed data based off
+!! the fileobj used, and also intializes the fileobj for subsequent IO operations.
+!!
+!! File mode is set to one of "read"/"write"/"overwrite"/"append":
 !!
 !!              io_success = open_file(fileobj, "filename", "overwrite", domain)
 !!
@@ -125,7 +176,7 @@ module fms2_io_mod
 end interface open_file
 
 
-!> @brief Creates a diskless netcdf or domain file
+!> @brief Creates a diskless netcdf or domain file. File is created in memory only.
 !!
 !> @return true if successful, false otherwise
 !!
@@ -168,7 +219,7 @@ module fms2_io_mod
   module procedure close_unstructured_domain_file
 end interface close_file
 
-!> @brief Add a dimension to a given file
+!> @brief Adds a dimension to a given netcdf file object.
 !!
 !> <br>Example usage:
 !!

fms2_io/fms_io_utils.F90

@@ -72,18 +72,48 @@ module fms_io_utils_mod
   type(char_linked_list), pointer :: head => null()
 endtype char_linked_list
 
+!> Reads in a ASCII file from a given path and populates a maskmap that corresponds to a domain decomposition.
+!!
+!! The logical array created will be the same shape as the domain layout, with each index representing a specific pe within the
+!  decomposition. This array is intended to be used as input for the mpp_define_domain routines maskmap argument,
+!! to essentially exclude certain pe's from a domain decomposition.
+!!
+!! Mask table format is as follows:
+!!
+!! <number-of-ranks-to-mask>
+!! <domain-layout>
+!! <rank-to-mask>
+!! <any-additional-ranks-to-mask>
+!!
+!! Example:
+!!
+!! 2
+!! 4,4
+!! 1,1
+!! 4,4
+!!
+!! For this mask table, 2 rank's would be masked: the top right corner (1,1) and the bottom left corner (4,4)
+!!
+!! This interface includes support for both 2D and 3D mask tables.
+!!
 !> @ingroup fms_io_utils_mod
 interface parse_mask_table
   module procedure parse_mask_table_2d
   module procedure parse_mask_table_3d
 end interface parse_mask_table
 
+!> Gets the file name to be used when utilizing a mosaic that contains multiple tiles.
+!! This is currenly used in the diag_manager and data_override in so that any output files
+!! will include the tile number of the writer. Uses the format "filename.tileN.nc" and requires
+!! the filename to already include the .nc suffix.
 !> @ingroup fms_io_utils_mod
 interface get_mosaic_tile_file
   module procedure get_mosaic_tile_file_sg
   module procedure get_mosaic_tile_file_ug
 end interface get_mosaic_tile_file
 
+!> Interface to allocate data real, integer, or character buffers for use in read/write routines.
+!! Not meant to be used externally.
 !> @ingroup fms_io_utils_mod
 interface allocate_array
   module procedure allocate_array_i4_kind_1d
@@ -115,6 +145,16 @@ module fms_io_utils_mod
 end interface allocate_array
 
 
+!> Utility routine that copies a smaller array into a larger one using the starting indices and sizes provided.
+!! Example usage:
+!!
+!!    call put_array_section( section, array, start, sizes )
+!!
+!!  Is equivalent to
+!!
+!!    array (start(1):start(1)+sizes(1), start(2):start(2)+sizes(2), ... ) = section(:,:,...)
+!!
+!!  This interface supports integers and reals for both 4 and 8 kind, and up to 5 dimensions.
 !> @ingroup fms_io_utils_mod
 interface put_array_section
   module procedure put_array_section_i4_kind_1d
@@ -140,6 +180,17 @@ module fms_io_utils_mod
 end interface put_array_section
 
 
+!> Utility routine that copies a subset of data from a larger array into a smaller one using the
+!! starting indices and sizes provided.
+!! Example usage:
+!!
+!!    call get_array_section( section, array, start, sizes )
+!!
+!!  Is equivalent to
+!!
+!!    section(:,:,...) = array (start(1):start(1)+sizes(1), start(2):start(2)+sizes(2), ... )
+!!
+!!  This interface supports integers and reals for both 4 and 8 kind, and up to 5 dimensions.
 !> @ingroup fms_io_utils_mod
 interface get_array_section
   module procedure get_array_section_i4_kind_1d
@@ -165,6 +216,17 @@ module fms_io_utils_mod
 end interface get_array_section
 
 
+!> Sets a string describing the datatype corresponding to the sdata argument
+!! Example:
+!!
+!!    call get_data_type_string(sdata, string)
+!!
+!!  String values and associated data types are:
+!!    - "int"     integers(kind=4)
+!!    - "i8_kind" integers(kind=8)
+!!    - "float"   real(kind=4)
+!!    - "double"  real(kind=8)
+!!    - "char"    character(len=*)
 !> @ingroup fms_io_utils_mod
 interface get_data_type_string
   module procedure get_data_type_string_0d

fms2_io/fms_netcdf_domain_io.F90

@@ -17,8 +17,13 @@
 !***********************************************************************
 !> @defgroup fms_netcdf_domain_io_mod fms_netcdf_domain_io_mod
 !> @ingroup fms2_io
-!> @brief Domain-specific I/O wrappers.
-
+!> @brief This module defines a derived type, FmsNetcdfDomainFile_t, and routines
+!! to handle calls to the netcdf library when using domain decomposition for a standard
+!! rectangular grid. See mpp_domains_mod for more information on domain decomposition.
+!!
+!! This module is not intended to be used externally, fms2_io_mod is intended to publicize the routines
+!! and types defined here to provide a single set of interfaces to be used acases.
+!!
 !> @addtogroup fms_netcdf_domain_io_mod
 !> @{
 module fms_netcdf_domain_io_mod
@@ -52,7 +57,18 @@ module fms_netcdf_domain_io_mod
 endtype DomainDimension_t
 
 
-!> @brief netcdf domain file type.
+!> @brief Type to represent a netCDF file when using a domain decomposition on a
+!! standard rectangular grid. Used to do distributed I/O across ranks,
+!! as determined by the io_layout. The io_layout is 2 integers set via mpp_set_io_domain
+!! and determines how many PEs will be performing IO operations within a given
+!! domain decompositon. The total number of writing PEs is equivalent to the
+!! product of the io_layout.
+!!
+!! For example, if domain's layout was (4,4) so 16 PEs total,
+!! then a io_layout of (2,2) would have 4 of the PEs performing I/O operations.
+!! When doing a read, each IO PE will receive a data portion from 3 of the non-IO PEs and then write the aggregate.
+!! When doing a write, each IO PE will read the data and then send a data portion to 3 of the non-IO PEs.
+!!
 !> @ingroup fms_netcdf_domain_io_mod
 type, extends(FmsNetcdfFile_t), public :: FmsNetcdfDomainFile_t
   type(domain2d) :: domain !< Two-dimensional domain.

fms2_io/fms_netcdf_unstructured_domain_io.F90

@@ -17,10 +17,16 @@
 !***********************************************************************
 !> @defgroup fms_netcdf_unstructured_domain_io_mod fms_netcdf_unstructured_domain_io_mod
 !> @ingroup fms2_io
-!> @brief Handles netcdf I/O for unstructured domains
+!> @brief This module defines a derived type, FmsNetcdfUnstructuredDomainFile_t, and routines to handle
+!! io operations when using a domain decomposition on a unstructured grid.
+!!
+!! An unstructured grid has custom axes defined to behave differently than standard cartesian grids. These
+!! grids are domain decomposed just like other grids, so global data is split amongst PEs.
+!! For more information, see the domainUG type defined in mpp_domains_mod.
+!!
+!! This module is not intended to be used externally, fms2_io_mod is intended to publicize the routines
+!! and types defined here to provide a single set of interfaces to be used across file types.
 !!
-!> Mainly routines for use via interfaces in @ref fms2_io_mod
-
 module fms_netcdf_unstructured_domain_io_mod
 use netcdf
 use mpp_domains_mod
@@ -30,7 +36,9 @@ module fms_netcdf_unstructured_domain_io_mod
 implicit none
 private
 
-!> @brief netcdf unstructured domain file type.
+!> @brief Type to represent a netCDF file when using a domain decompostion
+!! on a unstructured grid.
+!!
 !> @ingroup fms_netcdf_unstructured_domain_io_mod
 type, public, extends(FmsNetcdfFile_t) :: FmsNetcdfUnstructuredDomainFile_t
   type(domainug) :: domain !< Unstructured domain.

fms2_io/netcdf_io.F90

@@ -17,10 +17,16 @@
 !***********************************************************************
 !> @defgroup netcdf_io_mod netcdf_io_mod
 !> @ingroup fms2_io
-!> @brief Creates a basic netcdf type and routines to extend for other uses
+!> @brief This module defines a derived type, FmsNetcdfFile_t, and routines
+!! to handle calls to the netcdf library in order to read and write netcdf files within a MPI fortran program.
+!!
+!! This module is specifically for netcdf input/output when not using a domain decomposition to split up data between
+!! processors (see mpp_domains module). The FmsNetcdfFile_t type acts a base class that is extended by both
+!! FmsNetcdfDomainFile_t and FmsNetcdfUnstructuredDomainFile_t.
+!!
+!! This module is not intended to be used externally, fms2_io_mod is intended to publicize the routines
+!! and types defined here to provide a single set of interfaces between the three different file types.
 !!
-!> Handles type definitions and interfaces for netcdf I/O.
-
 !> @addtogroup netcdf_io_mod
 !> @{
 module netcdf_io_mod
@@ -134,7 +140,9 @@ module netcdf_io_mod
     procedure :: init
 endtype fmsOffloadingIn_type
 
-!> @brief Netcdf file type.
+!> @brief Type to represent a netCDF file. Can be used with multiple cores
+!! but only the root pe will perform any I/O operations, before sending the
+!! data to the other pes. 
 !> @ingroup netcdf_io_mod
 type, public :: FmsNetcdfFile_t
   character(len=FMS_PATH_LEN) :: path !< File path.