diff --git a/.readthedocs.yaml b/.readthedocs.yaml index b2113a9..82bb4c8 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -10,9 +10,9 @@ build: tools: python: "3.13" -# Build documentation in the "docs/" directory with Sphinx +# Build documentation in the "doc/" directory with Sphinx sphinx: - configuration: doc/UserGuide/source/conf.py + configuration: doc/conf.py # Optionally, but recommended, # declare the Python requirements required to build your documentation diff --git a/doc/Makefile b/doc/Makefile index 37ceb8b..370fc6d 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -1,11 +1,10 @@ -# Minimal makefile for Sphinx documentation -# +# Makefile for Sphinx documentation # You can set these variables from the command line, and also # from the environment for the first two. -SPHINXOPTS += -a -n #-W --keep-going +SPHINXOPTS += -a -n -W --keep-going SPHINXBUILD ?= sphinx-build -SOURCEDIR = source +SOURCEDIR = . BUILDDIR = build LINKCHECKDIR = $(BUILDDIR)/linkcheck @@ -17,8 +16,8 @@ help: doc: make clean - $(MAKE) html $(MAKE) linkcheck + $(MAKE) html linkcheck: $(SPHINXBUILD) -b linkcheck $(SPHINXOPTS) $(SOURCEDIR) $(LINKCHECKDIR) @@ -27,4 +26,4 @@ linkcheck: # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) -w $(BUILDDIR)/warnings.log \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) -w $(BUILDDIR)/warnings.log diff --git a/doc/UserGuide/source/QuickStart/QuickStartGuide.rst b/doc/UserGuide/source/QuickStart/QuickStartGuide.rst deleted file mode 100644 index 1b17089..0000000 --- a/doc/UserGuide/source/QuickStart/QuickStartGuide.rst +++ /dev/null @@ -1,148 +0,0 @@ -.. _QuickStart: - -*************************** -Running the UFS DA Workflow -*************************** - -This chapter provides instructions for building and running the Unified Forecast System (:term:`UFS`) DA Workflow System. - -.. include:: ../doc-snippets/gcblizzard-desc.rst - -.. attention:: - - These steps are designed for use on :ref:`Level 1 ` systems (e.g., Ursa, Orion, Hercules, Gaea-C6 and Derecho.) - -.. _GetCode: - -Get Code -*********** - -Clone the Land DA workflow repository. To clone the ``develop`` branch, run: - -.. code-block:: console - - git clone -b develop --recursive https://github.com/ufs-community/ufs-da-workflow - -.. _BuildandRun: - -Build and Run the DA Workflow -****************************** - -#. Navigate to the ``sorc`` directory. - - .. code-block:: console - - cd ${BASEDIR}/ufs-da-workflow/sorc - -#. Run the build script ``app_build.sh``: - -Note: Running and building the UFS Da Workflow options below will have different run styles based off the use of the JEDI-Bundle/GDAS and Workflow components and where '[APP]' is `S2SWA`, `S2SWAL`, `NG-GODAS`, `ATML`, or `ATM`. - -#. Option 1: Workflow components: YES, JEDI-bundle: NO - - .. code-block:: console - - ./app_build.sh -a=[APP] - -#. Option 2: Workflow components: YES, JEDI-bundle: YES - - .. code-block:: console - - ./app_build.sh -a=[APP] --jedi=bundle - -#. Option 3: Workflow components: NO, JEDI-bundle: YES - - .. code-block:: console - - ./app_build.sh -a=[APP] --jedi=bundle-only - -#. Option 4: Workflow components: YES, GDAS App: YES - - .. code-block:: console - - ./app_build.sh -a=[APP] --jedi=gdas - -#. Option 5: Workflow components: NO, JEDI-bundle: YES - - .. code-block:: console - - ./app_build.sh -a=[APP] --jedi=gdas-only - -.. _load-env: - -Load the Workflow Environment -****************************** - -To load the workflow environment: - -.. code-block:: console - - cd .. - module use modulefiles - module load wflow_[workflow_manager]_[machine] - -where `[workflow_manager]` is `ecflow`, `rocoto`, or `none`, and `[machine]` is `gaeac6`, `hercules`, `orion`, `ursa`, or `derecho`. - -Modify the Workflow Configuration YAML -************************************** - -Copy the experiment settings into ``config.yaml``: - -.. code-block:: console - - cd ${BASEDIR}/ufs-da-workflow/parm - cp config_samples/config.[sample_case].yaml config.yaml - -Users will need to configure the ``account`` variable in ``config.yaml``. - - * ``ACCOUNT:`` A valid account name. Most NOAA :term:`RDHPCS` systems require a valid account name; other systems may not (in which case, any value will do). - - -.. attention:: - - When regenerating an experiment from the same or similar ``config.yaml`` file, if the ``EXP_CASE_NAME`` remains the same, the old experiment directory with that name will be renamed with the ``*_old`` suffix, and the new experiment directory will use ``EXP_CASE_NAME``. However, the ``envir`` directory will **NOT** be regenerated unless the ``envir`` parameter is given a new name. If it keeps the same name, the previous ``ptmp/`` directory and everything in it will remain (rather than being renamed), and the experiment will continue from where it left off using the files from the previous directory. This can be helpful in certain cases but detrimental in others, so users need to make a conscious choice based on their use case. - -.. _generate-wflow: - -Set Up and Launch the Workflow -******************************* - -Generate the experiment directory by running: - -.. code-block:: console - - ./setup_wflow_env.py - -Note: Two options for running the workflow based off the desired WORKFLOW_MANAGER ('ecflow' or 'rocoto'.) - -#. Option 1: WORKFLOW_MANAGER: ecflow - -.. code-block:: console - - cd ../../exp_case/ecf_server - ./start_server.sh - cd ../[EXP_CASE_NAME]/ecf - ./begin_suite.sh - ecflow_ui & - -Where `[EXP_CASE_NAME]` is specified in the configuration file `config.yaml`. - -#. Option 2: WORKFLOW_MANAGER: rocoto - -.. code-block:: console - - cd ../../exp_case/[EXP_CASE_NAME] - ./automate_launch_script.py -i [time interval in seconds] - -Where the default value of `[time interval in seconds]` is 30. This means that the launch script `launch_rocoto_wflow.sh` is submitted every 30 seconds. - - -Check the result and log files -******************************* - -- `com_dir`: symlink to the directory containing the result files -- `log_dir`: symlink to the directory containing the log files -- `tmp_dir`: symlink to the directory containing the working directories - - - diff --git a/doc/UserGuide/source/QuickStart/index.rst b/doc/UserGuide/source/QuickStart/index.rst deleted file mode 100644 index 32a8473..0000000 --- a/doc/UserGuide/source/QuickStart/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _build-run-test-index: - -UFS DA Workflow -===================================== - -.. toctree:: - :maxdepth: 1 - - QuickStartGuide diff --git a/doc/UserGuide/source/_static/custom.css b/doc/UserGuide/source/_static/custom.css deleted file mode 100644 index c02df7f..0000000 --- a/doc/UserGuide/source/_static/custom.css +++ /dev/null @@ -1,25 +0,0 @@ -.red { color: red; } -.bolditalic { - font-family: "Courier New", Courier, monospace; - font-weight: bold; - font-style: italic; -} - -.underline { - text-decoration: underline; -} - -.bolditalic { - font-weight: bold; - font-style: italic; -} - -table.align-default { - margin-left: 0px; - margin-right: auto; -} - -table.align-center { - margin-left: 0px; - margin-right: auto; -} diff --git a/doc/UserGuide/source/index.rst b/doc/UserGuide/source/index.rst deleted file mode 100644 index 59ac536..0000000 --- a/doc/UserGuide/source/index.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. UFS Land DA documentation master file, created by - sphinx-quickstart on Fri Jan 20 10:35:26 2023. - -UFS DA Workflow User's Guide |release| -============================================ - -.. toctree:: - :numbered: - :maxdepth: 1 - :caption: Contents: - - QuickStart/index - diff --git a/doc/UsersGuide/BackgroundInfo/ComponentConfiguration.rst b/doc/UsersGuide/BackgroundInfo/ComponentConfiguration.rst new file mode 100644 index 0000000..059d89c --- /dev/null +++ b/doc/UsersGuide/BackgroundInfo/ComponentConfiguration.rst @@ -0,0 +1,309 @@ +.. _component-configuration: + +Componenet Configuration +======================== + +The UFS DA Workflow supports several forecast model application configurations. +Each application enables a different set of UFS Weather Model components. + +.. _component-configuration-s2swa: + +Coupling: ATM/OCN/ICE/WAV/CHM (APP=``S2SWA``) +---------------------------------------------- + +* Coupling option used for GFS and GEFS. GOCART is excluded in the GFS + configuration. + +.. list-table:: Component configuration of S2SWA in UFS Weather Model + :header-rows: 1 + :widths: 20 15 65 + + * - Model component + - Coupling + - Description + * - FMS + - ON + - Flexible Modeling System + * - FV3 + - ON + - Dynamical core of atmospheric model developed by NOAA GFDL + * - MPAS + - OFF + - Dynamical core of atmospheric model by Los Alamos Lab and NCAR + * - AQM + - OFF + - Air quality model (CMAQ) + * - GOCART + - ON + - Global Ozone Chemistry Aerosol Radiation and Transport + * - MOM6 + - ON + - Ocean model developed by NOAA GFDL + * - HYCOM + - OFF + - Ocean model developed by U.S. Navy + * - CICE6 + - ON + - Sea ice model + * - LM4 + - OFF + - Land model developed by NOAA GFDL + * - WW3 + - ON + - Wave model (WaveWatch III) + * - STOCK_PHYS + - ON + - Physics component of atmospheric model + * - CDEPS + - ON + - Community Data models for Earth Predictive Systems + * - CMEPS + - ON + - Community Mediator for Earth Prediction Systems + * - NOAHMP + - OFF + - Land surface model + * - FIRE_BEHAVIOR + - OFF + - Fire behavior model + +.. _component-configuration-s2swal: + +Coupling: ATM/OCN/ICE/WAV/CHM/LND (APP=``S2SWAL``) +-------------------------------------------------- + +* Fully coupled configuration. + +.. list-table:: Component configuration of S2SWAL in UFS Weather Model + :header-rows: 1 + :widths: 20 15 65 + + * - Model component + - Coupling + - Description + * - FMS + - ON + - Flexible Modeling System + * - FV3 + - ON + - Dynamical core of atmospheric model developed by NOAA GFDL + * - MPAS + - OFF + - Dynamical core of atmospheric model by Los Alamos Lab and NCAR + * - AQM + - OFF + - Air quality model (CMAQ) + * - GOCART + - ON + - Global Ozone Chemistry Aerosol Radiation and Transport + * - MOM6 + - ON + - Ocean model developed by NOAA GFDL + * - HYCOM + - OFF + - Ocean model developed by U.S. Navy + * - CICE6 + - ON + - Sea ice model + * - LM4 + - OFF + - Land model developed by NOAA GFDL + * - WW3 + - ON + - Wave model (WaveWatch III) + * - STOCK_PHYS + - ON + - Physics component of atmospheric model + * - CDEPS + - ON + - Community Data models for Earth Predictive Systems + * - CMEPS + - ON + - Community Mediator for Earth Prediction Systems + * - NOAHMP + - ON + - Land surface model + * - FIRE_BEHAVIOR + - OFF + - Fire behavior model + +.. _component-configuration-ng-godas: + +Coupling: DATM/OCN/ICE (APP=``NG-GODAS``) +----------------------------------------- + +* Coupling option used for NOAA/CPC global modeling. +* Next Generation Global Ocean Data Assimilation System (NG-GODAS). + +.. list-table:: Component configuration of NG-GODAS in UFS Weather Model + :header-rows: 1 + :widths: 20 15 65 + + * - Model component + - Coupling + - Description + * - FMS + - ON + - Flexible Modeling System + * - FV3 + - OFF + - Dynamical core of atmospheric model developed by NOAA GFDL + * - MPAS + - OFF + - Dynamical core of atmospheric model by Los Alamos Lab and NCAR + * - AQM + - OFF + - Air quality model (CMAQ) + * - GOCART + - OFF + - Global Ozone Chemistry Aerosol Radiation and Transport + * - MOM6 + - ON + - Ocean model developed by NOAA GFDL + * - HYCOM + - OFF + - Ocean model developed by U.S. Navy + * - CICE6 + - ON + - Sea ice model + * - LM4 + - OFF + - Land model developed by NOAA GFDL + * - WW3 + - OFF + - Wave model (WaveWatch III) + * - STOCK_PHYS + - ON + - Physics component of atmospheric model + * - CDEPS + - ON + - Community Data models for Earth Predictive Systems + * - CMEPS + - ON + - Community Mediator for Earth Prediction Systems + * - NOAHMP + - OFF + - Land surface model + * - FIRE_BEHAVIOR + - OFF + - Fire behavior model + +.. _component-configuration-atml: + +Coupling: ATM/LND (APP=``ATML``) +-------------------------------- + +* Coupling option to support the land-DA working group using Noah-MP. + +.. list-table:: Component configuration of ATML in UFS Weather Model + :header-rows: 1 + :widths: 20 15 65 + + * - Model component + - Coupling + - Description + * - FMS + - ON + - Flexible Modeling System + * - FV3 + - ON + - Dynamical core of atmospheric model developed by NOAA GFDL + * - MPAS + - OFF + - Dynamical core of atmospheric model by Los Alamos Lab and NCAR + * - AQM + - OFF + - Air quality model (CMAQ) + * - GOCART + - OFF + - Global Ozone Chemistry Aerosol Radiation and Transport + * - MOM6 + - OFF + - Ocean model developed by NOAA GFDL + * - HYCOM + - OFF + - Ocean model developed by U.S. Navy + * - CICE6 + - OFF + - Sea ice model + * - LM4 + - OFF + - Land model developed by NOAA GFDL + * - WW3 + - OFF + - Wave model (WaveWatch III) + * - STOCK_PHYS + - ON + - Physics component of atmospheric model + * - CDEPS + - OFF + - Community Data models for Earth Predictive Systems + * - CMEPS + - ON + - Community Mediator for Earth Prediction Systems + * - NOAHMP + - ON + - Land surface model + * - FIRE_BEHAVIOR + - OFF + - Fire behavior model + +.. _component-configuration-atm: + +Non-coupling: ATM (APP=``ATM``) +------------------------------- + +* Atmospheric model (FV3+CCPP) stand-alone. + +.. list-table:: Component configuration of ATM in UFS Weather Model + :header-rows: 1 + :widths: 20 15 65 + + * - Model component + - Coupling + - Description + * - FMS + - ON + - Flexible Modeling System + * - FV3 + - ON + - Dynamical core of atmospheric model developed by NOAA GFDL + * - MPAS + - OFF + - Dynamical core of atmospheric model by Los Alamos Lab and NCAR + * - AQM + - OFF + - Air quality model (CMAQ) + * - GOCART + - OFF + - Global Ozone Chemistry Aerosol Radiation and Transport + * - MOM6 + - OFF + - Ocean model developed by NOAA GFDL + * - HYCOM + - OFF + - Ocean model developed by U.S. Navy + * - CICE6 + - OFF + - Sea ice model + * - LM4 + - OFF + - Land model developed by NOAA GFDL + * - WW3 + - OFF + - Wave model (WaveWatch III) + * - STOCK_PHYS + - ON + - Physics component of atmospheric model + * - CDEPS + - OFF + - Community Data models for Earth Predictive Systems + * - CMEPS + - OFF + - Community Mediator for Earth Prediction Systems + * - NOAHMP + - OFF + - Land surface model + * - FIRE_BEHAVIOR + - OFF + - Fire behavior model diff --git a/doc/UsersGuide/BackgroundInfo/DirectoryStructure.rst b/doc/UsersGuide/BackgroundInfo/DirectoryStructure.rst new file mode 100644 index 0000000..8d3d3dc --- /dev/null +++ b/doc/UsersGuide/BackgroundInfo/DirectoryStructure.rst @@ -0,0 +1,316 @@ +.. _directory-structure: + +Directory Structure +=================== + +The :term:`UFS` DA Workflow follows the vertical directory structure and +environment variable conventions used by :term:`NCEP` Central Operations +(:term:`NCO`) :term:`WCOSS` Implementation Standards. The workflow keeps those +conventions while allowing large working and output files to reside under a +configurable ``PTMP`` location on NOAA :term:`HPC` systems. + +.. _nco-standard-variables: + +NCO Standard Variables +---------------------- + +The land-DA workflow uses the following NCO-style variables for its directory +layout and job environment. Variables set by the :term:`job card` provide the +experiment-wide context; variables set by the :term:`J-job ` layer are +derived for each workflow task. + +.. list-table:: Standard NCO environment variables + :header-rows: 1 + :widths: 20 60 20 + + * - Name + - Description + - Set by + * - ``COMROOT`` + - ``com`` root directory for input and output data on the current system. + - Job card + * - ``DATAROOT`` + - Directory containing job working directories, usually under + ``OPSROOT/tmp``. + - Job card + * - ``DBNROOT`` + - Root directory for data-alerting utilities. + - Job card + * - ``DCOMROOT`` + - ``dcom`` root directory. + - Job card + * - ``KEEPDATA`` + - ``YES``/``NO`` switch controlling whether the working directory is kept + after a job completes successfully. + - Job card + * - ``MAILTO`` + - Email recipients for job notifications. + - Job card + * - ``MAILCC`` + - Email recipients to copy on job notifications. + - Job card + * - ``OPSROOT`` + - Operations root directory, for example ``/lfs/$FS/ops/$envir``. + - Job card + * - ``PACKAGEROOT`` + - Root installation directory for the application. + - Job card + * - ``SENDCOM`` + - ``YES``/``NO`` switch controlling copies to ``COMOUT``. + - Job card + * - ``SENDECF`` + - ``YES``/``NO`` switch controlling ``ecflow_client`` child commands. + - Job card + * - ``SENDDBN`` + - ``YES``/``NO`` switch controlling whether products are sent off WCOSS2. + - Job card + * - ``SENDDBN_NTC`` + - ``YES``/``NO`` switch controlling whether products with WMO headers are + sent off WCOSS2. + - Job card + * - ``SENDWEB`` + - ``YES``/``NO`` switch controlling whether products are sent to a web + server, often NCORZDM. + - Job card + * - ``cyc`` + - Cycle hour in GMT, formatted as ``HH``. + - Job card + * - ``envir`` + - Runtime environment: usually ``test`` for initial testing, ``para`` for + parallel production testing, and ``prod`` for production. + - Job card + * - ``job`` + - Unique job name. + - Job card + * - ``jobid`` + - Unique job identifier. + - Job card + * - ``model_ver`` + - Three-digit package version number. + - Job card + * - ``subcyc`` + - Cycle minute in GMT, formatted as ``MM``. + - Job card + * - ``COMIN`` + - ``com`` directory for the current model's input data. + - J-job + * - ``COMOUT`` + - ``com`` directory for the current model's output data. + - J-job + * - ``COMIN[model]`` + - ``com`` directory for incoming data from ``[model]``. + - J-job + * - ``COMOUT[model]`` + - ``com`` directory for outgoing data from ``[model]``. + - J-job + * - ``DCOMIN`` + - ``dcom`` directory for the current model's input data. + - J-job + * - ``DCOMIN[datatype]`` + - ``dcom`` directory for incoming data from ``[datatype]``. + - J-job + * - ``DATA`` + - Job working directory, usually ``DATAROOT/jobid``. + - J-job + * - ``EXEC[model]`` + - Model executable directory, usually ``HOME[model]/exec``. + - J-job + * - ``FIX[model]`` + - Model static-data directory, usually ``HOME[model]/fix``. + - J-job + * - ``HOME[model]`` + - Application home directory. + - J-job + * - ``NET`` + - Model name, used as the first level of the ``com`` directory structure. + - J-job + * - ``PARM[model]`` + - Model parameter directory, usually ``HOME[model]/parm``. + - J-job + * - ``PDY`` + - Cycle date in ``YYYYMMDD`` format. + - J-job + * - ``PDYm#`` + - Previous dates in ``YYYYMMDD`` format; for example, ``PDYm1`` is the + previous day. + - J-job + * - ``PDYp#`` + - Future dates in ``YYYYMMDD`` format; for example, ``PDYp1`` is the next + day. + - J-job + * - ``RUN`` + - Model run name, used as the third level of the ``com`` directory + structure. + - J-job + * - ``USH[model]`` + - Model utility-script directory, usually ``HOME[model]/ush``. + - J-job + * - ``cycle`` + - Cycle time in GMT, formatted as ``tHHz`` or ``tHHMMz``. + - J-job + +.. _workflow-layer-structure: + +Workflow Layer Structure +------------------------ + +The UFS DA Workflow follows the NCO job-layer pattern. The J-job layer sets +task directories and common environment variables. Because +most of these settings are shared across workflow tasks, the UFS DA Workflow +centralizes that setup in ``${HOMEufsda}/parm/jjob_env_setup.sh`` instead of +maintaining a separate J-job script for every task. + +Each task job card is created in the experiment case directory when the setup +script builds the case. The job card sources the centralized J-job environment +setup and then calls the task's :term:`ex-script ` in +``${HOMEufsda}/scripts``. Utility Python and shell scripts called by the +ex-scripts are stored in +``${HOMEufsda}/ush``. + +.. figure:: ../../_static/ufsDAworkflowstructure.png + :alt: Structure of the UFS DA Workflow + :align: center + + Structure of the UFS DA Workflow + +.. _workflow-vertical-directory-structure: + +Workflow Vertical Directory Structure +------------------------------------- + +The ``ufs-da-workflow`` repository uses the following directory structure. + +.. code-block:: console + + ufs-da-workflow + ├── doc + │ ├── doc-snippets + │ ├── UsersGuide + │ ├── _static + │ ├── conf.py + │ ├── index.rst + │ ├── Makefile + │ ├── make.bat + │ ├── README_git + │ ├── references.bib + │ ├── requirements.in + │ └── requirements.txt + ├── ecf + │ ├── defs + │ ├── include + │ ├── template.begin_suite.sh + │ ├── template.control_suite.sh + │ ├── template.start_server.sh + │ └── stop_server.sh + ├── fix + ├── modulefiles + │ ├── tasks + │ │ ├── common + │ │ ├── derecho + │ │ ├── gaeac6 + │ │ ├── hercules + │ │ ├── orion + │ │ └── ursa + │ ├── ufs_common.lua + │ ├── ufsda_.intel.lua + │ └── wflow__.lua + ├── parm + │ ├── config_default + │ ├── config_samples + │ ├── jedi + │ │ ├── fieldmetadata + │ │ ├── fv3 + │ │ └── soca + │ ├── templates + │ │ ├── gocart + │ │ └── task_env + │ ├── automate_launch_script.py + │ ├── detect_platform.sh + │ ├── jcard_env_setup.sh + │ ├── jjob_env_setup.sh + │ └── setup_wflow_env.py + ├── scripts + │ ├── exufsda_analysis.sh + │ ├── exufsda_fcst_ic.sh + │ ├── exufsda_forecast.sh + │ ├── exufsda_plot_stats.sh + │ └── exufsda_prep_data.sh + ├── sorc + │ ├── CMakeLists.txt + │ ├── app_build.sh + │ ├── apply_incr.fd + │ ├── calcfIMS.fd + │ ├── jcb-algorithms + │ ├── jcb-gdas + │ ├── jedi-bundle + │ ├── tile2tile_converter.fd + │ ├── ufs_model.fd + │ ├── UFS_UTILS.fd + │ └── UFS_UTILS_nofrac.fd + ├── ush + │ ├── bkg_var_replace.py + │ ├── compare.py + │ ├── compare_nc_vars.py + │ ├── fill_jinja_template.py + │ ├── jcb_setup.py + │ ├── letkf_create_ens.py + │ ├── plot_*.py + │ └── *_ioda*.py + ├── versions + │ └── run.ver_ + ├── .gitignore + ├── .gitmodules + ├── .readthedocs.yaml + ├── LICENSE + └── README.md + +On WCOSS, operational products must set ``OPSROOT`` to +``/lfs/{FS}/ops/{envir}``. If ``PTMP`` is set to ``/lfs/{FS}/ops``, the UFS DA +Workflow directory structure aligns with the NCO implementation standards while +still allowing platforms to place large files on the correct disk space. + +The UFS DA Workflow also adds ``DATA_SHARE`` under ``tmp``. This directory +holds intermediate files that should not be stored in the ``COM`` directory. + +For comparison, the NCO implementation standards use the following vertical +directory structure: + +.. code-block:: console + + [envir] ([OPSROOT]) + ├── com ([COMROOT]) + │ ├── [NET] + │ │ └── [model_ver] + │ │ └── [RUN.PDY] ([COMIN]/[COMOUT]) + │ └── output + │ └── logs + ├── dcom ([DCOMROOT]) + ├── tmp ([DATAROOT]) + │ └── [jobid] ([DATA]) + └── packages ([PACKAGEROOT]) + └── model.vX.Y.Z ([HOMEmodel]) + ├── doc + ├── ecf + ├── exec ([EXECmodel]) + ├── fix + ├── jobs + ├── modulefiles + ├── parm ([PARMmodel]) + ├── scripts + ├── sorc + ├── ush ([USHmodel]) + └── versions + +.. _fix-directory-structure: + +Static Data Directory +--------------------- + +Static data, or :term:`FIX` files, are too large to store directly in the GitHub +repository. Instead, these files are kept in centralized locations on supported +HPC systems. During installation, ``sorc/app_build.sh`` creates symbolic links +from those centralized locations into the workflow ``fix`` directory. + +To use custom static data, remove the relevant symbolic link in ``fix`` and +place the custom data under the same directory structure expected by the +workflow. diff --git a/doc/UsersGuide/BackgroundInfo/Introduction.rst b/doc/UsersGuide/BackgroundInfo/Introduction.rst new file mode 100644 index 0000000..a3b986a --- /dev/null +++ b/doc/UsersGuide/BackgroundInfo/Introduction.rst @@ -0,0 +1,77 @@ +.. _Intro: + +**************** +Introduction +**************** + +Unified Forecast System (UFS) +============================= + +The UFS is a community-based, coupled, comprehensive Earth modeling system. It +includes :ufs:`multiple applications ` that support different +forecast durations and spatial domains. NOAA's operational model suite for +numerical weather prediction (:term:`NWP`) is transitioning to the UFS from +many different modeling systems. +The UFS is designed to enable research, development, and contribution +opportunities within the broader :term:`Weather Enterprise` (including +government, industry, and academia). For more information about the UFS, visit +the :ufs:`UFS Portal <>`. + +UFS DA Workflow +=============== + +The UFS Data Assimilation (DA) Workflow provides a configurable framework for +running :term:`data assimilation` experiments with the :term:`UFS` Weather +Model and the :term:`JCSDA` :term:`JEDI` system. It connects model forecasts, +observations, analysis applications, and workflow management tools so users can +prepare data, generate forecast backgrounds, run JEDI-based analyses, and cycle +updated model states through subsequent forecasts. + +The workflow is designed to support data assimilation across several UFS +Weather Model applications. These applications range from an atmosphere-only +configuration to coupled Earth system configurations that include ocean, sea +ice, wave, aerosol/chemistry, and land components. Supported application +configurations include: + +* ``S2SWA``: atmosphere (FV3+CCPP), ocean (MOM6), sea ice (CICE), waves + (WW3), and aerosol/chemistry (GOCART) +* ``S2SWAL``: ``S2SWA`` plus land (Noah-MP) +* ``NG-GODAS``: data atmosphere (DATM), ocean (MOM6), and sea ice (CICE) +* ``ATML``: atmosphere (FV3+CCPP) and land (Noah-MP) +* ``ATM``: stand-alone atmosphere (FV3+CCPP) + +By providing common configuration files, task scripts, and experiment templates, +the UFS DA Workflow helps users build reproducible DA experiments while +preserving flexibility for different model components, observation types, +analysis methods, and computing platforms. + +User Support and Documentation +============================== + +Questions +--------- + +The UFS DA Workflow's `GitHub Discussions forum `_ +provides online support for UFS users and developers to post questions and +exchange information. When users encounter difficulties running the UFS DA +Workflow, this is the place to post. Users can expect an initial response within +two business days. + +When posting a question, users are encouraged to provide the following +information: + +* The platform or system being used (e.g., Ursa, Orion, Derecho, Hercules, + GaeaC6) +* The version of the ufs-da-workflow being used (e.g., ``develop``, + ``release/public-v1.0.0``). To determine this, users can run ``git branch``; + the name of the branch with an asterisk (``*``) in front of it is the name of + the branch or tag they are working on. Note that the ufs-da-workflow version + and the documentation version should match, or users will run into + difficulties. +* Stage of the application when the issue appeared (i.e., build/compilation, + configuration, or forecast run) +* Contents of relevant configuration files +* Full error message (preferably in text form rather than a screenshot) +* Current shell (e.g., bash, csh) and modules loaded +* Compiler + MPI combination being used +* Run directory and code directory, if available on supported platforms diff --git a/doc/UsersGuide/BackgroundInfo/index.rst b/doc/UsersGuide/BackgroundInfo/index.rst new file mode 100644 index 0000000..3b9f9a2 --- /dev/null +++ b/doc/UsersGuide/BackgroundInfo/index.rst @@ -0,0 +1,11 @@ +.. _background-info: + +Background Information +====================== + +.. toctree:: + :maxdepth: 3 + + Introduction + ComponentConfiguration + DirectoryStructure diff --git a/doc/UsersGuide/BuildingRunningTesting/BuildingTesting.rst b/doc/UsersGuide/BuildingRunningTesting/BuildingTesting.rst new file mode 100644 index 0000000..23ef1af --- /dev/null +++ b/doc/UsersGuide/BuildingRunningTesting/BuildingTesting.rst @@ -0,0 +1,605 @@ +.. _build-ufs-da-workflow: + +Building and Testing the UFS DA Workflow +======================================== + +This chapter describes how to build the UFS DA Workflow and verify related +components. It is organized by task: build the workflow, add optional DA +dependencies, run tests, and review an optional global-workflow example. + +.. _build-workflow: + +Build the UFS DA Workflow +------------------------- + +Clone the Repository +^^^^^^^^^^^^^^^^^^^^ + +Retrieve the default ``develop`` branch from the +``ufs-community/ufs-da-workflow`` repository: + +.. code-block:: console + + git clone --recursive https://github.com/ufs-community/ufs-da-workflow + cd ufs-da-workflow + +The path to the ``ufs-da-workflow`` directory is referred to as +``${HOMEufsda}`` in this documentation. + +The workflow checkout contains the source, scripts, configuration files, and +build support files. Some directories, such as ``exec`` and ``lib64``, are +created by the build process. + +.. list-table:: UFS DA Workflow directories + :header-rows: 1 + :widths: 25 75 + + * - Directory or file + - Description + * - ``doc`` + - Documentation source for the user's guide. + * - ``ecf`` + - ecFlow scripts. + * - ``fix`` + - Fix fields, tables, and other static input data. + * - ``modulefiles`` + - Machine-specific module files used to build and run the workflow. + * - ``parm`` + - Parameter files, templates, default configuration files, and sample + configuration files. + * - ``scripts`` + - Ex-scripts, or main workflow scripts. + * - ``sorc`` + - Build-related scripts and external components. + * - ``ush`` + - Utility scripts called from ex-scripts. + * - ``versions`` + - Module version control files. + * - ``exec`` + - Workflow executables, created by the build process. + * - ``lib64`` + - Dynamic libraries, created by the build process. + +The workflow commonly sits beside generated or optional directories such as +``exp_case``, ``ptmp``, ``jedi``, and ``GDASApp``. + +.. _build-workflow-components: + +Build Workflow Components +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The main build interface is ``sorc/app_build.sh``. To build workflow +components: + +.. code-block:: console + + cd ${HOMEufsda}/sorc + ./app_build.sh -a=[APP] + +Replace ``[APP]`` with ``S2SWA``, ``S2SWAL``, ``NG-GODAS``, ``ATML``, or +``ATM``. If the application is not specified, the build script defaults to +``S2SWA``. + +The build script can auto-detect supported NOAA Tier-1 platforms. To specify +the platform explicitly: + +.. code-block:: console + + ./app_build.sh -p=[machine] -a=[APP] + +Replace ``[machine]`` with ``gaeac6``, ``hercules``, ``orion``, ``ursa``, or +``derecho``. + +The build script normally builds executables in ``sorc/build`` and then moves +them to ``${HOMEufsda}/exec``. To split those phases: + +.. code-block:: console + + # Build only + ./app_build.sh -a=[APP] --build + + # Move an existing build into the workflow installation + ./app_build.sh -a=[APP] --move + +When the build process completes, executables are installed in +``${HOMEufsda}/exec``. The UFS Weather Model executable is renamed with the +application suffix, for example ``ufs_model_s2swa``. + +.. note:: + + ``apply_incr.exe`` and ``calcfIMS.exe`` are used for snow DA and are not + installed for ``APP=NG-GODAS``. + +Create FIX Links +^^^^^^^^^^^^^^^^ + +To create symbolic links to FIX directories without building software: + +.. code-block:: console + + cd ${HOMEufsda}/sorc + ./app_build.sh --fix-only + +Existing symbolic links in ``fix`` are removed and regenerated. + +Build Script Options +^^^^^^^^^^^^^^^^^^^^ + +The most common ``app_build.sh`` options are listed below. Run +``./app_build.sh -h`` for the script's full usage message. + +.. list-table:: Common ``app_build.sh`` options + :header-rows: 1 + :widths: 20 55 25 + + * - Option + - Description + - Example + * - ``-h``, ``--help`` + - Shows the help guide. + - ``./app_build.sh -h`` + * - ``-p``, ``--platform`` + - Specifies the platform. If omitted, ``parm/detect_platform.sh`` is used + to detect supported Tier-1 platforms. + - ``./app_build.sh -p=ursa`` + * - ``-c``, ``--compiler`` + - Specifies the compiler. + - ``./app_build.sh -c=intel`` + * - ``-a``, ``--app`` + - Specifies the weather model application. + - ``./app_build.sh -a=S2SWA`` + * - ``--remove`` + - Removes existing build products and updates submodules. + - ``./app_build.sh --remove`` + * - ``--clean`` + - Runs ``make clean`` in the build directory. + - ``./app_build.sh --clean`` + * - ``--build`` + - Builds executables and libraries in ``sorc/build`` without moving them + to ``exec`` and ``lib64``. + - ``./app_build.sh --build`` + * - ``--move`` + - Moves pre-compiled executables and libraries from the build directory to + ``exec`` and ``lib64``. + - ``./app_build.sh --move`` + * - ``--da-utils-only`` + - Builds DA utilities only. + - ``./app_build.sh --da-utils-only`` + * - ``--fix-only`` + - Creates symbolic links to static FIX files and exits. + - ``./app_build.sh --fix-only`` + * - ``--jedi`` + - Selects the JEDI build option: ``off``, ``bundle``, ``gdas``, + ``bundle-only``, or ``gdas-only``. + - ``./app_build.sh --jedi=bundle`` + +When ``sorc/build`` already exists, the script prompts for an interactive +choice: + +.. list-table:: Interactive build options + :header-rows: 1 + :widths: 20 80 + + * - Option + - Description + * - ``R`` or ``r`` + - Remove the existing build directory and continue. + * - ``C`` or ``c`` + - Continue building in the existing build directory. + * - ``Q`` or ``q`` + - Quit the build process. + +Installed Executables +^^^^^^^^^^^^^^^^^^^^^ + +The following executables are installed in ``${HOMEufsda}/exec`` when their +source components are included in the selected build. + +.. list-table:: Workflow executables + :header-rows: 1 + :widths: 25 50 25 + + * - Name + - Description + - Source component + * - ``ufs_model_[app]`` + - UFS Weather Model executable for the selected application. + - ``ufs_model.fd`` + * - ``chgres_cube`` + - Creates cold-start initial condition files for the atmospheric model. + - ``UFS_UTILS.fd`` + * - ``cpld_gridgen`` + - Creates static and initial condition files for coupled configurations + such as ``S2SWA`` and ``S2SWAL``. + - ``UFS_UTILS.fd`` + * - ``oiprep`` + - Creates warm-start files for ocean and ice from component restart files. + - ``UFS_UTILS.fd`` + * - ``orog_gsl`` + - Creates GSL gravity wave drag fields required by the orographic drag + suite. + - ``UFS_UTILS.fd`` + * - ``apply_incr.exe`` + - Applies JEDI increments to ``sfc_data`` files. + - ``apply_incr.fd`` + * - ``calcfIMS.exe`` + - Creates IMS NetCDF files from ASCII data files. + - ``calcfIMS.fd`` + +.. _optional-da-dependencies: + +Build Optional DA Dependencies +------------------------------ + +JEDI-bundle with ``app_build.sh`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The JEDI-bundle synced with the GDAS App can be built through +``app_build.sh``: + +.. code-block:: console + + # Workflow components and JEDI-bundle + ./app_build.sh -a=[APP] --jedi=bundle + + # DA utilities and JEDI-bundle + ./app_build.sh --da-utils-only --jedi=bundle + + # JEDI-bundle only + ./app_build.sh --jedi=bundle-only + +The script copies ``sorc/jedi-bundle`` to ``../jedi`` relative to the workflow +checkout and installs it in ``../jedi/build`` by default. + +GDAS App with ``app_build.sh`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The GDAS App can also be built through ``app_build.sh``: + +.. code-block:: console + + # Workflow components and GDAS App + ./app_build.sh -a=[APP] --jedi=gdas + + # DA utilities and GDAS App + ./app_build.sh --da-utils-only --jedi=gdas + + # GDAS App only + ./app_build.sh --jedi=gdas-only + +By default, the GDAS App is installed in ``../GDASApp`` relative to the +workflow checkout. + +.. include:: ../../doc-snippets/gdas-derecho-note.rst + +Use Separately Installed JEDI +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you use separately installed JEDI executables, set the paths in +``config.yaml``: + +.. code-block:: yaml + + JEDI_BIN_PATH: /path/to/bin/directory/of/your/JEDI/ + JEDI_IODACONV_PATH: /path/to/python/directory/of/your/ioda_converter/ + +Manual Installation of JEDI-bundle +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The automated build script is preferred, but the JEDI-bundle can also be built +manually. + +#. Create and enter a parent directory: + + .. code-block:: console + + mkdir -p ${HOMEjedi} + cd ${HOMEjedi} + +#. Clone the ``sync_gdas`` branch: + + .. code-block:: console + + git clone -b sync_gdas https://github.com/NOAA-EPIC/jedi-bundle + +#. Reset loaded modules and enter the clone: + + .. code-block:: console + + cd jedi-bundle + module purge + + On Gaea-C6, use ``module reset`` instead of ``module purge``. + +#. Load the machine module and Git LFS: + + .. code-block:: console + + module use modulefiles/ + module load [machine].intel + module load git-lfs + +#. Configure and build: + + .. code-block:: console + + cd .. + mkdir -p build + cd build + ecbuild ${HOMEjedi}/jedi-bundle + make -j4 + +#. Check that Git LFS data files were retrieved correctly: + + .. code-block:: console + + cd ${HOMEjedi}/jedi-bundle/soca/test/Data + file godas_sst_bgerr.nc + + If this reports ``ASCII`` instead of a NetCDF file, Git LFS was not active + when the data was retrieved. + +Manual Installation of GDAS App +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. Move to the parent directory of the workflow checkout: + + .. code-block:: console + + cd ${HOMEufsda}/.. + + The workflow assumes by default that ``GDASApp`` is installed beside + ``ufs-da-workflow``. Users can override this path in ``config.yaml``. + +#. Clone GDAS App: + + .. code-block:: console + + git clone --recursive https://github.com/NOAA-EMC/GDASApp.git + +#. Build GDAS App: + + .. code-block:: console + + cd GDASApp + module load git-lfs + ./build.sh -f -a -d -t [machine] > build.log 2>&1 & + + Replace ``[machine]`` with ``gaeac6``, ``hercules``, ``orion``, or + ``ursa``. + +.. include:: ../../doc-snippets/gdas-derecho-note.rst + +.. _build-and-component-tests: + +Run Build and Component Tests +----------------------------- + +C-test of JEDI-bundle +^^^^^^^^^^^^^^^^^^^^^ + +#. Get interactive access on the target platform. Examples: + + .. code-block:: console + + # Gaea-C6 + salloc -A epic -t 0:30:00 -M c6 -p batch -N 1 + + # Hercules / Orion + salloc -A epic -t 0:30:00 -q debug --nodes=1 + + # Ursa + salloc -A epic -t 30 -p u1-compute -N 1 + + # Derecho + qinteractive -A nral0032 -l walltime=00:30:00 + +#. Load the JEDI-bundle environment: + + .. code-block:: console + + cd ${HOMEjedi}/jedi-bundle + module purge + module use modulefiles/ + module load [machine].intel + + On Gaea-C6, use ``module reset`` instead of ``module purge``. + +#. Move to the build directory and list available C-tests: + + .. code-block:: console + + cd ${HOMEjedi}/build + ctest -N + +#. Run tests. For example: + + .. code-block:: console + + ctest -R test_soca + + To run ``test_soca_3dvar`` alone, first run its prerequisite cases: + + .. code-block:: console + + ctest -R test_soca_gridgen + ctest -R test_soca_setcorscales + ctest -R test_soca_parameters_diffusion + ctest -R test_soca_3dvar + +#. Check the C-test log: + + .. code-block:: console + + cd ${HOMEjedi}/build/Testing/Temporary + vim LastTest.log + +#. Inspect test inputs, outputs, and working directories as needed: + + .. code-block:: console + + cd ${HOMEjedi}/build/soca/test/test_workdir/[case_name] + +#. Cancel the interactive allocation when finished: + + .. code-block:: console + + squeue -u [user_id] + scancel [JOBID] + + # Or, on PBS systems: + qstat -u [user_id] + qdel [JOBID] + +.. _ufs-weather-model-regression-tests: + +UFS Weather Model Regression Tests +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. Create a custom regression-test configuration: + + .. code-block:: console + + cd ~/ufs-weather-model/tests/ + cp rt.conf my_rt.conf + vim my_rt.conf + +#. Edit ``COMPILE`` and ``RUN`` lines for the cases of interest. For example: + + .. code-block:: text + + COMPILE | s2swa_32bit | intel | -DAPP=S2SWA -D32BIT=ON -DCCPP_SUITES=FV3_GFS_v17_coupled_p8_ugwpv1 | | fv3 | + + RUN | cpld_control_gfsv17 | - noaacloud | baseline | + RUN | cpld_restart_gfsv17 | - noaacloud | | cpld_control_gfsv17 + +#. Set machine-specific directories in ``rt.sh``: + + .. code-block:: console + + vim rt.sh + + Set ``dprefix``, ``STMP``, and ``PTMP`` to the intended parent directory. + +#. Run the regression test: + + .. code-block:: console + + ./rt.sh -a epic -kl my_rt.conf >& my_rt.out & + +.. list-table:: Selected ``rt.sh`` flags + :header-rows: 1 + :widths: 15 85 + + * - Flag + - Description + * - ``-c`` + - Create new baseline results. + * - ``-e`` + - Use ecFlow workflow manager. + * - ``-h`` + - Display help. + * - ``-k`` + - Keep run directory. + * - ``-l`` + - Run tests listed in a file. + * - ``-m`` + - Compare against new baseline results. + * - ``-n`` + - Run a single named test. + * - ``-r`` + - Use Rocoto workflow manager. + +After the test starts, inspect the case directory: + +.. code-block:: console + + cd {dprefix}/[user_id]/FV3_RT/rt_[test_id] + +The example produces directories for the executable build, the control run, and +the restart run. To modify inputs and rerun a case: + +.. code-block:: console + + cd cpld_control_gfsv17_intel + # Modify input files as needed + sbatch job_card + +.. _related-global-workflow-example: + +Related Global Workflow Example +------------------------------- + +This example is not necessary for building or running the UFS DA Workflow. It is +included only for users who want to compare UFS DA Workflow behavior with +global-workflow cases. + +#. Create test directories: + + .. code-block:: console + + mkdir global_workflow_test + cd global_workflow_test + mkdir HOMEDIR + mkdir STMP + +#. Clone global-workflow: + + .. code-block:: console + + module load git-lfs + git clone --recursive https://github.com/NOAA-EMC/global-workflow + cd global-workflow + +#. Build global-workflow components: + + .. code-block:: console + + cd sorc + ./build_all.sh -A [account] -c gfs gsi gdas + + ``[account]`` is the project account name on NOAA RDHPCS machines. The + ``-A [account] -c`` options build on compute nodes. To build on the login + node, omit those options. + +#. Link workflow files and enter the setup directory: + + .. code-block:: console + + ./link_workflow.sh + cd ../dev/workflow + +#. Update the host YAML file, such as ``hosts/ursa.yaml``, with the desired + ``HOMEDIR``, ``STMP``, ``PTMP``, and ``DO_ARCHCOM`` values. + +#. Generate a sample workflow: + + .. code-block:: console + + ./generate_workflows.sh -A [account] -y [sample_case] -t test /path/to/global_workflow_test/test_case + + Sample cases are listed in + ``global-workflow/dev/ci/gitlab-ci-hosts.yml``. + +#. Move to the experiment directory: + + .. code-block:: console + + cd /path/to/global_workflow_test/test_case/EXPDIR/[sample_case]_test + + To preserve working directories, set ``KEEPDATA`` in ``config.base``: + + .. code-block:: console + + export KEEPDATA="YES" + +#. Load Rocoto if needed, then run and monitor the workflow: + + .. code-block:: console + + rocotorun -w [sample_case]_test.xml -d [sample_case]_test.db + rocotostat -w [sample_case]_test.xml -d [sample_case]_test.db + +Output files are written under the configured ``COMROOT``. Log files are under +``COMROOT/[sample_case]_test/logs``, and temporary run directories are under the +configured ``STMP`` path. diff --git a/doc/UsersGuide/BuildingRunningTesting/Quickstart.rst b/doc/UsersGuide/BuildingRunningTesting/Quickstart.rst new file mode 100644 index 0000000..6791576 --- /dev/null +++ b/doc/UsersGuide/BuildingRunningTesting/Quickstart.rst @@ -0,0 +1,239 @@ +.. _QuickStart: + +***************** +Quick Start Guide +***************** + +This chapter provides quick-start instructions for building and running the +Unified Forecast System (UFS) DA Workflow. It summarizes the application +configurations and the common end-to-end run paths you will encounter in ufs-da-workflow. + +.. attention:: + + These steps are designed for Level 1 systems: Ursa, Orion, Hercules, + Gaea-C6, and Derecho. When using ecFlow on Ursa, run the ecFlow steps on + the ``uecflow01`` node. + +.. _app-configurations: + +Application Configurations +========================== + +Select the application with the ``APP`` setting in ``config.yaml`` and with the +``-a=[APP]`` option to ``app_build.sh`` when building workflow components. +See :ref:`component-configuration` for the full component tables. + +.. list-table:: Supported forecast model applications + :header-rows: 1 + :widths: 15 35 50 + + * - APP + - Coupling + - Description + * - ``S2SWA`` + - ATM/OCN/ICE/WAV/CHM + - Atmosphere, ocean, sea ice, wave, and chemistry configuration used for + GFS and GEFS-style applications. GOCART is excluded for GFS cases. + * - ``S2SWAL`` + - ATM/OCN/ICE/WAV/CHM/LND + - Fully coupled configuration with the Noah-MP land surface model. + * - ``NG-GODAS`` + - DATM/OCN/ICE + - Next Generation Global Ocean Data Assimilation System configuration. + * - ``ATML`` + - ATM/LND + - Atmosphere and land configuration used for Noah-MP land DA workflows. + * - ``ATM`` + - ATM + - Stand-alone atmospheric model configuration using FV3 and CCPP. + +.. _quickstart-analysis-forecast: + +Analysis and Forecast Workflow +============================== + +#. Retrieve the ``develop`` branch of the UFS DA Workflow: + + .. include:: ../../doc-snippets/clone-ufsda-sorc.rst + + .. note:: + + The path to the ``ufs-da-workflow`` directory is referred to as + ``${HOMEufsda}`` in the commands below. + +#. Build the executables. Select one of the following options: + + .. code-block:: console + + # Workflow components without JEDI-bundle or GDAS App + ./app_build.sh -a=[APP] + + # Workflow components with JEDI-bundle + ./app_build.sh -a=[APP] --jedi=bundle + + # JEDI-bundle only, without workflow components + ./app_build.sh --jedi=bundle-only + + # Workflow components with GDAS App + ./app_build.sh -a=[APP] --jedi=gdas + + # GDAS App only, without workflow components + ./app_build.sh --jedi=gdas-only + + # Link FIX directories only, without building + ./app_build.sh --fix-only + + Replace ``[APP]`` with ``S2SWA``, ``S2SWAL``, ``NG-GODAS``, ``ATML``, or + ``ATM``. + + .. include:: ../../doc-snippets/gdas-derecho-note.rst + +#. Load the modules and Python environment for the workflow: + + .. include:: ../../doc-snippets/load-workflow-env.rst + + Replace ``[workflow_manager]`` with ``ecflow`` or ``rocoto``. Replace + ``[machine]`` with ``gaeac6``, ``hercules``, ``orion``, ``ursa``, or + ``derecho``. + +#. Copy a sample configuration and edit it for the experiment: + + .. code-block:: console + + cd ${HOMEufsda}/parm + cp config_samples/config.[APP].[case_name].yaml config.yaml + vim config.yaml + + Update the parameter values as needed, especially ``ACCOUNT``. If JEDI is + installed in a separate location, set ``JEDI_BIN_PATH`` and the corresponding + IODA converter path in ``config.yaml``; otherwise, the workflow scripts use + the default paths. + +#. Set up the case-specific experiment directory: + + .. include:: ../../doc-snippets/setup-wflow-env.rst + + To specify the platform explicitly: + + .. code-block:: console + + ./setup_wflow_env.py -p [machine] + +#. Launch the workflow tasks. + + For ``WORKFLOW_MANAGER: ecflow``: + + .. include:: ../../doc-snippets/ecflow-launch.rst + + ``[EXP_CASE_NAME]`` is set in ``config.yaml``. If the ecFlow server is + already running, do not start it again. After all suites are complete, stop + the server: + + .. include:: ../../doc-snippets/ecflow-stop.rst + + For ``WORKFLOW_MANAGER: rocoto``: + + .. include:: ../../doc-snippets/rocoto-expdir.rst + + .. include:: ../../doc-snippets/rocoto-auto-launch.rst + + The automation script runs ``launch_rocoto_wflow.sh`` every 10 seconds by + default. To use a 30-second interval: + + .. include:: ../../doc-snippets/rocoto-auto-interval.rst + + To launch Rocoto manually instead: + + .. include:: ../../doc-snippets/rocoto-helper-launch.rst + + Submit the manual launch script sequentially until all tasks are complete. + +#. Check the generated result and log links: + + * ``com_dir``: symlink to the directory containing result files + * ``log_dir``: symlink to the directory containing log files + * ``tmp_dir``: symlink to the directory containing working directories + +.. _quickstart-analysis-only: + +Analysis-Only Workflow +====================== + +Use this path for a JEDI analysis workflow without the UFS Weather Model. + +#. Retrieve the code and enter ``sorc``: + + .. include:: ../../doc-snippets/clone-ufsda-sorc.rst + +#. Build DA utilities with either JEDI-bundle or GDAS App: + + .. code-block:: console + + ./app_build.sh --da-utils-only --jedi=bundle + ./app_build.sh --da-utils-only --jedi=gdas + + .. include:: ../../doc-snippets/gdas-derecho-note.rst + +#. Load the workflow environment: + + .. include:: ../../doc-snippets/load-workflow-env.rst + +#. Copy and edit an analysis-only configuration: + + .. code-block:: console + + cd ${HOMEufsda}/parm + cp config_samples/config.anal-only.[case_name].yaml config.yaml + vim config.yaml + +#. Set up and launch the experiment: + + .. include:: ../../doc-snippets/setup-wflow-env.rst + + Launch the analysis-only workflow with ecFlow: + + .. include:: ../../doc-snippets/ecflow-launch.rst + + Stop the ecFlow server after all relevant suites are complete: + + .. include:: ../../doc-snippets/ecflow-stop.rst + +.. _quickstart-forecast-only: + +Forecast-Only Workflow +====================== + +Use this path for a stand-alone UFS Weather Model forecast without JEDI-bundle +or GDAS App. + +#. Retrieve the code and enter ``sorc``: + + .. include:: ../../doc-snippets/clone-ufsda-sorc.rst + +#. Build the workflow components: + + .. code-block:: console + + ./app_build.sh -a=[APP] + +#. Load the workflow environment: + + .. include:: ../../doc-snippets/load-workflow-env.rst + +#. Copy and edit a forecast-only configuration: + + .. code-block:: console + + cd ${HOMEufsda}/parm + cp config_samples/config.[APP].free-fcst.cycle.ecflow.yaml config.yaml + vim config.yaml + +#. Set up and launch the experiment with ecFlow: + + .. include:: ../../doc-snippets/setup-wflow-env.rst + + .. include:: ../../doc-snippets/ecflow-launch.rst + + Stop the ecFlow server after all relevant suites are complete: + + .. include:: ../../doc-snippets/ecflow-stop.rst diff --git a/doc/UsersGuide/BuildingRunningTesting/index.rst b/doc/UsersGuide/BuildingRunningTesting/index.rst new file mode 100644 index 0000000..89c2d84 --- /dev/null +++ b/doc/UsersGuide/BuildingRunningTesting/index.rst @@ -0,0 +1,10 @@ +.. _build-run-test-index: + +Building, Running, and Testing the UFS DA Workflow +================================================== + +.. toctree:: + :maxdepth: 3 + + Quickstart + BuildingTesting diff --git a/doc/UsersGuide/CustomizingTheWorkflow/index.rst b/doc/UsersGuide/CustomizingTheWorkflow/index.rst new file mode 100644 index 0000000..3c73f07 --- /dev/null +++ b/doc/UsersGuide/CustomizingTheWorkflow/index.rst @@ -0,0 +1,9 @@ +.. _customizing-workflow: + +Customizing the Workflow +======================== + +More info coming soon. + +.. toctree:: + :maxdepth: 3 diff --git a/doc/UsersGuide/Reference/FAQ.rst b/doc/UsersGuide/Reference/FAQ.rst new file mode 100644 index 0000000..e5f65c6 --- /dev/null +++ b/doc/UsersGuide/Reference/FAQ.rst @@ -0,0 +1,6 @@ +.. _faq: + +Frequently Asked Questions (FAQ) +================================ + +This page will collect frequently asked questions for the UFS DA Workflow. diff --git a/doc/UsersGuide/Reference/Glossary.rst b/doc/UsersGuide/Reference/Glossary.rst new file mode 100644 index 0000000..d1c01e9 --- /dev/null +++ b/doc/UsersGuide/Reference/Glossary.rst @@ -0,0 +1,456 @@ +.. _Glossary: + +Glossary +======== + +.. glossary:: + + 3D-Var + Three-Dimensional Variational data assimilation. + + ATM + The Weather Model configuration that runs only the standalone atmospheric + model. + + ATML + The ATML configuration uses the Noah-MP :term:`land component` from the + UFS :term:`Weather Model` with an active :term:`FV3` atmospheric component + (`ufsatm `_). + + CCPP + The `Common Community Physics Package `_ + is a forecast-model agnostic, vetted collection of code containing + atmospheric physical parameterizations and suites of parameterizations for + use in Numerical Weather Prediction (NWP) along with a framework that + connects the physics to the host forecast model. + + CDEPS + The `Community Data Models for Earth Predictive Systems `_ + repository (CDEPS) contains a set of :term:`NUOPC`-compliant data + components and :term:`ESMF`-based "stream" code that selectively removes + feedback in coupled model systems. In essence, CDEPS handles the static + Data Atmosphere (:term:`DATM`) integration with dynamic coupled model + components (e.g., :term:`MOM6`). The CDEPS data models perform the basic + function of reading external data files, modifying those data, and then + sending the data back to the :term:`CMEPS` :term:`mediator`. The fields + sent to the mediator are the same as those that would be sent by an active + component. This takes advantage of the fact that the mediator and other + CMEPS-compliant model components have no fundamental knowledge of whether + another component is fully active or just a data component. More + information about DATM is available in the CDEPS + `Documentation `_. + + CESM + The `Community Earth System Model `_ (CESM) + is a fully-coupled global climate model developed at the National Center + for Atmospheric Research (:term:`NCAR`) in collaboration with colleagues in + the research community. + + CMEPS + The `Community Mediator for Earth Prediction Systems `_ + (CMEPS) is a :term:`NUOPC`-compliant :term:`mediator` used for coupling + Earth system model components. It is currently being used in NCAR's + Community Earth System Model (:term:`CESM`) and NOAA's + subseasonal-to-seasonal (S2S) coupled system. More information is + available in the + `CMEPS Documentation `_. + + coldstart + cold start + A coldstart forecast initializes a model using data from a different + source (e.g., climatology data, forecast data from a different model, + analysis files) to "spin up," or start, the forecast. + + container + `Docker `_ describes a + container as "a standard unit of software that packages up code and all + its dependencies so the application runs quickly and reliably from one + computing environment to another." + + cron + cron job + crontab + cron table + Cron is a job scheduler accessed through the command-line on UNIX-like + operating systems. It is useful for automating tasks such as regression + testing. Cron periodically checks a cron table (aka crontab) to see if any + tasks are ready to execute. If so, it runs them. + + cycle + An hour of the day on which a forecast is started. In the UFS DA Workflow, + it usually follows ``YYYYMMDD-HHmmss`` format. + + data assimilation + DA + One of the major sources of error in weather and climate forecasts is + uncertainty related to the initial conditions that are used to generate + future predictions. Even the most precise instruments have a small range + of unavoidable measurement error, which means that tiny measurement errors + can compound over time. These small differences result in very similar + forecasts in the short term, but they cause widely divergent forecasts in + the long term. Errors in weather and climate forecasts can also arise + because models are imperfect representations of reality. Data assimilation + systems seek to mitigate these problems by combining the most timely + observational data with a "first guess" of the atmospheric state, usually + a previous forecast, and other sources of data to provide a "best guess" + analysis of the atmospheric state to start a weather or climate + simulation. When combined with an ensemble of model runs, data assimilation + helps predict a range of possible atmospheric states, giving an overall + measure of uncertainty in a given forecast. + + DA-fcst + The analysis (DA) result updates the restart files for the forecast run + (in contrast to a :term:`free-fcst`). + + DA increment + Analysis increment + A DA increment, or analysis increment, is the difference between a "first + guess" of the state of the system, usually a previous model forecast, and + the "best guess" of what the actual initial state of the system is, the + analysis produced by the DA system. When introducing a new starting state + for the model integration, care must be taken to ensure that the initial + state is balanced and realistic according to the model equations; + otherwise, the forecast may be poor or even unstable. + + Refer to the linked articles for more information on + `3D-Var `_ + and `LETKF-OI `_. + + DATM + DATM is the Data Atmosphere component of :term:`CDEPS`. It uses static + atmospheric forcing files, derived from observations or previous + atmospheric model runs, instead of output from an active atmospheric + model. This reduces the complexity and computational cost associated with + coupling to an active atmospheric model. The Data Atmosphere component is + particularly useful when employing computationally intensive Data + Assimilation (DA) techniques to update ocean and/or sea ice fields in a + coupled model. In general, use of DATM in place of :term:`ATM` can be + appropriate when users are running a coupled model and only want certain + components of the model to be active. More information about DATM is + available in the + `CDEPS Documentation `_. + + dycore + dynamical core + Global atmospheric model based on fluid dynamics principles, including + Euler's equations of motion. + + ECMWF + The `European Centre for Medium-Range Weather Forecasts `_ + (ECMWF). + + ERA5 + The ECMWF Reanalysis v5 + (`ERA5 `_) + dataset is the fifth-generation ECMWF atmospheric reanalysis. + + ESMF + `Earth System Modeling Framework `_. + The ESMF defines itself as "a suite of software tools for developing + high-performance, multi-component Earth science modeling applications." It + is a community-developed software infrastructure for building and coupling + models. + + ex-scripts + Scripting layer, contained in ``ufs-da-workflow/scripts/``, that should be + called by a :term:`J-job ` for each workflow component to run a + specific task or sub-task in the workflow. The different scripting layers + are described in detail in the :nco:`NCO Implementation Standards document `. + + FMS + The Flexible Modeling System (`FMS `_) is + a software framework for supporting the efficient development, + construction, execution, and scientific interpretation of atmospheric, + oceanic, and climate system models. + + FIX + FIX files + Static input data, tables, and fields required by workflow applications. + In the UFS DA Workflow, FIX files are usually stored in a centralized HPC + location and linked into the repository's ``fix`` directory. + + forcing data + In coupled mode, data that are generated by one component of a model can + be fed into another component to provide required input describing the + state of the Earth system. When models are run in offline, or "uncoupled" + mode, the model does not receive this input from another active component, + so forcing files are provided. These files may consist of observational + data or data gathered when running other components separately, and they + contain values for the required input fields. + + free-fcst + Analysis (DA) result and forecast are independent; the analysis result + does not impact the forecast run (in contrast to a :term:`DA-fcst`). + + FV3 + The Finite-Volume Cubed-Sphere dynamical core (dycore). Developed at + NOAA's `Geophysical Fluid Dynamics Laboratory `__, + it is a scalable and flexible dycore capable of both hydrostatic and + non-hydrostatic atmospheric simulations. It is the dycore used in the UFS + Weather Model. + + GDAS + The Global Data Assimilation System + (`GDAS `_) + is the system used by the Global Forecast System (:term:`GFS`) model to + place observations into a gridded model space for the purpose of starting, + or initializing, weather forecasts with observed data. + + GFS + The Global Forecast System + (`GFS `_) + is an :term:`NCEP` model that generates atmospheric and land-soil + variables and couples atmosphere, ocean, land/soil, and sea ice models. + + GHCN + The Global Historical Climatology Network + (`GHCN `_) + is a database of daily climate summaries from land surface stations. + + GSWP3 + The `Global Soil Wetness Project Phase 3 `_ + dataset documents several hydro-energy-ecosystem variables. + + HPC + High-Performance Computing. + + ICs + Initial conditions. + + job card + job cards + Batch-submission scripts generated for workflow tasks. A job card sets + task-level scheduler options and environment variables before the + workflow enters the J-job and ex-script layers. + + IMS + The `Interactive Multisensor Snow and Ice Mapping System `_ + (IMS) is an operational snow and ice mapping product. + + IODA + The :term:`JEDI` Interface for Observation Data Access + (:jedi:`IODA `) provides + observation formatting and processing tools for data assimilation + applications. + + J-jobs + Scripts, contained in ``ufs-da-workflow/jobs/``, that should be directly + called for each workflow component, either on the command line or by the + workflow manager, to run a specific task in the workflow. The different + scripting layers are described in detail in the + :nco:`NCO Implementation Standards document `. + + JCB + JEDI Configuration Builder + The JEDI Configuration Builder (JCB) is a Python package used to assemble + information on :term:`JEDI` algorithms and data assimilation types into a + convenient YAML file for use in data assimilation applications. + + JCSDA + The Joint Center for Satellite Data Assimilation. + + JEDI + The Joint Effort for Data assimilation Integration + (`JEDI `_) is a unified and + versatile data assimilation (DA) system for Earth System Prediction. It + aims to enable efficient research and accelerated transition from research + to operations by providing a framework that takes into account all + components of the Earth system in a consistent manner. The JEDI software + package can run on a variety of platforms and for a variety of purposes, + and it is designed to readily accommodate new atmospheric and oceanic + models and new observation systems. The + `JEDI User's Guide `_ + contains extensive information on the software. + + JEDI is developed and distributed by the + `Joint Center for Satellite Data Assimilation `_, + a multi-agency research center hosted by the University Corporation for + Atmospheric Research (`UCAR `_). + + jedi-bundle + :term:`JCSDA`'s `jedi-bundle `_ + repository provides an integrated Earth System data assimilation + capability. It combines a variety of :term:`JEDI` components, including + :term:`OOPS`, :term:`IODA`, and :term:`UFO`. + + LND + The LND experiment configuration uses the Noah-MP :term:`land component` + with the :term:`DATM` component. + + land component + The Noah Multi-Physics (Noah-MP) land surface model (LSM) is an + open-source, community-developed LSM that has been incorporated into the + UFS Weather Model (WM). It is the UFS WM's land component. + + LETKF + Local Ensemble Transform Kalman Filter (LETKF) data assimilation. + + LETKF-OI + Local Ensemble Transform Kalman Filter-Optimal Interpolation. + + Mediator + mediator + A mediator, sometimes called a coupler, is a software component that + includes code for representing component interactions. Typical operations + include merging data fields, ensuring consistent treatment of coastlines, + computing fluxes, and temporal averaging. + + MOM + MOM6 + Modular Ocean Model + MOM6 is the latest generation of the Modular Ocean Model. It is numerical + model code for simulating the ocean general circulation. MOM6 was + originally developed by the + `Geophysical Fluid Dynamics Laboratory `__. + Currently, `MOM6 code `_ and an + `extensive suite of test cases `_ + are available under an open-development software framework. Although there + are many public forks of MOM6, the + `NOAA EMC fork `_ is used in the UFS + Weather Model. + + MPI + MPI stands for Message Passing Interface. An MPI is a standardized + communication system used in parallel programming. It establishes portable + and efficient syntax for the exchange of messages and data between + multiple processors that are used by a single computer program. An MPI is + required for high-performance computing (HPC) systems. + + NCAR + The `National Center for Atmospheric Research `_. + + netCDF + NetCDF (`Network Common Data Form `_) + is a file format and community standard for storing multidimensional + scientific data. It includes a set of software libraries and + machine-independent data formats that support the creation, access, and + sharing of array-oriented scientific data. + + NCEP + National Centers for Environmental Prediction (NCEP) is an arm of the + National Weather Service consisting of nine centers. More information can + be found at https://www.weather.gov/ncep/. + + NCO + :term:`NCEP` Central Operations. Visit the + `NCO website `_ for more information. + + NUOPC + National Unified Operational Prediction Capability + The `National Unified Operational Prediction Capability `_ + (NUOPC) is a consortium of Navy, NOAA, and Air Force modelers and their + research partners. It aims to advance the weather modeling systems used by + meteorologists, mission planners, and decision makers. NUOPC partners are + working toward a common model architecture, a standard way of building + models, in order to make it easier to collaboratively build modeling + systems. + + Noah-MP + The land component of the UFS WM used in the UFS DA Workflow. + + NUOPC Layer + The :term:`NUOPC` Layer defines conventions and a set of generic + components for building coupled models using the Earth System Modeling + Framework (:term:`ESMF`). NUOPC applications are built on four generic + components: driver, model, mediator, and connector. For more information, + visit the `NUOPC website `_. + + NUOPC Cap + NUOPC Model Cap + A NUOPC "cap" is an interface between a given model component and the rest + of a coupled model system. It is a small software layer that sits on top of + the component model, making calls into it. + + NWP + Numerical Weather Prediction (NWP) takes current observations of weather + and processes them with computer models to forecast the future state of + the weather. + + OOPS + The :term:`JEDI` Object-Oriented Prediction System + (:jedi:`OOPS `) includes data + assimilation algorithms for use in data assimilation applications. + + RDHPCS + `Research and Development High-Performance Computing Systems `_. + + Rocoto + A workflow manager used to define, submit, monitor, and control dependent + workflow tasks. The UFS DA Workflow can generate Rocoto XML and database + files when ``WORKFLOW_MANAGER`` is set to ``rocoto``. + + SFCSNO + Global Telecommunication System data available from :term:`GDAS`/:term:`GFS`. + + SMAP + `Soil Moisture Active Passive Data (SMAP) `_. + + SMOPS + The `Soil Moisture Operational Products System (SMOPS) `_ + combines soil moisture data collected from several satellites to provide a + global soil moisture map. + + Spack + `Spack `_ is a package + management tool designed to support multiple versions and configurations + of software on a wide variety of platforms and environments. It was + designed for large supercomputing centers where many users and application + teams share common installations of software on clusters with exotic + architectures. + + spack-stack + The `spack-stack `_ is a + collaborative effort between the NOAA Environmental Modeling Center (EMC), + the UCAR Joint Center for Satellite Data Assimilation (JCSDA), and the + Earth Prediction Innovation Center (EPIC). *spack-stack* is a repository + that provides a :term:`Spack`-based method for building the software stack + required for numerical weather prediction (NWP) tools such as the + :ufs:`Unified Forecast System (UFS) <>` and the :jedi:`Joint Effort for Data assimilation Integration (JEDI) <>` + framework. *spack-stack* uses the Spack package manager along with custom + Spack configuration files and Python scripts to simplify installation of + the libraries required to run various applications. The *spack-stack* can + be installed on a range of platforms and comes pre-configured for many + systems. Users can install the necessary packages for a particular + application and later add the missing packages for another application + without having to rebuild the entire stack. + + UFO + The :term:`JEDI` Unified Forward Operator + (:jedi:`UFO `) is used to compare + model forecasts and observations in data assimilation applications. + + UFS + The Unified Forecast System (UFS) is a community-based, coupled, + comprehensive Earth modeling system consisting of several applications + (apps). These apps span regional to global domains and sub-hourly to + seasonal time scales. The UFS is designed to support the + :term:`Weather Enterprise` and to be the source system for NOAA's + operational numerical weather prediction applications. For more + information, visit the :ufs:`UFS Portal <>`. + + Umbrella repository + A repository that houses external code, or "externals," from additional + repositories. + + warmstart + warm start + A warmstart forecast uses saved fields from a recent forecast of the same + model, often provided via RESTART files, to populate certain variables. + This is in contrast to a :term:`coldstart`. + + Weather Enterprise + Individuals and organizations from public, private, and academic sectors + that contribute to the research, development, and production of weather + forecast products; primary consumers of these weather forecast products. + + Weather Model + WM + A prognostic model that can be used for short- and medium-range research + and operational forecasts. It can be run as an atmosphere-only model or as + an atmospheric model coupled with one or more additional components, such + as a wave or ocean model. + + WCOSS + WCOSS2 + Weather and Climate Operational Supercomputing System, NOAA's operational + supercomputing environment for NCEP production workflows. diff --git a/doc/UsersGuide/Reference/Rocoto.rst b/doc/UsersGuide/Reference/Rocoto.rst new file mode 100644 index 0000000..1844571 --- /dev/null +++ b/doc/UsersGuide/Reference/Rocoto.rst @@ -0,0 +1,186 @@ +.. _rocoto: + +Rocoto Introductory Information +=============================== + +Rocoto is one of the workflow managers supported by the UFS DA Workflow. When +``WORKFLOW_MANAGER`` is set to ``rocoto`` in ``config.yaml``, the experimental +case directory is set up to run workflow tasks with the Rocoto workflow manager. + +In the examples below, ``${HOMEufsda}`` refers to the full path of the cloned +``ufs-da-workflow`` repository. + +.. _rocoto-select: + +Select Rocoto +------------- + +Load the workflow environment: + +.. code-block:: console + + cd ${HOMEufsda} + module use modulefiles + module load wflow_[workflow_manager]_[machine] + +Replace ``[workflow_manager]`` with ``rocoto`` and replace ``[machine]`` with +``gaeac6``, ``hercules``, ``orion``, ``ursa``, or ``derecho``. + +In ``config.yaml``, set: + +.. code-block:: yaml + + parm: + WORKFLOW_MANAGER: rocoto + +Based on the ``WORKFLOW_MANAGER`` parameter in the configuration file, a Rocoto +XML file is generated in the experimental directory. + +.. _rocoto-automated-launch: + +Automated Launch +---------------- + +Move to the generated experiment directory: + +.. include:: ../../doc-snippets/rocoto-expdir.rst + +Run the automation script: + +.. include:: ../../doc-snippets/rocoto-auto-launch.rst + +The automation script runs ``launch_rocoto_wflow.sh`` every 10 seconds by +default. To use a 30-second interval: + +.. include:: ../../doc-snippets/rocoto-auto-interval.rst + +or: + +.. code-block:: console + + ./automate_launch_script.py --interval 30 + +The script prints status information to the terminal, for example: + +.. code-block:: text + + Using log file: log.rocoto_launch + Running ./launch_rocoto_wflow.sh ... + Cycles: 0 out of 2 completed. + Detected wflow_status = IN PROGRESS + Waiting 10 seconds before next run... + +Press ``Ctrl+C`` to stop the automation loop. To resume the loop, run +``./automate_launch_script.py`` again from the experiment directory. + +.. _rocoto-manual-helper-launch: + +Manual Launch With the Helper Script +------------------------------------ + +Users can run the launch helper directly: + +.. include:: ../../doc-snippets/rocoto-expdir.rst + +.. include:: ../../doc-snippets/rocoto-helper-launch.rst + +Run ``launch_rocoto_wflow.sh`` sequentially until all workflow tasks complete. +The script calls ``rocotorun``, checks ``rocotostat`` output, writes +``log.rocoto_launch``, and reports whether the workflow is still in progress, +has succeeded, or has failed. + +.. _rocoto-manual-commands: + +Manual Launch With Rocoto Commands +---------------------------------- + +If needed, load Rocoto directly for the target platform: + +.. list-table:: Rocoto module commands + :header-rows: 1 + :widths: 20 80 + + * - Platform + - Commands + * - Ursa + - ``module load rocoto`` + * - Orion + - ``module load contrib``; ``module load rocoto`` + * - Hercules + - ``module load contrib``; ``module load rocoto`` + * - Gaea-C6 + - ``module use /ncrc/proj/epic/rocoto/modulefiles/``; + ``module load rocoto`` + * - Derecho + - ``module use /glade/work/epicufsrt/contrib/derecho/modulefiles``; + ``module load rocoto/1.3.7`` + +Launch the workflow manually: + +.. include:: ../../doc-snippets/rocoto-expdir.rst + +.. code-block:: console + + rocotorun -w land_analysis.xml -d land_analysis.db -v 10 + +Check task status: + +.. code-block:: console + + rocotostat -w land_analysis.xml -d land_analysis.db -v 10 + +.. _rocoto-restart-dead-task: + +Restart a DEAD Task +------------------- + +When a task fails, ``rocotostat`` may report it in the ``DEAD`` state: + +.. code-block:: text + + CYCLE TASK JOBID STATE EXIT STATUS TRIES DURATION + ======================================================================== + 200001030000 analysis 16805206 SUCCEEDED 0 1 69.0 + 200001030000 forecast 16805240 DEAD 5 1 - + +After resolving the underlying issue, rewind the failed task: + +.. code-block:: console + + rocotorewind -w land_analysis.xml -d land_analysis.db -c 200001030000 -t forecast + +The cycle supplied with ``-c`` must include minutes in ``YYYYMMDDHHmm`` format. +To rewind multiple tasks: + +.. code-block:: console + + rocotorewind -w land_analysis.xml -d land_analysis.db -c 200001030000 -t 'analysis,forecast' + +Submit the rewound task again: + +.. code-block:: console + + rocotoboot -w land_analysis.xml -d land_analysis.db -c 200001030000 -t forecast + +.. _rocoto-command-flags: + +Useful Rocoto Command Flags +--------------------------- + +.. list-table:: Common Rocoto flags + :header-rows: 1 + :widths: 20 80 + + * - Flag + - Description + * - ``-w`` + - Path to the Rocoto workflow XML file. + * - ``-d`` + - Path to the Rocoto workflow database file. + * - ``-v`` + - Run Rocoto in verbose mode at the specified level. + * - ``-c`` + - Cycle or cycle list. Values may be a single cycle, comma-separated + cycles, a cycle range, or ``all``. + * - ``-t`` + - Task or comma-separated task list. diff --git a/doc/UsersGuide/Reference/index.rst b/doc/UsersGuide/Reference/index.rst new file mode 100644 index 0000000..2e5702b --- /dev/null +++ b/doc/UsersGuide/Reference/index.rst @@ -0,0 +1,11 @@ +.. _reference: + +Reference +========= + +.. toctree:: + :maxdepth: 3 + + Rocoto + FAQ + Glossary diff --git a/doc/UsersGuide/index.rst b/doc/UsersGuide/index.rst new file mode 100644 index 0000000..6db28d1 --- /dev/null +++ b/doc/UsersGuide/index.rst @@ -0,0 +1,14 @@ +:orphan: + +.. _users-guide: + +User's Guide +============ + +.. toctree:: + :maxdepth: 3 + + BackgroundInfo/index + BuildingRunningTesting/index + CustomizingTheWorkflow/index + Reference/index diff --git a/doc/UserGuide/source/_static/theme_overrides.css b/doc/_static/theme_overrides.css similarity index 100% rename from doc/UserGuide/source/_static/theme_overrides.css rename to doc/_static/theme_overrides.css diff --git a/doc/_static/ufsDAworkflowstructure.png b/doc/_static/ufsDAworkflowstructure.png new file mode 100644 index 0000000..7b0541c Binary files /dev/null and b/doc/_static/ufsDAworkflowstructure.png differ diff --git a/doc/UserGuide/source/conf.py b/doc/conf.py similarity index 98% rename from doc/UserGuide/source/conf.py rename to doc/conf.py index dd5b20d..7bf5528 100644 --- a/doc/UserGuide/source/conf.py +++ b/doc/conf.py @@ -39,7 +39,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] +exclude_patterns = ['doc-snippets/*'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' @@ -113,7 +113,6 @@ html_context = {} def setup(app): - app.add_css_file('custom.css') # may also be an URL app.add_css_file('theme_overrides.css') # may also be a URL # -- Options for intersphinx extension --------------------------------------- diff --git a/doc/doc-snippets/clone-ufsda-sorc.rst b/doc/doc-snippets/clone-ufsda-sorc.rst new file mode 100644 index 0000000..1f22e3a --- /dev/null +++ b/doc/doc-snippets/clone-ufsda-sorc.rst @@ -0,0 +1,4 @@ +.. code-block:: console + + git clone --recursive https://github.com/ufs-community/ufs-da-workflow + cd ufs-da-workflow/sorc diff --git a/doc/doc-snippets/ecflow-launch.rst b/doc/doc-snippets/ecflow-launch.rst new file mode 100644 index 0000000..3aa405d --- /dev/null +++ b/doc/doc-snippets/ecflow-launch.rst @@ -0,0 +1,7 @@ +.. code-block:: console + + cd ../../exp_case/ecf_server + ./start_server.sh + cd ../../exp_case/[EXP_CASE_NAME]/ecf + ./begin_suite.sh + ecflow_ui & diff --git a/doc/doc-snippets/ecflow-stop.rst b/doc/doc-snippets/ecflow-stop.rst new file mode 100644 index 0000000..37cbf02 --- /dev/null +++ b/doc/doc-snippets/ecflow-stop.rst @@ -0,0 +1,4 @@ +.. code-block:: console + + cd ../../exp_case/ecf_server + ./stop_server.sh diff --git a/doc/doc-snippets/gdas-derecho-note.rst b/doc/doc-snippets/gdas-derecho-note.rst new file mode 100644 index 0000000..45b41d3 --- /dev/null +++ b/doc/doc-snippets/gdas-derecho-note.rst @@ -0,0 +1,3 @@ +.. note:: + + GDAS App is not available on Derecho. diff --git a/doc/doc-snippets/load-workflow-env.rst b/doc/doc-snippets/load-workflow-env.rst new file mode 100644 index 0000000..e15c867 --- /dev/null +++ b/doc/doc-snippets/load-workflow-env.rst @@ -0,0 +1,5 @@ +.. code-block:: console + + cd ${HOMEufsda} + module use modulefiles + module load wflow_[workflow_manager]_[machine] diff --git a/doc/doc-snippets/rocoto-auto-interval.rst b/doc/doc-snippets/rocoto-auto-interval.rst new file mode 100644 index 0000000..4db1abc --- /dev/null +++ b/doc/doc-snippets/rocoto-auto-interval.rst @@ -0,0 +1,3 @@ +.. code-block:: console + + ./automate_launch_script.py -i 30 diff --git a/doc/doc-snippets/rocoto-auto-launch.rst b/doc/doc-snippets/rocoto-auto-launch.rst new file mode 100644 index 0000000..acea953 --- /dev/null +++ b/doc/doc-snippets/rocoto-auto-launch.rst @@ -0,0 +1,3 @@ +.. code-block:: console + + ./automate_launch_script.py diff --git a/doc/doc-snippets/rocoto-expdir.rst b/doc/doc-snippets/rocoto-expdir.rst new file mode 100644 index 0000000..70b08b8 --- /dev/null +++ b/doc/doc-snippets/rocoto-expdir.rst @@ -0,0 +1,3 @@ +.. code-block:: console + + cd ${HOMEufsda}/../exp_case/[EXP_CASE_NAME] diff --git a/doc/doc-snippets/rocoto-helper-launch.rst b/doc/doc-snippets/rocoto-helper-launch.rst new file mode 100644 index 0000000..6646e39 --- /dev/null +++ b/doc/doc-snippets/rocoto-helper-launch.rst @@ -0,0 +1,4 @@ +.. code-block:: console + + ./launch_rocoto_wflow.sh + vim log.rocoto_launch diff --git a/doc/doc-snippets/setup-wflow-env.rst b/doc/doc-snippets/setup-wflow-env.rst new file mode 100644 index 0000000..3c4ace6 --- /dev/null +++ b/doc/doc-snippets/setup-wflow-env.rst @@ -0,0 +1,3 @@ +.. code-block:: console + + ./setup_wflow_env.py diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 0000000..4fd6f19 --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,17 @@ +.. UFS DA Workflow documentation master file, created by + sphinx-quickstart on Fri Jan 20 10:35:26 2023. + +.. _ufs-da-workflow-docs: + +UFS DA Workflow Documentation (|version|) +========================================= + +.. toctree:: + :numbered: + :maxdepth: 1 + + UsersGuide/BackgroundInfo/index + UsersGuide/BuildingRunningTesting/index + UsersGuide/CustomizingTheWorkflow/index + UsersGuide/Reference/index + diff --git a/doc/make.bat b/doc/make.bat index dc1312a..207019d 100644 --- a/doc/make.bat +++ b/doc/make.bat @@ -7,7 +7,7 @@ REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) -set SOURCEDIR=source +set SOURCEDIR=. set BUILDDIR=build %SPHINXBUILD% >NUL 2>NUL diff --git a/doc/references.bib b/doc/references.bib new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/doc/references.bib @@ -0,0 +1 @@ +