Fix doc generation and improve on structure + some content

docs/CMakeLists.txt

@@ -1,31 +1,58 @@
 find_package(Doxygen 1.9 REQUIRED)
 find_package(Sphinx REQUIRED)
 
-file(GLOB_RECURSE FDB_PUBLIC_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../src/fdb5/api/*.h)
-file(GLOB_RECURSE PYFDB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/../src/pyfdb/*.py)
-file(GLOB_RECURSE Z3FDB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/../src/z3fdb/*.py)
-file(GLOB_RECURSE SPHINX_INPUT_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.rst)
-set(DOXYGEN_INDEX_FILE ${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/index.html)
-set(SPHINX_INDEX_FILE ${CMAKE_CURRENT_BINARY_DIR}/sphinx/index.html)
-set(DOXYFILE ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile)
+# --- Paths ---
+set(FDB_API_HEADER_DIR "${PROJECT_SOURCE_DIR}/src/fdb5/api")
+set(DOXYGEN_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/doxygen")
+set(SPHINX_OUTPUT_DIR  "${CMAKE_CURRENT_BINARY_DIR}/sphinx")
+set(DOXYGEN_XML_DIR    "${DOXYGEN_OUTPUT_DIR}/xml")
+set(DOXYGEN_INPUT_DIR  "${FDB_API_HEADER_DIR}")
+set(FDB_VERSION        "${PROJECT_VERSION}")
 
-file(GLOB_RECURSE PYFDB_DOC_TEST_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/../docs/pyfdb/example.rst)
+# --- Input file globs ---
+file(GLOB_RECURSE FDB_PUBLIC_HEADERS CONFIGURE_DEPENDS "${FDB_API_HEADER_DIR}/*.h")
+file(GLOB_RECURSE PYFDB_SRCS CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/pyfdb/*.py")
+file(GLOB_RECURSE Z3FDB_SRCS CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/z3fdb/*.py")
+file(GLOB_RECURSE SPHINX_INPUT_FILES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/*.rst")
+list(FILTER SPHINX_INPUT_FILES EXCLUDE REGEX "_build")
 
+# --- Output stamps ---
+set(DOXYGEN_INDEX_FILE "${DOXYGEN_OUTPUT_DIR}/html/index.html")
+set(SPHINX_INDEX_FILE  "${SPHINX_OUTPUT_DIR}/index.html")
+
+# --- Configure Doxyfile ---
+configure_file(
+    "${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile"
+    @ONLY
+)
+
+# --- Doxygen ---
+add_custom_command(
+    OUTPUT  ${DOXYGEN_INDEX_FILE}
+    COMMAND ${CMAKE_COMMAND} -E make_directory "${DOXYGEN_OUTPUT_DIR}"
+    COMMAND ${DOXYGEN_EXECUTABLE} "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile"
+    DEPENDS ${FDB_PUBLIC_HEADERS} "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile"
+    MAIN_DEPENDENCY "${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in"
+    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+    COMMENT "Running Doxygen on FDB API headers"
+)
+
+# --- Sphinx ---
 add_custom_command(
-    DEPENDS 
-        ${FDB_PUBLIC_HEADERS}
-        ${PYFDB_SRCS}
-        ${Z3FDB_SRCS}
-        ${SPHINX_INPUT_FILES}
-        conf.py
+    OUTPUT  ${SPHINX_INDEX_FILE}
     COMMAND
-        DOXYGEN_EXECUTABLE=${DOXYGEN_EXECUTABLE}
-        SPHINX_EXECUTABLE=${SPHINX_EXECUTABLE}
-        DOCBUILD_OUTPUT=${CMAKE_CURRENT_BINARY_DIR}
-        ${CMAKE_CURRENT_SOURCE_DIR}/build_docs.sh
-    MAIN_DEPENDENCY ${DOXYFILE}
-    COMMENT "Generating documentation"
-    OUTPUT ${SPHINX_INDEX_FILE} ${DOXYGEN_INDEX_FILE}
+        ${SPHINX_EXECUTABLE} -j auto -E -a -T
+        -Dbreathe_projects.FDB=${DOXYGEN_XML_DIR}
+        -Dversion=${FDB_VERSION}
+        -Drelease=${FDB_VERSION}
+        "${CMAKE_CURRENT_SOURCE_DIR}"
+        "${SPHINX_OUTPUT_DIR}"
+    DEPENDS ${DOXYGEN_INDEX_FILE} ${SPHINX_INPUT_FILES}
+            ${PYFDB_SRCS} ${Z3FDB_SRCS}
+            "${CMAKE_CURRENT_SOURCE_DIR}/conf.py"
+    COMMENT "Running Sphinx to build HTML documentation"
 )
 
+# --- Target ---
 add_custom_target(fdb-doc ALL DEPENDS ${SPHINX_INDEX_FILE} ${DOXYGEN_INDEX_FILE})

docs/Doxyfile → docs/Doxyfile.in

@@ -48,7 +48,7 @@ PROJECT_NAME           = "Fields Database - FDB"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         =
+PROJECT_NUMBER         = @FDB_VERSION@
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewers a
@@ -74,7 +74,7 @@ PROJECT_ICON           =
 # entered, it will be relative to the location where Doxygen was started. If
 # left blank the current directory will be used.
 
-OUTPUT_DIRECTORY       = $(DOXYGEN_OUTPUT_DIR)
+OUTPUT_DIRECTORY       = @DOXYGEN_OUTPUT_DIR@
 
 # If the CREATE_SUBDIRS tag is set to YES then Doxygen will create up to 4096
 # sub-directories (in 2 levels) under the output directory of each output format
@@ -991,7 +991,7 @@ WARN_LOGFILE           =
 # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = $(DOXYGEN_INPUT_DIR)
+INPUT                  = @DOXYGEN_INPUT_DIR@
 
 # This tag can be used to specify the character encoding of the source files
 # that Doxygen parses. Internally Doxygen uses the UTF-8 encoding. Doxygen uses

docs/build_docs.sh

@@ -1,25 +1,28 @@
 #!/usr/bin/env bash
 set -ex
 
-script_dir="$(dirname "$(readlink -f "$0")")"
-doxyfile="${script_dir}/Doxyfile"
-doxygen_executable="${DOXYGEN_EXECUTABLE:-doxygen}"
-sphinx_executable="${SPHINX_EXECUTABLE:-sphinx-build}"
-version_str=$(cat "${script_dir}/../VERSION")
+script_dir="$(cd "$(dirname "$0")" && pwd)"
+repo_root="$(cd "${script_dir}/.." && pwd)"
+version_str=$(cat "${repo_root}/VERSION")
 output_dir=${DOCBUILD_OUTPUT:-doc-build}
 doxygen_output_dir="${output_dir}/doxygen"
-doxygen_input_dir="${script_dir}/../src/fdb5/api"
+doxygen_input_dir="${repo_root}/src/fdb5/api"
 
-mkdir -p "${doxygen_output_dir}"
+doxygen_executable="${DOXYGEN_EXECUTABLE:-doxygen}"
+sphinx_executable="${SPHINX_EXECUTABLE:-sphinx-build}"
 
-echo "PWD=$(pwd)"
+# Substitute CMake @VARIABLE@ placeholders in Doxyfile.in
+mkdir -p "${doxygen_output_dir}"
+sed \
+    -e "s|@DOXYGEN_OUTPUT_DIR@|${doxygen_output_dir}|g" \
+    -e "s|@DOXYGEN_INPUT_DIR@|${doxygen_input_dir}|g" \
+    -e "s|@FDB_VERSION@|${version_str}|g" \
+    "${script_dir}/Doxyfile.in" > "${output_dir}/Doxyfile"
 
-DOXYGEN_OUTPUT_DIR="${doxygen_output_dir}" \
-    DOXYGEN_INPUT_DIR="${doxygen_input_dir}" \
-    ${doxygen_executable} ${doxyfile}
+${doxygen_executable} "${output_dir}/Doxyfile"
 
 "${sphinx_executable}" -j auto -E -a -T \
-    -Dbreathe_projects.FDB=$(readlink -f ${doxygen_output_dir}/xml) \
-    -Dversion=${version_str} \
-    -Drelease=${version_str} \
+    -Dbreathe_projects.FDB="$(cd "${doxygen_output_dir}/xml" && pwd)" \
+    -Dversion="${version_str}" \
+    -Drelease="${version_str}" \
     "${script_dir}" "${output_dir}/sphinx"

docs/conf.py

@@ -3,6 +3,7 @@
 # For the full list of built-in configuration values, see the documentation:
 # https://www.sphinx-doc.org/en/master/usage/configuration.html
 import datetime
+import os
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
@@ -85,6 +86,7 @@
 html_static_path = ["_static"]
 
 # -- Breathe configuration ---------------------------------------------------
+breathe_projects = {"FDB": os.environ.get("DOXYGEN_XML_DIR", "doxygen/xml")}
 breathe_default_project = "FDB"
 
 # -- autosectionlabel configuration ------------------------------------------

docs/fdb/cli_tools/compare.rst

@@ -1,4 +1,4 @@
-fdb-compare
+fdb compare
 ===========
 
 | Compares data of two different FDBs or different data version within one FDB.
@@ -11,12 +11,16 @@ Usage
 -----
 
 Compares two entire FDBs for MARS id equality:
+
 ::
+
     % fdb-compare --config1=<path/to/config1.yaml> --config2=<path/to/config2.yaml> [options...]
 
 
 Compare different data in the default (production) FDB:
+
 ::
+
     % fdb-compare --request1="..." --request2="..." [options...]
 
 
@@ -26,7 +30,7 @@ Options
 +----------------------------------------+-----------------------------------------------------------------------------------------+
 | ``--config1=string``                   | Path to a FDB config. If not passed, FDB5 specific                                      |
 |                                        | environment variables like FDB5_CONFIG_FILE will be evaluated. If ``--config1``         |
-|                                        | is not provided, ``--request1`` must be provided.i                                      |
+|                                        | is not provided, ``--request1`` must be provided.                                       |
 +----------------------------------------+-----------------------------------------------------------------------------------------+
 | ``--config2=string``                   | Optional: Path to a second different FDB config. Default: Same as ``--config1``         |
 |                                        | (if passed), otherwise FDB5 specific environment variables are evaluated.               |
@@ -38,18 +42,19 @@ Options
 +----------------------------------------+-----------------------------------------------------------------------------------------+
 | ``--request2=string``                  | Optional: Specialized mars keys to request different data from the second FDB,          |
 |                                        | e.g. ``--request2="class=od,expver=abcd".``                                             |
-|                                        | Allows comparing different MARS subtrees. Only valid if ``--request1` has ben specified.| 
+|                                        | Allows comparing different MARS subtrees.                                               |
+|                                        | Only valid if ``--request1`` has been specified.                                        |
 +----------------------------------------+-----------------------------------------------------------------------------------------+
-| ``--scope=string``                     | Optional - Values: ``[mars (default)|header-only|all]``                                 |
-|                                        | The FDBs can be compared in different scopes,                                           |
-|                                        |  1) ``[mars]`` Mars Metadata only (default),                                            |
-|                                        |  2) ``[header-only]`` includes Mars Key comparison                                      |
-|                                        |     and the comparison of the data headers (e.g. grib headers).                         |
-|                                        |     Note: with ``grib-comparison-type=bit-identical``, the full message is              |
-|                                        |     still compared byte-for-byte; ``header-only`` filtering only applies                |
-|                                        |     to ``grib-keys`` comparison.                                                        |
-|                                        |  3) ``[all]`` includes Mars key and data header comparison but also                     |
-|                                        |     the data sections up to a defined floating point tolerance.                         |
+| ``--scope=string``                     | Optional - Values: ``[mars (default) / header-only / all]``                             |
+|                                        | The FDBs can be compared in different scopes:                                           |
+|                                        | 1) ``mars`` Mars Metadata only (default),                                               |
+|                                        | 2) ``header-only`` includes Mars Key comparison and the comparison                      |
+|                                        | of the data headers (e.g. grib headers). Note: with                                     |
+|                                        | ``grib-comparison-type=bit-identical``, the full message is still                       |
+|                                        | compared byte-for-byte; ``header-only`` filtering only applies to                       |
+|                                        | ``grib-keys`` comparison.                                                               |
+|                                        | 3) ``all`` includes Mars key and data header comparison but also                        |
+|                                        | the data sections up to a defined floating point tolerance.                             |
 +----------------------------------------+-----------------------------------------------------------------------------------------+
 | ``--grib-comparison-type=string``      | Optional - Values: ``[hash-keys|grib-keys(default)|bit-identical]``                     |
 |                                        | Comparing two Grib messages can be done via either (``grib-comparison-type=hash-keys``) |
@@ -79,21 +84,29 @@ Options
 Examples
 --------
 
-Compares two entire FDBs for equality. Using the `--scope=all` options, not only the headers but also the data is compared:
+Compares two entire FDBs for equality. Using the ``--scope=all`` option, not only the headers but also the data is compared:
+
 ::
+
     % fdb-compare --config1=<path/to/config1.yaml> --config2=<path/to/config2.yaml> --scope=all
 
 Compares a specific part (class=rd,expver=1234) of two different FDBs for equality:
+
 ::
+
     % fdb-compare --config1=<path/to/config1.yaml> --config2=<path/to/config2.yaml> --request1="class=rd,expver=1234"
 
 
-Compares different data versions within one FDB using ``--request1`` and ``--request2``. In that case the only usable ``--grib-comparison-type`` is `grib-keys` (the default option):
+Compares different data versions within one FDB using ``--request1`` and ``--request2``. In that case the only usable ``--grib-comparison-type`` is ``grib-keys`` (the default option):
+
 ::
+
     % fdb-compare --config1=<path/to/config1.yaml> --request1="class=od,expver=abcd" --request2="class=rd,expver=1234"  --grib-comparison-type="grib-keys" | tee out
 
 Compares different data versions within the production FDB if executed on ATOS (the FDB5 config will be inferred from the environment):
+
 ::
+
     % fdb-compare --request1="class=od,expver=abcd" --request2="class=rd,expver=1234"  --grib-comparison-type="grib-keys" | tee out
 
 
@@ -107,7 +120,7 @@ The tool exits with code **0** on success (all compared entries match) and **non
 Notes
 -----
 
-When comparing within a single FDB (using ``--config1`` is enough).
-Minimum arguments are either ``--config1`` or ``--request1``. This is explicitly checked, 
+When comparing within a single FDB, using ``--config1`` is enough.
+Minimum arguments are either ``--config1`` or ``--request1``. This is explicitly checked,
 otherwise the default behaviour would result in a comparison of the default FDB (prodfdb on ATOS) with itself.
 

docs/fdb/content/DebugTools/dump-index.rst → docs/fdb/cli_tools/dump-index.rst

@@ -4,6 +4,7 @@ fdb dump-index
 Dump the contents of a particular index file for debugging purposes.
 
 Note that one index file can (and likely will) contain multiple indexes, which will be dumped sequentially.
+
 Usage
 -----
 
@@ -12,6 +13,8 @@ Usage
 Options
 -------
 
+This tool has no tool-specific options. It accepts only positional path arguments.
+
 Examples
 --------
 

docs/fdb/content/DebugTools/dump-toc.rst → docs/fdb/cli_tools/dump-toc.rst

@@ -14,11 +14,13 @@ Usage
 Options
 -------
 
-+----------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| ``--walk``                             | Walk through tocs any linked subtocs in logically correct view. This displays the contents presented by any linked subtocs, and masks any masked entries, to show the contents as would be presented to the user.|
-+----------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-| ``--config=string``                    | FDB configuration filename.                                                                                                                                                                                      |
-+----------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
++----------------------------------------+-----------------------------------------------------------------------------------------+
+| ``--walk``                             | | Walk through tocs and any linked subtocs in logically correct view.                   |
+|                                        | | This displays the contents presented by any linked subtocs, and masks any             |
+|                                        | | masked entries, to show the contents as would be presented to the user.               |
++----------------------------------------+-----------------------------------------------------------------------------------------+
+| ``--structure``                        | Add toc record lengths and offsets to output                                            |
++----------------------------------------+-----------------------------------------------------------------------------------------+
 
 Examples
 --------

docs/fdb/content/DebugTools/dump.rst → docs/fdb/cli_tools/dump.rst

@@ -14,7 +14,7 @@ Options
 -------
 
 +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+
-| ``--simple``                           | Also print the location of each field                                                                               |
+| ``--simple``                           | Provide a more concise, single-line output                                                                          |
 +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+
 | ``--ignore-errors``                    | Ignore errors (report them as warnings) and continue processing wherever possible                                   |
 +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+

docs/fdb/content/GeneralPurposeTools/fdb-stats.rst → docs/fdb/cli_tools/fdb-stats.rst

@@ -48,7 +48,7 @@ You may pass a partial request (as a key) that will print information on all FDB
   Size of fields                  : 3,542,829,840 (3.29952 Gbytes)
   Duplicated fields               : 1,056
   Size of duplicates              : 3,464,100,288 (3.22619 Gbytes)
-  Reacheable fields               : 24
+  Reachable fields               : 24
   Reachable size                  : 78,729,552 (75.0824 Mbytes)
   TOC records                     : 46
   Size of TOC files               : 47,104 (46 Kbytes)
@@ -77,7 +77,7 @@ The --details flag prints a report per database that is visited, as well as the
   Size of fields                  : 20,784,601,728 (19.3572 Gbytes)
   Duplicated fields               : 6,240
   Size of duplicates              : 20,469,683,520 (19.0639 Gbytes)
-  Reacheable fields               : 96
+  Reachable fields               : 96
   Reachable size                  : 314,918,208 (300.329 Mbytes)
   TOC records                     : 268
   Size of TOC files               : 274,432 (268 Kbytes)