diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 114bf9377..4526457d3 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -1,31 +1,58 @@ find_package(Doxygen 1.9 REQUIRED) find_package(Sphinx REQUIRED) -file(GLOB_RECURSE FDB_PUBLIC_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../src/fdb5/api/*.h) -file(GLOB_RECURSE PYFDB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/../src/pyfdb/*.py) -file(GLOB_RECURSE Z3FDB_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/../src/z3fdb/*.py) -file(GLOB_RECURSE SPHINX_INPUT_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.rst) -set(DOXYGEN_INDEX_FILE ${CMAKE_CURRENT_BINARY_DIR}/doxygen/html/index.html) -set(SPHINX_INDEX_FILE ${CMAKE_CURRENT_BINARY_DIR}/sphinx/index.html) -set(DOXYFILE ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile) +# --- Paths --- +set(FDB_API_HEADER_DIR "${PROJECT_SOURCE_DIR}/src/fdb5/api") +set(DOXYGEN_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/doxygen") +set(SPHINX_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/sphinx") +set(DOXYGEN_XML_DIR "${DOXYGEN_OUTPUT_DIR}/xml") +set(DOXYGEN_INPUT_DIR "${FDB_API_HEADER_DIR}") +set(FDB_VERSION "${PROJECT_VERSION}") -file(GLOB_RECURSE PYFDB_DOC_TEST_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/../docs/pyfdb/example.rst) +# --- Input file globs --- +file(GLOB_RECURSE FDB_PUBLIC_HEADERS CONFIGURE_DEPENDS "${FDB_API_HEADER_DIR}/*.h") +file(GLOB_RECURSE PYFDB_SRCS CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/pyfdb/*.py") +file(GLOB_RECURSE Z3FDB_SRCS CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/z3fdb/*.py") +file(GLOB_RECURSE SPHINX_INPUT_FILES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/*.rst") +list(FILTER SPHINX_INPUT_FILES EXCLUDE REGEX "_build") +# --- Output stamps --- +set(DOXYGEN_INDEX_FILE "${DOXYGEN_OUTPUT_DIR}/html/index.html") +set(SPHINX_INDEX_FILE "${SPHINX_OUTPUT_DIR}/index.html") + +# --- Configure Doxyfile --- +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in" + "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile" + @ONLY +) + +# --- Doxygen --- +add_custom_command( + OUTPUT ${DOXYGEN_INDEX_FILE} + COMMAND ${CMAKE_COMMAND} -E make_directory "${DOXYGEN_OUTPUT_DIR}" + COMMAND ${DOXYGEN_EXECUTABLE} "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile" + DEPENDS ${FDB_PUBLIC_HEADERS} "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile" + MAIN_DEPENDENCY "${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + COMMENT "Running Doxygen on FDB API headers" +) + +# --- Sphinx --- add_custom_command( - DEPENDS - ${FDB_PUBLIC_HEADERS} - ${PYFDB_SRCS} - ${Z3FDB_SRCS} - ${SPHINX_INPUT_FILES} - conf.py + OUTPUT ${SPHINX_INDEX_FILE} COMMAND - DOXYGEN_EXECUTABLE=${DOXYGEN_EXECUTABLE} - SPHINX_EXECUTABLE=${SPHINX_EXECUTABLE} - DOCBUILD_OUTPUT=${CMAKE_CURRENT_BINARY_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/build_docs.sh - MAIN_DEPENDENCY ${DOXYFILE} - COMMENT "Generating documentation" - OUTPUT ${SPHINX_INDEX_FILE} ${DOXYGEN_INDEX_FILE} + ${SPHINX_EXECUTABLE} -j auto -E -a -T + -Dbreathe_projects.FDB=${DOXYGEN_XML_DIR} + -Dversion=${FDB_VERSION} + -Drelease=${FDB_VERSION} + "${CMAKE_CURRENT_SOURCE_DIR}" + "${SPHINX_OUTPUT_DIR}" + DEPENDS ${DOXYGEN_INDEX_FILE} ${SPHINX_INPUT_FILES} + ${PYFDB_SRCS} ${Z3FDB_SRCS} + "${CMAKE_CURRENT_SOURCE_DIR}/conf.py" + COMMENT "Running Sphinx to build HTML documentation" ) +# --- Target --- add_custom_target(fdb-doc ALL DEPENDS ${SPHINX_INDEX_FILE} ${DOXYGEN_INDEX_FILE}) diff --git a/docs/Doxyfile b/docs/Doxyfile.in similarity index 99% rename from docs/Doxyfile rename to docs/Doxyfile.in index e2e2ce271..0eccace39 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile.in @@ -48,7 +48,7 @@ PROJECT_NAME = "Fields Database - FDB" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = +PROJECT_NUMBER = @FDB_VERSION@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewers a @@ -74,7 +74,7 @@ PROJECT_ICON = # entered, it will be relative to the location where Doxygen was started. If # left blank the current directory will be used. -OUTPUT_DIRECTORY = $(DOXYGEN_OUTPUT_DIR) +OUTPUT_DIRECTORY = @DOXYGEN_OUTPUT_DIR@ # If the CREATE_SUBDIRS tag is set to YES then Doxygen will create up to 4096 # sub-directories (in 2 levels) under the output directory of each output format @@ -991,7 +991,7 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = $(DOXYGEN_INPUT_DIR) +INPUT = @DOXYGEN_INPUT_DIR@ # This tag can be used to specify the character encoding of the source files # that Doxygen parses. Internally Doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/docs/build_docs.sh b/docs/build_docs.sh index 00738c2e3..fd02b987c 100755 --- a/docs/build_docs.sh +++ b/docs/build_docs.sh @@ -1,25 +1,28 @@ #!/usr/bin/env bash set -ex -script_dir="$(dirname "$(readlink -f "$0")")" -doxyfile="${script_dir}/Doxyfile" -doxygen_executable="${DOXYGEN_EXECUTABLE:-doxygen}" -sphinx_executable="${SPHINX_EXECUTABLE:-sphinx-build}" -version_str=$(cat "${script_dir}/../VERSION") +script_dir="$(cd "$(dirname "$0")" && pwd)" +repo_root="$(cd "${script_dir}/.." && pwd)" +version_str=$(cat "${repo_root}/VERSION") output_dir=${DOCBUILD_OUTPUT:-doc-build} doxygen_output_dir="${output_dir}/doxygen" -doxygen_input_dir="${script_dir}/../src/fdb5/api" +doxygen_input_dir="${repo_root}/src/fdb5/api" -mkdir -p "${doxygen_output_dir}" +doxygen_executable="${DOXYGEN_EXECUTABLE:-doxygen}" +sphinx_executable="${SPHINX_EXECUTABLE:-sphinx-build}" -echo "PWD=$(pwd)" +# Substitute CMake @VARIABLE@ placeholders in Doxyfile.in +mkdir -p "${doxygen_output_dir}" +sed \ + -e "s|@DOXYGEN_OUTPUT_DIR@|${doxygen_output_dir}|g" \ + -e "s|@DOXYGEN_INPUT_DIR@|${doxygen_input_dir}|g" \ + -e "s|@FDB_VERSION@|${version_str}|g" \ + "${script_dir}/Doxyfile.in" > "${output_dir}/Doxyfile" -DOXYGEN_OUTPUT_DIR="${doxygen_output_dir}" \ - DOXYGEN_INPUT_DIR="${doxygen_input_dir}" \ - ${doxygen_executable} ${doxyfile} +${doxygen_executable} "${output_dir}/Doxyfile" "${sphinx_executable}" -j auto -E -a -T \ - -Dbreathe_projects.FDB=$(readlink -f ${doxygen_output_dir}/xml) \ - -Dversion=${version_str} \ - -Drelease=${version_str} \ + -Dbreathe_projects.FDB="$(cd "${doxygen_output_dir}/xml" && pwd)" \ + -Dversion="${version_str}" \ + -Drelease="${version_str}" \ "${script_dir}" "${output_dir}/sphinx" diff --git a/docs/conf.py b/docs/conf.py index df31fd6a7..55d947576 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -3,6 +3,7 @@ # For the full list of built-in configuration values, see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html import datetime +import os # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information @@ -85,6 +86,7 @@ html_static_path = ["_static"] # -- Breathe configuration --------------------------------------------------- +breathe_projects = {"FDB": os.environ.get("DOXYGEN_XML_DIR", "doxygen/xml")} breathe_default_project = "FDB" # -- autosectionlabel configuration ------------------------------------------ diff --git a/docs/fdb/c_api.rst b/docs/fdb/c_api.rst deleted file mode 100644 index 2f0a77605..000000000 --- a/docs/fdb/c_api.rst +++ /dev/null @@ -1,48 +0,0 @@ -C API -===== - -Library Initialisation ----------------------- -.. doxygengroup:: Initialisation - :content-only: - -Version Information -------------------- -.. doxygengroup:: Version - :content-only: - -FDB API -------- -.. doxygengroup:: FDB - :content-only: - -Error Handling --------------- -.. doxygengroup:: Error - :content-only: - -Key ---- -.. doxygengroup:: Key - :content-only: - -Split-Key ---------- -.. doxygengroup:: SplitKey - :content-only: - -Request -------- -.. doxygengroup:: Request - :content-only: - -List Iterator -------------- -.. doxygengroup:: ListIterator - :content-only: - -Data Reader ------------ -.. doxygengroup:: DataReader - :content-only: - diff --git a/docs/fdb/cli_tools/compare.rst b/docs/fdb/cli_tools/compare.rst index 935002305..214c2979e 100644 --- a/docs/fdb/cli_tools/compare.rst +++ b/docs/fdb/cli_tools/compare.rst @@ -1,4 +1,4 @@ -fdb-compare +fdb compare =========== | Compares data of two different FDBs or different data version within one FDB. @@ -11,12 +11,16 @@ Usage ----- Compares two entire FDBs for MARS id equality: + :: + % fdb-compare --config1= --config2= [options...] Compare different data in the default (production) FDB: + :: + % fdb-compare --request1="..." --request2="..." [options...] @@ -26,7 +30,7 @@ Options +----------------------------------------+-----------------------------------------------------------------------------------------+ | ``--config1=string`` | Path to a FDB config. If not passed, FDB5 specific | | | environment variables like FDB5_CONFIG_FILE will be evaluated. If ``--config1`` | -| | is not provided, ``--request1`` must be provided.i | +| | is not provided, ``--request1`` must be provided. | +----------------------------------------+-----------------------------------------------------------------------------------------+ | ``--config2=string`` | Optional: Path to a second different FDB config. Default: Same as ``--config1`` | | | (if passed), otherwise FDB5 specific environment variables are evaluated. | @@ -38,18 +42,19 @@ Options +----------------------------------------+-----------------------------------------------------------------------------------------+ | ``--request2=string`` | Optional: Specialized mars keys to request different data from the second FDB, | | | e.g. ``--request2="class=od,expver=abcd".`` | -| | Allows comparing different MARS subtrees. Only valid if ``--request1` has ben specified.| +| | Allows comparing different MARS subtrees. | +| | Only valid if ``--request1`` has been specified. | +----------------------------------------+-----------------------------------------------------------------------------------------+ -| ``--scope=string`` | Optional - Values: ``[mars (default)|header-only|all]`` | -| | The FDBs can be compared in different scopes, | -| | 1) ``[mars]`` Mars Metadata only (default), | -| | 2) ``[header-only]`` includes Mars Key comparison | -| | and the comparison of the data headers (e.g. grib headers). | -| | Note: with ``grib-comparison-type=bit-identical``, the full message is | -| | still compared byte-for-byte; ``header-only`` filtering only applies | -| | to ``grib-keys`` comparison. | -| | 3) ``[all]`` includes Mars key and data header comparison but also | -| | the data sections up to a defined floating point tolerance. | +| ``--scope=string`` | Optional - Values: ``[mars (default) / header-only / all]`` | +| | The FDBs can be compared in different scopes: | +| | 1) ``mars`` Mars Metadata only (default), | +| | 2) ``header-only`` includes Mars Key comparison and the comparison | +| | of the data headers (e.g. grib headers). Note: with | +| | ``grib-comparison-type=bit-identical``, the full message is still | +| | compared byte-for-byte; ``header-only`` filtering only applies to | +| | ``grib-keys`` comparison. | +| | 3) ``all`` includes Mars key and data header comparison but also | +| | the data sections up to a defined floating point tolerance. | +----------------------------------------+-----------------------------------------------------------------------------------------+ | ``--grib-comparison-type=string`` | Optional - Values: ``[hash-keys|grib-keys(default)|bit-identical]`` | | | Comparing two Grib messages can be done via either (``grib-comparison-type=hash-keys``) | @@ -79,21 +84,29 @@ Options Examples -------- -Compares two entire FDBs for equality. Using the `--scope=all` options, not only the headers but also the data is compared: +Compares two entire FDBs for equality. Using the ``--scope=all`` option, not only the headers but also the data is compared: + :: + % fdb-compare --config1= --config2= --scope=all Compares a specific part (class=rd,expver=1234) of two different FDBs for equality: + :: + % fdb-compare --config1= --config2= --request1="class=rd,expver=1234" -Compares different data versions within one FDB using ``--request1`` and ``--request2``. In that case the only usable ``--grib-comparison-type`` is `grib-keys` (the default option): +Compares different data versions within one FDB using ``--request1`` and ``--request2``. In that case the only usable ``--grib-comparison-type`` is ``grib-keys`` (the default option): + :: + % fdb-compare --config1= --request1="class=od,expver=abcd" --request2="class=rd,expver=1234" --grib-comparison-type="grib-keys" | tee out Compares different data versions within the production FDB if executed on ATOS (the FDB5 config will be inferred from the environment): + :: + % fdb-compare --request1="class=od,expver=abcd" --request2="class=rd,expver=1234" --grib-comparison-type="grib-keys" | tee out @@ -107,7 +120,7 @@ The tool exits with code **0** on success (all compared entries match) and **non Notes ----- -When comparing within a single FDB (using ``--config1`` is enough). -Minimum arguments are either ``--config1`` or ``--request1``. This is explicitly checked, +When comparing within a single FDB, using ``--config1`` is enough. +Minimum arguments are either ``--config1`` or ``--request1``. This is explicitly checked, otherwise the default behaviour would result in a comparison of the default FDB (prodfdb on ATOS) with itself. diff --git a/docs/fdb/content/DebugTools/dump-index.rst b/docs/fdb/cli_tools/dump-index.rst similarity index 93% rename from docs/fdb/content/DebugTools/dump-index.rst rename to docs/fdb/cli_tools/dump-index.rst index 62c25a069..eeae83c17 100644 --- a/docs/fdb/content/DebugTools/dump-index.rst +++ b/docs/fdb/cli_tools/dump-index.rst @@ -4,6 +4,7 @@ fdb dump-index Dump the contents of a particular index file for debugging purposes. Note that one index file can (and likely will) contain multiple indexes, which will be dumped sequentially. + Usage ----- @@ -12,6 +13,8 @@ Usage Options ------- +This tool has no tool-specific options. It accepts only positional path arguments. + Examples -------- diff --git a/docs/fdb/content/DebugTools/dump-toc.rst b/docs/fdb/cli_tools/dump-toc.rst similarity index 73% rename from docs/fdb/content/DebugTools/dump-toc.rst rename to docs/fdb/cli_tools/dump-toc.rst index 664a823a0..991dbf395 100644 --- a/docs/fdb/content/DebugTools/dump-toc.rst +++ b/docs/fdb/cli_tools/dump-toc.rst @@ -14,11 +14,13 @@ Usage Options ------- -+----------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| ``--walk`` | Walk through tocs any linked subtocs in logically correct view. This displays the contents presented by any linked subtocs, and masks any masked entries, to show the contents as would be presented to the user.| -+----------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| ``--config=string`` | FDB configuration filename. | -+----------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++----------------------------------------+-----------------------------------------------------------------------------------------+ +| ``--walk`` | | Walk through tocs and any linked subtocs in logically correct view. | +| | | This displays the contents presented by any linked subtocs, and masks any | +| | | masked entries, to show the contents as would be presented to the user. | ++----------------------------------------+-----------------------------------------------------------------------------------------+ +| ``--structure`` | Add toc record lengths and offsets to output | ++----------------------------------------+-----------------------------------------------------------------------------------------+ Examples -------- diff --git a/docs/fdb/content/DebugTools/dump.rst b/docs/fdb/cli_tools/dump.rst similarity index 96% rename from docs/fdb/content/DebugTools/dump.rst rename to docs/fdb/cli_tools/dump.rst index 7a1cfdc2e..e4e127df7 100644 --- a/docs/fdb/content/DebugTools/dump.rst +++ b/docs/fdb/cli_tools/dump.rst @@ -14,7 +14,7 @@ Options ------- +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--simple`` | Also print the location of each field | +| ``--simple`` | Provide a more concise, single-line output | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ | ``--ignore-errors`` | Ignore errors (report them as warnings) and continue processing wherever possible | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ diff --git a/docs/fdb/content/GeneralPurposeTools/fdb-stats.rst b/docs/fdb/cli_tools/fdb-stats.rst similarity index 98% rename from docs/fdb/content/GeneralPurposeTools/fdb-stats.rst rename to docs/fdb/cli_tools/fdb-stats.rst index 3ab639af8..1abb983de 100644 --- a/docs/fdb/content/GeneralPurposeTools/fdb-stats.rst +++ b/docs/fdb/cli_tools/fdb-stats.rst @@ -48,7 +48,7 @@ You may pass a partial request (as a key) that will print information on all FDB Size of fields : 3,542,829,840 (3.29952 Gbytes) Duplicated fields : 1,056 Size of duplicates : 3,464,100,288 (3.22619 Gbytes) - Reacheable fields : 24 + Reachable fields : 24 Reachable size : 78,729,552 (75.0824 Mbytes) TOC records : 46 Size of TOC files : 47,104 (46 Kbytes) @@ -77,7 +77,7 @@ The --details flag prints a report per database that is visited, as well as the Size of fields : 20,784,601,728 (19.3572 Gbytes) Duplicated fields : 6,240 Size of duplicates : 20,469,683,520 (19.0639 Gbytes) - Reacheable fields : 96 + Reachable fields : 96 Reachable size : 314,918,208 (300.329 Mbytes) TOC records : 268 Size of TOC files : 274,432 (268 Kbytes) diff --git a/docs/fdb/cli_tools/fdb.rst b/docs/fdb/cli_tools/fdb.rst new file mode 100644 index 000000000..7a98f36b7 --- /dev/null +++ b/docs/fdb/cli_tools/fdb.rst @@ -0,0 +1,53 @@ +fdb +=== + +The ``fdb`` command is a wrapper that dispatches to the individual ``fdb-`` tools +documented in this section. Rather than invoking each tool binary directly, you can use +``fdb `` as a single entry point. + +Usage +----- + +.. code-block:: text + + fdb [options] ... + +Dispatching +^^^^^^^^^^^ + +``fdb `` locates the corresponding ``fdb-`` executable in the same +directory and forwards all remaining arguments to it. For example: + +.. code-block:: bash + + fdb list class=od,stream=oper # equivalent to: fdb-list class=od,stream=oper + fdb write input.grib # equivalent to: fdb-write input.grib + +If ```` does not match a built-in or an installed ``fdb-`` binary, +``fdb`` prints a usage summary and exits with an error. + +Built-in Commands +^^^^^^^^^^^^^^^^^ + +A small number of commands are handled directly by the wrapper without dispatching to a +separate binary: + +``fdb help`` + Print a usage summary listing available commands. + +``fdb version`` + Print the FDB library version string. + +``fdb home`` + Print the FDB home directory (queries ``fdb-info --home`` internally). + +Example +------- + +.. code-block:: bash + + % fdb list class=od,stream=oper,date=20151004 + {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=700,param=155} + {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=850,param=129} + {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=850,param=130} + ... diff --git a/docs/fdb/content/GeneralPurposeTools/grib2fdb.rst b/docs/fdb/cli_tools/grib2fdb.rst similarity index 87% rename from docs/fdb/content/GeneralPurposeTools/grib2fdb.rst rename to docs/fdb/cli_tools/grib2fdb.rst index a4a6019c5..96e9ff453 100644 --- a/docs/fdb/content/GeneralPurposeTools/grib2fdb.rst +++ b/docs/fdb/cli_tools/grib2fdb.rst @@ -1,7 +1,7 @@ grib2fdb5 ========= -Inserts data into the FDB, creating a new databases if needed. +Inserts data into the FDB, creating new databases if needed. Note that the interface is designed to be the same as the older grib2fdb tool for use with fdb4. @@ -14,15 +14,15 @@ Options ------- +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--c`` | Check that the class of the supplied data matches | +| ``-c`` | Check that the class of the supplied data matches | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--e`` | Check that the expver of the supplied data matches | +| ``-e`` | Check that the expver of the supplied data matches | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--T`` | Check that the type of the supplied data matches | +| ``-T`` | Check that the type of the supplied data matches | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--s`` | Check that the stream of the supplied data matches | +| ``-s`` | Check that the stream of the supplied data matches | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--f`` | The grib file to archive | +| ``-f`` | The grib file to archive | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ | ``--config=string`` | FDB configuration filename | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ diff --git a/docs/fdb/content/DebugTools/hammer.rst b/docs/fdb/cli_tools/hammer.rst similarity index 95% rename from docs/fdb/content/DebugTools/hammer.rst rename to docs/fdb/cli_tools/hammer.rst index 2c40a22c1..447352626 100644 --- a/docs/fdb/content/DebugTools/hammer.rst +++ b/docs/fdb/cli_tools/hammer.rst @@ -11,7 +11,7 @@ This tool takes a sample GRIB file containing one field, and repeatedly resets t Usage ----- -``fdb hammer [--statistics] [--read] [–nensembles=] [--number=] --nsteps= --nlevels= --nparams= --expver= --class= `` +``fdb hammer [--statistics] [--read] [--nensembles=] [--number=] --nsteps= --nlevels= --nparams= --expver= --class= `` Options ------- diff --git a/docs/fdb/content/SpecialPurposeTools/hide.rst b/docs/fdb/cli_tools/hide.rst similarity index 100% rename from docs/fdb/content/SpecialPurposeTools/hide.rst rename to docs/fdb/cli_tools/hide.rst diff --git a/docs/fdb/cli_tools/index.rst b/docs/fdb/cli_tools/index.rst index a7c0488ca..2876f80d8 100644 --- a/docs/fdb/cli_tools/index.rst +++ b/docs/fdb/cli_tools/index.rst @@ -1,23 +1,104 @@ CLI Tools ========= +FDB provides CLI tools for archiving and retrieving data, administrative maintenance, and debugging. + +All tools can be invoked through the :doc:`fdb ` wrapper command or directly as ``fdb-``. + .. toctree:: :hidden: - write + fdb + compare + fdb-stats + grib2fdb + info list + purge read - mars + schema + where + wipe + write + hide + move + overlay + root + dump + dump-index + dump-toc + hammer + patch + reconsolidate-toc + +General Purpose Tools +--------------------- + +.. list-table:: + :widths: 25 75 + :header-rows: 1 + + * - Command + - Description + * - :doc:`fdb compare ` + - Compare data between two FDBs or different data versions within one FDB. + * - :doc:`fdb info ` + - Show information about the FDB configuration and binaries. + * - :doc:`fdb list ` + - List the contents of FDB databases. + * - :doc:`fdb purge ` + - Purge duplicate entries and remove associated data. + * - :doc:`fdb read ` + - Read data from the FDB and write it to a file. + * - :doc:`fdb schema ` + - Print the schema used by an FDB instance or database. + * - :doc:`fdb stats ` + - Print aggregated information about FDB databases. + * - :doc:`fdb where ` + - Print the location of an FDB database. + * - :doc:`fdb wipe ` + - Delete FDB databases and the data they contain. + * - :doc:`fdb write ` + - Insert data into the FDB, creating databases as needed. + * - :doc:`grib2fdb5 ` + - Insert GRIB data into the FDB (legacy tool). + +Special Purpose Tools +--------------------- + +.. list-table:: + :widths: 25 75 + :header-rows: 1 -Most common tools are the following: + * - Command + - Description + * - :doc:`fdb hide ` + - Mask all entries in a database so they are permanently inaccessible, without destroying data. + * - :doc:`fdb move ` + - Lock and duplicate a database to another root. + * - :doc:`fdb overlay ` + - Overlay entries from one database onto another. Advanced use only. + * - :doc:`fdb root ` + - Find the on-disk location of specified databases. -:fdb-write_: Inserts data into the FDB, creating a new databases if needed. -:fdb-list_: Lists the contents of the FDB databases. Data are filtered according a MARS_ request -:fdb-read_: Read data from the FDB and write this data into a specified target file. +Debug Tools +----------- +.. list-table:: + :widths: 25 75 + :header-rows: 1 -.. _fdb-write: -.. _fdb-list: list.rst -.. _fdb-read: read.rst -.. _fdb-compare: compare.rst -.. _MARS: mars.rst + * - Command + - Description + * - :doc:`fdb dump ` + - Dump the structural contents of the FDB (e.g. Table of Contents entries). + * - :doc:`fdb dump-index ` + - Dump the contents of a particular index file. + * - :doc:`fdb dump-toc ` + - Dump the structural contents of a TOC or subtoc file. + * - :doc:`fdb hammer ` + - Simulate the I/O load of a writing process within a forecast. + * - :doc:`fdb patch ` + - Re-archive data whilst modifying the class or expver of the database key. + * - :doc:`fdb reconsolidate-toc ` + - Repair databases with poorly written or fragmented TOC entries. diff --git a/docs/fdb/content/GeneralPurposeTools/info.rst b/docs/fdb/cli_tools/info.rst similarity index 99% rename from docs/fdb/content/GeneralPurposeTools/info.rst rename to docs/fdb/cli_tools/info.rst index 1edb3be7b..dade67690 100644 --- a/docs/fdb/content/GeneralPurposeTools/info.rst +++ b/docs/fdb/cli_tools/info.rst @@ -60,5 +60,5 @@ Get location of FDB schema file in an easily parsable form: Get location of current FDB configuration file in an easily parsable form: :: - % fdb info --config + % fdb info --config-file /testcases/fdb5/fdb5_simple.yaml \ No newline at end of file diff --git a/docs/fdb/cli_tools/list.rst b/docs/fdb/cli_tools/list.rst index 6646d7b4d..8ec2ff6d8 100644 --- a/docs/fdb/cli_tools/list.rst +++ b/docs/fdb/cli_tools/list.rst @@ -1,9 +1,10 @@ -fdb-list +fdb list ======== -Lists the contents of the FDB databases. -In the body of the output, one line is given per field that has been archived. These (by default) present the fields that are available and will be retrievable - i.e. masked duplicates are skipped. +Lists the contents of the FDB databases. +In the body of the output, one line is given per field that has been archived. These (by default) present the fields that are available and will be retrievable - i.e. masked duplicates are skipped. The lines are broken into three segments, which represent the hierarchical nature of the schema: + * The first component identifies the FDB database containing the data * The second component identifies the (set of) indexes * The third component identifies entries collocated within an index @@ -11,7 +12,7 @@ The lines are broken into three segments, which represent the hierarchical natur Usage ----- -``fdb-list [options] [request1] [request2] ...`` +``fdb list [options] [request1] [request2] ...`` Options ------- @@ -37,6 +38,18 @@ Options +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ | ``--all`` | **(Debug and testing only)** Visit all FDB databases | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ +| ``--config=string`` | FDB configuration filename | ++----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ +| ``--timestamp`` | Also print the timestamp when the field was indexed | ++----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ +| ``--length`` | Also print the field size | ++----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ +| ``--only-duplicates`` | List only the duplicated (older) entries | ++----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ +| ``--compact`` | Aggregate available fields in MARS requests | ++----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ +| ``--depth=integer`` | Output entries up to 'depth' levels deep [1-3] | ++----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ Examples @@ -46,14 +59,14 @@ You may pass a partial request (as a key) that will list all the fields in the F Note that this is a global search through all the databases of the FDB that match this key. :: - % fdb-list class=od,expver=0001,stream=oper,date=20151004 - + % fdb list class=od,expver=0001,stream=oper,date=20151004 + retrieve, class=od, expver=0001, stream=oper, date=20151004 - + {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=700,param=155} {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=850,param=129} {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=850,param=130} @@ -63,32 +76,30 @@ Note that this is a global search through all the databases of the FDB that matc A JSON listing may be obtained for use in tools that parse the available data :: - % fdb-list --json class=od,expver=0001,stream=oper,date=20151004 - + % fdb list --json class=od,expver=0001,stream=oper,date=20151004 + [{"class":"od","stream":"oper","date":"20151004","time":"1200","domain":"g","type":"an","levtype":"pl","step":"0","levelist":"700","param":"155"},{...},...] -The ``--location`` option can be useful to identify exactly where the field is located within the database. +The ``--location`` option can be useful to identify exactly where the field is located within the database. :: - % fdb-list --location class=od,expver=0001,stream=oper,date=20151004 - + % fdb list --location class=od,expver=0001,stream=oper,date=20151004 + retrieve, class=od, expver=0001, stream=oper, date=20151004 - + {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=850,param=130} (/data/mars_p_d17_d17_1_15/fdb/od:0001:oper:20151004:1200:g/an:pl.20161103.120238.dhs1213.ecmwf.int.1739461754885.data,13121592,3280398) ... The ``--porcelain`` option gives stable output for use in scripts and as input to other simple tools. This will only print (exactly) one line per entry, with no extraneous output. The output of this option will remain stable across versions. :: - % fdb-list --porcelain class=od,expver=0001,stream=oper,date=20151004 + % fdb list --porcelain class=od,expver=0001,stream=oper,date=20151004 {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=700,param=155} {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=850,param=129} {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=850,param=130} ... - - diff --git a/docs/fdb/cli_tools/mars.rst b/docs/fdb/cli_tools/mars.rst deleted file mode 100644 index 355361962..000000000 --- a/docs/fdb/cli_tools/mars.rst +++ /dev/null @@ -1,45 +0,0 @@ -MARS request -============ - -A **MARS request** is the way to specify an action on a set of fields or observations. The directives specified in a MARS request have the following syntax: -:: - - verb, - keyword1 = value 1, - ... = ..., - keywordN = value N - -Where: - -:verb: identifies the action to be taken, e.g. retrieve or list. -:keyword: is a known MARS variable, e.g. type or date -:value: is the value assigned to the keyword, e.g. Analysis or temperature. - -| Keywords may be assigned a **single value**, a **list of values** or a **range of values**. -| A **list** is indicated by the separator **/** -| A **range** is indicated by using the keywords **to** as well as **/** and **by**. -| Examples of different formats for values are given in the table below. - -=============== =============== -Format Example -=============== =============== -single value ``param = temperature/SSRD`` -list of values ``step = 12/24/48`` -range of values ``date = 19990104/to/19990110/by/2`` -=============== =============== - -The following example shows a retrieve request with a mix of single, list and range of values: -:: - - retrieve, - class = od, - stream = oper, - expver = 1, - date = -1, - type = analysis, - levtype = model levels, - levelist = 1/to/91, - param = temperature, - grid = 0.1/0.1 - -Retrieve requests have to specify, at least, directives to identify the data. The ones to identify fields are defined in the MARS Catalogue. Data manipulation (post-processing, such as **grid**) directives are optional, depending on user needs. diff --git a/docs/fdb/content/SpecialPurposeTools/move.rst b/docs/fdb/cli_tools/move.rst similarity index 87% rename from docs/fdb/content/SpecialPurposeTools/move.rst rename to docs/fdb/cli_tools/move.rst index ea56601f9..e0784c681 100644 --- a/docs/fdb/content/SpecialPurposeTools/move.rst +++ b/docs/fdb/cli_tools/move.rst @@ -1,9 +1,9 @@ fdb move ======== -Move the content of one FDB database. This locks the source database, make it possible to create a second database in another root, duplicates all data. Source data are not automatically removed. +Move the content of one FDB database. This locks the source database, making it possible to create a second database in another root, duplicates all data. Source data are not automatically removed. -This tool is envisaged for migrating data from two different data store (i.e. from a performance to a capacity data store) +This tool is envisaged for migrating data from two different data stores (i.e. from a performance to a capacity data store) Usage ----- @@ -35,6 +35,8 @@ Database request must match exactly one database. +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ | ``--config=string`` | FDB configuration filename | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ +| ``--transport=string`` | Distributed data move (MPI based) | ++----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ Example diff --git a/docs/fdb/content/SpecialPurposeTools/overlay.rst b/docs/fdb/cli_tools/overlay.rst similarity index 98% rename from docs/fdb/content/SpecialPurposeTools/overlay.rst rename to docs/fdb/cli_tools/overlay.rst index 1f2c5e2c9..b8da2a26f 100644 --- a/docs/fdb/content/SpecialPurposeTools/overlay.rst +++ b/docs/fdb/cli_tools/overlay.rst @@ -22,7 +22,7 @@ Usage Options ------- -At least one of class or expver is required is required +At least one of class or expver is required. +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ | ``--force`` | Apply overlay even if the target database already exists | diff --git a/docs/fdb/content/DebugTools/patch.rst b/docs/fdb/cli_tools/patch.rst similarity index 84% rename from docs/fdb/content/DebugTools/patch.rst rename to docs/fdb/cli_tools/patch.rst index 4f6cdc3eb..96af73988 100644 --- a/docs/fdb/content/DebugTools/patch.rst +++ b/docs/fdb/cli_tools/patch.rst @@ -13,22 +13,22 @@ Usage Options ------- -At least one of class or expver is required is required +At least one of class or expver is required. +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--expver=string`` | Specifies the new class to use for the copied data | +| ``--expver=string`` | Specifies the new experiment version to use for the copied data | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--class=string`` | Specifies the new experiment version to use for the copied data | +| ``--class=string`` | Specifies the new class to use for the copied data | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ | ``--ignore-errors`` | Ignore errors (report them as warnings) and continue processing wherever possible | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ | ``--raw`` | | Don't apply (contextual) expansion and checking on requests. This prevents the use of named parameters | -| | | (such as t rather than param=130), dates (such as date=-1), or similar. Keys and values passed must match those | +| | | (such as t rather than param=130), dates (such as date=-1), or similar. Keys and values passed must match those | | | | used internally to the FDB exactly. | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--minimum-keys`` | Default is class,expver | -| | Define the minimum set of keys that must be specified. Prevents inadvertently exploring and copying the entire FDB. | +| ``--minimum-keys`` | | Default is class,expver | +| | | Define the minimum set of keys that must be specified. Prevents inadvertent exploration of the entire FDB. | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ | ``--all`` | (Debug and testing only) Visit all FDB databases | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ diff --git a/docs/fdb/content/GeneralPurposeTools/purge.rst b/docs/fdb/cli_tools/purge.rst similarity index 100% rename from docs/fdb/content/GeneralPurposeTools/purge.rst rename to docs/fdb/cli_tools/purge.rst diff --git a/docs/fdb/cli_tools/read.rst b/docs/fdb/cli_tools/read.rst index fb644b04a..09022ba46 100644 --- a/docs/fdb/cli_tools/read.rst +++ b/docs/fdb/cli_tools/read.rst @@ -1,4 +1,4 @@ -fdb-read +fdb read ======== Read data from the FDB and write this data into a specified target file. This may involve visiting multiple databases if required by the request. @@ -7,8 +7,9 @@ Usage ----- :: - fdb-read request.mars target.grib - fdb-read --extract source.grib target.grib + fdb read request.mars target.grib + fdb read --raw request.mars target.grib + fdb read --extract source.grib target.grib Options ------- @@ -18,19 +19,23 @@ Options +----------------------------------------+-----------------------------------------------------------------------------------------+ | ``--statistics`` | Report timing statistics | +----------------------------------------+-----------------------------------------------------------------------------------------+ +| ``--config=string`` | FDB configuration filename | ++----------------------------------------+-----------------------------------------------------------------------------------------+ +| ``--raw`` | Uses the raw request without expansion | ++----------------------------------------+-----------------------------------------------------------------------------------------+ Examples -------- -Specify the [MARS request](MARS.md) in a plain text file. +Specify the MARS request in a plain text file. Note that this MARS request must be fully expanded. For example, it may not contain the **/to/** or **/by/** statements that the MARS client is able to expand. :: % cat myrequest retrieve,class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g,type=an,levtype=pl,step=0,levelist=700,param=155 - + # this will retrieve 2 fields - % fdb-read myrequest foo.grib + % fdb read myrequest foo.grib retrieve,class=od,date=20151004,domain=g,expver=0001,levelist=500/700,levtype=pl,param=155,step=0,stream=oper,time=1200,type=an Compress handle: 3.3e-05 second elapsed, 3.2e-05 second cpu Compress handle: 2e-06 second elapsed, 2e-06 second cpu @@ -42,7 +47,7 @@ Note that this MARS request must be fully expanded. For example, it may not cont Obtain data from the FDB using the MARS request that would be implied by an existing GRIB file. In this example, foo.grib contains 2 fields that identify what needs to be retrieved. :: - % fdb-read --extract foo.grib out.grib + % fdb read --extract foo.grib out.grib retrieve,class=od,date=20151004,domain=g,expver=0001,levelist=500,levtype=pl,param=155,step=0,stream=oper,time=1200,type=an ... Compress handle: 3.3e-05 second elapsed, 3.2e-05 second cpu diff --git a/docs/fdb/content/DebugTools/reconsolidate-toc.rst b/docs/fdb/cli_tools/reconsolidate-toc.rst similarity index 100% rename from docs/fdb/content/DebugTools/reconsolidate-toc.rst rename to docs/fdb/cli_tools/reconsolidate-toc.rst diff --git a/docs/fdb/content/SpecialPurposeTools/root.rst b/docs/fdb/cli_tools/root.rst similarity index 100% rename from docs/fdb/content/SpecialPurposeTools/root.rst rename to docs/fdb/cli_tools/root.rst diff --git a/docs/fdb/content/GeneralPurposeTools/schema.rst b/docs/fdb/cli_tools/schema.rst similarity index 100% rename from docs/fdb/content/GeneralPurposeTools/schema.rst rename to docs/fdb/cli_tools/schema.rst diff --git a/docs/fdb/content/GeneralPurposeTools/where.rst b/docs/fdb/cli_tools/where.rst similarity index 100% rename from docs/fdb/content/GeneralPurposeTools/where.rst rename to docs/fdb/cli_tools/where.rst diff --git a/docs/fdb/content/GeneralPurposeTools/wipe.rst b/docs/fdb/cli_tools/wipe.rst similarity index 90% rename from docs/fdb/content/GeneralPurposeTools/wipe.rst rename to docs/fdb/cli_tools/wipe.rst index 4b19758ad..2d19a0092 100644 --- a/docs/fdb/content/GeneralPurposeTools/wipe.rst +++ b/docs/fdb/cli_tools/wipe.rst @@ -29,7 +29,7 @@ Options | ``--ignore-no-data`` | Do not return error if there is no data to delete. | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ | ``--raw`` | | Don't apply (contextual) expansion and checking on requests. | -| | | Keys and values passed must match those used internally to the FDB exactly. | +| | | Keys and values passed must match those used internally to the FDB exactly. | | | | This prevents the use of named parameters (such as t rather than param=130), dates (such as date=-1), or similar. | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ | ``--minimum-keys=string,string`` | | Default is class,expver,stream,date,time | @@ -37,6 +37,10 @@ Options +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ | ``--all`` | (Debug and testing only) Visit all FDB databases | +----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ +| ``--unsafe-wipe-all`` | Wipe all (unowned) contents of an unclean database | ++----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ +| ``--config=string`` | FDB configuration filename | ++----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ Example 1 --------- diff --git a/docs/fdb/cli_tools/write.rst b/docs/fdb/cli_tools/write.rst index 9de36e1e1..d24e931a3 100644 --- a/docs/fdb/cli_tools/write.rst +++ b/docs/fdb/cli_tools/write.rst @@ -1,30 +1,36 @@ -fdb-write +fdb write ========= -| Inserts data into the FDB, creating a new databases if needed. -| The data is copied into the FDB, and the tool reports the location where it was inserted. -| This process is atomic and can be run concurrently to other processes reading or writing to the same FDB databases. +Inserts data into the FDB, creating a new database if needed. +The data is copied into the FDB, and the tool reports the location where it was inserted. +This process is atomic and can be run concurrently to other processes reading or writing to the same FDB databases. Usage ----- -``fdb-write [options] [gribfile2] ...`` +``fdb write [--include-filter=...] [--exclude-filter=...] [path2] ...`` Options ------- +----------------------------------------+-----------------------------------------------------------------------------------------+ -| ``--verbose`` | Prints more information, namely the key of each datum and the | +| ``--verbose`` | Prints more information, namely the key of each datum and the | | | information of which data was filtered out | +----------------------------------------+-----------------------------------------------------------------------------------------+ | ``--statistics`` | Report timing statistics | +----------------------------------------+-----------------------------------------------------------------------------------------+ -| ``--include-filter=string=string,...`` | | Filter out any data that **does not match** this key-value pairs. | +| ``--include-filter=string,...`` | | Filter out any data that **does not match** this key-value pairs. | | | | Key-value pairs can be in the form of a MARS request, e.g.: ``step=141/to/240/by/3`` | +----------------------------------------+-----------------------------------------------------------------------------------------+ -| ``--exclude-filter=string=string,...`` | | Filter out any data that **does match** this key-value pair. | +| ``--exclude-filter=string,...`` | | Filter out any data that **does match** this key-value pair. | | | | Key-value pairs can be in the form of a MARS request, e.g.: ``levelist=850/1000`` | +----------------------------------------+-----------------------------------------------------------------------------------------+ +| ``--config=string`` | FDB configuration filename | ++----------------------------------------+-----------------------------------------------------------------------------------------+ +| ``--modifiers=string`` | List of comma separated key-values of modifiers to each message in input data | ++----------------------------------------+-----------------------------------------------------------------------------------------+ +| ``--archivers=integer`` | Number of archivers (default is 1). For testing only. | ++----------------------------------------+-----------------------------------------------------------------------------------------+ Examples -------- @@ -32,19 +38,19 @@ Examples You may pass multiple grib files. The tool will insert them sequentially. :: - % fdb-write data.grib - + % fdb write data.grib + Processing data.grib FDB archive 12 fields, size 37.5412 Mbytes, in 0.088939 second (422.091 Mbytes per second) fdb::service::archive: 0.089006 second elapsed, 0.089005 second cpu -Use ``--include-filter`` you have a large data set of which you want to write a small sub-set easily identifiable from a few keys ... +Use ``--include-filter`` when you have a large data set of which you want to write a small sub-set easily identifiable from a few keys ... :: - % fdb-write --verbose --include-filter=time=0000 data.grib - + % fdb write --verbose --include-filter=time=0000 data.grib + Processing data.grib Archiving {class=rd,date=20160402,domain=g,expver=xxxx,levelist=1000,levtype=pl,param=138,step=0,stream=oper,time=0000,type=an} Archiving {class=rd,date=20160402,domain=g,expver=xxxx,levelist=1000,levtype=pl,param=155,step=0,stream=oper,time=0000,type=an} @@ -61,11 +67,11 @@ Use ``--include-filter`` you have a large data set of which you want to write a -Use ``--exclude-filter`` you have a large data set of which you want to filter out a small sub-set easily identifiable from a few keys ... +Use ``--exclude-filter`` when you have a large data set of which you want to filter out a small sub-set easily identifiable from a few keys ... :: - % fdb-write --verbose --exclude-filter=time=1200,levelist=1000 data.grib - + % fdb write --verbose --exclude-filter=time=1200,levelist=1000 data.grib + Processing data.grib Archiving {class=rd,date=20160402,domain=g,expver=xxxx,levelist=1000,levtype=pl,param=138,step=0,stream=oper,time=0000,type=an} Archiving {class=rd,date=20160402,domain=g,expver=xxxx,levelist=1000,levtype=pl,param=155,step=0,stream=oper,time=0000,type=an} diff --git a/docs/fdb/content/DebugTools.rst b/docs/fdb/content/DebugTools.rst deleted file mode 100644 index 55a1cb373..000000000 --- a/docs/fdb/content/DebugTools.rst +++ /dev/null @@ -1,10 +0,0 @@ -Debug Tools -=========== - -The debugging tools are found here: - -.. toctree:: - :maxdepth: 1 - :glob: - - DebugTools/* diff --git a/docs/fdb/content/DebugTools/read.rst b/docs/fdb/content/DebugTools/read.rst deleted file mode 100644 index 469041a80..000000000 --- a/docs/fdb/content/DebugTools/read.rst +++ /dev/null @@ -1,57 +0,0 @@ -fdb read -======== - -Read data from the FDB and write this data into a specified target file. This may involve visiting multiple databases if required by the request. - -Usage ------ -:: - - fdb read request.mars target.grib - fdb read --raw request.mars target.grib - fdb read --extract source.grib target.grib - -Options -------- - -+----------------------------------------+-----------------------------------------------------------------------------------------+ -| ``--extract`` | Extract the request(s) from an existing GRIB file | -+----------------------------------------+-----------------------------------------------------------------------------------------+ -| ``--statistics`` | Report timing statistics | -+----------------------------------------+-----------------------------------------------------------------------------------------+ -| ``--config=string`` | FDB configuration filename. | -+----------------------------------------+-----------------------------------------------------------------------------------------+ -| ``--raw`` | Uses the raw request without expansion | -+----------------------------------------+-----------------------------------------------------------------------------------------+ - -Examples --------- - -Specify the MARS request in a plain text file. -Note that this MARS request must be fully expanded. For example, it may not contain the **/to/** or **/by/** statements that the MARS client is able to expand. -:: - - % cat myrequest - retrieve,class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g,type=an,levtype=pl,step=0,levelist=700,param=155 - - # this will retrieve 2 fields - % fdb read myrequest foo.grib - retrieve,class=od,date=20151004,domain=g,expver=0001,levelist=500/700,levtype=pl,param=155,step=0,stream=oper,time=1200,type=an - Compress handle: 3.3e-05 second elapsed, 3.2e-05 second cpu - Compress handle: 2e-06 second elapsed, 2e-06 second cpu - Read rate: 4.7575 Gbytes per second - Write rate: 2.54081 Gbytes per second - Save into: 0.022224 second elapsed, 0.022182 second cpu - - -Obtain data from the FDB using the MARS request that would be implied by an existing GRIB file. In this example, foo.grib contains 2 fields that identify what needs to be retrieved. -:: - - % fdb read --extract foo.grib out.grib - retrieve,class=od,date=20151004,domain=g,expver=0001,levelist=500,levtype=pl,param=155,step=0,stream=oper,time=1200,type=an - ... - Compress handle: 3.3e-05 second elapsed, 3.2e-05 second cpu - Compress handle: 2e-06 second elapsed, 2e-06 second cpu - Read rate: 4.7575 Gbytes per second - Write rate: 2.54081 Gbytes per second - Save into: 0.022224 second elapsed, 0.022182 second cpu diff --git a/docs/fdb/content/DebugTools/write.rst b/docs/fdb/content/DebugTools/write.rst deleted file mode 100644 index 5ed793863..000000000 --- a/docs/fdb/content/DebugTools/write.rst +++ /dev/null @@ -1,86 +0,0 @@ -fdb write -========= - -Inserts data into the FDB, creating a new databases if needed. -The data is copied into the FDB, and the tool reports the location where it was inserted. -This process is atomic and can be run concurrently to other processes reading or writing to the same FDB databases. - -Usage ------ - -``fdb write [--filter-include=...] [--filter-exclude=...] [path2] ...`` - -Options -------- - -+----------------------------------------+-----------------------------------------------------------------------------------------+ -| ``--verbose`` | Prints more information, namely the key of each datum and the | -| | information of which data was filtered out | -+----------------------------------------+-----------------------------------------------------------------------------------------+ -| ``--statistics`` | Report timing statistics | -+----------------------------------------+-----------------------------------------------------------------------------------------+ -| ``--include-filter=string,...`` | | Filter out any data that **does not match** this key-value pairs. | -| | | Key-value pairs can be in the form of a MARS request, e.g.: ``step=141/to/240/by/3`` | -+----------------------------------------+-----------------------------------------------------------------------------------------+ -| ``--exclude-filter=string,...`` | | Filter out any data that **does match** this key-value pair. | -| | | Key-value pairs can be in the form of a MARS request, e.g.: ``levelist=850/1000`` | -+----------------------------------------+-----------------------------------------------------------------------------------------+ -| ``--config=string`` | FDB configuration filename. | -+----------------------------------------+-----------------------------------------------------------------------------------------+ -| ``--modifiers=string`` | List of comma separated key-values of modifiers to each message int input data | -+----------------------------------------+-----------------------------------------------------------------------------------------+ - -Examples --------- - -You may pass multiple grib files. The tool will insert them sequentially. -:: - - % fdb write data.grib - - Processing data.grib - FDB archive 12 fields, size 37.5412 Mbytes, in 0.088939 second (422.091 Mbytes per second) - fdb::service::archive: 0.089006 second elapsed, 0.089005 second cpu - - - -Use ``--include-filter`` you have a large data set of which you want to write a small sub-set easily identifiable from a few keys ... -:: - - % fdb write --verbose --include-filter=time=0000 data.grib - - Processing data.grib - Archiving {class=rd,date=20160402,domain=g,expver=xxxx,levelist=1000,levtype=pl,param=138,step=0,stream=oper,time=0000,type=an} - Archiving {class=rd,date=20160402,domain=g,expver=xxxx,levelist=1000,levtype=pl,param=155,step=0,stream=oper,time=0000,type=an} - Archiving {class=rd,date=20160402,domain=g,expver=xxxx,levelist=850,levtype=pl,param=138,step=0,stream=oper,time=0000,type=an} - Archiving {class=rd,date=20160402,domain=g,expver=xxxx,levelist=850,levtype=pl,param=155,step=0,stream=oper,time=0000,type=an} - Include key {time=0000} filtered out datum {class=rd,date=20160402,domain=g,expver=xxxx,levelist=1000,levtype=pl,param=138,step=0,stream=oper,time=1200,type=an} - Include key {time=0000} filtered out datum {class=rd,date=20160402,domain=g,expver=xxxx,levelist=1000,levtype=pl,param=155,step=0,stream=oper,time=1200,type=an} - Include key {time=0000} filtered out datum {class=rd,date=20160402,domain=g,expver=xxxx,levelist=850,levtype=pl,param=138,step=0,stream=oper,time=1200,type=an} - Include key {time=0000} filtered out datum {class=rd,date=20160402,domain=g,expver=xxxx,levelist=850,levtype=pl,param=155,step=0,stream=oper,time=1200,type=an} - Archiving {class=rd,date=20160401,domain=g,expver=xxxx,levelist=1000,levtype=pl,param=138,step=0,stream=oper,time=0000,type=an} - ... - FDB archive 8 fields, size 25.0275 Mbytes, in 0.129475 second (193.301 Mbytes per seconds) - fdb::service::archive: 0.129522 second elapsed, 0.129514 second cpu - - - -Use ``--exclude-filter`` you have a large data set of which you want to filter out a small sub-set easily identifiable from a few keys ... -:: - - % fdb write --verbose --exclude-filter=time=1200,levelist=1000 data.grib - - Processing data.grib - Archiving {class=rd,date=20160402,domain=g,expver=xxxx,levelist=1000,levtype=pl,param=138,step=0,stream=oper,time=0000,type=an} - Archiving {class=rd,date=20160402,domain=g,expver=xxxx,levelist=1000,levtype=pl,param=155,step=0,stream=oper,time=0000,type=an} - Archiving {class=rd,date=20160402,domain=g,expver=xxxx,levelist=850,levtype=pl,param=138,step=0,stream=oper,time=0000,type=an} - Archiving {class=rd,date=20160402,domain=g,expver=xxxx,levelist=850,levtype=pl,param=155,step=0,stream=oper,time=0000,type=an} - Exclude key {time=1200,levelist=1000} filtered out datum {class=rd,date=20160402,domain=g,expver=xxxx,levelist=1000,levtype=pl,param=138,step=0,stream=oper,time=1200,type=an} - Exclude key {time=1200,levelist=1000} filtered out datum {class=rd,date=20160402,domain=g,expver=xxxx,levelist=1000,levtype=pl,param=155,step=0,stream=oper,time=1200,type=an} - Archiving {class=rd,date=20160402,domain=g,expver=xxxx,levelist=850,levtype=pl,param=138,step=0,stream=oper,time=1200,type=an} - Archiving {class=rd,date=20160401,domain=g,expver=xxxx,levelist=850,levtype=pl,param=155,step=0,stream=oper,time=0000,type=an} - Exclude key {time=1200,levelist=1000} filtered out datum {class=rd,date=20160401,domain=g,expver=xxxx,levelist=1000,levtype=pl,param=138,step=0,stream=oper,time=1200,type=an} - Exclude key {time=1200,levelist=1000} filtered out datum {class=rd,date=20160401,domain=g,expver=xxxx,levelist=1000,levtype=pl,param=155,step=0,stream=oper,time=1200,type=an} - ... - FDB archive 12 fields, size 37.5412 Mbytes, in 0.160719 second (233.584 Mbytes per seconds) - fdb::service::archive: 0.160764 second elapsed, 0.160724 second cpu diff --git a/docs/fdb/content/GeneralPurposeTools.rst b/docs/fdb/content/GeneralPurposeTools.rst deleted file mode 100644 index 119c48635..000000000 --- a/docs/fdb/content/GeneralPurposeTools.rst +++ /dev/null @@ -1,10 +0,0 @@ -General Purpose Tools -===================== - -The General Purpose tools are found here: - -.. toctree:: - :maxdepth: 1 - :glob: - - GeneralPurposeTools/* \ No newline at end of file diff --git a/docs/fdb/content/GeneralPurposeTools/fdb.rst b/docs/fdb/content/GeneralPurposeTools/fdb.rst deleted file mode 100644 index 38abb3add..000000000 --- a/docs/fdb/content/GeneralPurposeTools/fdb.rst +++ /dev/null @@ -1,26 +0,0 @@ -fdb -=== - -Description ------------ - -A wrapper command for the tools described in this section (FDB5 Tools). - -In general, the usage "fdb command arg1 arg2" will execute the tool fdb-command with arguments arg1 and arg2 - -Usage ------ - -``fdb command [options]`` - -Example -------- - -fdb list example -:: - - % fdb list class="od",stream=oper,date=20151004 - {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=700,param=155} - {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=850,param=129} - {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=850,param=130} - ... diff --git a/docs/fdb/content/GeneralPurposeTools/list.rst b/docs/fdb/content/GeneralPurposeTools/list.rst deleted file mode 100644 index 72d764c53..000000000 --- a/docs/fdb/content/GeneralPurposeTools/list.rst +++ /dev/null @@ -1,96 +0,0 @@ -fdb list -======== - -Lists the contents of the FDB databases. -In the body of the output, one line is given per field that has been archived. These (by default) present the fields that are available and will be retrievable - i.e. masked duplicates are skipped. -The lines are broken into three segments, which represent the hierarchical nature of the schema: -* The first component identifies the FDB database containing the data -* The second component identifies the (set of) indexes -* The third component identifies entries collocated within an index - -Usage ------ - -``fdb list [options] [request1] [request2] ...`` - -Options -------- - -+----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--location`` | Also print the location of each field | -+----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--ignore-errors`` | Ignore errors (report them as warnings) and continue processing wherever possible | -+----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--json`` | Output in JSON form | -+----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--porcelain`` | Streamlined and stable output for input into other tools | -+----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--raw`` | | Don't apply (contextual) expansion and checking on requests. | -| | | Keys and values passed must match those used internally to the FDB exactly. | -| | | This prevents the use of named parameters (such as t rather than param=130), dates (such as date=-1), or similar | -+----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--full`` | Include all entries (including masked duplicates) | -+----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--minimum-keys=string,string`` | | Default is ``class,expver`` | -| | | Define the minimum set of keys that must be specified. This prevents inadvertently exploring the entire FDB. | -| | | **Note: Use this carefully as it may trigger exploring the entire FDB.** | -+----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--all`` | **(Debug and testing only)** Visit all FDB databases | -+----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ -| ``--config=string`` | FDB configuration filename | -+----------------------------------------+---------------------------------------------------------------------------------------------------------------------+ - - -Examples --------- - -You may pass a partial request (as a key) that will list all the fields in the FDB that match that key. -Note that this is a global search through all the databases of the FDB that match this key. -:: - - % fdb list class=od,expver=0001,stream=oper,date=20151004 - - retrieve, - class=od, - expver=0001, - stream=oper, - date=20151004 - - {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=700,param=155} - {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=850,param=129} - {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=850,param=130} - ... - - -A JSON listing may be obtained for use in tools that parse the available data -:: - - % fdb list --json class=od,expver=0001,stream=oper,date=20151004 - - [{"class":"od","stream":"oper","date":"20151004","time":"1200","domain":"g","type":"an","levtype":"pl","step":"0","levelist":"700","param":"155"},{...},...] - - -The ``--location`` option can be useful to identify exactly where the field is located within the database. -:: - - % fdb list --location class=od,expver=0001,stream=oper,date=20151004 - - retrieve, - class=od, - expver=0001, - stream=oper, - date=20151004 - - {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=850,param=130} (/data/mars_p_d17_d17_1_15/fdb/od:0001:oper:20151004:1200:g/an:pl.20161103.120238.dhs1213.ecmwf.int.1739461754885.data,13121592,3280398) - ... - -The ``--porcelain`` option gives stable output for use in scripts and as input to other simple tools. This will only print (exactly) one line per entry, with no extraneous output. The output of this option will remain stable across versions. -:: - - % fdb list --porcelain class=od,expver=0001,stream=oper,date=20151004 - {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=700,param=155} - {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=850,param=129} - {class=od,expver=0001,stream=oper,date=20151004,time=1200,domain=g}{type=an,levtype=pl}{step=0,levelist=850,param=130} - ... - - diff --git a/docs/fdb/content/SpecialPurposeTools.rst b/docs/fdb/content/SpecialPurposeTools.rst deleted file mode 100644 index baadc2da5..000000000 --- a/docs/fdb/content/SpecialPurposeTools.rst +++ /dev/null @@ -1,10 +0,0 @@ -Special Purpose Tools -===================== - -The Special Purpose tools are found here: - -.. toctree:: - :maxdepth: 1 - :glob: - - SpecialPurposeTools/* \ No newline at end of file diff --git a/docs/fdb/content/api.rst b/docs/fdb/content/api.rst new file mode 100644 index 000000000..85e6eb97a --- /dev/null +++ b/docs/fdb/content/api.rst @@ -0,0 +1,70 @@ +.. index:: Reference; C++ API, Reference; C API + :name: api-reference + +============= +API Reference +============= + +C++ API +------- + +.. doxygenclass:: fdb5::FDB + :members: + :undoc-members: + +C API +----- + +Initialisation +^^^^^^^^^^^^^^ + +.. doxygengroup:: Initialisation + :content-only: + +Version Accessors +^^^^^^^^^^^^^^^^^ + +.. doxygengroup:: Version Accessors + :content-only: + +Error Handling +^^^^^^^^^^^^^^ + +.. doxygengroup:: Error Handling + :content-only: + +Key +^^^ + +.. doxygengroup:: Key + :content-only: + +Request +^^^^^^^ + +.. doxygengroup:: Request + :content-only: + +SplitKey +^^^^^^^^ + +.. doxygengroup:: SplitKey + :content-only: + +ListIterator +^^^^^^^^^^^^ + +.. doxygengroup:: ListIterator + :content-only: + +DataReader +^^^^^^^^^^ + +.. doxygengroup:: DataReader + :content-only: + +FDB API +^^^^^^^ + +.. doxygengroup:: FDB API + :content-only: diff --git a/docs/fdb/content/config-schema.rst b/docs/fdb/content/config-schema.rst index 694844ca8..ffe9ccff2 100644 --- a/docs/fdb/content/config-schema.rst +++ b/docs/fdb/content/config-schema.rst @@ -2,7 +2,7 @@ Setting up FDB ############## -Seting up FDB requires the user to write a configuration and a schema file. +Setting up FDB requires the user to write a configuration and a schema file. *************************** Writing a FDB Configuration @@ -22,7 +22,7 @@ The following is of the form local: roots: - path: /path/to/fdb/root -There a number of different types such as local, remote, distributed, and +There are a number of different types such as local, remote, distributed, and select. *Local* implements the passage of data from the frontend to storage backend, talk @@ -39,18 +39,18 @@ Select Type: type: select fdbs: - select: class=od - type:local + type: local spaces: roots: - -path: /path/to/fdb/od - -select: class=rd,expver=xx.?.? + - path: /path/to/fdb/od + - select: class=rd,expver=xx.?.? type: local spaces: roots: - path: /path/to/fdb/rd -The *remote* type handles access to the remote FDB vis TCP/IP. It talks to the -FDB server using an asynchronous protocol. It only handles the transition. not +The *remote* type handles access to the remote FDB via TCP/IP. It talks to the +FDB server using an asynchronous protocol. It only handles the transition, not the distribution of data. Remote type: @@ -70,16 +70,16 @@ Dist type: type: dist lanes: - -type: remote + - type: remote host: fdb-minus-1 port: 36604 - -type: remote + - type: remote host: fdb-minus-2 port: 36604 These types can be composed together in the config file when using FDB. -## TODO: Get this reviewed and add more information. +.. TODO: Get this reviewed and add more information. *************** Database Schema @@ -120,7 +120,7 @@ type ``Double`` values that cannot be parsed as double values are rejected and errors emitted. .. code-block:: text - :caption: Redfining datatypes for keywords + :caption: Redefining datatypes for keywords param: Param; step: Step; @@ -186,14 +186,14 @@ In this example the data may contain the ``levelist`` keyword. [stream, date, time [type, levtype - [step, levellist?, param]]] + [step, levelist?, param]]] .. attention:: To retrieve data matching all possible values including messages with - omitted keyword you will need to write two requests. One request specifiying - all possible values for the keayword and a second request with the keyword - ommitted. + omitted keyword you will need to write two requests. One request specifying + all possible values for the keyword and a second request with the keyword + omitted. Matching on Keyword Values ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -248,7 +248,7 @@ used to form a key. Since types change how a key is formed specific types can be used to truncate values. A specific example for this is ``ClimateMonthly`` which requires the value to be in date format and creates a three letter - code for a month out of it, effectively truncating to date to just 'month'. + code for a month out of it, effectively truncating the date to just 'month'. .. list-table:: Types :widths: 25 75 @@ -335,7 +335,7 @@ Forecast Data Example The primary characteristic of forecast runs is that there are discrete chunks of data produced by a large number of processes in parallel. But these produce a well-defined, self-contained dataset in which all the data are produced in a relatively narrow time window, in which the -bulk of data accesses to this data are occur at the same times and separate from access to +bulk of data accesses to this data occur at the same times and separate from access to earlier or later forecast runs, and that the data are typically deleted together as the forecast run becomes older and less valuable. @@ -345,7 +345,7 @@ through the simulation is measured by the keyword step. And the combination of t is required to obtain the real-time valid time for the resultant output. These are treated very differently to each other. -The first axes of time is included in the first level of the schema. As a result, each forecast +The first axis of time is included in the first level of the schema. As a result, each forecast run produces its own, self-contained dataset. The second axis of time is included in the third level as a 'fast moving' key along side concepts such as the level in the atmosphere, and the parameter being output. @@ -410,8 +410,8 @@ are also very poor for typical usage. As such, we suggest a change to use a hierarchical scheme, where notably components of date appear at multiple levels. The datasets are created per year of simulated output, -indexing structures per month, and then date itself is treated analagously to step -in typical forecast ouput (i.e. as a fast-changing key). +indexing structures per month, and then date itself is treated analogously to step +in typical forecast output (i.e. as a fast-changing key). .. code-block:: text :caption: Example Configuration for Climate Data diff --git a/docs/fdb/content/genindex.rst b/docs/fdb/content/genindex.rst deleted file mode 100644 index 9e530fa2f..000000000 --- a/docs/fdb/content/genindex.rst +++ /dev/null @@ -1,2 +0,0 @@ -Index -===== diff --git a/docs/fdb/content/installation.rst b/docs/fdb/content/installation.rst index 38290b160..bdff3dfa6 100644 --- a/docs/fdb/content/installation.rst +++ b/docs/fdb/content/installation.rst @@ -22,7 +22,7 @@ Now proceed with installation as follows: cd $builddir # 2. Run CMake - ecbuild --prefix=$installdir -- -DCMAKE_INSTALL_PREFIX= $srcdir + ecbuild --prefix=$installdir -- $srcdir # 3. Compile / Install make -j10 diff --git a/docs/fdb/content/introduction.rst b/docs/fdb/content/introduction.rst new file mode 100644 index 000000000..46de0a516 --- /dev/null +++ b/docs/fdb/content/introduction.rst @@ -0,0 +1,22 @@ +============ +Introduction +============ + +|Licence| + +FDB (Fields DataBase) is a domain-specific object store developed at ECMWF for storing, indexing and retrieving GRIB data. Each GRIB message is stored as a field and indexed through semantic metadata (i.e. physical variables such as temperature, pressure, ...). +A set of fields can be retrieved specifying a request using a specific language developed for accessing :doc:`mars` Archive. + +FDB exposes a C++ API as well as CLI :doc:`../cli_tools/index`. + +.. toctree:: + :maxdepth: 1 + + requirements + installation + reference + + +.. |Licence| image:: https://img.shields.io/badge/License-Apache%202.0-blue.svg + :target: https://github.com/ecmwf/fdb/blob/develop/LICENSE + :alt: Apache Licence diff --git a/docs/fdb/content/mars.rst b/docs/fdb/content/mars.rst index 187dd73f9..318bf0e36 100644 --- a/docs/fdb/content/mars.rst +++ b/docs/fdb/content/mars.rst @@ -24,7 +24,7 @@ Where: =============== =============== Format Example =============== =============== -single value ``param = temperature/SSRD`` +single value ``param = temperature`` list of values ``step = 12/24/48`` range of values ``date = 19990104/to/19990110/by/2`` =============== =============== diff --git a/docs/fdb/content/requirements.rst b/docs/fdb/content/requirements.rst index f33fc1fe9..795af6277 100644 --- a/docs/fdb/content/requirements.rst +++ b/docs/fdb/content/requirements.rst @@ -11,4 +11,4 @@ Runtime dependencies: Build dependencies: :CMake: For use and installation see http://www.cmake.org/ -:ecbuild: ECMWF library of CMake macros () \ No newline at end of file +:ecbuild: ECMWF library of CMake macros (https://github.com/ecmwf/ecbuild) \ No newline at end of file diff --git a/docs/fdb/content/tools.rst b/docs/fdb/content/tools.rst deleted file mode 100644 index fe155222e..000000000 --- a/docs/fdb/content/tools.rst +++ /dev/null @@ -1,26 +0,0 @@ -FDB Tools -========= - -FDB has a set of general purpose CLI tools for archiving and retrieving user data, some special purpose tools for administrative maintenance and for developer tools for debugging. - -The debugging tools are: - -.. toctree:: - :maxdepth: 1 - - DebugTools - -The general purpose tools are: - -.. toctree:: - :maxdepth: 1 - - GeneralPurposeTools - -The special purpose tools are: - -.. toctree:: - :maxdepth: 1 - - SpecialPurposeTools - diff --git a/docs/fdb/cpp_api.rst b/docs/fdb/cpp_api.rst deleted file mode 100644 index 092767f43..000000000 --- a/docs/fdb/cpp_api.rst +++ /dev/null @@ -1,7 +0,0 @@ -C++ API -======= - -.. doxygenclass:: fdb5::FDB - :members: - :undoc-members: - diff --git a/docs/fdb/index.rst b/docs/fdb/index.rst index 6d18adad0..0826ccf02 100644 --- a/docs/fdb/index.rst +++ b/docs/fdb/index.rst @@ -1,13 +1,13 @@ .. _FDB_Introduction: -FDB -=================================== +Fields DataBase - FDB Documentation +==================================== :Version: |version| The FDB is a domain-specific object store developed at ECMWF for storing, indexing and retrieving GRIB data. Each GRIB message is stored as a -field and indexed trough semantic metadata (i.e., physical variables such as +field and indexed through semantic metadata (i.e. physical variables such as temperature, pressure, …). A set of fields can be retrieved specifying a request using a specific language developed for accessing MARS Archive. @@ -17,16 +17,13 @@ request using a specific language developed for accessing MARS Archive. :maxdepth: 2 :caption: Contents - content/requirements - content/installation - content/reference - content/tools + content/introduction + content/mars content/config-schema - content/license cli_tools/index - cpp_api - c_api - content/genindex + content/api + content/license + genindex .. |Licence| image:: https://img.shields.io/badge/License-Apache%202.0-blue.svg :target: https://github.com/ecmwf/fdb/blob/develop/LICENSE