From 78cd0caa641d0b4f2c8ba07b7410437c5e47f9ee Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 5 Aug 2025 15:34:27 -0400 Subject: [PATCH 01/69] DOC: Document using `TWINED_SERVICES_TOPIC_NAME` envvar --- docs/source/asking_questions.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/source/asking_questions.rst b/docs/source/asking_questions.rst index 45536df23..6c20deb71 100644 --- a/docs/source/asking_questions.rst +++ b/docs/source/asking_questions.rst @@ -26,6 +26,12 @@ Questions are always asked to a *revision* of a service. You can ask a service a Asking a question ================= +.. important:: + If you're using an environment other than the ``main`` environment, then before asking any questions to your Twined + services, set the ``TWINED_SERVICES_TOPIC_NAME`` environment variable to the name of the Twined services Pub/Sub + topic (this is set when :ref:`deploying a service network `). It will be in the form + ``.octue.twined.services`` + .. code-block:: python from octue.twined.resources import Child From 42694181486a74bba768a2c641519d565d3a5759 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 5 Aug 2025 15:35:22 -0400 Subject: [PATCH 02/69] REF: Move `start` CLI subcommand under `octue twined service` command BREAKING CHANGE: Use `octue twined service start` instead of `octue twined start` --- docs/source/creating_services.rst | 2 +- docs/source/running_services_locally.rst | 2 +- octue/cli.py | 7 ++++++- pyproject.toml | 2 +- tests/test_cli.py | 7 +++++-- tests/twined/templates/test_template_apps.py | 2 ++ 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/source/creating_services.rst b/docs/source/creating_services.rst index 636806339..aac37d98a 100644 --- a/docs/source/creating_services.rst +++ b/docs/source/creating_services.rst @@ -126,7 +126,7 @@ See :ref:`here ` for service naming requirements. - Set in: - ``OCTUE_SERVICE_REVISION_TAG`` environment variable - - If using ``octue twined start`` command, the ``--revision-tag`` option (takes priority) + - If using ``octue twined service start`` command, the ``--revision-tag`` option (takes priority) Template apps diff --git a/docs/source/running_services_locally.rst b/docs/source/running_services_locally.rst index 52c22c8f9..1c037ba94 100644 --- a/docs/source/running_services_locally.rst +++ b/docs/source/running_services_locally.rst @@ -61,7 +61,7 @@ Via the CLI .. code-block:: shell - octue twined start + octue twined service start This will run the service as a child waiting for questions until you press ``Ctrl + C`` or an error is encountered. The service will be available to be questioned by other services at the service ID ``organisation/name`` as specified in diff --git a/octue/cli.py b/octue/cli.py index 090ec00eb..87b3e4ea0 100644 --- a/octue/cli.py +++ b/octue/cli.py @@ -575,7 +575,12 @@ def diagnostics(cloud_path, local_path, download_datasets): # child.cancel(question_uuid=question_uuid, event_store_table_id=service_configuration.event_store_table_id) -@twined.command() +@twined.group() +def service(): + """Start or manage a Twined service.""" + + +@service.command() @click.option( "-c", "--service-config", diff --git a/pyproject.toml b/pyproject.toml index 5905874bf..9e1c7ebc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "octue" -version = "0.67.0" +version = "0.68.0" description = "A package providing template applications for data services, and a python SDK to the Octue API." readme = "README.md" authors = ["Marcus Lugg ", "Thomas Clark "] diff --git a/tests/test_cli.py b/tests/test_cli.py index a16d894a5..1ae688d6b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -665,7 +665,7 @@ def test_start_command(self): ): with ServicePatcher(): with self.assertLogs(level=logging.INFO) as logging_context: - result = CliRunner().invoke(octue_cli, ["twined", "start", "--timeout=0"]) + result = CliRunner().invoke(octue_cli, ["twined", "service", "start", "--timeout=0"]) self.assertEqual(logging_context.records[1].message, "Starting .") self.assertIsNone(result.exception) @@ -684,7 +684,10 @@ def test_start_command_with_revision_tag_override_when_revision_tag_environment_ ): with ServicePatcher(): with self.assertLogs() as logging_context: - result = CliRunner().invoke(octue_cli, ["twined", "start", "--revision-tag=hello", "--timeout=0"]) + result = CliRunner().invoke( + octue_cli, + ["twined", "service", "start", "--revision-tag=hello", "--timeout=0"], + ) self.assertEqual( logging_context.records[1].message, diff --git a/tests/twined/templates/test_template_apps.py b/tests/twined/templates/test_template_apps.py index 4d3e5bc59..cce2d531e 100644 --- a/tests/twined/templates/test_template_apps.py +++ b/tests/twined/templates/test_template_apps.py @@ -80,6 +80,7 @@ def test_child_services_template(self): sys.executable, cli_path, "twined", + "service", "start", f"--service-config={os.path.join(elevation_service_path, 'octue.yaml')}", ], @@ -95,6 +96,7 @@ def test_child_services_template(self): sys.executable, cli_path, "twined", + "service", "start", f"--service-config={os.path.join(wind_speed_service_path, 'octue.yaml')}", ], From 81abef7355f534bf31388271a9f81bbc501dbdb9 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Thu, 7 Aug 2025 15:32:01 -0400 Subject: [PATCH 03/69] REV: Revert "REF: Remove `google_crc32c` warning suppression" This reverts commit 8f108d8eec4843a362443b32c8ff953293ed3113. skipci --- octue/cloud/storage/client.py | 5 ++++- octue/mixins/hashable.py | 5 ++++- octue/resources/datafile.py | 7 ++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/octue/cloud/storage/client.py b/octue/cloud/storage/client.py index de0759f6a..cbdf7d362 100644 --- a/octue/cloud/storage/client.py +++ b/octue/cloud/storage/client.py @@ -14,13 +14,16 @@ from google.cloud.storage import Client from google.cloud.storage.constants import _DEFAULT_TIMEOUT from google.cloud.storage.retry import DEFAULT_RETRY -from google_crc32c import Checksum from octue.cloud import storage from octue.exceptions import CloudStorageBucketNotFound from octue.utils.decoders import OctueJSONDecoder from octue.utils.encoders import OctueJSONEncoder +with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=RuntimeWarning, module="google_crc32c") + from google_crc32c import Checksum + logger = logging.getLogger(__name__) diff --git a/octue/mixins/hashable.py b/octue/mixins/hashable.py index 59ded1348..184e9c810 100644 --- a/octue/mixins/hashable.py +++ b/octue/mixins/hashable.py @@ -1,8 +1,11 @@ import base64 import collections.abc import datetime +import warnings -from google_crc32c import Checksum +with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=RuntimeWarning, module="google_crc32c") + from google_crc32c import Checksum EMPTY_STRING_HASH_VALUE = "AAAAAA==" diff --git a/octue/resources/datafile.py b/octue/resources/datafile.py index 4c1e88447..ea0369348 100644 --- a/octue/resources/datafile.py +++ b/octue/resources/datafile.py @@ -7,8 +7,8 @@ import os import shutil import tempfile +import warnings -from google_crc32c import Checksum import requests from octue.cloud import storage @@ -32,6 +32,11 @@ except (ModuleNotFoundError, ImportError): pass +with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=RuntimeWarning, module="google_crc32c") + from google_crc32c import Checksum + + logger = logging.getLogger(__name__) From abb512e986c044b608f1bb1f516cafe45bfcb575 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 15 Aug 2025 16:22:45 -0400 Subject: [PATCH 04/69] DEP: Add `mkdocs-material` --- poetry.lock | 268 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 266 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 604e77af0..ee329a43d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -112,6 +112,26 @@ files = [ [package.extras] dev = ["backports.zoneinfo ; python_version < \"3.9\"", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata ; sys_platform == \"win32\""] +[[package]] +name = "backrefs" +version = "5.9" +description = "A wrapper around re and regex that adds additional back references." +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "backrefs-5.9-py310-none-any.whl", hash = "sha256:db8e8ba0e9de81fcd635f440deab5ae5f2591b54ac1ebe0550a2ca063488cd9f"}, + {file = "backrefs-5.9-py311-none-any.whl", hash = "sha256:6907635edebbe9b2dc3de3a2befff44d74f30a4562adbb8b36f21252ea19c5cf"}, + {file = "backrefs-5.9-py312-none-any.whl", hash = "sha256:7fdf9771f63e6028d7fee7e0c497c81abda597ea45d6b8f89e8ad76994f5befa"}, + {file = "backrefs-5.9-py313-none-any.whl", hash = "sha256:cc37b19fa219e93ff825ed1fed8879e47b4d89aa7a1884860e2db64ccd7c676b"}, + {file = "backrefs-5.9-py314-none-any.whl", hash = "sha256:df5e169836cc8acb5e440ebae9aad4bf9d15e226d3bad049cf3f6a5c20cc8dc9"}, + {file = "backrefs-5.9-py39-none-any.whl", hash = "sha256:f48ee18f6252b8f5777a22a00a09a85de0ca931658f1dd96d4406a34f3748c60"}, + {file = "backrefs-5.9.tar.gz", hash = "sha256:808548cb708d66b82ee231f962cb36faaf4f2baab032f2fbb783e9c2fdddaa59"}, +] + +[package.extras] +extras = ["regex"] + [[package]] name = "beautifulsoup4" version = "4.13.4" @@ -301,7 +321,7 @@ version = "8.2.0" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["main", "dev"] files = [ {file = "click-8.2.0-py3-none-any.whl", hash = "sha256:6b303f0b2aa85f1cb4e5303078fadcbcd4e476f114fab9b5007005711839325c"}, {file = "click-8.2.0.tar.gz", hash = "sha256:f5452aeddd9988eefa20f90f05ab66f17fce1ee2a36907fd30b05bbb5953814d"}, @@ -321,7 +341,7 @@ files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -markers = {main = "platform_system == \"Windows\"", dev = "sys_platform == \"win32\""} +markers = {main = "platform_system == \"Windows\""} [[package]] name = "coolname" @@ -592,6 +612,24 @@ files = [ fs = "2.4.16" google-crc32c = "1.3.0" +[[package]] +name = "ghp-import" +version = "2.1.0" +description = "Copy your docs directly to the gh-pages branch." +optional = false +python-versions = "*" +groups = ["dev"] +files = [ + {file = "ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343"}, + {file = "ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619"}, +] + +[package.dependencies] +python-dateutil = ">=2.8.1" + +[package.extras] +dev = ["flake8", "markdown", "twine", "wheel"] + [[package]] name = "google-api-core" version = "2.24.2" @@ -1169,6 +1207,22 @@ files = [ [package.dependencies] referencing = ">=0.31.0" +[[package]] +name = "markdown" +version = "3.8.2" +description = "Python implementation of John Gruber's Markdown." +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "markdown-3.8.2-py3-none-any.whl", hash = "sha256:5c83764dbd4e00bdd94d85a19b8d55ccca20fe35b2e678a1422b380324dd5f24"}, + {file = "markdown-3.8.2.tar.gz", hash = "sha256:247b9a70dd12e27f67431ce62523e675b866d254f900c4fe75ce3dda62237c45"}, +] + +[package.extras] +docs = ["mdx_gh_links (>=0.2)", "mkdocs (>=1.6)", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-nature (>=0.6)", "mkdocs-section-index", "mkdocstrings[python]"] +testing = ["coverage", "pyyaml"] + [[package]] name = "markupsafe" version = "3.0.2" @@ -1240,6 +1294,109 @@ files = [ {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, ] +[[package]] +name = "mergedeep" +version = "1.3.4" +description = "A deep merge function for 🐍." +optional = false +python-versions = ">=3.6" +groups = ["dev"] +files = [ + {file = "mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307"}, + {file = "mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8"}, +] + +[[package]] +name = "mkdocs" +version = "1.6.1" +description = "Project documentation with Markdown." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e"}, + {file = "mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2"}, +] + +[package.dependencies] +click = ">=7.0" +colorama = {version = ">=0.4", markers = "platform_system == \"Windows\""} +ghp-import = ">=1.0" +jinja2 = ">=2.11.1" +markdown = ">=3.3.6" +markupsafe = ">=2.0.1" +mergedeep = ">=1.3.4" +mkdocs-get-deps = ">=0.2.0" +packaging = ">=20.5" +pathspec = ">=0.11.1" +pyyaml = ">=5.1" +pyyaml-env-tag = ">=0.1" +watchdog = ">=2.0" + +[package.extras] +i18n = ["babel (>=2.9.0)"] +min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4) ; platform_system == \"Windows\"", "ghp-import (==1.0)", "importlib-metadata (==4.4) ; python_version < \"3.10\"", "jinja2 (==2.11.1)", "markdown (==3.3.6)", "markupsafe (==2.0.1)", "mergedeep (==1.3.4)", "mkdocs-get-deps (==0.2.0)", "packaging (==20.5)", "pathspec (==0.11.1)", "pyyaml (==5.1)", "pyyaml-env-tag (==0.1)", "watchdog (==2.0)"] + +[[package]] +name = "mkdocs-get-deps" +version = "0.2.0" +description = "MkDocs extension that lists all dependencies according to a mkdocs.yml file" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "mkdocs_get_deps-0.2.0-py3-none-any.whl", hash = "sha256:2bf11d0b133e77a0dd036abeeb06dec8775e46efa526dc70667d8863eefc6134"}, + {file = "mkdocs_get_deps-0.2.0.tar.gz", hash = "sha256:162b3d129c7fad9b19abfdcb9c1458a651628e4b1dea628ac68790fb3061c60c"}, +] + +[package.dependencies] +mergedeep = ">=1.3.4" +platformdirs = ">=2.2.0" +pyyaml = ">=5.1" + +[[package]] +name = "mkdocs-material" +version = "9.6.17" +description = "Documentation that simply works" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "mkdocs_material-9.6.17-py3-none-any.whl", hash = "sha256:221dd8b37a63f52e580bcab4a7e0290e4a6f59bd66190be9c3d40767e05f9417"}, + {file = "mkdocs_material-9.6.17.tar.gz", hash = "sha256:48ae7aec72a3f9f501a70be3fbd329c96ff5f5a385b67a1563e5ed5ce064affe"}, +] + +[package.dependencies] +babel = ">=2.10,<3.0" +backrefs = ">=5.7.post1,<6.0" +click = "<8.2.2" +colorama = ">=0.4,<1.0" +jinja2 = ">=3.1,<4.0" +markdown = ">=3.2,<4.0" +mkdocs = ">=1.6,<2.0" +mkdocs-material-extensions = ">=1.3,<2.0" +paginate = ">=0.5,<1.0" +pygments = ">=2.16,<3.0" +pymdown-extensions = ">=10.2,<11.0" +requests = ">=2.26,<3.0" + +[package.extras] +git = ["mkdocs-git-committers-plugin-2 (>=1.1,<3)", "mkdocs-git-revision-date-localized-plugin (>=1.2.4,<2.0)"] +imaging = ["cairosvg (>=2.6,<3.0)", "pillow (>=10.2,<11.0)"] +recommended = ["mkdocs-minify-plugin (>=0.7,<1.0)", "mkdocs-redirects (>=1.2,<2.0)", "mkdocs-rss-plugin (>=1.6,<2.0)"] + +[[package]] +name = "mkdocs-material-extensions" +version = "1.3.1" +description = "Extension pack for Python Markdown and MkDocs Material." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31"}, + {file = "mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443"}, +] + [[package]] name = "more-itertools" version = "10.7.0" @@ -1480,6 +1637,22 @@ files = [ {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, ] +[[package]] +name = "paginate" +version = "0.5.7" +description = "Divides large result sets into pages for easier browsing" +optional = false +python-versions = "*" +groups = ["dev"] +files = [ + {file = "paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591"}, + {file = "paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945"}, +] + +[package.extras] +dev = ["pytest", "tox"] +lint = ["black"] + [[package]] name = "pandas" version = "2.2.3" @@ -1567,6 +1740,18 @@ sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-d test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] xml = ["lxml (>=4.9.2)"] +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + [[package]] name = "platformdirs" version = "4.3.8" @@ -1718,6 +1903,25 @@ files = [ [package.extras] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pymdown-extensions" +version = "10.16.1" +description = "Extension pack for Python Markdown." +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "pymdown_extensions-10.16.1-py3-none-any.whl", hash = "sha256:d6ba157a6c03146a7fb122b2b9a121300056384eafeec9c9f9e584adfdb2a32d"}, + {file = "pymdown_extensions-10.16.1.tar.gz", hash = "sha256:aace82bcccba3efc03e25d584e6a22d27a8e17caa3f4dd9f207e49b787aa9a91"}, +] + +[package.dependencies] +markdown = ">=3.6" +pyyaml = "*" + +[package.extras] +extra = ["pygments (>=2.19.1)"] + [[package]] name = "pytest" version = "7.4.4" @@ -1846,6 +2050,21 @@ files = [ {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] +[[package]] +name = "pyyaml-env-tag" +version = "1.1" +description = "A custom YAML tag for referencing environment variables in YAML files." +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04"}, + {file = "pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff"}, +] + +[package.dependencies] +pyyaml = "*" + [[package]] name = "referencing" version = "0.36.2" @@ -2708,6 +2927,49 @@ platformdirs = ">=3.9.1,<5" docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8) ; platform_python_implementation == \"PyPy\" or platform_python_implementation == \"GraalVM\" or platform_python_implementation == \"CPython\" and sys_platform == \"win32\" and python_version >= \"3.13\"", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10) ; platform_python_implementation == \"CPython\""] +[[package]] +name = "watchdog" +version = "6.0.0" +description = "Filesystem events monitoring" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1cdb490583ebd691c012b3d6dae011000fe42edb7a82ece80965b42abd61f26"}, + {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc64ab3bdb6a04d69d4023b29422170b74681784ffb9463ed4870cf2f3e66112"}, + {file = "watchdog-6.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c897ac1b55c5a1461e16dae288d22bb2e412ba9807df8397a635d88f671d36c3"}, + {file = "watchdog-6.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6eb11feb5a0d452ee41f824e271ca311a09e250441c262ca2fd7ebcf2461a06c"}, + {file = "watchdog-6.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef810fbf7b781a5a593894e4f439773830bdecb885e6880d957d5b9382a960d2"}, + {file = "watchdog-6.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:afd0fe1b2270917c5e23c2a65ce50c2a4abb63daafb0d419fde368e272a76b7c"}, + {file = "watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948"}, + {file = "watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860"}, + {file = "watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0"}, + {file = "watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c"}, + {file = "watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134"}, + {file = "watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b"}, + {file = "watchdog-6.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e6f0e77c9417e7cd62af82529b10563db3423625c5fce018430b249bf977f9e8"}, + {file = "watchdog-6.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:90c8e78f3b94014f7aaae121e6b909674df5b46ec24d6bebc45c44c56729af2a"}, + {file = "watchdog-6.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e7631a77ffb1f7d2eefa4445ebbee491c720a5661ddf6df3498ebecae5ed375c"}, + {file = "watchdog-6.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:c7ac31a19f4545dd92fc25d200694098f42c9a8e391bc00bdd362c5736dbf881"}, + {file = "watchdog-6.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9513f27a1a582d9808cf21a07dae516f0fab1cf2d7683a742c498b93eedabb11"}, + {file = "watchdog-6.0.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7a0e56874cfbc4b9b05c60c8a1926fedf56324bb08cfbc188969777940aef3aa"}, + {file = "watchdog-6.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:e6439e374fc012255b4ec786ae3c4bc838cd7309a540e5fe0952d03687d8804e"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2"}, + {file = "watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a"}, + {file = "watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680"}, + {file = "watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f"}, + {file = "watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282"}, +] + +[package.extras] +watchmedo = ["PyYAML (>=3.10)"] + [[package]] name = "webencodings" version = "0.5.1" @@ -2835,4 +3097,4 @@ hdf5 = ["h5py"] [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "dca07830c0cbbbd9cdcd2b1e29514a276be8a83ce95d7d266eac249ff6b8f7df" +content-hash = "24be32340ed09272c80d6a6e6d033d841fa19f25e390d29fe2c7af7e19a258b3" diff --git a/pyproject.toml b/pyproject.toml index 9e1c7ebc6..35d0d4d51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ sphinx-rtd-theme = ">=1,<2" sphinx-tabs = ">=3.4.0,<4" sphinx-toolbox = "^3" ruff = "^0.6.9" +mkdocs-material = "^9.6.17" [tool.ruff] line-length = 120 From 19cc299c2a75f808dd14cff080f5fda19a507e07 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 15 Aug 2025 16:31:00 -0400 Subject: [PATCH 05/69] WIP: Disable sphinx pre-commit step --- .pre-commit-config.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a582b072e..62f0b4be1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,18 +31,18 @@ repos: hooks: - id: prettier - - repo: https://github.com/thclark/pre-commit-sphinx - rev: 0.0.3 - hooks: - - id: build-docs - language_version: python3 - additional_dependencies: - - "poetry==1.2.0b2" - - "Sphinx>=5,<8" - - "sphinx-rtd-theme>=1,<2" - - "sphinx-tabs>=3,<4" - - "sphinx-toolbox>=3" - - "git+https://github.com/octue/octue-sdk-python.git@main" + # - repo: https://github.com/thclark/pre-commit-sphinx + # rev: 0.0.3 + # hooks: + # - id: build-docs + # language_version: python3 + # additional_dependencies: + # - "poetry==1.2.0b2" + # - "Sphinx>=5,<8" + # - "sphinx-rtd-theme>=1,<2" + # - "sphinx-tabs>=3,<4" + # - "sphinx-toolbox>=3" + # - "git+https://github.com/octue/octue-sdk-python.git@main" - repo: https://github.com/windpioneers/pre-commit-hooks rev: 0.0.5 From 3a3a322736879fea10082bc59d7c72206c091e24 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 15 Aug 2025 16:31:16 -0400 Subject: [PATCH 06/69] DOC: Add basic setup for Material for Mkdocs --- docs/docs/index.md | 17 +++++++++++++++++ docs/mkdocs.yml | 4 ++++ 2 files changed, 21 insertions(+) create mode 100644 docs/docs/index.md create mode 100644 docs/mkdocs.yml diff --git a/docs/docs/index.md b/docs/docs/index.md new file mode 100644 index 000000000..327a4edc2 --- /dev/null +++ b/docs/docs/index.md @@ -0,0 +1,17 @@ +# Welcome to MkDocs + +For full documentation visit [mkdocs.org](https://www.mkdocs.org). + +## Commands + +- `mkdocs new [dir-name]` - Create a new project. +- `mkdocs serve` - Start the live-reloading docs server. +- `mkdocs build` - Build the documentation site. +- `mkdocs -h` - Print help message and exit. + +## Project layout + + mkdocs.yml # The configuration file. + docs/ + index.md # The documentation homepage. + ... # Other markdown pages, images and other files. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml new file mode 100644 index 000000000..becdde37b --- /dev/null +++ b/docs/mkdocs.yml @@ -0,0 +1,4 @@ +site_name: Octue Twined +site_url: https://docs.twined.octue.com +theme: + name: material From 1710a74c137eab68e645adb2c6614d23ceb6f609 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 15 Aug 2025 16:42:49 -0400 Subject: [PATCH 07/69] DOC: Enable mkdocs admonitions and privacy plugin --- docs/mkdocs.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index becdde37b..a9f5a0d91 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -2,3 +2,9 @@ site_name: Octue Twined site_url: https://docs.twined.octue.com theme: name: material +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.superfences +plugins: + - privacy From 8ad32aeb7ff2dc08107d3773f3901b5679337511 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 15 Aug 2025 16:49:27 -0400 Subject: [PATCH 08/69] DOC: Add light/dark mode toggle with system preference default skipci --- docs/mkdocs.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index a9f5a0d91..473bb7abe 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -2,6 +2,21 @@ site_name: Octue Twined site_url: https://docs.twined.octue.com theme: name: material + palette: + # Palette toggle for light mode + - media: "(prefers-color-scheme: light)" + scheme: default + toggle: + icon: material/weather-night + name: Switch to dark mode + + # Palette toggle for dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + toggle: + icon: material/weather-sunny + name: Switch to light mode + markdown_extensions: - admonition - pymdownx.details From c597ca4c85993b5a7f2bb631a9ac4058e6cc9958 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 15 Aug 2025 17:12:49 -0400 Subject: [PATCH 09/69] DOC: Add index page --- docs/docs/index.md | 64 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/docs/docs/index.md b/docs/docs/index.md index 327a4edc2..4ae9d741d 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -1,17 +1,57 @@ -# Welcome to MkDocs +# Introduction -For full documentation visit [mkdocs.org](https://www.mkdocs.org). +The python SDK for [Octue](https://octue.com) Twined scientific data services and digital twins - get faster data +groundwork so you have more time for the science! -## Commands +!!! info "Definition: Twined service" -- `mkdocs new [dir-name]` - Create a new project. -- `mkdocs serve` - Start the live-reloading docs server. -- `mkdocs build` - Build the documentation site. -- `mkdocs -h` - Print help message and exit. + A data service or digital twin built with the Twined framework that can be asked questions, process them, and + return answers. Twined services can communicate with each other with minimal extra setup. -## Project layout +## Key features - mkdocs.yml # The configuration file. - docs/ - index.md # The documentation homepage. - ... # Other markdown pages, images and other files. +**Unified cloud/local file, dataset, and manifest operations** + +- Create and build datasets easily +- Organise them with timestamps, labels, and tags +- Filter and combine them using this metadata +- Store them locally or in the cloud (or both for low-latency reading/writing with cloud-guaranteed data availability) +- Use internet/cloud-based datasets as if they were local e.g. + - `https://example.com/important_dataset.dat` + - `gs://example-bucket/important_dataset.dat` +- Create manifests (a set of datasets needed for a particular analysis) to modularise your dataset input/output + +**Ask existing services questions from anywhere** + +- Send them data to process from anywhere +- Automatically have their logs, monitor messages, and any errors forwarded to you and displayed as if they were local +- Receive their output data as JSON +- Receive a manifest of any output datasets they produce for you to download or access as you wish + +**Create, run, and deploy your apps as services** + +- No need to change your app - just wrap it +- Use the `octue` CLI to run your service locally or deploy it to Google Kubernetes Engine (GKE) +- Create JSON-schema interfaces to explicitly define the form of configuration, input, and output data +- Ask other services questions as part of your app (i.e. build trees of services) +- Automatically display readable, colourised logs, or use your own log handler +- Avoid time-consuming and confusing devops, cloud configuration, and backend maintenance + +**High standards, quick responses, and good intentions** + +- Open-source and transparent on GitHub - anyone can see the code and raise an issue +- Automated testing, standards, releases, and deployment +- High test coverage +- Works on MacOS, Linux, and Windows +- Developed not-for-profit for the renewable energy industry + +## Need help, found a bug, or want to request a new feature? + +We use [GitHub Issues](https://github.com/octue/octue-sdk-python/issues) to manage: + +- Bug reports +- Feature requests +- Support requests + +Bug reports, feature requests and support requests, may also be made directly to your Octue support contact, or via the +[support pages](https://www.octue.com/contact). From 8761b2480c6c36dd055aa8f7e250805427cd14d0 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 15 Aug 2025 17:14:20 -0400 Subject: [PATCH 10/69] DOC: Add asking questions page skipci --- docs/docs/asking_questions.md | 417 ++++++++++++++++++++++++++++++++++ 1 file changed, 417 insertions(+) create mode 100644 docs/docs/asking_questions.md diff --git a/docs/docs/asking_questions.md b/docs/docs/asking_questions.md new file mode 100644 index 000000000..4d7a85585 --- /dev/null +++ b/docs/docs/asking_questions.md @@ -0,0 +1,417 @@ +# Asking services questions + +## What is a question? + +A question is a set of data (input values and/or an input manifest) sent +to a child for processing/analysis. Questions can be: + +- **Synchronous ("ask-and-wait"):** A question whose answer is waited + for in real time + +- **Asynchronous ("fire-and-forget"):** A question whose answer is not + waited for and is instead retrieved later. There are two types: + + - **Regular:** Responses to these questions are automatically stored + in an event store where they can be + [retrieved using the Octue SDK](#retrieving-answers-to-asynchronous-questions) + + - **Push endpoint:** Responses to these questions are pushed to an + HTTP endpoint for asynchronous handling using Octue's + [django-twined](https://django-twined.readthedocs.io/en/latest/) + or custom logic in your own webserver. + +Questions are always asked to a _revision_ of a service. You can ask a +service a question if you have its [SRUID](services.md/#service-names), project ID, and the necessary permissions. + +## Asking a question + +```python +from octue.twined.resources import Child + +child = Child( + id="my-organisation/my-service:2.1.7", + backend={"name": "GCPPubSubBackend", "project_id": "my-project"}, +) + +answer, question_uuid = child.ask( + input_values={"height": 32, "width": 3}, + input_manifest=manifest, +) + +answer["output_values"] +>>> {"some": "data"} + +answer["output_manifest"]["my_dataset"].files +>>> , })> +``` + +!!! warning + + If you're using an environment other than the `main` environment, then before asking any questions to your Twined + services, set the `TWINED_SERVICES_TOPIC_NAME` environment variable to the name of the Twined services Pub/Sub topic + (this is set when [deploying a service network](deploying_services.md/#deploying-services-advanced-developers-guide). + It will be in the form `.octue.twined.services` + +!!! note + + Not including a service revision tag will cause your question to be sent to the default revision of the service + (usually the latest deployed version). This is determined by making a request to a + [service registry](https://django-twined.readthedocs.io/en/latest/) if one or more + [registries are defined](#using-a-service-registry). If none of the service registries contain an entry for this + service, a specific service revision tag must be used. + +You can also set the following options when you call `Child.ask`: + +- `children` - If the child has children of its own (i.e. grandchildren of the parent), this optional argument can be + used to override the child's "default" children. This allows you to specify particular versions of grandchildren to + use (see [this subsection below](#overriding-a-childs-children). +- `subscribe_to_logs` - if true, the child will forward its logs to you +- `allow_local_files` - if true, local files/datasets are allowed in any input manifest you supply +- `handle_monitor_message` - if provided a function, it will be called on any monitor messages from the child +- `record_events` -- if true, events received from the parent while it processes the question are saved to the + `Child.received_events` property +- `save_diagnostics` - must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if + turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just + if the analysis fails +- `question_uuid` - if provided, the question will use this UUID instead of a generated one +- `push_endpoint` - if provided, the result and other events produced during the processing of the question will be + pushed to this HTTP endpoint (a URL) +- `asynchronous` - if true, don't wait for an answer to the question (the result and other events can be + [retrieved from the event store later](#retrieving-answers-to-asynchronous-questions) +- `cpus` - the number of CPUs to request for the question; defaults to the number set by the child service +- `memory` - the amount of memory to request for the question e.g. "256Mi" or "1Gi"; defaults to the amount set by the + child service +- `ephemeral_storage` - the amount of ephemeral storage to request for the question e.g. "256Mi" or "1Gi"; defaults to + the amount set by the child service +- `timeout` - how long in seconds to wait for an answer (`None` by default - i.e. don't time out) + +If the question fails: + +- If `raise_errors=False`, the unraised error is returned +- If `raise_errors=False` and `max_retries > 0`, the question is retried up to this number of times +- If `raise_errors=False`, `max_retries > 0`, and `prevent_retries_when` is a list of exception types, the question is + retried unless the error type is in the list +- If `raise_errors=False`, `log_errors=True`, and the question fails after its final retry, the error is logged + +### Exceptions raised by a child + +If a child raises an exception while processing your question, the +exception will always be forwarded and re-raised in your local service +or python session. You can handle exceptions in whatever way you like. + +### Timeouts + +If setting a timeout, bear in mind that the question has to reach the +child, the child has to run its analysis on the inputs sent to it (this +will most likely make up the dominant part of the wait time), and the +answer has to be sent back to the parent. If you're not sure how long a +particular analysis might take, it's best to set the timeout to `None` +initially or ask the owner/maintainer of the child for an estimate. + +## Retrieving answers to asynchronous questions + +To retrieve results and other events from the processing of a question +later, make sure you have the permissions to access the event store and +run: + +```python +from octue.twined.cloud.pub_sub.bigquery import get_events + +events = get_events(question_uuid="53353901-0b47-44e7-9da3-a3ed59990a71") +``` + +**Options** + +- `table_id` - If you're not using the standard deployment, you can + specify a different table here +- `question_uuid` - Retrieve events from this specific question +- `parent_question_uuid` - Retrieve events from questions triggered by + the same parent question (this doesn't include the parent question's + events) +- `originator_question_uuid` - Retrieve events for the entire tree of + questions triggered by an originator question (a question asked + manually through `Child.ask`; this does include the originator + question's events) +- `kind` - Only retrieve this kind of event if present (e.g. "result") +- `include_backend_metadata` - If `True`, retrieve information about the + service backend that produced the event +- `limit` - If set to a positive integer, limit the number of events + returned to this + +!!! note + + Only one of `question_uuid`, `parent_question_uuid`, and `originator_question_uuid` can be provided at one time. + +??? example "See an example output here..." + + ``` python + >>> events + [ + { + "event": { + "kind": "delivery_acknowledgement", + }, + }, + { + "event": { + "kind": "log_record", + "log_record": { + "args": null, + "created": 1709739861.5949728, + "exc_info": null, + "exc_text": null, + "filename": "app.py", + "funcName": "run", + "levelname": "INFO", + "levelno": 20, + "lineno": 28, + "module": "app", + "msecs": 594.9728488922119, + "msg": "Finished example analysis.", + "name": "app", + "pathname": "/workspace/example_service_cloud_run/app.py", + "process": 2, + "processName": "MainProcess", + "relativeCreated": 8560.13798713684, + "stack_info": null, + "thread": 68328473233152, + "threadName": "ThreadPoolExecutor-0_2" + } + }, + }, + { + "event": { + "kind": "heartbeat", + }, + }, + { + "event": { + "kind": "result", + "output_manifest": { + "datasets": { + "example_dataset": { + "files": [ + "gs://octue-sdk-python-test-bucket/example_output_datasets/example_dataset/output.dat" + ], + "id": "419bff6b-08c3-4c16-9eb1-5d1709168003", + "labels": [], + "name": "divergent-strange-gharial-of-pizza", + "path": "https://storage.googleapis.com/octue-sdk-python-test-bucket/example_output_datasets/example_dataset/.signed_metadata_files/divergent-strange-gharial-of-pizza", + "tags": {} + } + }, + "id": "a13713ae-f207-41c6-9e29-0a848ced6039", + "name": null + }, + "output_values": [1, 2, 3, 4, 5] + }, + }, + ] + ``` + +--- + +## Asking multiple questions in parallel + +You can also ask multiple questions to a service in parallel - just +provide questions as dictionaries of `Child.ask` arguments: + +```python +child.ask_multiple( + {"input_values": {"height": 32, "width": 3}}, + {"input_values": {"height": 12, "width": 10}}, + {"input_values": {"height": 7, "width": 32}}, +) +>>> [ + ({"output_values": {"some": "output"}, "output_manifest": None}, '2681ef4e-4ab7-4cf9-8783-aad982d5e324'), + ({"output_values": {"another": "result"}, "output_manifest": None}, '474923bd-14b6-4f4c-9bfe-8148358f35cd'), + ({"output_values": {"different": "result"}, "output_manifest": None}, '9a50daae-2328-4728-9ddd-b2252474f118'), + ] +``` + +This method uses multithreading, allowing all the questions to be asked +at once instead of one after another. + +!!! hint + + The maximum number of threads that can be used to ask questions in + parallel can be set via the `max_workers` argument. It has no effect on + the total number of questions that can be asked, just how many can be in + progress at once. + +## Asking a question within a service + +If you have +[created your own Twined service](/creating_services) and want to ask children questions, you can do this more +easily than above. Children are accessible from the `analysis` object by +the keys you give them in the +[service configuration](/creating_services/#octueyaml) file. For example, you can ask an `elevation` service a +question like this: + +```python +answer, question_uuid = analysis.children["elevation"].ask( + input_values={"longitude": 0, "latitude": 1} +) +``` + +if these values are in your service configuration file: + +```json +{ + "children": [ + { + "key": "wind_speed", + "id": "template-child-services/wind-speed-service:2.1.1", + "backend": { + "name": "GCPPubSubBackend", + "project_id": "my-project" + } + }, + { + "key": "elevation", + "id": "template-child-services/elevation-service:3.1.9", + "backend": { + "name": "GCPPubSubBackend", + "project_id": "my-project" + } + } + ] +} +``` + +and your `twine.json` file includes the child keys in its `children` +field: + +```json +{ + "children": [ + { + "key": "wind_speed", + "purpose": "A service that returns the average wind speed for a given latitude and longitude." + }, + { + "key": "elevation", + "purpose": "A service that returns the elevation for a given latitude and longitude." + } + ] +} +``` + +See the parent service's [service +configuration](https://github.com/octue/octue-sdk-python/blob/main/octue/twined/templates/template-child-services/parent_service/octue.yaml) +and [app.py +file](https://github.com/octue/octue-sdk-python/blob/main/octue/twined/templates/template-child-services/parent_service/app.py) +in the [child-services app +template](https://github.com/octue/octue-sdk-python/tree/main/octue/twined/templates/template-child-services) +to see this in action. + +## Overriding a child's children + +If the child you're asking a question to has its own children (static +children), you can override these by providing the IDs of the children +you want it to use (dynamic children) to the +`Child.ask` method. Questions that would have gone to the static +children will instead go to the dynamic children. Note that: + +- You must provide the children in the same format as they're provided + in the [service configuration](/creating_services/#octueyaml) +- If you override one static child, you must override others, too +- The dynamic children must have the same keys as the static children + (so the child knows which service to ask which questions) +- You should ensure the dynamic children you provide are compatible with + and appropriate for questions from the child service + +For example, if the child requires these children in its service +configuration: + +```json +[ + { + "key": "wind_speed", + "id": "template-child-services/wind-speed-service:2.1.1", + "backend": { + "name": "GCPPubSubBackend", + "project_id": "octue-sdk-python" + } + }, + { + "key": "elevation", + "id": "template-child-services/elevation-service:3.1.9", + "backend": { + "name": "GCPPubSubBackend", + "project_id": "octue-sdk-python" + } + } +] +``` + +then you can override them like this: + +```python +answer, question_uuid = child.ask( + input_values={"height": 32, "width": 3}, + children=[ + { + "key": "wind_speed", + "id": "my/own-service:1.0.0", + "backend": { + "name": "GCPPubSubBackend", + "project_id": "octue-sdk-python" + }, + }, + { + "key": "elevation", + "id": "organisation/another-service:0.1.0", + "backend": { + "name": "GCPPubSubBackend", + "project_id": "octue-sdk-python" + }, + }, + ], +) +``` + +### Overriding beyond the first generation + +It's an intentional choice to only go one generation deep with +overriding children. If you need to be able to specify a whole tree of +children, grandchildren, and so on, please [upvote this +issue.](https://github.com/octue/octue-sdk-python/issues/528) + +## Using a service registry + +When asking a question, you can optionally specify one or more [service +registries](https://django-twined.readthedocs.io/en/latest/) to resolve +SRUIDs against. This checks if the service revision exists (good for +catching typos in SRUIDs) and raises an error if it doesn't. Service +registries can also get the default revision of a service if you don't +provide a revision tag. Asking a question if without specifying a +registry will bypass these checks. + +### Specifying service registries + +You can specify service registries in two ways: + +1. For all questions asked inside a service. In the service + configuration (`octue.yaml` file): + + > ```yaml + > services: + > - namespace: my-organisation + > name: my-app + > service_registries: + > - name: my-registry + > endpoint: blah.com/services + > ``` + +2. For questions to a specific child, inside or outside a service: + + > ```python + > child = Child( + > id="my-organisation/my-service:1.1.0", + > backend={"name": "GCPPubSubBackend", "project_id": "my-project"}, + > service_registries=[ + > {"name": "my-registry", "endpoint": "blah.com/services"}, + > ] + > ) + > ``` From e11a255c51fa417a328849515b92b5511a1e2f2a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 15 Aug 2025 17:35:28 -0400 Subject: [PATCH 11/69] DOC: Set navigation order --- docs/mkdocs.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 473bb7abe..7163ff11f 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -21,5 +21,28 @@ markdown_extensions: - admonition - pymdownx.details - pymdownx.superfences + plugins: - privacy + +nav: + - index.md + - installation.md + - data_containers.md + - datafile.md + - dataset.md + - manifest.md + - services.md + - asking_questions.md + - creating_services.md + - updating_services.md + - running_services_locally.md + - deploying_services.md + - testing_services.md + - troubleshooting_services.md + - logging.md + - authentication.md + - inter_service_compatibility.md + # - api.md + - license.md + - version_history.md From ad30ca05148cb9eafccd07ee0c3844a50e8db4d2 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 15 Aug 2025 17:37:29 -0400 Subject: [PATCH 12/69] DOC: Add installation page --- docs/docs/installation.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 docs/docs/installation.md diff --git a/docs/docs/installation.md b/docs/docs/installation.md new file mode 100644 index 000000000..4c0632b0a --- /dev/null +++ b/docs/docs/installation.md @@ -0,0 +1,27 @@ +# Installation + +## Pip + +```shell +pip install octue==x.y.z +``` + +## Poetry + +Read more about Poetry [here](https://python-poetry.org). + +```shell +poetry add octue=x.y.z +``` + +## Add to your dependencies + +To use a specific version of the Octue SDK in your python application, +simply add: + +```shell +octue==x.y.z +``` + +to your `requirements.txt` or `setup.py` file, where `x.y.z` is your +preferred version of the SDK (we recommend the latest stable version). From ffd6d00cb45434d9b49483df021aa6f6b19b2887 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 15 Aug 2025 17:40:12 -0400 Subject: [PATCH 13/69] DOC: Add data containers page --- docs/docs/data_containers.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 docs/docs/data_containers.md diff --git a/docs/docs/data_containers.md b/docs/docs/data_containers.md new file mode 100644 index 000000000..7e816b73b --- /dev/null +++ b/docs/docs/data_containers.md @@ -0,0 +1,10 @@ +# Datafiles, datasets, and manifests + +One of the main features of Twined is making using, creating, and +sharing scientific datasets easy. There are three main data classes in +the that do this. + +- **Datafile** - [a single local or cloud file](/datafile) and its metadata. +- **Dataset** - [a set of related datafiles](/dataset) that exist in the same location, plus metadata. +- **Manifest** - [a set of related datasets](/manifest) that exist anywhere, plus metadata. Typically produced by or for + one analysis. From 019f2d15bc3784dbea330d686f19ec78717fa582 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Fri, 15 Aug 2025 17:47:55 -0400 Subject: [PATCH 14/69] DOC: Configure mkdocs navigation skipci --- docs/mkdocs.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 7163ff11f..4b2a7c0ab 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,7 +1,16 @@ site_name: Octue Twined site_url: https://docs.twined.octue.com + theme: name: material + features: + - navigation.top + - navigation.instant + - navigation.instant.progress + - navigation.instant.prefetch + - navigation.tracking + - navigation.path + palette: # Palette toggle for light mode - media: "(prefers-color-scheme: light)" From 38a288c628f6856c665523ee7695d230bd96ad41 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 16:02:00 -0400 Subject: [PATCH 15/69] DEP: Add `mkdocs-glightbox` to provide image zooming to docs --- poetry.lock | 14 +++++++++++++- pyproject.toml | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index ee329a43d..f14730f5b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1354,6 +1354,18 @@ mergedeep = ">=1.3.4" platformdirs = ">=2.2.0" pyyaml = ">=5.1" +[[package]] +name = "mkdocs-glightbox" +version = "0.4.0" +description = "MkDocs plugin supports image lightbox with GLightbox." +optional = false +python-versions = "*" +groups = ["dev"] +files = [ + {file = "mkdocs-glightbox-0.4.0.tar.gz", hash = "sha256:392b34207bf95991071a16d5f8916d1d2f2cd5d5bb59ae2997485ccd778c70d9"}, + {file = "mkdocs_glightbox-0.4.0-py3-none-any.whl", hash = "sha256:e0107beee75d3eb7380ac06ea2d6eac94c999eaa49f8c3cbab0e7be2ac006ccf"}, +] + [[package]] name = "mkdocs-material" version = "9.6.17" @@ -3097,4 +3109,4 @@ hdf5 = ["h5py"] [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "24be32340ed09272c80d6a6e6d033d841fa19f25e390d29fe2c7af7e19a258b3" +content-hash = "1093362b1bfb65a7d8bcd257d7fea3d02d712d5b979e996953912880f44b873e" diff --git a/pyproject.toml b/pyproject.toml index 35d0d4d51..3d0f59d88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ sphinx-tabs = ">=3.4.0,<4" sphinx-toolbox = "^3" ruff = "^0.6.9" mkdocs-material = "^9.6.17" +mkdocs-glightbox = "^0.4.0" [tool.ruff] line-length = 120 From f0e5dadbb1e3c9a4ac988c570792b5e72b95e74e Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 16:02:34 -0400 Subject: [PATCH 16/69] DOC: Improve image support in docs --- docs/mkdocs.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 4b2a7c0ab..f57dccc2b 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -3,6 +3,7 @@ site_url: https://docs.twined.octue.com theme: name: material + features: - navigation.top - navigation.instant @@ -30,9 +31,13 @@ markdown_extensions: - admonition - pymdownx.details - pymdownx.superfences + - attr_list + - md_in_html + - pymdownx.blocks.caption plugins: - privacy + - glightbox nav: - index.md From a0d4816ed731b77e13719558df80c915fc8cda59 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 16:03:43 -0400 Subject: [PATCH 17/69] DOC: Move images into new docs directory skipci --- .../images/213_purple-fruit-snake-transparent.gif | Bin docs/{source => docs}/images/coloured_logs.png | Bin docs/{source => docs}/images/datafile_use_cases.png | Bin .../add_roles_to_service_account.png | Bin .../choose_dockerfile.png | Bin .../choose_repository.png | Bin .../deploying_services_advanced/create_service.png | Bin .../create_service_account.png | Bin .../deploying_services_advanced/create_trigger.png | Bin .../low_carbon_regions.png | Bin .../deploying_services_advanced/save_and_create.png | Bin .../service_name_and_region.png | Bin .../deploying_services_advanced/set_traffic.png | Bin .../set_up_with_cloud_build.png | Bin .../images/digital_twin_component_basic.svg | 0 .../digital_twin_component_for_simulation.svg | 0 .../images/digital_twin_hierarchy.svg | 0 .../images/digital_twin_hierarchy_extended.svg | 0 .../{source => docs}/images/schema_form_example.png | Bin .../images/updating_services/checks.png | Bin .../images/updating_services/deployment.png | Bin .../images/updating_services/diff.png | Bin .../images/updating_services/pytest.png | Bin 23 files changed, 0 insertions(+), 0 deletions(-) rename docs/{source => docs}/images/213_purple-fruit-snake-transparent.gif (100%) rename docs/{source => docs}/images/coloured_logs.png (100%) rename docs/{source => docs}/images/datafile_use_cases.png (100%) rename docs/{source => docs}/images/deploying_services_advanced/add_roles_to_service_account.png (100%) rename docs/{source => docs}/images/deploying_services_advanced/choose_dockerfile.png (100%) rename docs/{source => docs}/images/deploying_services_advanced/choose_repository.png (100%) rename docs/{source => docs}/images/deploying_services_advanced/create_service.png (100%) rename docs/{source => docs}/images/deploying_services_advanced/create_service_account.png (100%) rename docs/{source => docs}/images/deploying_services_advanced/create_trigger.png (100%) rename docs/{source => docs}/images/deploying_services_advanced/low_carbon_regions.png (100%) rename docs/{source => docs}/images/deploying_services_advanced/save_and_create.png (100%) rename docs/{source => docs}/images/deploying_services_advanced/service_name_and_region.png (100%) rename docs/{source => docs}/images/deploying_services_advanced/set_traffic.png (100%) rename docs/{source => docs}/images/deploying_services_advanced/set_up_with_cloud_build.png (100%) rename docs/{source => docs}/images/digital_twin_component_basic.svg (100%) rename docs/{source => docs}/images/digital_twin_component_for_simulation.svg (100%) rename docs/{source => docs}/images/digital_twin_hierarchy.svg (100%) rename docs/{source => docs}/images/digital_twin_hierarchy_extended.svg (100%) rename docs/{source => docs}/images/schema_form_example.png (100%) rename docs/{source => docs}/images/updating_services/checks.png (100%) rename docs/{source => docs}/images/updating_services/deployment.png (100%) rename docs/{source => docs}/images/updating_services/diff.png (100%) rename docs/{source => docs}/images/updating_services/pytest.png (100%) diff --git a/docs/source/images/213_purple-fruit-snake-transparent.gif b/docs/docs/images/213_purple-fruit-snake-transparent.gif similarity index 100% rename from docs/source/images/213_purple-fruit-snake-transparent.gif rename to docs/docs/images/213_purple-fruit-snake-transparent.gif diff --git a/docs/source/images/coloured_logs.png b/docs/docs/images/coloured_logs.png similarity index 100% rename from docs/source/images/coloured_logs.png rename to docs/docs/images/coloured_logs.png diff --git a/docs/source/images/datafile_use_cases.png b/docs/docs/images/datafile_use_cases.png similarity index 100% rename from docs/source/images/datafile_use_cases.png rename to docs/docs/images/datafile_use_cases.png diff --git a/docs/source/images/deploying_services_advanced/add_roles_to_service_account.png b/docs/docs/images/deploying_services_advanced/add_roles_to_service_account.png similarity index 100% rename from docs/source/images/deploying_services_advanced/add_roles_to_service_account.png rename to docs/docs/images/deploying_services_advanced/add_roles_to_service_account.png diff --git a/docs/source/images/deploying_services_advanced/choose_dockerfile.png b/docs/docs/images/deploying_services_advanced/choose_dockerfile.png similarity index 100% rename from docs/source/images/deploying_services_advanced/choose_dockerfile.png rename to docs/docs/images/deploying_services_advanced/choose_dockerfile.png diff --git a/docs/source/images/deploying_services_advanced/choose_repository.png b/docs/docs/images/deploying_services_advanced/choose_repository.png similarity index 100% rename from docs/source/images/deploying_services_advanced/choose_repository.png rename to docs/docs/images/deploying_services_advanced/choose_repository.png diff --git a/docs/source/images/deploying_services_advanced/create_service.png b/docs/docs/images/deploying_services_advanced/create_service.png similarity index 100% rename from docs/source/images/deploying_services_advanced/create_service.png rename to docs/docs/images/deploying_services_advanced/create_service.png diff --git a/docs/source/images/deploying_services_advanced/create_service_account.png b/docs/docs/images/deploying_services_advanced/create_service_account.png similarity index 100% rename from docs/source/images/deploying_services_advanced/create_service_account.png rename to docs/docs/images/deploying_services_advanced/create_service_account.png diff --git a/docs/source/images/deploying_services_advanced/create_trigger.png b/docs/docs/images/deploying_services_advanced/create_trigger.png similarity index 100% rename from docs/source/images/deploying_services_advanced/create_trigger.png rename to docs/docs/images/deploying_services_advanced/create_trigger.png diff --git a/docs/source/images/deploying_services_advanced/low_carbon_regions.png b/docs/docs/images/deploying_services_advanced/low_carbon_regions.png similarity index 100% rename from docs/source/images/deploying_services_advanced/low_carbon_regions.png rename to docs/docs/images/deploying_services_advanced/low_carbon_regions.png diff --git a/docs/source/images/deploying_services_advanced/save_and_create.png b/docs/docs/images/deploying_services_advanced/save_and_create.png similarity index 100% rename from docs/source/images/deploying_services_advanced/save_and_create.png rename to docs/docs/images/deploying_services_advanced/save_and_create.png diff --git a/docs/source/images/deploying_services_advanced/service_name_and_region.png b/docs/docs/images/deploying_services_advanced/service_name_and_region.png similarity index 100% rename from docs/source/images/deploying_services_advanced/service_name_and_region.png rename to docs/docs/images/deploying_services_advanced/service_name_and_region.png diff --git a/docs/source/images/deploying_services_advanced/set_traffic.png b/docs/docs/images/deploying_services_advanced/set_traffic.png similarity index 100% rename from docs/source/images/deploying_services_advanced/set_traffic.png rename to docs/docs/images/deploying_services_advanced/set_traffic.png diff --git a/docs/source/images/deploying_services_advanced/set_up_with_cloud_build.png b/docs/docs/images/deploying_services_advanced/set_up_with_cloud_build.png similarity index 100% rename from docs/source/images/deploying_services_advanced/set_up_with_cloud_build.png rename to docs/docs/images/deploying_services_advanced/set_up_with_cloud_build.png diff --git a/docs/source/images/digital_twin_component_basic.svg b/docs/docs/images/digital_twin_component_basic.svg similarity index 100% rename from docs/source/images/digital_twin_component_basic.svg rename to docs/docs/images/digital_twin_component_basic.svg diff --git a/docs/source/images/digital_twin_component_for_simulation.svg b/docs/docs/images/digital_twin_component_for_simulation.svg similarity index 100% rename from docs/source/images/digital_twin_component_for_simulation.svg rename to docs/docs/images/digital_twin_component_for_simulation.svg diff --git a/docs/source/images/digital_twin_hierarchy.svg b/docs/docs/images/digital_twin_hierarchy.svg similarity index 100% rename from docs/source/images/digital_twin_hierarchy.svg rename to docs/docs/images/digital_twin_hierarchy.svg diff --git a/docs/source/images/digital_twin_hierarchy_extended.svg b/docs/docs/images/digital_twin_hierarchy_extended.svg similarity index 100% rename from docs/source/images/digital_twin_hierarchy_extended.svg rename to docs/docs/images/digital_twin_hierarchy_extended.svg diff --git a/docs/source/images/schema_form_example.png b/docs/docs/images/schema_form_example.png similarity index 100% rename from docs/source/images/schema_form_example.png rename to docs/docs/images/schema_form_example.png diff --git a/docs/source/images/updating_services/checks.png b/docs/docs/images/updating_services/checks.png similarity index 100% rename from docs/source/images/updating_services/checks.png rename to docs/docs/images/updating_services/checks.png diff --git a/docs/source/images/updating_services/deployment.png b/docs/docs/images/updating_services/deployment.png similarity index 100% rename from docs/source/images/updating_services/deployment.png rename to docs/docs/images/updating_services/deployment.png diff --git a/docs/source/images/updating_services/diff.png b/docs/docs/images/updating_services/diff.png similarity index 100% rename from docs/source/images/updating_services/diff.png rename to docs/docs/images/updating_services/diff.png diff --git a/docs/source/images/updating_services/pytest.png b/docs/docs/images/updating_services/pytest.png similarity index 100% rename from docs/source/images/updating_services/pytest.png rename to docs/docs/images/updating_services/pytest.png From b09dfaffdf5548ca0235c649b808c56e68e32603 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 16:14:46 -0400 Subject: [PATCH 18/69] DOC: Add dataset docs --- docs/docs/dataset.md | 230 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 docs/docs/dataset.md diff --git a/docs/docs/dataset.md b/docs/docs/dataset.md new file mode 100644 index 000000000..02442c875 --- /dev/null +++ b/docs/docs/dataset.md @@ -0,0 +1,230 @@ +# Dataset + +!!! info "Definitions" + + **Dataset** + + A set of related datafiles that exist in the same location, dataset metadata, and helper methods. + + **Locality** + + A dataset has one of these localities: + - **Cloud-based:** it exists only in the cloud + - **Local:** it exists only on your local filesystem + +!!! tip + + Use a dataset if you want to: + + - Group together a set of files that naturally relate to each other e.g. a timeseries that's been split into + multiple files. + - Add metadata to it for future sorting and filtering + - Include it in a manifest with other datasets and send them to a Twined service for processing + +## Key features + +### Work with local and cloud datasets + +Working with a dataset is the same whether it's local or cloud-based. + +```python +from octue.resources import Dataset + +# Local dataset +dataset = Dataset(path="path/to/dataset") + +# Dataset in cloud bucket +dataset = Dataset(path="gs://my-bucket/path/to/dataset") +``` + +!!! warning + + Datasets recurse all subdirectories by default unless `recursive=False` + is set. + +### Upload a dataset + +Back up and share your datasets for collaboration. + +```python +dataset.upload("gs://my-bucket/path/to/upload") +``` + +### Download a dataset + +Use a shared or public dataset or retrieve a backup. + +```python +dataset.download("path/to/download") +``` + +### Easy and expandable custom metadata + +Find the needle in the haystack by making your data searchable. You can +set the following metadata on a dataset: + +- Name +- Labels (a set of lowercase strings) +- Tags (a dictionary of key-value pairs) + +This metadata is stored locally in a `.octue` file in the same directory +as the dataset and is used during `Dataset` instantiation. It can be +accessed like this: + +```python +dataset.name +>>> "my-dataset" + +dataset.labels +>>> {"processed"} + +dataset.tags +>>> {"organisation": "octue", "energy": "renewable"} +``` + +You can update the metadata by setting it on the instance while inside +the `Dataset` context manager. + +```python +with dataset: + datafile.labels.add("updated") +``` + +You can do this outside the context manager too, but you then need to +call the update method: + +```python +dataset.labels.add("updated") +dataset.update_metadata() +``` + +### Get dataset and metadata hashes + +Make your analysis reproducible: guarantee a dataset contains exactly +the same data as before by checking its hash. + +```python +dataset.hash_value +>>> 'uvG7TA==' +``` + +!!! note + + A dataset's hash is a function of its datafiles' hashes. Datafile and + dataset metadata do not affect it. + +You can also check that dataset metadata is the same. + +```python +dataset.metadata_hash_value +>>> 'DIgCHg==' +``` + +### Immutable ID + +Each dataset has an immutable UUID: + +```python +dataset.id +>>> '9a1f9b26-6a48-4f2d-be80-468d3270d79c' +``` + +### Check a dataset's locality + +Is this dataset local or in the cloud? + +```python +dataset.exists_locally +>>> True + +dataset.exists_in_cloud +>>> False +``` + +A dataset can only return `True` for one of these at a time. + +### Filter datasets + +Narrow down a dataset to just the files you want to avoiding extra +downloading and processing. + +Datafiles in a dataset are stored in a +`FilterSet`, meaning they can be easily filtered by any attribute of the +datafiles contained e.g. name, extension, ID, timestamp, tags, labels, +size. The filtering syntax is similar to Django's i.e. + +```shell +# Get datafiles that have an attribute that satisfies the filter. +dataset.files.filter(__=) + +# Or, if your filter is a simple equality filter: +dataset.files.filter(=) +``` + +Here's an example: + +```python +# Make a dataset. +dataset = Dataset( + path="blah", + files=[ + Datafile(path="my_file.csv", labels=["one", "a", "b" "all"]), + Datafile(path="your_file.txt", labels=["two", "a", "b", "all"), + Datafile(path="another_file.csv", labels=["three", "all"]), + ] +) + +# Filter it! +dataset.files.filter(name__starts_with="my") +>>> })> + +dataset.files.filter(extension="csv") +>>> , })> + +dataset.files.filter(labels__contains="a") +>>> , })> +``` + +You can iterate through the filtered files: + +```python +for datafile in dataset.files.filter(labels__contains="a"): + print(datafile.name) +>>> 'my_file.csv' + 'your_file.txt' +``` + +If there's just one result, get it via the `FilterSet.one` method: + +```python +dataset.files.filter(name__starts_with="my").one() +>>> +``` + +You can also chain filters or specify them all at the same time - these +two examples produce the same result: + +```python +# Chaining multiple filters. +dataset.files.filter(extension="csv").filter(labels__contains="a") +>>> })> + +# Specifying multiple filters at once. +dataset.files.filter(extension="csv", labels__contains="a") +>>> })> +``` + +For the full list of available filters, [click here](/available_filters). + +### Order datasets + +A dataset can also be ordered by any of the attributes of its datafiles: + +```python +dataset.files.order_by("name") +>>> , , ])> +``` + +The ordering can also be carried out in reverse (i.e. descending order) +by passing `reverse=True` as a second argument to the +`FilterSet.order_by` method. From ce31cb74a4b11096688fe704f6aa6e9434e3f4e5 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 16:17:28 -0400 Subject: [PATCH 19/69] DOC: Add datafile docs --- docs/docs/datafile.md | 312 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 312 insertions(+) create mode 100644 docs/docs/datafile.md diff --git a/docs/docs/datafile.md b/docs/docs/datafile.md new file mode 100644 index 000000000..c605656cb --- /dev/null +++ b/docs/docs/datafile.md @@ -0,0 +1,312 @@ +# Datafile + +!!! info "Definitions" + + **Datafile** + + A single local or cloud file, its metadata, and helper methods. + + **Locality** + + A datafile has one of these localities: + + - **Cloud-based:** it exists only in the cloud + - **Local:** it exists only on your local filesystem + - **Cloud-based and local:** it's cloud-based but has been + downloaded for low-latency reading/writing + +!!! tip + + Use a datafile to work with a file if you want to: + + - Read/write to local and cloud files in the same way + - Include it in a `dataset `{.interpreted-text role="doc"} that + can be sent to a Twined service for processing + - Add metadata to it for future sorting and filtering + +## Key features + +### Work with local and cloud data + +Working with a datafile is the same whether it's local or cloud-based. +It's also almost identical to using [python built-in open +function](https://docs.python.org/3/library/functions.html#open). For +example, to write to a datafile: + +```python +from octue.resources import Datafile + +datafile = Datafile("path/to/file.dat") + +# Or: + +datafile = Datafile("gs://my-bucket/path/to/file.dat") + +with datafile.open("w") as f: + f.write("Some data") + datafile.labels.add("processed") +``` + +All the same file modes you'd use with the [python built-in open +context manager](https://docs.python.org/3/library/functions.html#open) +are available for datafiles e.g. `"r"` and `"a"`. + +### Automatic lazy downloading + +Save time and bandwidth by only downloading when necessary. + +Downloading data from cloud datafiles is automatic and lazy so you get +both low-latency content read/write and quick metadata reads. This makes +viewing and filtering by the metadata of cloud datasets and datafiles +quick and avoids unnecessary data transfer, energy usage, and costs. + +Datafile content isn't downloaded until you: + +- Try to read or write its contents using the `Datafile.open` context manager +- Call its `download` method +- Use its `local_path` property + +Read more about downloading files [here](/downloading_datafiles). + +### CLI command friendly + +Use any command line tool on your datafiles. Datafiles are python +objects, but they represent real files that can be fed to any CLI +command you like + +```python +import subprocess +output = subprocess.check_output(["openfast", datafile.local_path]) +``` + +### Easy and expandable custom metadata + +Find the needle in the haystack by making your data searchable. You can +set the following metadata on a datafile: + +- Timestamp +- Labels (a set of lowercase strings) +- Tags (a dictionary of key-value pairs) + +This metadata is stored locally in a `.octue` file for local datafiles +or on the cloud objects for cloud datafiles and is used during +`Datafile` instantiation. It can be accessed like this: + +```python +datafile.timestamp +>>> datetime.datetime(2022, 5, 4, 17, 57, 57, 136739) + +datafile.labels +>>> {"processed"} + +datafile.tags +>>> {"organisation": "octue", "energy": "renewable"} +``` + +You can update the metadata by setting it on the instance while inside +the `Datafile.open` context manager. + +```python +with datafile.open("a"): + datafile.labels.add("updated") +``` + +You can do this outside the context manager too, but you then need to +call the update method: + +```python +datafile.labels.add("updated") +datafile.update_metadata() +``` + +### Upload an existing local datafile + +Back up and share your datafiles for collaboration. You can upload an +existing local datafile to the cloud without using the +`Datafile.open` context manager if you don't need to modify its contents: + +```python +datafile.upload("gs://my-bucket/my_datafile.dat", update_metadata=True) +``` + +### Get file and metadata hashes + +Make your analysis reproducible: guarantee a datafile contains exactly +the same data as before by checking its hash. + +```python +datafile.hash_value +>>> 'mnG7TA==' +``` + +You can also check that any metadata is the same. + +```python +datafile.metadata_hash_value +>>> 'DIgCHg==' +``` + +### Immutable ID + +Each datafile has an immutable UUID: + +```python +datafile.id +>>> '9a1f9b26-6a48-4f2d-be80-468d3270d79b' +``` + +### Check a datafile's locality + +Is this datafile local or in the cloud? + +```python +datafile.exists_locally +>>> True + +datafile.exists_in_cloud +>>> False +``` + +A cloud datafile that has been downloaded will return `True` for both of +these properties. + +### Represent HDF5 files + +Support fast I/O processing and storage. + +!!! warning + + If you want to represent HDF5 files with a `Datafile`, you must include + the extra requirements provided by the `hdf5` key at installation i.e. + + ``` shell + pip install octue[hdf5] + ``` + + or + + ``` shell + poetry add octue -E hdf5 + ``` + +## Usage examples + +The `Datafile` class can be used functionally or as a context manager. +When used as a context manager, it is analogous with the [python +built-in open +function](https://docs.python.org/3/library/functions.html#open). On +exiting the context (the `with` block), it closes the datafile locally +and, if the datafile also exists in the cloud, updates the cloud object +with any data or metadata changes. + +![image](images/datafile_use_cases.png) + +### Example A + +**Scenario:** Download a cloud object, calculate Octue metadata from its +contents, and add the new metadata to the cloud object + +**Starting point:** Object in cloud with or without Octue metadata + +**Goal:** Object in cloud with updated metadata + +```python +from octue.resources import Datafile + + +datafile = Datafile("gs://my-bucket/path/to/data.csv") + +with datafile.open() as f: + data = f.read() + new_metadata = metadata_calculating_function(data) + + datafile.timestamp = new_metadata["timestamp"] + datafile.tags = new_metadata["tags"] + datafile.labels = new_metadata["labels"] +``` + +### Example B + +**Scenario:** Add or update Octue metadata on an existing cloud object +_without downloading its content_ + +**Starting point:** A cloud object with or without Octue metadata + +**Goal:** Object in cloud with updated metadata + +```python +from datetime import datetime +from octue.resources import Datafile + + +datafile = Datafile("gs://my-bucket/path/to/data.csv") + +datafile.timestamp = datetime.now() +datafile.tags = {"manufacturer": "Vestas", "output": "1MW"} +datafile.labels = {"new"} + +datafile.upload(update_metadata=True) # Or, datafile.update_metadata() +``` + +### Example C + +**Scenario:** Read in the data and Octue metadata of an existing cloud +object without intent to update it in the cloud + +**Starting point:** A cloud object with Octue metadata + +**Goal:** Cloud object data (contents) and metadata held locally in +local variables + +```python +from octue.resources import Datafile + + +datafile = Datafile("gs://my-bucket/path/to/data.csv") + +with datafile.open() as f: + data = f.read() + +metadata = datafile.metadata() +``` + +### Example D + +**Scenario:** Create a new cloud object from local data, adding Octue +metadata + +**Starting point:** A file-like locally (or content data in local +variable) with Octue metadata stored in local variables + +**Goal:** A new object in the cloud with data and Octue metadata + +For creating new data in a new local file: + +```python +from octue.resources import Datafile + + +datafile = Datafile( + "path/to/local/file.dat", + tags={"cleaned": True, "type": "linear"}, + labels={"Vestas"} +) + +with datafile.open("w") as f: + f.write("This is some cleaned data.") + +datafile.upload("gs://my-bucket/path/to/data.dat") +``` + +For existing data in an existing local file: + +```python +from octue.resources import Datafile + + +tags = {"cleaned": True, "type": "linear"} +labels = {"Vestas"} + +datafile = Datafile(path="path/to/local/file.dat", tags=tags, labels=labels) +datafile.upload("gs://my-bucket/path/to/data.dat") +``` From 46ed231407a9c553144a5b522155223773cfee9c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 16:17:41 -0400 Subject: [PATCH 20/69] DOC: Adjust definition format in docs intro page --- docs/docs/index.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/docs/index.md b/docs/docs/index.md index 4ae9d741d..e10ad91e5 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -3,7 +3,9 @@ The python SDK for [Octue](https://octue.com) Twined scientific data services and digital twins - get faster data groundwork so you have more time for the science! -!!! info "Definition: Twined service" +!!! info "Definition" + + **Twined service** A data service or digital twin built with the Twined framework that can be asked questions, process them, and return answers. Twined services can communicate with each other with minimal extra setup. From 22b4496e98694f4b9e1f8be0f3ce48302225367d Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 16:22:05 -0400 Subject: [PATCH 21/69] DOC: Add manifest doc --- docs/docs/manifest.md | 114 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 docs/docs/manifest.md diff --git a/docs/docs/manifest.md b/docs/docs/manifest.md new file mode 100644 index 000000000..3951bea1b --- /dev/null +++ b/docs/docs/manifest.md @@ -0,0 +1,114 @@ +# Manifest + +!!! info "Definition" + + **Manifest** + + A set of related cloud and/or local [datasets](/dataset), metadata, and helper methods. Typically produced by or + needed for processing by a Twined service. + +!!! tip + + Use a manifest to send [datasets](/dataset) to a Twined service as a question (for processing) - the + service will send an output manifest back with its answer if the answer includes output datasets. + +## Key features + +### Group related datasets together + +Make a clear grouping of datasets needed for a particular analysis. + +```python +from octue.resources import Manifest + +manifest = Manifest( + datasets={ + "my_dataset_0": "gs://my-bucket/my_dataset_0", + "my_dataset_1": "gs://my-bucket/my_dataset_1", + "my_dataset_2": "gs://another-bucket/my_dataset_2", + } +) +``` + +### Send datasets to a service + +Get a Twined service to analyse data for you as part of a larger +analysis. + +```python +from octue.twined.resources import Child + +child = Child( + id="octue/wind-speed:2.1.0", + backend={"name": "GCPPubSubBackend", "project_id": "my-project"}, +) + +answer, question_uuid = child.ask(input_manifest=manifest) +``` + +See [here](asking_questions.md) for more information. + +### Receive datasets from a service + +Access output datasets from a Twined service from the cloud when you're +ready. + +```python +manifest = answer["output_manifest"] +manifest["an_output_dataset"].files +>>> , })> +``` + +!!! hint + + Datasets in an output manifest are stored in the cloud. You'll need to + keep a reference to where they are to access them - the output manifest + is this reference. You'll need to use it straight away or save it to + make use of it. + +### Download all datasets from a manifest + +Download all or a subset of datasets from a manifest. + +```python +manifest.download() +>>> { + "my_dataset": "/path/to/dataset" +} +``` + +!!! note + + Datasets are downloaded to a temporary directory if no paths are given. + +## Further information + +### Manifests of local datasets + +You can include local datasets in your manifest if you can guarantee all +services that need them can access them. A use case for this is, for +example, a supercomputer cluster running several Twined services +locally that process and transfer large amounts of data. It is much +faster to store and access the required datasets locally than upload +them to the cloud and then download them again for each service (as +would happen with cloud datasets). + +!!! warning + + If you want to ask a child a question that includes a manifest containing one or more local datasets, you must + include the `allow_local_files` parameter. For example, if you have an analysis object with a child called "wind_speed": + + ``` python + input_manifest = Manifest( + datasets={ + "my_dataset_0": "gs://my-bucket/my_dataset_0", + "my_dataset_1": "local/path/to/my_dataset_1", + } + ) + + answer, question_uuid = analysis.children["wind_speed"].ask( + input_values=analysis.input_values, + input_manifest=analysis.input_manifest, + allow_local_files=True, + ) + ``` From 6a854311a9258a52032afe2fdf8b9566df407b6b Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 16:51:09 -0400 Subject: [PATCH 22/69] DOC: Add services doc skipci --- docs/docs/services.md | 125 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 docs/docs/services.md diff --git a/docs/docs/services.md b/docs/docs/services.md new file mode 100644 index 000000000..26ccf77d6 --- /dev/null +++ b/docs/docs/services.md @@ -0,0 +1,125 @@ +# Twined services + +There's a growing range of live [services](/) in the +Twined ecosystem that you can query, mostly related to wind energy and +other renewables. Here's a quick glossary of terms before we tell you +more: + +!!! info "Definitions" + + **Twined service** + + See [here](/). + + **Child** + + A Twined service that can be asked a question. This name reflects + the tree structure of services (specifically, [a + DAG](https://en.wikipedia.org/wiki/Directed_acyclic_graph)) formed + by the service asking the question (the parent), the child it asks + the question to, any children that the child asks questions to as + part of forming its answer, and so on. + + **Parent** + + A Twined service that asks a question to another Twined service (a + child). + + **Asking a question** + + Sending data (input values and/or an input manifest) to a child for + processing/analysis. + + **Receiving an answer** + + Receiving data (output values and/or an output manifest) from a + child you asked a question to. + + **Twined ecosystem** + + The set of services running the Octue SDK as their backend. These + services guarantee: + + - Defined input/output JSON schemas and validation + - An easy and consistent interface for asking them questions and + receiving their answers + - Logs, exceptions, and monitor messages forwarded to you + - High availability (if deployed in the cloud) + +## Service names + +Questions are always asked to a _revision_ of a service. Services +revisions are named in a similar way to docker images. They look like +`namespace/name:tag` where the tag is often a semantic version (but +doesn't have to be). + +!!! info "Definitions" + + **Service revision** + + A specific instance of a Twined service that can be individually + addressed. The revision could correspond to a version of the + service, a dynamic development branch for it, or a deliberate + duplication or variation of it. + + **Service revision unique identifier (SRUID)** + + The combination of a service revision's namespace, name, and + revision tag that uniquely identifies it. For example, + `octue/my-service:1.3.0` where the namespace is `octue`, the name is + `my-service`, and the revision tag is `1.3.0`. + + **Service namespace** + + The group to which the service belongs e.g. your name or your + organisation's name. If in doubt, use the GitHub handle of the user + or organisation publishing the services. + + Namespaces must be lower kebab case (i.e. they may contain the + letters \[a-z\], numbers \[0-9\], and hyphens \[-\]). They may not + begin or end with hyphens. + + **Service name** + + A name to uniquely identify the service within its namespace. This + usually corresponds to the name of the GitHub repository for the + service. Names must be lower kebab case (i.e. they may contain the + letters \[a-z\], numbers \[0-9\] and hyphens \[-\]). They may not + begin or end with hyphens. + + **Service revision tag** + + A tag that uniquely identifies a particular revision of a service. + The revision tag could be a: + + - Commit hash (e.g. `a3eb45`) + - Semantic version (e.g. `0.12.4`) + - Branch name (e.g. `development`) + - Particular environment the service is deployed in (e.g. + `production`) + - Combination of these (e.g. `0.12.4-production`) + + Tags may contain lowercase and uppercase letters, numbers, + underscores, periods, and hyphens, but can't start with a period or + a dash. They can contain a maximum of 128 characters. These + requirements are the same as the [Docker tag + format](https://docs.docker.com/engine/reference/commandline/tag/). + + **Service ID** + + The SRUID is a special case of a service ID. A service ID can be an + SRUID or just the service namespace and name. It can be used to ask + a question to a service without specifying a specific revision of + it. This enables asking questions to, for example, the service + `octue/my-service` and automatically having them routed to its + default (usually latest) revision. + [See here for more info](/asking_questions/#asking-a-question) + +## Service communication standard + +Twined services communicate according to the service communication +standard. The JSON schema defining this can be found +[here](https://strands.octue.com/octue/service-communication). Messages +received by services are validated against it and invalid messages are +rejected. The schema is in beta, so (rare) breaking changes are +reflected in the minor version number. From dc26bd7931d3b43c66dbe1c62b47ecd2b12643d7 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 17:03:14 -0400 Subject: [PATCH 23/69] DOC: Update CLI printout in readme --- README.md | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index cbd2e8ead..19af0d04b 100644 --- a/README.md +++ b/README.md @@ -36,30 +36,20 @@ octue --help ```text Usage: octue [OPTIONS] COMMAND [ARGS]... - The CLI for the Octue SDK. Use it to start an Octue data service or digital - twin locally or run an analysis on one locally. + The CLI for Octue SDKs and APIs, most notably Twined. Read more in the docs: https://octue-python-sdk.readthedocs.io/en/latest/ Options: - --id UUID UUID of the analysis being undertaken. None - (for local use) will cause a unique ID to be - generated. --logger-uri TEXT Stream logs to a websocket at the given URI. --log-level [debug|info|warning|error] Log level used for the analysis. [default: info] - --force-reset / --no-force-reset - Forces a reset of analysis cache and outputs - [For future use, currently not implemented] - [default: force-reset] --version Show the version and exit. -h, --help Show this message and exit. Commands: - deploy Deploy a python app to the cloud as an Octue service or digital... - run Run an analysis on the given input data using an Octue service... - start Start an Octue service or digital twin locally as a child so it... + twined The Twined CLI. ``` ## Deprecated code From 6f69f0517a2154550476163a6cb51b4a63bc046b Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 17:03:49 -0400 Subject: [PATCH 24/69] DOC: Replace "Octue SDK" with "Twined" in docs --- docs/docs/asking_questions.md | 2 +- docs/docs/installation.md | 2 +- docs/docs/services.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/docs/asking_questions.md b/docs/docs/asking_questions.md index 4d7a85585..33589c198 100644 --- a/docs/docs/asking_questions.md +++ b/docs/docs/asking_questions.md @@ -13,7 +13,7 @@ to a child for processing/analysis. Questions can be: - **Regular:** Responses to these questions are automatically stored in an event store where they can be - [retrieved using the Octue SDK](#retrieving-answers-to-asynchronous-questions) + [retrieved using the Twined CLI](#retrieving-answers-to-asynchronous-questions) - **Push endpoint:** Responses to these questions are pushed to an HTTP endpoint for asynchronous handling using Octue's diff --git a/docs/docs/installation.md b/docs/docs/installation.md index 4c0632b0a..0fef71cb0 100644 --- a/docs/docs/installation.md +++ b/docs/docs/installation.md @@ -16,7 +16,7 @@ poetry add octue=x.y.z ## Add to your dependencies -To use a specific version of the Octue SDK in your python application, +To use a specific version of Twined in your python application, simply add: ```shell diff --git a/docs/docs/services.md b/docs/docs/services.md index 26ccf77d6..02eb209d9 100644 --- a/docs/docs/services.md +++ b/docs/docs/services.md @@ -37,7 +37,7 @@ more: **Twined ecosystem** - The set of services running the Octue SDK as their backend. These + The set of services running Twined as their backend. These services guarantee: - Defined input/output JSON schemas and validation From b6744a1bb15e920dff2864c2d65c4b50d413ba98 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 18:34:28 -0400 Subject: [PATCH 25/69] DOC: Add creating services doc --- docs/docs/creating_services.md | 156 +++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 docs/docs/creating_services.md diff --git a/docs/docs/creating_services.md b/docs/docs/creating_services.md new file mode 100644 index 000000000..4d74e3751 --- /dev/null +++ b/docs/docs/creating_services.md @@ -0,0 +1,156 @@ +# Creating services + +One of the main features of Twined is to allow you to easily +create services that can accept questions and return answers. They can +run locally on any machine or be deployed to the cloud. Currently: + +- The backend communication between twins uses Google Pub/Sub whether + they're local or deployed +- Services are deployed to a Kubernetes/Kueue cluster +- The language of the entrypoint must by `python3` (you can call + processes using other languages within this though) + +## Anatomy of a Twined service + +A Twined service is defined by the following files (located in the +repository root by default). + +### app.py + +This is the entrypoint into your code - read more [here](/creating_apps). + +### twine.json + +This file defines the schema for the service's configuration, input, +and output data. Read more +[here](https://twined.readthedocs.io/en/latest/) and see an example +[here](https://twined.readthedocs.io/en/latest/quick_start_create_your_first_twine.html). + +### Dependencies file + +A file specifying your app's dependencies. This is a [setup.py +file](https://docs.python.org/3/distutils/setupscript.html), a +[requirements.txt +file](https://learnpython.com/blog/python-requirements-file/), or a +[pyproject.toml file](https://python-poetry.org/docs/pyproject/) listing +all the python packages your app depends on and the version ranges that +are supported. + +### octue.yaml + +??? example "This describes the service configuration - read more..." + + This file defines the basic structure of your service. It must contain + at least: + + ``` yaml + services: + - namespace: my-organisation + name: my-app + ``` + It may also need the following key-value pairs: + + - `app_source_path: ` - if your `app.py` file is not in the repository root + + All paths should be relative to the repository root. Other valid + entries can be found in the `ServiceConfiguration` constructor. + + !!! warning + + Currently, only one service can be defined per repository, but it must + still appear as a list item of the "services" key. At some point, it + will be possible to define multiple services in one repository. + + If a service's app needs any configuration, asks questions to any + other Twined services, or produces output datafiles/datasets, you will + need to provide some or all of the following values for that service: + + - `configuration_values` + - `configuration_manifest` + - `children` + - `output_location` + - `use_signed_urls_for_output_datasets` + +### Dockerfile (optional) + +??? example "Provide this if your needs exceed the default Octue Dockerfile - read more..." + + Twined services run in a Docker container if they are deployed. They + can also run this way locally. The SDK provides a default `Dockerfile` + for these purposes that will work for most cases: + + - For deploying to [Kubernetes](https://github.com/octue/octue-sdk-python/blob/main/octue/twined/cloud/deployment/dockerfiles/Dockerfile-python313) + + However, you may need to write and provide your own `Dockerfile` if + your app requires: + + - Non-python or system dependencies (e.g. `openfast`, `wget`) + - Python dependencies that aren't installable via `pip` + - Private python packages + + Here are two examples of a custom `Dockerfile` that use different base + images: + + - [A TurbSim service](https://github.com/octue/turbsim-service/blob/main/Dockerfile) + - [An OpenFAST service](https://github.com/octue/openfast-service/blob/main/Dockerfile) + + If you do provide one, you must provide its path relative to your + repository to the [build-twined-services] GitHub Actions [workflow](https://github.com/octue/workflows/blob/main/.github/workflows/build-twined-service.yml). + + As always, if you need help with this, feel free to drop us a message or raise an issue! + + +### Where to specify the namespace, name, and revision tag + +See [here](services.md/#service-names) for service naming requirements. + +**Namespace** + +- Required: yes +- Set in: + - `octue.yaml` + - `OCTUE_SERVICE_NAMESPACE` environment variable (takes priority) + +**Name** + +- Required: yes +- Set in: + - `octue.yaml` + - `OCTUE_SERVICE_NAME` environment variable (takes priority) + +**Revision tag** + +- Required: no +- Default: a random "coolname" (e.g. `hungry-hippo`) +- Set in: + - `OCTUE_SERVICE_REVISION_TAG` environment variable + - If using `octue twined service start` command, the `--revision-tag` option (takes priority) + +## Template apps + +We've created some template apps for you to look at and play around +with. We recommend going through them in this order: + +1. The [fractal app template](https://github.com/octue/octue-sdk-python/tree/main/octue/twined/templates/template-fractal) - introduces a basic Twined service that returns output values to its + parent. +2. The [using-manifests app template](https://github.com/octue/octue-sdk-python/tree/main/octue/twined/templates/template-using-manifests) - introduces using a manifest of output datasets to return output + files to its parent. +3. The [child-services app template](https://github.com/octue/octue-sdk-python/tree/main/octue/twined/templates/template-child-services) - introduces asking questions to child services and using their + answers to form an output to return to its parent. + +## Deploying services automatically + +Automated deployment with Octue means: + +- Your service runs in Google Kubernetes Engine (GKE), ready to accept + questions from and return answers to other services. +- You don't need to do anything to update your deployed service with + new code changes - the service simply gets rebuilt and re-deployed + each time you push a commit to your `main` branch, or merge a pull + request into it (other branches and deployment strategies are + available, but this is the default). +- Serverless is the default - your service only runs when questions from + other services are sent to it, meaning there are minimal costs to + having it deployed but not in use. + +If you'd like help deploying services, contact us. To do it yourself, see [here](/deploying_services). From 5551838547c0a08cf29a0f35fd1ba438d3a20008 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 18:37:48 -0400 Subject: [PATCH 26/69] DOC: Add updating services doc --- docs/docs/updating_services.md | 109 +++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 docs/docs/updating_services.md diff --git a/docs/docs/updating_services.md b/docs/docs/updating_services.md new file mode 100644 index 000000000..f6ca83a63 --- /dev/null +++ b/docs/docs/updating_services.md @@ -0,0 +1,109 @@ +# Updating a Twined service + +This page describes how to update an existing, deployed Twined service - +in other words, how to deploy a new Twined service revision. + +We assume that: + +- Your service's repository is on GitHub and you have push access to it +- The [standard Twined service deployment GitHub Actions + workflow](https://github.com/octue/workflows/blob/main/.github/workflows/build-twined-service.yml) + is set up in the repository and being used to build and push the + service image to the artifact registry on merge of a pull request into + the `main` branch (see an example + [here](https://github.com/octue/example-service-kueue/blob/main/.github/workflows/release.yml)) +- A release workflow is set up that will tag and release the new service + revision on GitHub (see an example + [here](https://github.com/octue/example-service-kueue/blob/main/.github/workflows/release.yml)) + +## Instructions + +1. Check out and pull the `main` branch to make sure you're up to date + with the latest changes + + ```shell + git checkout main + git pull + ``` + +2. Install your service locally so you can run the tests and your + development environment can lint the code etc.: + + ```shell + poetry install + ``` + +3. Set up [pre-commit](https://pre-commit.com/) to enforce code + quality: + + ```shell + pre-commit install && pre-commit install -t commit-msg + ``` + +4. Check out a new branch so you can work independently of any other + work on the code happening at the same time + + ```shell + git checkout -b my-new-feature + ``` + +5. Add and make changes to your app's code as needed, committing each + self-contained change. Use the [Conventional + Commits](https://www.conventionalcommits.org/en/v1.0.0/) commit + message format so the new version for your service can be + automatically calculated. + + ```shell + git add a-new-file another-new-file + git commit -m "Your commit message" + ...repeat... + ``` + + Push your commits frequently so your work is backed up on GitHub + + ```shell + git push + ``` + +6. Write any new tests you need to verify your code works and update + any old tests as needed + +7. Run the tests locally using `pytest` and fix anything that makes + them fail + + ![image](images/updating_services/pytest.png) + +8. Update the [semantic version](https://semver.org/) of your app. This + communicates to anyone updating from a previous version of the + service whether they can use it as before or if there might be + changes they need to make to their own code or data first. + + - `poetry version patch` for a bug fix or small non-code change + - `poetry version minor` for a new feature + - `poetry version major` for a breaking change + + Don't forget to commit this change, too. + +9. When you're ready to review the changes, head to GitHub and open a + pull request of your branch into `main`. This makes it easy for you + and anyone else to see what's changed. Check the "Files Changed" + tab to make sure everything's there and consistent (it's easy to + forget to push a commit). Ask your colleagues to review the code if + required. + + ![image](images/updating_services/diff.png) + +10. When you're ready to release the new version of your service, check + that the GitHub checks have passed. These ensure code quality, that + the tests pass, and that the new version number is correct. + +> ![image](images/updating_services/checks.png) + +11. Merge the pull request into `main`. This will run the deployment + workflow (usually called `cd` - continuous deployment), making the + new version of the service available to everyone. +12. Check that the deployment workflow has run successfully (this can + take a few minutes). You can check the progress in the "Actions" + tab of the GitHub repository + +> ![image](images/updating_services/deployment.png) From 168466db508ed9c1b6032013ee50d7c689a380be Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 18:39:39 -0400 Subject: [PATCH 27/69] DOC: Add local service running docs --- docs/docs/running_services_locally.md | 72 +++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 docs/docs/running_services_locally.md diff --git a/docs/docs/running_services_locally.md b/docs/docs/running_services_locally.md new file mode 100644 index 000000000..e547b9ec3 --- /dev/null +++ b/docs/docs/running_services_locally.md @@ -0,0 +1,72 @@ +# Running services locally + +Services can be operated locally (e.g. for testing or ad-hoc data +processing). You can: + +- Run your service once (i.e. run one analysis): + - Via the `octue twined` CLI + - By using the `octue` library in a python script +- Start your service as a child, allowing it to answer any number of + questions from any other Twined service: + - Via the CLI + +## Running a service once + +### Via the CLI + +1. Ensure you've created a valid + [`octue.yaml`](/creating_services/#octueyaml) + file for your service + +2. Run: + + > ```shell + > octue twined question ask local --input-values='{"some": "input"}' + > ``` + +The output values and/or manifest will be printed to `stdout` but are +also stored in the event store. + +### Via a python script + +Imagine we have a simple app that calculates the area of a square. It +could be run locally on a given height and width like this: + +```python +from octue.twined.runner import Runner + +runner = Runner(app_src="path/to/app.py", twine="path/to/twine.json") +analysis = runner.run(input_values={"height": 5, "width": 10}) + +analysis.output_values +>>> {"area": 50} + +analysis.output_manifest +>>> None +``` + +See the `Runner` API documentation for more advanced usage including providing configuration, children, and an input manifest. + +## Starting a service as a child + +### Via the CLI + +1. Ensure you've created a valid + [`octue.yaml`](/creating_services/#octueyaml) + file for your service + +2. Run: + + > ```shell + > octue twined service start + > ``` + +This will run the service as a child waiting for questions until you +press `Ctrl + C` or an error is encountered. The service will be +available to be questioned by other services at the service ID +`organisation/name` as specified in the `octue.yaml` file. + +!!! tip + + You can use the `--timeout` option to stop the service after a given + number of seconds. From 244ab1ba16220a4ab1572eb8cb206a9defdcae91 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 18:41:08 -0400 Subject: [PATCH 28/69] DOC: Add deploying services doc --- docs/docs/deploying_services.md | 79 +++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 docs/docs/deploying_services.md diff --git a/docs/docs/deploying_services.md b/docs/docs/deploying_services.md new file mode 100644 index 000000000..826d29f69 --- /dev/null +++ b/docs/docs/deploying_services.md @@ -0,0 +1,79 @@ +# Deploying services (developer's guide) {#deploying_services_advanced} + +This is a guide for developers that want to deploy Twined services +themselves - it is not needed if Octue manages your services for you or +if you are only asking questions to existing Twined services. + +## What is deployment? + +Deploying a Twined service means the service: + +- Is a docker image that is spun up and down in a Kubernetes cluster on + demand +- Is ready at any time to answer questions from users and other Twined + services in the service network +- Can ask questions to any other Twined service in the service network +- Will automatically spin down after it has finished answering a + question +- Will automatically build and redeploy after a relevant code change + (e.g. on push or merge into `main`) + +We can split deployment into service deployment and infrastructure +deployment. + +## Deploying a service + +Assuming the service network infrastructure already exists, a service +can be deployed by building and pushing its docker image to the service +network's Artifact Registry repository. We recommend pushing a new +image for each release of the code e.g. on merge into the `main` branch. +Each new image is the deployment of a new service revision. This can be +done automatically: + +- Follow the + [instructions](https://github.com/octue/workflows#deploying-a-kuberneteskueue-octue-twined-service-revision) + to add the + [build-twined-service](https://github.com/octue/workflows/blob/main/.github/workflows/build-twined-service.yml) + GitHub Actions workflow to your service's GitHub repository. Set its + trigger to merge or push to `main` (see example below) +- This needs to be done **once for every service** you want to deploy +- A live example can be [found + here](https://github.com/octue/example-service-kueue/blob/main/.github/workflows/release.yml) + including automated pre-deployment testing and creation of a GitHub + release + +You can now [ask your service some questions](asking_questions.md)! It will be available in the service network as +`/:` (e.g. `octue/example-service-kueue:0.1.1`). + +## Deploying the infrastructure + +### Prerequisites + +Twined services are currently deployable to Google Cloud Platform (GCP). +You must have "owner" level access to the GCP project you're +deploying to and billing must be set up for it. + +### Deploying step-by-step + +There are two steps to deploying the infrastructure: + +1. Deploy the core infrastructure (storage bucket, event store, IAM + service accounts and roles) +2. Deploy the Kubernetes cluster, event handler, service registry, and + Pub/Sub topic + +#### 1. Deploy core infrastructure + +- Follow [the + instructions](https://github.com/octue/terraform-octue-twined-core) to + deploy the resources in the `terraform-octue-twined-core` Terraform + module +- This only needs to be done once per service network + +#### 2. Deploy Kubernetes cluster + +- Follow the + [instructions](https://github.com/octue/terraform-octue-twined-cluster) + to deploy the resources in the `terraform-octue-twined-cluster` + Terraform module +- This only needs to be done once per service network From f3c0d8c6c7e4ea33cd2aadeec558eb476ff83d90 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 18:54:09 -0400 Subject: [PATCH 29/69] DOC: Add testing services doc --- docs/docs/testing_services.md | 290 ++++++++++++++++++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 docs/docs/testing_services.md diff --git a/docs/docs/testing_services.md b/docs/docs/testing_services.md new file mode 100644 index 000000000..32fe14408 --- /dev/null +++ b/docs/docs/testing_services.md @@ -0,0 +1,290 @@ +# Testing services {#testing_services} + +We recommend writing automated tests for your service so anyone who +wants to use it can have confidence in its quality and reliability at a +glance. [Here's an example +test](https://github.com/octue/example-service-kueue/blob/main/tests/test_app.py) +for our example service. + +## Emulating children + +If your app has children, you should emulate them in your tests instead +of communicating with the real ones. This makes your tests: + +- **Independent** of anything external to your app code - i.e. + independent of the remote child, your internet connection, and + communication between your app and the child (Google Pub/Sub). +- **Much faster** - the emulation will complete in a few milliseconds as + opposed to the time it takes the real child to actually run an + analysis, which could be minutes, hours, or days. Tests for our [child + services template + app](https://github.com/octue/octue-sdk-python/tree/main/octue/twined/templates/template-child-services) + run around **900 times faster** when the children are emulated. + +### The Child Emulator + +We've written a child emulator that takes a list of events and returns +them to the parent for handling in the order given - without contacting +the real child or using Pub/Sub. Any events a real child can produce are +supported. `Child` instances can be mocked like-for-like by +`ChildEmulator` instances without the parent knowing. + +### Event kinds + +You can emulate any event that your app (the parent) can handle. The +table below shows what these are. + +| Event kind | Number of events supported | Example | +| -------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `delivery_acknowledgement` | One | `{"event": {"kind": "delivery_acknowledgement"}, "attributes": {"question_uuid": "79192e90-9022-4797-b6c7-82dc097dacdb", ...}}` | +| `heartbeat` | Any number | `{"event": {"kind": "heartbeat"}, "attributes": {"question_uuid": "79192e90-9022-4797-b6c7-82dc097dacdb", ...}` | +| `log_record` | Any number | `{"event": {"kind": "log_record": "log_record": {"msg": "Starting analysis."}}, "attributes": {"question_uuid": "79192e90-9022-4797-b6c7-82dc097dacdb", ...}` | +| `monitor_message` | Any number | `{"event": {"kind": "monitor_message": "data": '{"progress": "35%"}'}, "attributes": {"question_uuid": "79192e90-9022-4797-b6c7-82dc097dacdb", ...}` | +| `exception` | One | `{"event": {"kind": "exception", "exception_type": "ValueError", "exception_message": "x cannot be less than 10."}, "attributes": {"question_uuid": "79192e90-9022-4797-b6c7-82dc097dacdb", ...}` | +| `result` | One | `{"event": {"kind": "result", "output_values": {"my": "results"}, "output_manifest": None}, "attributes": {"question_uuid": "79192e90-9022-4797-b6c7-82dc097dacdb", ...}` | + +**Notes** + +- Event formats and contents must conform with the [service + communication + schema](https://strands.octue.com/octue/service-communication). +- Every event must be accompanied with the required event attributes +- The `log_record` key of a `log_record` event is any dictionary that + the `logging.makeLogRecord` function can convert into a log record. +- The `data` key of a `monitor_message` event must be a JSON-serialised + string +- Any events after a `result` or `exception` event won't be passed to + the parent because execution of the child emulator will have ended. + +### Instantiating a child emulator + +```python +events = [ + { + { + "event": { + "kind": "log_record", + "log_record": {"msg": "Starting analysis."}, + ... # Left out for brevity. + }, + "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", + "retry_count": 0, + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "parent_question_uuid": "5776ad74-52a6-46f7-a526-90421d91b8b2", + "originator_question_uuid": "86dc55b2-4282-42bd-92d0-bd4991ae7356", + "parent": "octue/test-service:1.0.0", + "originator": "octue/test-service:1.0.0", + "sender": "octue/test-service:1.0.0", + "sender_type": "CHILD", + "sender_sdk_version": "0.51.0", + "recipient": "octue/another-service:3.2.1" + }, + }, + }, + { + "event": { + "kind": "monitor_message", + "data": '{"progress": "35%"}', + }, + "attributes": { + ... # Left out for brevity. + }, + }, + { + "event": { + "kind": "log_record", + "log_record": {"msg": "Finished analysis."}, + ... # Left out for brevity. + }, + "attributes": { + ... # Left out for brevity. + }, + }, + { + "event": { + "kind": "result", + "output_values": [1, 2, 3, 4, 5], + }, + "attributes": { + ... # Left out for brevity. + }, + }, +] + +child_emulator = ChildEmulator(events) + +def handle_monitor_message(message): + ... + +result, question_uuid = child_emulator.ask( + input_values={"hello": "world"}, + handle_monitor_message=handle_monitor_message, +) +>>> {"output_values": [1, 2, 3, 4, 5], "output_manifest": None} +``` + +### Using the child emulator + +To emulate your children in tests, patch the `Child` class with the `ChildEmulator` class. + +```python +from unittest.mock import patch + +from octue.twined.runner import Runner +from octue.twined.cloud.emulators import ChildEmulator + + +app_directory_path = "path/to/directory_containing_app" + +# You can explicitly specify your children here as shown or +# read the same information in from your service configuration file. +children = [ + { + "key": "my_child", + "id": "octue/my-child-service:2.1.0", + "backend": { + "name": "GCPPubSubBackend", + "project_id": "my-project" + } + }, +] + +runner = Runner( + app_src=app_directory_path, + twine=os.path.join(app_directory_path, "twine.json"), + children=children, + service_id="your-org/your-service:2.1.0", +) + +emulated_children = [ + ChildEmulator( + events=[ + { + "event": { + "kind": "result", + "output_values": [300], + }, + "attributes": { + "datetime": "2024-04-11T10:46:48.236064", + "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", + "retry_count": 0, + "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", + "parent_question_uuid": "5776ad74-52a6-46f7-a526-90421d91b8b2", + "originator_question_uuid": "86dc55b2-4282-42bd-92d0-bd4991ae7356", + "parent": "you/your-service:2.1.0", + "originator": "you/your-service:2.1.0", + "sender": "octue/my-child-service:2.1.0", + "sender_type": "CHILD", + "sender_sdk_version": "0.56.0", + "recipient": "you/your-service:2.1.0" + }, + }, + ], + ) +] + +with patch("octue.runner.Child", side_effect=emulated_children): + analysis = runner.run(input_values={"some": "input"}) + +analysis.output_values +>>> [300] + +analysis.output_manifest +>>> None +``` + +**Notes** + +- If your app uses more than one child, provide more child emulators in + the `emulated_children` list in the order they're asked questions in + your app. +- If a given child is asked more than one question, provide a child + emulator for each question asked in the same order the questions are + asked. + +## Creating a test fixture + +Since the child is _emulated_, it doesn't actually do any calculation - +if you change the inputs, the outputs won't change correspondingly (or +at all). So, it's up to you to define a set of realistic inputs and +corresponding outputs (the list of emulated events) to test your +service. These are called **test fixtures**. + +!!! note + + Unlike a real child, the **inputs** given to the emulator aren't + validated against the schema in the child's twine -this is because the + twine is only available to the real child. This is ok - you're testing + your service, not the child your service contacts. The events given to + the emulator are still validated against the service communication + schema, though. + +You can create test fixtures manually or by using the +`Child.received_events` property after questioning a real child. + +```python +import json +from octue.twined.resources import Child + + +child = Child( + id="octue/my-child:2.1.0", + backend={"name": "GCPPubSubBackend", "project_id": "my-project"}, +) + +result, question_uuid = child.ask(input_values=[1, 2, 3, 4]) + +child.received_events +>>> [ + { + "event": { + 'kind': 'delivery_acknowledgement', + }, + "attributes": { + ... # Left out for brevity. + }, + }, + { + "event": { + 'kind': 'log_record', + 'log_record': { + 'msg': 'Finished analysis.', + 'args': None, + 'levelname': 'INFO', + ... # Left out for brevity. + }, + }, + "attributes": { + ... # Left out for brevity. + }, + }, + { + "event": { + 'kind': 'result', + 'output_values': {"some": "results"}, + }, + "attributes": { + ... # Left out for brevity. + }, + }, + ] +``` + +You can then feed these into a child emulator to emulate one possible +response of the child: + +```python +from octue.twined.cloud.emulators import ChildEmulator + + +child_emulator = ChildEmulator(events=child.received_events) +result, question_uuid = child_emulator.ask(input_values=[1, 2, 3, 4]) + +result +>>> {"some": "results"} +``` + +You can also create test fixtures from +[downloaded service crash diagnostics](/troubleshooting_services/#creating-test-fixtures-from-diagnostics) From 50076fa957e7119c0c39ca86a99fc0bd2a528780 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 18:55:57 -0400 Subject: [PATCH 30/69] DOC: Add troubleshooting doc --- docs/docs/troubleshooting_services.md | 131 ++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 docs/docs/troubleshooting_services.md diff --git a/docs/docs/troubleshooting_services.md b/docs/docs/troubleshooting_services.md new file mode 100644 index 000000000..f9de86687 --- /dev/null +++ b/docs/docs/troubleshooting_services.md @@ -0,0 +1,131 @@ +# Troubleshooting services + +## Diagnostics + +Services save the following data to the cloud if they crash while +processing a question (the default), or when they finish processing a +question successfully if diagnostics are permanently turned on (not the +default): + +- Input values +- Input manifest and datasets +- Child configuration values +- Child configuration manifest and datasets +- Inputs to and events received in response to each question the service + asked its children (if it has any). These are stored in the order the + questions were asked. + +!!! important + + For this feature to be enabled, the child must have the + `diagnostics_cloud_path` field in its service configuration + ([`octue.yaml`](/creating_services/#octueyaml) file) set to a Google Cloud Storage path. + +## Accessing diagnostics + +If diagnostics are enabled, a service will upload the diagnostics and +send the upload path to the parent as a log message. A user with +credentials to access this path can use the `octue` CLI to retrieve the +diagnostics data: + +```shell +octue twined question diagnostics +``` + +More information on the command: + +``` +>>> octue twined question diagnostics -h + +Usage: octue twined question diagnostics [OPTIONS] CLOUD_PATH + + Download diagnostics for an analysis from the given directory in + Google Cloud Storage. The cloud path should end in the analysis ID. + + CLOUD_PATH: The path to the directory in Google Cloud Storage containing the + diagnostics data. + +Options: + --local-path DIRECTORY The path to a directory to store the directory of + diagnostics data in. Defaults to the current working + directory. + --download-datasets If provided, download any datasets from the + diagnostics and update their paths in their + manifests to the new local paths. + -h, --help Show this message and exit. +``` + +## Creating test fixtures from diagnostics {#test_fixtures_from_diagnostics} + +You can create test fixtures directly from diagnostics, allowing you to +recreate the exact conditions that caused your service to fail. + +```python +from unittest.mock import patch + +from octue.twined.runner import Runner +from octue.twined.utils.testing import load_test_fixture_from_diagnostics + + +( + configuration_values, + configuration_manifest, + input_values, + input_manifest, + child_emulators, +) = load_test_fixture_from_diagnostics(path="path/to/downloaded/diagnostics") + +# You can explicitly specify your children here as shown or +# read the same information in from your service configuration file. +children = [ + { + "key": "my_child", + "id": "octue/my-child-service:2.1.0", + "backend": { + "name": "GCPPubSubBackend", + "project_id": "my-project", + } + }, + { + "key": "another_child", + "id": "octue/another-child-service:2.1.0", + "backend": { + "name": "GCPPubSubBackend", + "project_id": "my-project", + } + } +] + +runner = Runner( + app_src="path/to/directory_containing_app", + twine="twine.json", + children=children, + configuration_values=configuration_values, + configuration_manifest=configuration_manifest, + service_id="your-org/your-service:2.1.0", +) + +with patch("octue.twined.runner.Child", side_effect=child_emulators): + analysis = runner.run(input_values=input_values, input_manifest=input_manifest) +``` + +## Disabling diagnostics + +When asking a question to a child, parents can disable diagnostics +upload in the child on a question-by-question basis by setting +`save_diagnostics` to `"SAVE_DIAGNOSTICS_OFF"` in `Child.ask`. For example: + +```python +from octue.twined.resources import Child + + +child = Child( + id="my-organisation/my-service:2.1.0", + backend={"name": "GCPPubSubBackend", "project_id": "my-project"}, +) + +answer, question_uuid = child.ask( + input_values={"height": 32, "width": 3}, + save_diagnostics="SAVE_DIAGNOSTICS_OFF", +) +``` From 0ca8676eef22378ec8714131318568dcdacf7146 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 18:58:54 -0400 Subject: [PATCH 31/69] DOC: Add logging doc --- docs/docs/logging.md | 55 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 docs/docs/logging.md diff --git a/docs/docs/logging.md b/docs/docs/logging.md new file mode 100644 index 000000000..db5ce7f88 --- /dev/null +++ b/docs/docs/logging.md @@ -0,0 +1,55 @@ +# Logging + +By default, `octue` streams your logs to `stderr` in a nice, readable +format so your log messages are immediately visible when you start +developing without any extra configuration. If you prefer to use your +own handlers or formatters, simply set `USE_OCTUE_LOG_HANDLER=0` in the +environment running your app. + +## Readable logs + +Some advantages of the Octue log handler are: + +- Its readable format +- Its clear separation of log **context** from log **message**. + +Below, the context is on the left and includes: + +- The time +- Log level +- Module producing the log +- Octue analysis ID + +This is followed by the actual log message on the right: + +``` +[2021-07-10 20:03:12,713 | INFO | octue.runner | 102ee7d5-4b94-4f8a-9dcd-36dbd00662ec] Hello! The child services template app is running! +``` + +## Colourised services + +Another advantage to using the Octue log handler is that each Twined +service is coloured according to its position in the tree, making it +much easier to read log messages from multiple levels of children. + +![image](images/coloured_logs.png) + +In this example: + +- The log context is in blue +- Anything running in the root parent service's app is labeled with the + analysis ID in green +- Anything running in the immediate child services (`elevation` and + `wind_speed`) are labelled with the analysis ID in yellow +- Any children further down the tree (i.e. children of the child + services and so on) will have their own labels in other colours + consistent to their level + +## Add extra information + +You can add certain log record attributes to the logging context by also +providing the following environment variables: + +- `INCLUDE_LINE_NUMBER_IN_LOGS=1` - include the line number +- `INCLUDE_PROCESS_NAME_IN_LOGS=1` - include the process name +- `INCLUDE_THREAD_NAME_IN_LOGS=1` - include the thread name From 4ffa941adeeccb89c7f3d9a669a6ee9efcf39a3c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 19:00:46 -0400 Subject: [PATCH 32/69] DOC: Add authentication doc skipci --- docs/docs/authentication.md | 53 +++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 docs/docs/authentication.md diff --git a/docs/docs/authentication.md b/docs/docs/authentication.md new file mode 100644 index 000000000..32eb069d1 --- /dev/null +++ b/docs/docs/authentication.md @@ -0,0 +1,53 @@ +# Authentication + +You need authentication while using `octue` to: + +- Access data from Google Cloud Storage +- Use, run, or deploy Twined services + +Authentication is provided by a GCP service account. + +## Creating a service account + +By setting up your Twined service network with the +[Twined Terraform modules](/deploying_services), a set of maintainer service accounts have already been +created with the required permissions. + +## Using a service account + +### Locally + +1. Access your service accounts + [here](https://console.cloud.google.com/iam-admin/serviceaccounts), + making sure the correct project is selected +2. Click on the relevant service account, go to the "Keys" tab, and + create (download) a JSON key for it - it will be called + `-XXXXX.json`. + + !!! danger + + It's best not to store this in your project to prevent accidentally + committing it or building it into a docker image layer. Instead, keep it + somewhere else on your local system with any other service account keys + you already have. + + If you must keep within your project, it's good practice to name the + file `gcp-credentials.json` and make sure that `gcp-cred*` is in your + `.gitignore` and `.dockerignore` files. + +3. If you're developing in a container (like a VSCode `devcontainer`), + mount the file into the container. You can make gcloud available + too - check out [this + tutorial](https://medium.com/datamindedbe/application-default-credentials-477879e31cb5). +4. Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the + absolute path of the key file. If using a `devcontainer`, make sure + this is the path inside the container and not the path on your local + machine. + +### On GCP infrastructure / deployed services + +- Credentials are automatically provided when running code or services + on GCP infrastructure, including the Kubernetes cluster +- `octue` uses these when when running on these platforms, so there's + no need to upload a service account key or include one in service + docker images From ff79d81285fa22fa0f3ce659515fbccba31a5855 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 19:04:04 -0400 Subject: [PATCH 33/69] ENH: Update version compatibility table script for markdown format --- scripts/print_twined_version_compatibility_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/print_twined_version_compatibility_table.py b/scripts/print_twined_version_compatibility_table.py index 41b0cb0f5..5673569ee 100644 --- a/scripts/print_twined_version_compatibility_table.py +++ b/scripts/print_twined_version_compatibility_table.py @@ -25,4 +25,4 @@ # Transpose the dataframe so parents are the rows and children the columns. df = pd.DataFrame.from_dict(sorted_data).transpose() -print(tabulate.tabulate(df, headers="keys", tablefmt="grid")) +print(tabulate.tabulate(df, headers="keys", tablefmt="pipe")) From d61005d0e651c7b99a3aa89d4df64295a3c75efb Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 19:04:43 -0400 Subject: [PATCH 34/69] DOC: Add inter-service compatibility doc --- docs/docs/inter_service_compatibility.md | 76 ++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 docs/docs/inter_service_compatibility.md diff --git a/docs/docs/inter_service_compatibility.md b/docs/docs/inter_service_compatibility.md new file mode 100644 index 000000000..df6215a54 --- /dev/null +++ b/docs/docs/inter_service_compatibility.md @@ -0,0 +1,76 @@ +# Inter-service compatibility + +Twined services acting as parents and children communicate with each +other according to the [services communication +schema](https://strands.octue.com/octue/service-communication). The +table below shows which `octue` versions parents can run (rows) to send +questions compatible with versions children are running (columns). Note +that this table does not display whether children's responses are +compatible with the parent, just that a child is able to accept a +question. + +**Key** + +- `0` = incompatible +- `1` = compatible + +| | 0.67.0 | 0.66.1 | 0.66.0 | 0.65.0 | 0.64.0 | 0.63.0 | 0.62.1 | 0.62.0 | 0.61.2 | 0.61.1 | 0.61.0 | 0.60.2 | 0.60.1 | 0.60.0 | 0.59.1 | 0.59.0 | 0.58.0 | 0.57.2 | 0.57.1 | 0.57.0 | 0.56.0 | 0.55.0 | 0.54.0 | 0.53.0 | 0.52.2 | 0.52.1 | 0.52.0 | 0.51.0 | 0.50.1 | 0.50.0 | 0.49.2 | 0.49.1 | 0.49.0 | 0.48.0 | 0.47.2 | 0.47.1 | 0.47.0 | 0.46.3 | 0.46.2 | 0.46.1 | 0.46.0 | 0.45.0 | 0.44.0 | 0.43.7 | 0.43.6 | 0.43.5 | 0.43.4 | 0.43.3 | 0.43.2 | 0.43.1 | 0.43.0 | 0.42.1 | 0.42.0 | 0.41.1 | 0.41.0 | 0.40.2 | 0.40.1 | 0.40.0 | +| :----- | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | +| 0.67.0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.66.1 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.66.0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.65.0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.64.0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.63.0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.62.1 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.62.0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.61.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.61.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.61.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.60.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.60.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.60.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.59.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.59.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.58.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.57.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.57.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.57.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.56.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.55.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.54.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.53.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.52.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.52.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.52.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.51.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.50.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.50.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.49.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.49.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.49.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.48.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.47.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.47.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.47.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.46.3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.46.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.46.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.46.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.45.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.44.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.43.7 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.43.6 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.43.5 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.43.4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.43.3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.43.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.43.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.43.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.42.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.42.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.41.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.41.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.40.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.40.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.40.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | From a098a4eec1b3d23b1ab986572ed13c01c396ce1c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 19:06:10 -0400 Subject: [PATCH 35/69] DOC: Add licence and version history docs --- docs/docs/license.md | 10 ++++++++++ docs/docs/version_history.md | 25 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 docs/docs/license.md create mode 100644 docs/docs/version_history.md diff --git a/docs/docs/license.md b/docs/docs/license.md new file mode 100644 index 000000000..8fdf9f723 --- /dev/null +++ b/docs/docs/license.md @@ -0,0 +1,10 @@ +# License + +## The Boring Bit + +See [the octue-sdk-python license](https://github.com/octue/octue-sdk-python/blob/main/LICENSE). + +## Third Party Libraries + +**octue-sdk-python** includes or is linked against code from third party +libraries - see [our attributions page](https://github.com/octue/octue-sdk-python/blob/main/ATTRIBUTIONS.md). diff --git a/docs/docs/version_history.md b/docs/docs/version_history.md new file mode 100644 index 000000000..8068e4209 --- /dev/null +++ b/docs/docs/version_history.md @@ -0,0 +1,25 @@ +# Version History {#chapter-version-history} + +See our [releases on GitHub.](https://github.com/octue/octue-sdk-python/releases) + +## Semantic versioning + +We use [semantic versioning](https://semver.org/) so you can see when +new releases make breaking changes or just add new features or bug +fixes. Breaking changes are highlighted in our pull request descriptions +and release notes. + +!!! important + + Note that `octue` is still in beta, so its major version number remains + at 0 (i.e. `0.y.z`). This means that, for now, both breaking changes and + new features are denoted by an increase in the minor version number (`y` + in `x.y.z`). When we come out of beta, breaking changes will be denoted + by an increase in the major version number (`x` in `x.y.z`). + +## Deprecated code + +When code is deprecated, it will still work but a deprecation warning +will be issued with a suggestion on how to update it. After an +adjustment period, deprecations will be removed from the codebase +according to the [code removal schedule](https://github.com/octue/octue-sdk-python/issues/415). This constitutes a breaking change. From b49146432b6d2ee5ca77314b9549c99629682ae4 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 19:09:31 -0400 Subject: [PATCH 36/69] DOC: Add downloading datafiles doc --- docs/docs/downloading_datafiles.md | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 docs/docs/downloading_datafiles.md diff --git a/docs/docs/downloading_datafiles.md b/docs/docs/downloading_datafiles.md new file mode 100644 index 000000000..38a5aa883 --- /dev/null +++ b/docs/docs/downloading_datafiles.md @@ -0,0 +1,33 @@ +# More information on downloading datafiles + +- To avoid unnecessary data transfer and costs, cloud datafiles are not + downloaded locally [until necessary](/datafile/#automatic-lazy-downloading) +- When downloaded, they are downloaded by default to a temporary local + file that will exist at least as long as the python session is running +- Calling `Datafile.download` or using `Datafile.local_path` again will + not re-download the file +- Any changes made to the datafile via the `Datafile.open` method are + made to the local copy and then synced with the cloud object + +!!! warning + + External changes to cloud files will not be synced locally unless the + datafile is re-instantiated. + +- If you want a cloud datafile to be downloaded to a permanent location, + you can do one of: + + ```python + datafile.download(local_path="my/local/path.csv") + + datafile.local_path = "my/local/path.csv" + ``` + +- To pre-set a permanent download location on instantiation, run: + + ```python + datafile = Datafile( + "gs://my-bucket/path/to/file.dat", + local_path="my/local/path.csv", + ) + ``` From f3f7bd641d0bc0d68dba5e50f5d472c57c8c8f3a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 19:12:12 -0400 Subject: [PATCH 37/69] DOC: Add creating apps doc --- docs/docs/creating_apps.md | 167 +++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 docs/docs/creating_apps.md diff --git a/docs/docs/creating_apps.md b/docs/docs/creating_apps.md new file mode 100644 index 000000000..c9cb0ff26 --- /dev/null +++ b/docs/docs/creating_apps.md @@ -0,0 +1,167 @@ +# app.py file {#creating_apps} + +The `app.py` file is, as you might expect, the entrypoint to your app. +It can contain any valid python including imports and use of any number +of external packages or internal/local packages. + +## Structure + +The `app.py` file must contain exactly one of the `octue` python app +interfaces to serve as the entrypoint to your code. These take a single +`Analysis` instance as a parameter or attribute: + +- **Option 1:** A function named `run` with the following signature: + + ```python + def run(analysis): + """A function that uses input and configuration from an ``Analysis`` + instance and stores any output values and output manifests on it. + It shouldn't return anything. + + :param octue.resources.Analysis analysis: + :return None: + """ + ... + ``` + +- **Option 2:** A class named `App` with the following signature: + + ```python + class App: + """A class that takes an ``Analysis`` instance and any number of + other parameters in its constructor. It can have any number of + methods but must always have a ``run`` method with the signature + shown below. + + :param octue.resources.Analysis analysis: + :return None: + """ + + def __init__(self, analysis): + self.analysis = analysis + ... + + def run(self): + """A method that that uses input and configuration from an + ``Analysis`` instance and stores any output values and + output manifests on it. It shouldn't return anything. + + :return None: + """ + ... + + ... + ``` + +## Accessing inputs and storing outputs + +Your app must access configuration and input data from and store output +data on the `analysis` parameter (for function-based apps) or attribute (for +class-based apps). This allows standardised configuration/input/output +validation against the twine and interoperability of all Twined services +while leaving you freedom to do any kind of computation. To access the +data, use the following attributes on the `analysis` +parameter/attribute: + +- Configuration values: `analysis.configuration_values` +- Configuration manifest: `analysis.configuration_manifest` +- Input values: `analysis.input_values` +- Input manifest: `analysis.input_manifest` +- Output values: `analysis.output_values` +- Output manifest: `analysis.output_manifest` + +## Sending monitor messages + +As well as sending the final result of the analysis your app produces to +the parent, you can send monitor messages as computation progresses. +This functionality can be used to update a plot or database in real time +as the analysis is in progress. + +```python +def run(analysis): + some_data = {"x": 0, "y", 0.1, "z": 2} + analysis.send_monitor_message(data=some_data) +``` + +Before sending monitor messages, the `monitor_message_schema` field must +be provided in `twine.json`. For example: + +```json +{ + ... + "monitor_message_schema": { + "x": { + "description": "Real component", + "type": "number" + }, + "y": { + "description": "Imaginary component", + "type": "number" + }, + "z": { + "description": "Number of iterations before divergence", + "type": "number", + "minimum": 0 + } + }, + ... +} +``` + +Monitor messages can also be set up to send periodically in time. + +```python +def run(analysis): + + # Define a data structure whose attributes can be accessed in real + # time as they're updated during the analysis. + class DataStructure: + def __init__(self): + self.x = 0 + self.y = 0 + self.z = 0 + + def as_dict(self): + """Add a method that provides the data in the format + required for the monitor messages. + + :return dict: + """ + return {"x": self.x, "y": self.y, "z": self.z} + + # Create an instance of the data structure. + my_updating_data = DataStructure() + + # Use the `as_dict` method to provide up-to-date data from the data + # structure to send as monitor messages every 60s. + analysis.set_up_periodic_monitor_message( + create_monitor_message=my_updating_data.as_dict, + period=60, + ) + + # Run long-running computations on the data structure that update its + # "x", "y", and "z" attributes in real time. The periodic monitor + # message will always send the current values of x, y, and z. + some_function(my_updating_data) +``` + +## Finalising the analysis + +When the analysis has finished, it is automatically finalised. This +means: + +- The output values and manifest are validated against `twine.json` to + ensure they're in the format promised by the service. +- If the app produced an output manifest and the `output_location` field + is set to a cloud directory path in the app configuration, the output + datasets are uploaded to this location. + +!!! note + + You can manually call + `analysis.finalise` if you want to upload any output datasets to a different + location to the one specified in the service configuration. If you do + this, the analysis will not be finalised again - make sure you only call + it when your output data is ready. Note that the `output_location` and + `use_signed_urls_for_output_datasets` settings in the service + configuration are ignored if you call it manually. From d1917cda564fab9ebc36d012f9dd541364645d08 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 19:14:36 -0400 Subject: [PATCH 38/69] DOC: Add available filters doc skipci --- docs/docs/available_filters.md | 110 +++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 docs/docs/available_filters.md diff --git a/docs/docs/available_filters.md b/docs/docs/available_filters.md new file mode 100644 index 000000000..df59c77fb --- /dev/null +++ b/docs/docs/available_filters.md @@ -0,0 +1,110 @@ +# Available filters + +Lots of filters are available when using the `Dataset.files.filter` +method. We've broken them down by the type of attribute the datafiles +are being filtered by: + +- Numbers (e.g. `int`, `float`): + + - `is` + - `is_not` + - `equals` + - `not_equals` + - `lt` + - `lte` + - `gt` + - `gte` + - `in_range` + - `not_in_range` + +- Iterables (e.g. `list`, `set`, `tuple`, `dictionary`): + + - `is` + - `is_not` + - `equals` + - `not_equals` + - `contains` + - `not_contains` + - `icontains` + - `not_icontains` + +- `bool` + + - `is` + - `is_not` + +- `str` + + - `is` + - `is_not` + - `equals` + - `not_equals` + - `iequals` + - `not_iequals` + - `lt` (less than) + - `lte` (less than or equal) + - `gt` (greater than) + - `gte` (greater than or equal) + - `contains` + - `not_contains` + - `icontains` (case-insensitive contains) + - `not_icontains` + - `starts_with` + - `not_starts_with` + - `ends_with` + - `not_ends_with` + - `in_range` + - `not_in_range` + +- `NoneType` + + - `is` + - `is_not` + +- `LabelSet` + + - `is` + - `is_not` + - `equals` + - `not_equals` + - `contains` + - `not_contains` + - `any_label_contains` + - `not_any_label_contains` + - `any_label_starts_with` + - `not_any_label_starts_with` + - `any_label_ends_with` + - `not_any_label_ends_with` + +- `datetime.datetime` + + - `is` + - `is_not` + - `equals` + - `not_equals` + - `lt` (less than) + - `lte` (less than or equal) + - `gt` (greater than) + - `gte` (greater than or equal) + - `in_range` + - `not_in_range` + - `year_equals` + - `year_in` + - `month_equals` + - `month_in` + - `day_equals` + - `day_in` + - `weekday_equals` + - `weekday_in` + - `iso_weekday_equals` + - `iso_weekday_in` + - `time_equals` + - `time_in` + - `hour_equals` + - `hour_in` + - `minute_equals` + - `minute_in` + - `second_equals` + - `second_in` + - `in_date_range` + - `in_time_range` From 05b5378400763942d487750eff01aafa8e852993 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 19:17:39 -0400 Subject: [PATCH 39/69] DOC: Remove `.rst` based docs --- docs/source/_ext/googleanalytics.py | 29 - docs/source/_ext/sphinx_accordion/README.md | 19 - docs/source/_ext/sphinx_accordion/__init__.py | 0 .../_ext/sphinx_accordion/accordion.css | 11 - .../source/_ext/sphinx_accordion/accordion.js | 48 -- .../source/_ext/sphinx_accordion/accordion.py | 259 -------- .../semantic-ui-2.4.2/.versions | 3 - .../semantic-ui-2.4.2/accordion.css | 247 -------- .../semantic-ui-2.4.2/accordion.js | 578 ------------------ .../semantic-ui-2.4.2/accordion.min.css | 170 ------ .../semantic-ui-2.4.2/accordion.min.js | 391 ------------ docs/source/api.rst | 79 --- docs/source/asking_questions.rst | 393 ------------ docs/source/authentication.rst | 48 -- docs/source/available_filters.rst | 103 ---- docs/source/conf.py | 196 ------ docs/source/creating_apps.rst | 162 ----- docs/source/creating_services.rst | 155 ----- docs/source/data_containers.rst | 14 - docs/source/datafile.rst | 308 ---------- docs/source/dataset.rst | 232 ------- docs/source/deploying_services.rst | 67 -- docs/source/downloading_datafiles.rst | 29 - docs/source/index.rst | 98 --- docs/source/installation.rst | 34 -- docs/source/inter_service_compatibility.rst | 133 ---- docs/source/license.rst | 13 - docs/source/logging.rst | 53 -- docs/source/manifest.rst | 119 ---- docs/source/running_services_locally.rst | 72 --- docs/source/services.rst | 98 --- docs/source/testing_services.rst | 288 --------- docs/source/troubleshooting_services.rst | 128 ---- docs/source/twined/about.rst | 21 - docs/source/twined/about_digital_twins.rst | 55 -- .../twined/about_introducing_json_schema.rst | 132 ---- .../twined/about_other_considerations.rst | 105 ---- docs/source/twined/about_requirements.rst | 29 - docs/source/twined/anatomy.rst | 101 --- docs/source/twined/anatomy_children.rst | 9 - docs/source/twined/anatomy_credentials.rst | 78 --- docs/source/twined/anatomy_manifest.rst | 369 ----------- docs/source/twined/anatomy_monitors.rst | 55 -- docs/source/twined/anatomy_values.rst | 128 ---- docs/source/twined/deployment.rst | 66 -- docs/source/twined/examples.rst | 192 ------ docs/source/twined/favicon.ico | Bin 1150 -> 0 bytes docs/source/twined/index.rst | 106 ---- docs/source/twined/license.rst | 85 --- docs/source/twined/lifecycle.rst | 30 - docs/source/twined/quick_start.rst | 11 - .../quick_start_create_your_first_twine.rst | 111 ---- .../twined/quick_start_installation.rst | 41 -- docs/source/twined/version_history.rst | 41 -- docs/source/updating_services.rst | 95 --- docs/source/version_history.rst | 25 - 56 files changed, 6462 deletions(-) delete mode 100644 docs/source/_ext/googleanalytics.py delete mode 100644 docs/source/_ext/sphinx_accordion/README.md delete mode 100644 docs/source/_ext/sphinx_accordion/__init__.py delete mode 100644 docs/source/_ext/sphinx_accordion/accordion.css delete mode 100644 docs/source/_ext/sphinx_accordion/accordion.js delete mode 100644 docs/source/_ext/sphinx_accordion/accordion.py delete mode 100755 docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/.versions delete mode 100755 docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/accordion.css delete mode 100755 docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/accordion.js delete mode 100755 docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/accordion.min.css delete mode 100755 docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/accordion.min.js delete mode 100644 docs/source/api.rst delete mode 100644 docs/source/asking_questions.rst delete mode 100644 docs/source/authentication.rst delete mode 100644 docs/source/available_filters.rst delete mode 100644 docs/source/conf.py delete mode 100644 docs/source/creating_apps.rst delete mode 100644 docs/source/creating_services.rst delete mode 100644 docs/source/data_containers.rst delete mode 100644 docs/source/datafile.rst delete mode 100644 docs/source/dataset.rst delete mode 100644 docs/source/deploying_services.rst delete mode 100644 docs/source/downloading_datafiles.rst delete mode 100644 docs/source/index.rst delete mode 100644 docs/source/installation.rst delete mode 100644 docs/source/inter_service_compatibility.rst delete mode 100644 docs/source/license.rst delete mode 100644 docs/source/logging.rst delete mode 100644 docs/source/manifest.rst delete mode 100644 docs/source/running_services_locally.rst delete mode 100644 docs/source/services.rst delete mode 100644 docs/source/testing_services.rst delete mode 100644 docs/source/troubleshooting_services.rst delete mode 100644 docs/source/twined/about.rst delete mode 100644 docs/source/twined/about_digital_twins.rst delete mode 100644 docs/source/twined/about_introducing_json_schema.rst delete mode 100644 docs/source/twined/about_other_considerations.rst delete mode 100644 docs/source/twined/about_requirements.rst delete mode 100644 docs/source/twined/anatomy.rst delete mode 100644 docs/source/twined/anatomy_children.rst delete mode 100644 docs/source/twined/anatomy_credentials.rst delete mode 100644 docs/source/twined/anatomy_manifest.rst delete mode 100644 docs/source/twined/anatomy_monitors.rst delete mode 100644 docs/source/twined/anatomy_values.rst delete mode 100644 docs/source/twined/deployment.rst delete mode 100644 docs/source/twined/examples.rst delete mode 100644 docs/source/twined/favicon.ico delete mode 100644 docs/source/twined/index.rst delete mode 100644 docs/source/twined/license.rst delete mode 100644 docs/source/twined/lifecycle.rst delete mode 100644 docs/source/twined/quick_start.rst delete mode 100644 docs/source/twined/quick_start_create_your_first_twine.rst delete mode 100644 docs/source/twined/quick_start_installation.rst delete mode 100644 docs/source/twined/version_history.rst delete mode 100644 docs/source/updating_services.rst delete mode 100644 docs/source/version_history.rst diff --git a/docs/source/_ext/googleanalytics.py b/docs/source/_ext/googleanalytics.py deleted file mode 100644 index 556033413..000000000 --- a/docs/source/_ext/googleanalytics.py +++ /dev/null @@ -1,29 +0,0 @@ -from sphinx.errors import ExtensionError - - -def add_ga_javascript(app, pagename, templatename, context, doctree): - if app.config.googleanalytics_enabled: - id = app.config.googleanalytics_id - metatags = context.get("metatags", "") - metatags += "\n" - metatags += f'\n' - metatags += "\n" - context["metatags"] = metatags - - -def check_config(app): - if not app.config.googleanalytics_id: - raise ExtensionError("'googleanalytics_id' config value must be set for ga statistics to function properly.") - - -def setup(app): - app.add_config_value("googleanalytics_id", "", "html") - app.add_config_value("googleanalytics_enabled", True, "html") - app.connect("html-page-context", add_ga_javascript) - app.connect("builder-inited", check_config) - return {"version": "0.1"} diff --git a/docs/source/_ext/sphinx_accordion/README.md b/docs/source/_ext/sphinx_accordion/README.md deleted file mode 100644 index c35001278..000000000 --- a/docs/source/_ext/sphinx_accordion/README.md +++ /dev/null @@ -1,19 +0,0 @@ -``` -extensions = [ - ... - 'sphinx_accordion.accordion' - ... -] -``` - -``` -.. accordion:: - - .. accordion-row:: The Title - - The Contents - - .. accordion-row:: The Second Title - - The Contents 2 -``` diff --git a/docs/source/_ext/sphinx_accordion/__init__.py b/docs/source/_ext/sphinx_accordion/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/docs/source/_ext/sphinx_accordion/accordion.css b/docs/source/_ext/sphinx_accordion/accordion.css deleted file mode 100644 index cedc5fec4..000000000 --- a/docs/source/_ext/sphinx_accordion/accordion.css +++ /dev/null @@ -1,11 +0,0 @@ -.sphinx-accordion.accordion { - margin-bottom: 1.75em; -} - -.sphinx-accordion.title p { - display: inline-block; - margin-top: 8px; - margin-right: 0px; - margin-bottom: 8px; - margin-left: 0px; -} diff --git a/docs/source/_ext/sphinx_accordion/accordion.js b/docs/source/_ext/sphinx_accordion/accordion.js deleted file mode 100644 index 0814c90c4..000000000 --- a/docs/source/_ext/sphinx_accordion/accordion.js +++ /dev/null @@ -1,48 +0,0 @@ -// if (!String.prototype.startsWith) { -// Object.defineProperty(String.prototype, 'startsWith', { -// value: function(search, pos) { -// pos = !pos || pos < 0 ? 0 : +pos; -// return this.substring(pos, pos + search.length) === search; -// } -// }); -// } - -$(document).ready(function () { - console.log("FFS"); -}); - -$(function () { - console.log("SOMETHING HAPPENS MAYBE"); - - // We store the data-row values as sphinx-data- - // Add data-row attribute with the extracted value - $(".sphinx-accordion.title").each(function () { - const this1 = $(this); - const prefix = "sphinx-accordion-title-"; - const classes = this1.attr("class").split(/\s+/); - $.each(classes, function (idx, clazz) { - if (clazz.startsWith(prefix)) { - this1.attr("data-row", clazz.substring(prefix.length)); - } - }); - - const data_row = this1.attr("data-row"); - - this1.on("click", function () { - // Find offset in view - const offset = this1.offset().top - $(window).scrollTop(); - - // Toggle active class on this subsequent sibling - if (this1.hasClass("active")) { - this1.removeClass("active"); - this1.next().removeClass("active"); - } else { - this1.addClass("active"); - this1.next().addClass("active"); - } - - // Keep tab with the original view offset - $(window).scrollTop(this1.offset().top - offset); - }); - }); -}); diff --git a/docs/source/_ext/sphinx_accordion/accordion.py b/docs/source/_ext/sphinx_accordion/accordion.py deleted file mode 100644 index 93e277a9a..000000000 --- a/docs/source/_ext/sphinx_accordion/accordion.py +++ /dev/null @@ -1,259 +0,0 @@ -"""Accordion dropdown for Sphinx, with HTML builder""" - -import json -import os -import posixpath - -from docutils import nodes -from docutils.parsers.rst import Directive -from pkg_resources import resource_filename -from pygments.lexers import get_all_lexers -from sphinx.util import logging -from sphinx.util.osutil import copyfile - -FILES = [ - "semantic-ui-2.4.2/accordion.css", - "semantic-ui-2.4.2/accordion.js", - "accordion.css", - "accordion.js", -] - - -LEXER_MAP = {} -for lexer in get_all_lexers(): - for short_name in lexer[1]: - LEXER_MAP[short_name] = lexer[0] - - -def get_compatible_builders(app): - builders = [ - "html", - "singlehtml", - "dirhtml", - "readthedocs", - "readthedocsdirhtml", - "readthedocssinglehtml", - "readthedocssinglehtmllocalmedia", - "spelling", - ] - builders.extend(app.config["sphinx_tabs_valid_builders"]) - return builders - - -class AccordionDirective(Directive): - """Top-level accordion directive""" - - has_content = True - - def run(self): - """Parse an accordion directive""" - self.assert_has_content() - env = self.state.document.settings.env - - node = nodes.container() - node["classes"] = ["sphinx-accordion", "ui", "styled", "fluid", "accordion"] - - if "next_accordion_id" not in env.temp_data: - env.temp_data["next_accordion_id"] = 0 - if "accordion_stack" not in env.temp_data: - env.temp_data["accordion_stack"] = [] - - accordion_id = env.temp_data["next_accordion_id"] - accordion_key = "accordion_%d" % accordion_id - env.temp_data["next_accordion_id"] += 1 - env.temp_data["accordion_stack"].append(accordion_id) - - env.temp_data[accordion_key] = {} - env.temp_data[accordion_key]["row_ids"] = [] - env.temp_data[accordion_key]["row_titles"] = [] - env.temp_data[accordion_key]["is_first_row"] = True - - self.state.nested_parse(self.content, self.content_offset, node) - - if env.app.builder.name in get_compatible_builders(env.app): - title_nodes = [] - row_ids = env.temp_data[accordion_key]["row_ids"] - row_titles = env.temp_data[accordion_key]["row_titles"] - for idx, [data_row, row_name] in enumerate(row_titles): - title_node = nodes.container() - title_node.tagname = "div" - title_node["classes"] = ["sphinx-accordion", "title"] - title_node["classes"].append(f"sphinx-accordion-title-{accordion_id}-{row_ids[idx]}") - title_node += row_name.children - icon_node = nodes.inline() - icon_node.tagname = "i" - icon_node["classes"] = ["dropdown", "icon"] - # Access the first child, we don't want the container that somehow gets generated - title_node.children.insert(0, icon_node) - title_nodes.append(title_node) - - node.children = [child for pair in zip(title_nodes, node.children) for child in pair] - - env.temp_data["accordion_stack"].pop() - return [node] - - -class AccordionRowDirective(Directive): - """AccordionRow directive, for adding a row to an accordion""" - - has_content = True - - def run(self): - """Parse a row directive""" - self.assert_has_content() - env = self.state.document.settings.env - - accordion_id = env.temp_data["accordion_stack"][-1] - accordion_key = "accordion_%d" % accordion_id - - args = self.content[0].strip() - if args.startswith("{"): - try: - args = json.loads(args) - self.content.trim_start(1) - except ValueError: - args = {} - else: - args = {} - - row_name = nodes.container() - self.state.nested_parse(self.content[:1], self.content_offset, row_name) - args["row_name"] = row_name - - include_accordion_id_in_data_row = False - if "row_id" not in args: - args["row_id"] = env.new_serialno(accordion_key) - include_accordion_id_in_data_row = True - i = 1 - while args["row_id"] in env.temp_data[accordion_key]["row_ids"]: - args["row_id"] = "%s-%d" % (args["row_id"], i) - i += 1 - env.temp_data[accordion_key]["row_ids"].append(args["row_id"]) - - data_row = str(args["row_id"]) - if include_accordion_id_in_data_row: - data_row = "%d-%s" % (accordion_id, data_row) - data_row = "sphinx-accordion-content-{}".format(data_row) - - env.temp_data[accordion_key]["row_titles"].append((data_row, args["row_name"])) - - text = "\n".join(self.content) - node = nodes.container(text) - classes = "sphinx-accordion content" - node["classes"] = classes.split(" ") - node["classes"].extend(args.get("classes", [])) - node["classes"].append(data_row) - - self.state.nested_parse(self.content[2:], self.content_offset, node) - - if env.app.builder.name not in get_compatible_builders(env.app): - outer_node = nodes.container() - row = nodes.container() - row.tagname = "a" - row["classes"] = ["item"] - row += row_name - outer_node.append(row) - outer_node.append(node) - return [outer_node] - - return [node] - - -class _FindAccordionDirectiveVisitor(nodes.NodeVisitor): - """Visitor pattern than looks for a sphinx accordion directive in a document""" - - def __init__(self, document): - nodes.NodeVisitor.__init__(self, document) - self._found = False - - def unknown_visit(self, node): - if ( - not self._found - and isinstance(node, nodes.container) - and "classes" in node - and isinstance(node["classes"], list) - ): - self._found = "sphinx-accordion" in node["classes"] - - @property - def found_accordion_directive(self): - """Return whether a sphinx accordion directive was found""" - return self._found - - -def update_context(app, pagename, templatename, context, doctree): - """Remove sphinx-accordion CSS and JS asset files if not used in a page""" - if doctree is None: - return - visitor = _FindAccordionDirectiveVisitor(doctree) - doctree.walk(visitor) - if not visitor.found_accordion_directive: - paths = [posixpath.join("_static", "sphinx_accordion/" + f) for f in FILES] - if "css_files" in context: - context["css_files"] = context["css_files"][:] - for path in paths: - if path.endswith(".css") and path in context["css_files"]: - context["css_files"].remove(path) - if "script_files" in context: - context["script_files"] = context["script_files"][:] - for path in paths: - if path.endswith(".js") and path in context["script_files"]: - context["script_files"].remove(path) - - -def copy_assets(app, exception): - """Copy asset files to the output""" - if "getLogger" in dir(logging): - log = logging.getLogger(__name__).info - warn = logging.getLogger(__name__).warning - else: - log = app.info - warn = app.warning - builders = get_compatible_builders(app) - if exception: - return - if app.builder.name not in builders: - if not app.config["sphinx_accordion_nowarn"]: - warn("Not copying accordion assets! Not compatible with %s builder" % app.builder.name) - return - - log("Copying accordion assets") - - installdir = os.path.join(app.builder.outdir, "_static", "sphinx_accordion") - - for path in FILES: - source = resource_filename("sphinx_accordion", path) - dest = os.path.join(installdir, path) - destdir = os.path.dirname(dest) - if not os.path.exists(destdir): - os.makedirs(destdir) - - copyfile(source, dest) - - -def setup(app): - """Set up the plugin""" - app.add_config_value("sphinx_accordion_nowarn", False, "") - app.add_config_value("sphinx_accordion_valid_builders", [], "") - app.add_directive("accordion", AccordionDirective) - app.add_directive("accordion-row", AccordionRowDirective) - - for path in ["sphinx_accordion/" + f for f in FILES]: - if path.endswith(".css"): - if "add_css_file" in dir(app): - app.add_css_file(path) - else: - app.add_stylesheet(path) - if path.endswith(".js"): - if "add_script_file" in dir(app): - app.add_script_file(path) - else: - app.add_js_file(path) - - app.connect("html-page-context", update_context) - app.connect("build-finished", copy_assets) - - return { - "parallel_read_safe": True, - "parallel_write_safe": True, - } diff --git a/docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/.versions b/docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/.versions deleted file mode 100755 index 01b3d8231..000000000 --- a/docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/.versions +++ /dev/null @@ -1,3 +0,0 @@ -meteor@1.1.6 -semantic:ui-accordion@2.1.3 -underscore@1.0.3 diff --git a/docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/accordion.css b/docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/accordion.css deleted file mode 100755 index fd6afb827..000000000 --- a/docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/accordion.css +++ /dev/null @@ -1,247 +0,0 @@ -/*! - * # Semantic UI 2.4.1 - Accordion - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */ - -/******************************* - Accordion -*******************************/ - -.ui.accordion, -.ui.accordion .accordion { - max-width: 100%; -} -.ui.accordion .accordion { - margin: 1em 0em 0em; - padding: 0em; -} - -/* Title */ -.ui.accordion .title, -.ui.accordion .accordion .title { - cursor: pointer; -} - -/* Default Styling */ -.ui.accordion .title:not(.ui) { - padding: 0.5em 0em; - font-family: "Lato", "Helvetica Neue", Arial, Helvetica, sans-serif; - font-size: 1em; - color: rgba(0, 0, 0, 0.87); -} - -/* Content */ -.ui.accordion .title ~ .content, -.ui.accordion .accordion .title ~ .content { - display: none; -} - -/* Default Styling */ -.ui.accordion:not(.styled) .title ~ .content:not(.ui), -.ui.accordion:not(.styled) .accordion .title ~ .content:not(.ui) { - margin: ""; - padding: 0.5em 0em 1em; -} -.ui.accordion:not(.styled) .title ~ .content:not(.ui):last-child { - padding-bottom: 0em; -} - -/* Arrow */ -.ui.accordion .title .dropdown.icon, -.ui.accordion .accordion .title .dropdown.icon { - display: inline-block; - float: none; - opacity: 1; - width: 1.25em; - height: 1em; - margin: 0em 0.25rem 0em 0rem; - padding: 0em; - font-size: 1em; - -webkit-transition: opacity 0.1s ease, -webkit-transform 0.1s ease; - transition: opacity 0.1s ease, -webkit-transform 0.1s ease; - transition: transform 0.1s ease, opacity 0.1s ease; - transition: transform 0.1s ease, opacity 0.1s ease, - -webkit-transform 0.1s ease; - vertical-align: baseline; - -webkit-transform: none; - transform: none; -} - -/*-------------- - Coupling ----------------*/ - -/* Menu */ -.ui.accordion.menu .item .title { - display: block; - padding: 0em; -} -.ui.accordion.menu .item .title > .dropdown.icon { - float: right; - margin: 0.21425em 0em 0em 1em; - -webkit-transform: rotate(180deg); - transform: rotate(180deg); -} - -/* Header */ -.ui.accordion .ui.header .dropdown.icon { - font-size: 1em; - margin: 0em 0.25rem 0em 0rem; -} - -/******************************* - States -*******************************/ - -.ui.accordion .active.title .dropdown.icon, -.ui.accordion .accordion .active.title .dropdown.icon { - -webkit-transform: rotate(90deg); - transform: rotate(90deg); -} -.ui.accordion.menu .item .active.title > .dropdown.icon { - -webkit-transform: rotate(90deg); - transform: rotate(90deg); -} - -/******************************* - Types -*******************************/ - -/*-------------- - Styled ----------------*/ - -.ui.styled.accordion { - width: 600px; -} -.ui.styled.accordion, -.ui.styled.accordion .accordion { - border-radius: 0.28571429rem; - background: #ffffff; - -webkit-box-shadow: 0px 1px 2px 0 rgba(34, 36, 38, 0.15), - 0px 0px 0px 1px rgba(34, 36, 38, 0.15); - box-shadow: 0px 1px 2px 0 rgba(34, 36, 38, 0.15), - 0px 0px 0px 1px rgba(34, 36, 38, 0.15); -} -.ui.styled.accordion .title, -.ui.styled.accordion .accordion .title { - margin: 0em; - padding: 0.75em 1em; - color: rgba(0, 0, 0, 0.4); - font-weight: bold; - border-top: 1px solid rgba(34, 36, 38, 0.15); - -webkit-transition: background 0.1s ease, color 0.1s ease; - transition: background 0.1s ease, color 0.1s ease; -} -.ui.styled.accordion > .title:first-child, -.ui.styled.accordion .accordion .title:first-child { - border-top: none; -} - -/* Content */ -.ui.styled.accordion .content, -.ui.styled.accordion .accordion .content { - margin: 0em; - padding: 0.5em 1em 1.5em; -} -.ui.styled.accordion .accordion .content { - padding: 0em; - padding: 0.5em 1em 1.5em; -} - -/* Hover */ -.ui.styled.accordion .title:hover, -.ui.styled.accordion .active.title, -.ui.styled.accordion .accordion .title:hover, -.ui.styled.accordion .accordion .active.title { - background: transparent; - color: rgba(0, 0, 0, 0.87); -} -.ui.styled.accordion .accordion .title:hover, -.ui.styled.accordion .accordion .active.title { - background: transparent; - color: rgba(0, 0, 0, 0.87); -} - -/* Active */ -.ui.styled.accordion .active.title { - background: transparent; - color: rgba(0, 0, 0, 0.95); -} -.ui.styled.accordion .accordion .active.title { - background: transparent; - color: rgba(0, 0, 0, 0.95); -} - -/******************************* - States -*******************************/ - -/*-------------- - Active ----------------*/ - -.ui.accordion .active.content, -.ui.accordion .accordion .active.content { - display: block; -} - -/******************************* - Variations -*******************************/ - -/*-------------- - Fluid ----------------*/ - -.ui.fluid.accordion, -.ui.fluid.accordion .accordion { - width: 100%; -} - -/*-------------- - Inverted ----------------*/ - -.ui.inverted.accordion .title:not(.ui) { - color: rgba(255, 255, 255, 0.9); -} - -/******************************* - Theme Overrides -*******************************/ - -@font-face { - font-family: "Accordion"; - src: url(data:application/x-font-ttf;charset=utf-8;base64,AAEAAAALAIAAAwAwT1MvMggjB5AAAAC8AAAAYGNtYXAPfOIKAAABHAAAAExnYXNwAAAAEAAAAWgAAAAIZ2x5Zryj6HgAAAFwAAAAyGhlYWT/0IhHAAACOAAAADZoaGVhApkB5wAAAnAAAAAkaG10eAJuABIAAAKUAAAAGGxvY2EAjABWAAACrAAAAA5tYXhwAAgAFgAAArwAAAAgbmFtZfC1n04AAALcAAABPHBvc3QAAwAAAAAEGAAAACAAAwIAAZAABQAAAUwBZgAAAEcBTAFmAAAA9QAZAIQAAAAAAAAAAAAAAAAAAAABEAAAAAAAAAAAAAAAAAAAAABAAADw2gHg/+D/4AHgACAAAAABAAAAAAAAAAAAAAAgAAAAAAACAAAAAwAAABQAAwABAAAAFAAEADgAAAAKAAgAAgACAAEAIPDa//3//wAAAAAAIPDZ//3//wAB/+MPKwADAAEAAAAAAAAAAAAAAAEAAf//AA8AAQAAAAAAAAAAAAIAADc5AQAAAAABAAAAAAAAAAAAAgAANzkBAAAAAAEAAAAAAAAAAAACAAA3OQEAAAAAAQASAEkAtwFuABMAADc0PwE2FzYXFh0BFAcGJwYvASY1EgaABQgHBQYGBQcIBYAG2wcGfwcBAQcECf8IBAcBAQd/BgYAAAAAAQAAAEkApQFuABMAADcRNDc2MzIfARYVFA8BBiMiJyY1AAUGBwgFgAYGgAUIBwYFWwEACAUGBoAFCAcFgAYGBQcAAAABAAAAAQAAqWYls18PPPUACwIAAAAAAM/9o+4AAAAAz/2j7gAAAAAAtwFuAAAACAACAAAAAAAAAAEAAAHg/+AAAAIAAAAAAAC3AAEAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAAQAAAAC3ABIAtwAAAAAAAAAKABQAHgBCAGQAAAABAAAABgAUAAEAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAADgCuAAEAAAAAAAEADAAAAAEAAAAAAAIADgBAAAEAAAAAAAMADAAiAAEAAAAAAAQADABOAAEAAAAAAAUAFgAMAAEAAAAAAAYABgAuAAEAAAAAAAoANABaAAMAAQQJAAEADAAAAAMAAQQJAAIADgBAAAMAAQQJAAMADAAiAAMAAQQJAAQADABOAAMAAQQJAAUAFgAMAAMAAQQJAAYADAA0AAMAAQQJAAoANABaAHIAYQB0AGkAbgBnAFYAZQByAHMAaQBvAG4AIAAxAC4AMAByAGEAdABpAG4AZ3JhdGluZwByAGEAdABpAG4AZwBSAGUAZwB1AGwAYQByAHIAYQB0AGkAbgBnAEYAbwBuAHQAIABnAGUAbgBlAHIAYQB0AGUAZAAgAGIAeQAgAEkAYwBvAE0AbwBvAG4ALgADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA) - format("truetype"), - url(data:application/font-woff;charset=utf-8;base64,d09GRk9UVE8AAASwAAoAAAAABGgAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAABDRkYgAAAA9AAAAS0AAAEtFpovuE9TLzIAAAIkAAAAYAAAAGAIIweQY21hcAAAAoQAAABMAAAATA984gpnYXNwAAAC0AAAAAgAAAAIAAAAEGhlYWQAAALYAAAANgAAADb/0IhHaGhlYQAAAxAAAAAkAAAAJAKZAedobXR4AAADNAAAABgAAAAYAm4AEm1heHAAAANMAAAABgAAAAYABlAAbmFtZQAAA1QAAAE8AAABPPC1n05wb3N0AAAEkAAAACAAAAAgAAMAAAEABAQAAQEBB3JhdGluZwABAgABADr4HAL4GwP4GAQeCgAZU/+Lix4KABlT/4uLDAeLa/iU+HQFHQAAAHkPHQAAAH4RHQAAAAkdAAABJBIABwEBBw0PERQZHnJhdGluZ3JhdGluZ3UwdTF1MjB1RjBEOXVGMERBAAACAYkABAAGAQEEBwoNVp38lA78lA78lA77lA773Z33bxWLkI2Qj44I9xT3FAWOj5CNkIuQi4+JjoePiI2Gi4YIi/uUBYuGiYeHiIiHh4mGi4aLho2Ijwj7FPcUBYeOiY+LkAgO+92L5hWL95QFi5CNkI6Oj4+PjZCLkIuQiY6HCPcU+xQFj4iNhouGi4aJh4eICPsU+xQFiIeGiYaLhouHjYePiI6Jj4uQCA74lBT4lBWLDAoAAAAAAwIAAZAABQAAAUwBZgAAAEcBTAFmAAAA9QAZAIQAAAAAAAAAAAAAAAAAAAABEAAAAAAAAAAAAAAAAAAAAABAAADw2gHg/+D/4AHgACAAAAABAAAAAAAAAAAAAAAgAAAAAAACAAAAAwAAABQAAwABAAAAFAAEADgAAAAKAAgAAgACAAEAIPDa//3//wAAAAAAIPDZ//3//wAB/+MPKwADAAEAAAAAAAAAAAAAAAEAAf//AA8AAQAAAAEAADfYOJZfDzz1AAsCAAAAAADP/aPuAAAAAM/9o+4AAAAAALcBbgAAAAgAAgAAAAAAAAABAAAB4P/gAAACAAAAAAAAtwABAAAAAAAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAEAAAAAtwASALcAAAAAUAAABgAAAAAADgCuAAEAAAAAAAEADAAAAAEAAAAAAAIADgBAAAEAAAAAAAMADAAiAAEAAAAAAAQADABOAAEAAAAAAAUAFgAMAAEAAAAAAAYABgAuAAEAAAAAAAoANABaAAMAAQQJAAEADAAAAAMAAQQJAAIADgBAAAMAAQQJAAMADAAiAAMAAQQJAAQADABOAAMAAQQJAAUAFgAMAAMAAQQJAAYADAA0AAMAAQQJAAoANABaAHIAYQB0AGkAbgBnAFYAZQByAHMAaQBvAG4AIAAxAC4AMAByAGEAdABpAG4AZ3JhdGluZwByAGEAdABpAG4AZwBSAGUAZwB1AGwAYQByAHIAYQB0AGkAbgBnAEYAbwBuAHQAIABnAGUAbgBlAHIAYQB0AGUAZAAgAGIAeQAgAEkAYwBvAE0AbwBvAG4ALgADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA) - format("woff"); - font-weight: normal; - font-style: normal; -} - -/* Dropdown Icon */ -.ui.accordion .title .dropdown.icon, -.ui.accordion .accordion .title .dropdown.icon { - font-family: Accordion; - line-height: 1; - -webkit-backface-visibility: hidden; - backface-visibility: hidden; - font-weight: normal; - font-style: normal; - text-align: center; -} -.ui.accordion .title .dropdown.icon:before, -.ui.accordion .accordion .title .dropdown.icon:before { - content: "\f0da" /*rtl:'\f0d9'*/; -} - -/******************************* - User Overrides -*******************************/ diff --git a/docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/accordion.js b/docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/accordion.js deleted file mode 100755 index 971edbb7c..000000000 --- a/docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/accordion.js +++ /dev/null @@ -1,578 +0,0 @@ -/*! - * # Semantic UI 2.4.1 - Accordion - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */ - -(function ($, window, document, undefined) { - "use strict"; - - window = - typeof window != "undefined" && window.Math == Math - ? window - : typeof self != "undefined" && self.Math == Math - ? self - : Function("return this")(); - - $.fn.accordion = function (parameters) { - var $allModules = $(this), - time = new Date().getTime(), - performance = [], - query = arguments[0], - methodInvoked = typeof query == "string", - queryArguments = [].slice.call(arguments, 1), - requestAnimationFrame = - window.requestAnimationFrame || - window.mozRequestAnimationFrame || - window.webkitRequestAnimationFrame || - window.msRequestAnimationFrame || - function (callback) { - setTimeout(callback, 0); - }, - returnedValue; - $allModules.each(function () { - var settings = $.isPlainObject(parameters) - ? $.extend(true, {}, $.fn.accordion.settings, parameters) - : $.extend({}, $.fn.accordion.settings), - className = settings.className, - namespace = settings.namespace, - selector = settings.selector, - error = settings.error, - eventNamespace = "." + namespace, - moduleNamespace = "module-" + namespace, - moduleSelector = $allModules.selector || "", - $module = $(this), - $title = $module.find(selector.title), - $content = $module.find(selector.content), - element = this, - instance = $module.data(moduleNamespace), - observer, - module; - - module = { - initialize: function () { - module.debug("Initializing", $module); - module.bind.events(); - if (settings.observeChanges) { - module.observeChanges(); - } - module.instantiate(); - }, - - instantiate: function () { - instance = module; - $module.data(moduleNamespace, module); - }, - - destroy: function () { - module.debug("Destroying previous instance", $module); - $module.off(eventNamespace).removeData(moduleNamespace); - }, - - refresh: function () { - $title = $module.find(selector.title); - $content = $module.find(selector.content); - }, - - observeChanges: function () { - if ("MutationObserver" in window) { - observer = new MutationObserver(function (mutations) { - module.debug("DOM tree modified, updating selector cache"); - module.refresh(); - }); - observer.observe(element, { - childList: true, - subtree: true, - }); - module.debug("Setting up mutation observer", observer); - } - }, - - bind: { - events: function () { - module.debug("Binding delegated events"); - $module.on( - settings.on + eventNamespace, - selector.trigger, - module.event.click - ); - }, - }, - - event: { - click: function () { - module.toggle.call(this); - }, - }, - - toggle: function (query) { - var $activeTitle = - query !== undefined - ? typeof query === "number" - ? $title.eq(query) - : $(query).closest(selector.title) - : $(this).closest(selector.title), - $activeContent = $activeTitle.next($content), - isAnimating = $activeContent.hasClass(className.animating), - isActive = $activeContent.hasClass(className.active), - isOpen = isActive && !isAnimating, - isOpening = !isActive && isAnimating; - module.debug("Toggling visibility of content", $activeTitle); - if (isOpen || isOpening) { - if (settings.collapsible) { - module.close.call($activeTitle); - } else { - module.debug( - "Cannot close accordion content collapsing is disabled" - ); - } - } else { - module.open.call($activeTitle); - } - }, - - open: function (query) { - var $activeTitle = - query !== undefined - ? typeof query === "number" - ? $title.eq(query) - : $(query).closest(selector.title) - : $(this).closest(selector.title), - $activeContent = $activeTitle.next($content), - isAnimating = $activeContent.hasClass(className.animating), - isActive = $activeContent.hasClass(className.active), - isOpen = isActive || isAnimating; - if (isOpen) { - module.debug("Accordion already open, skipping", $activeContent); - return; - } - module.debug("Opening accordion content", $activeTitle); - settings.onOpening.call($activeContent); - settings.onChanging.call($activeContent); - if (settings.exclusive) { - module.closeOthers.call($activeTitle); - } - $activeTitle.addClass(className.active); - $activeContent.stop(true, true).addClass(className.animating); - if (settings.animateChildren) { - if ( - $.fn.transition !== undefined && - $module.transition("is supported") - ) { - $activeContent.children().transition({ - animation: "fade in", - queue: false, - useFailSafe: true, - debug: settings.debug, - verbose: settings.verbose, - duration: settings.duration, - }); - } else { - $activeContent.children().stop(true, true).animate( - { - opacity: 1, - }, - settings.duration, - module.resetOpacity - ); - } - } - $activeContent.slideDown( - settings.duration, - settings.easing, - function () { - $activeContent - .removeClass(className.animating) - .addClass(className.active); - module.reset.display.call(this); - settings.onOpen.call(this); - settings.onChange.call(this); - } - ); - }, - - close: function (query) { - var $activeTitle = - query !== undefined - ? typeof query === "number" - ? $title.eq(query) - : $(query).closest(selector.title) - : $(this).closest(selector.title), - $activeContent = $activeTitle.next($content), - isAnimating = $activeContent.hasClass(className.animating), - isActive = $activeContent.hasClass(className.active), - isOpening = !isActive && isAnimating, - isClosing = isActive && isAnimating; - if ((isActive || isOpening) && !isClosing) { - module.debug("Closing accordion content", $activeContent); - settings.onClosing.call($activeContent); - settings.onChanging.call($activeContent); - $activeTitle.removeClass(className.active); - $activeContent.stop(true, true).addClass(className.animating); - if (settings.animateChildren) { - if ( - $.fn.transition !== undefined && - $module.transition("is supported") - ) { - $activeContent.children().transition({ - animation: "fade out", - queue: false, - useFailSafe: true, - debug: settings.debug, - verbose: settings.verbose, - duration: settings.duration, - }); - } else { - $activeContent.children().stop(true, true).animate( - { - opacity: 0, - }, - settings.duration, - module.resetOpacity - ); - } - } - $activeContent.slideUp( - settings.duration, - settings.easing, - function () { - $activeContent - .removeClass(className.animating) - .removeClass(className.active); - module.reset.display.call(this); - settings.onClose.call(this); - settings.onChange.call(this); - } - ); - } - }, - - closeOthers: function (index) { - var $activeTitle = - index !== undefined - ? $title.eq(index) - : $(this).closest(selector.title), - $parentTitles = $activeTitle - .parents(selector.content) - .prev(selector.title), - $activeAccordion = $activeTitle.closest(selector.accordion), - activeSelector = - selector.title + "." + className.active + ":visible", - activeContent = - selector.content + "." + className.active + ":visible", - $openTitles, - $nestedTitles, - $openContents; - if (settings.closeNested) { - $openTitles = $activeAccordion - .find(activeSelector) - .not($parentTitles); - $openContents = $openTitles.next($content); - } else { - $openTitles = $activeAccordion - .find(activeSelector) - .not($parentTitles); - $nestedTitles = $activeAccordion - .find(activeContent) - .find(activeSelector) - .not($parentTitles); - $openTitles = $openTitles.not($nestedTitles); - $openContents = $openTitles.next($content); - } - if ($openTitles.length > 0) { - module.debug( - "Exclusive enabled, closing other content", - $openTitles - ); - $openTitles.removeClass(className.active); - $openContents.removeClass(className.animating).stop(true, true); - if (settings.animateChildren) { - if ( - $.fn.transition !== undefined && - $module.transition("is supported") - ) { - $openContents.children().transition({ - animation: "fade out", - useFailSafe: true, - debug: settings.debug, - verbose: settings.verbose, - duration: settings.duration, - }); - } else { - $openContents.children().stop(true, true).animate( - { - opacity: 0, - }, - settings.duration, - module.resetOpacity - ); - } - } - $openContents.slideUp( - settings.duration, - settings.easing, - function () { - $(this).removeClass(className.active); - module.reset.display.call(this); - } - ); - } - }, - - reset: { - display: function () { - module.verbose("Removing inline display from element", this); - $(this).css("display", ""); - if ($(this).attr("style") === "") { - $(this).attr("style", "").removeAttr("style"); - } - }, - - opacity: function () { - module.verbose("Removing inline opacity from element", this); - $(this).css("opacity", ""); - if ($(this).attr("style") === "") { - $(this).attr("style", "").removeAttr("style"); - } - }, - }, - - setting: function (name, value) { - module.debug("Changing setting", name, value); - if ($.isPlainObject(name)) { - $.extend(true, settings, name); - } else if (value !== undefined) { - if ($.isPlainObject(settings[name])) { - $.extend(true, settings[name], value); - } else { - settings[name] = value; - } - } else { - return settings[name]; - } - }, - internal: function (name, value) { - module.debug("Changing internal", name, value); - if (value !== undefined) { - if ($.isPlainObject(name)) { - $.extend(true, module, name); - } else { - module[name] = value; - } - } else { - return module[name]; - } - }, - debug: function () { - if (!settings.silent && settings.debug) { - if (settings.performance) { - module.performance.log(arguments); - } else { - module.debug = Function.prototype.bind.call( - console.info, - console, - settings.name + ":" - ); - module.debug.apply(console, arguments); - } - } - }, - verbose: function () { - if (!settings.silent && settings.verbose && settings.debug) { - if (settings.performance) { - module.performance.log(arguments); - } else { - module.verbose = Function.prototype.bind.call( - console.info, - console, - settings.name + ":" - ); - module.verbose.apply(console, arguments); - } - } - }, - error: function () { - if (!settings.silent) { - module.error = Function.prototype.bind.call( - console.error, - console, - settings.name + ":" - ); - module.error.apply(console, arguments); - } - }, - performance: { - log: function (message) { - var currentTime, executionTime, previousTime; - if (settings.performance) { - currentTime = new Date().getTime(); - previousTime = time || currentTime; - executionTime = currentTime - previousTime; - time = currentTime; - performance.push({ - Name: message[0], - Arguments: [].slice.call(message, 1) || "", - Element: element, - "Execution Time": executionTime, - }); - } - clearTimeout(module.performance.timer); - module.performance.timer = setTimeout( - module.performance.display, - 500 - ); - }, - display: function () { - var title = settings.name + ":", - totalTime = 0; - time = false; - clearTimeout(module.performance.timer); - $.each(performance, function (index, data) { - totalTime += data["Execution Time"]; - }); - title += " " + totalTime + "ms"; - if (moduleSelector) { - title += " '" + moduleSelector + "'"; - } - if ( - (console.group !== undefined || console.table !== undefined) && - performance.length > 0 - ) { - console.groupCollapsed(title); - if (console.table) { - console.table(performance); - } else { - $.each(performance, function (index, data) { - console.log( - data["Name"] + ": " + data["Execution Time"] + "ms" - ); - }); - } - console.groupEnd(); - } - performance = []; - }, - }, - invoke: function (query, passedArguments, context) { - var object = instance, - maxDepth, - found, - response; - passedArguments = passedArguments || queryArguments; - context = element || context; - if (typeof query == "string" && object !== undefined) { - query = query.split(/[\. ]/); - maxDepth = query.length - 1; - $.each(query, function (depth, value) { - var camelCaseValue = - depth != maxDepth - ? value + - query[depth + 1].charAt(0).toUpperCase() + - query[depth + 1].slice(1) - : query; - if ( - $.isPlainObject(object[camelCaseValue]) && - depth != maxDepth - ) { - object = object[camelCaseValue]; - } else if (object[camelCaseValue] !== undefined) { - found = object[camelCaseValue]; - return false; - } else if ($.isPlainObject(object[value]) && depth != maxDepth) { - object = object[value]; - } else if (object[value] !== undefined) { - found = object[value]; - return false; - } else { - module.error(error.method, query); - return false; - } - }); - } - if ($.isFunction(found)) { - response = found.apply(context, passedArguments); - } else if (found !== undefined) { - response = found; - } - if ($.isArray(returnedValue)) { - returnedValue.push(response); - } else if (returnedValue !== undefined) { - returnedValue = [returnedValue, response]; - } else if (response !== undefined) { - returnedValue = response; - } - return found; - }, - }; - if (methodInvoked) { - if (instance === undefined) { - module.initialize(); - } - module.invoke(query); - } else { - if (instance !== undefined) { - instance.invoke("destroy"); - } - module.initialize(); - } - }); - return returnedValue !== undefined ? returnedValue : this; - }; - - $.fn.accordion.settings = { - name: "Accordion", - namespace: "accordion", - - silent: false, - debug: false, - verbose: false, - performance: true, - - on: "click", // event on title that opens accordion - - observeChanges: true, // whether accordion should automatically refresh on DOM insertion - - exclusive: true, // whether a single accordion content panel should be open at once - collapsible: true, // whether accordion content can be closed - closeNested: false, // whether nested content should be closed when a panel is closed - animateChildren: true, // whether children opacity should be animated - - duration: 350, // duration of animation - easing: "easeOutQuad", // easing equation for animation - - onOpening: function () {}, // callback before open animation - onClosing: function () {}, // callback before closing animation - onChanging: function () {}, // callback before closing or opening animation - - onOpen: function () {}, // callback after open animation - onClose: function () {}, // callback after closing animation - onChange: function () {}, // callback after closing or opening animation - - error: { - method: "The method you called is not defined", - }, - - className: { - active: "active", - animating: "animating", - }, - - selector: { - accordion: ".accordion", - title: ".title", - trigger: ".title", - content: ".content", - }, - }; - - // Adds easing - $.extend($.easing, { - easeOutQuad: function (x, t, b, c, d) { - return -c * (t /= d) * (t - 2) + b; - }, - }); -})(jQuery, window, document); diff --git a/docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/accordion.min.css b/docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/accordion.min.css deleted file mode 100755 index 023b2443a..000000000 --- a/docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/accordion.min.css +++ /dev/null @@ -1,170 +0,0 @@ -/*! - * # Semantic UI 2.4.1 - Accordion - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */ -.ui.accordion, -.ui.accordion .accordion { - max-width: 100%; -} -.ui.accordion .accordion { - margin: 1em 0 0; - padding: 0; -} -.ui.accordion .accordion .title, -.ui.accordion .title { - cursor: pointer; -} -.ui.accordion .title:not(.ui) { - padding: 0.5em 0; - font-family: Lato, "Helvetica Neue", Arial, Helvetica, sans-serif; - font-size: 1em; - color: rgba(0, 0, 0, 0.87); -} -.ui.accordion .accordion .title ~ .content, -.ui.accordion .title ~ .content { - display: none; -} -.ui.accordion:not(.styled) .accordion .title ~ .content:not(.ui), -.ui.accordion:not(.styled) .title ~ .content:not(.ui) { - margin: ""; - padding: 0.5em 0 1em; -} -.ui.accordion:not(.styled) .title ~ .content:not(.ui):last-child { - padding-bottom: 0; -} -.ui.accordion .accordion .title .dropdown.icon, -.ui.accordion .title .dropdown.icon { - display: inline-block; - float: none; - opacity: 1; - width: 1.25em; - height: 1em; - margin: 0 0.25rem 0 0; - padding: 0; - font-size: 1em; - -webkit-transition: opacity 0.1s ease, -webkit-transform 0.1s ease; - transition: opacity 0.1s ease, -webkit-transform 0.1s ease; - transition: transform 0.1s ease, opacity 0.1s ease; - transition: transform 0.1s ease, opacity 0.1s ease, - -webkit-transform 0.1s ease; - vertical-align: baseline; - -webkit-transform: none; - transform: none; -} -.ui.accordion.menu .item .title { - display: block; - padding: 0; -} -.ui.accordion.menu .item .title > .dropdown.icon { - float: right; - margin: 0.21425em 0 0 1em; - -webkit-transform: rotate(180deg); - transform: rotate(180deg); -} -.ui.accordion .ui.header .dropdown.icon { - font-size: 1em; - margin: 0 0.25rem 0 0; -} -.ui.accordion .accordion .active.title .dropdown.icon, -.ui.accordion .active.title .dropdown.icon { - -webkit-transform: rotate(90deg); - transform: rotate(90deg); -} -.ui.accordion.menu .item .active.title > .dropdown.icon { - -webkit-transform: rotate(90deg); - transform: rotate(90deg); -} -.ui.styled.accordion { - width: 600px; -} -.ui.styled.accordion, -.ui.styled.accordion .accordion { - border-radius: 0.28571429rem; - background: #fff; - -webkit-box-shadow: 0 1px 2px 0 rgba(34, 36, 38, 0.15), - 0 0 0 1px rgba(34, 36, 38, 0.15); - box-shadow: 0 1px 2px 0 rgba(34, 36, 38, 0.15), - 0 0 0 1px rgba(34, 36, 38, 0.15); -} -.ui.styled.accordion .accordion .title, -.ui.styled.accordion .title { - margin: 0; - padding: 0.75em 1em; - color: rgba(0, 0, 0, 0.4); - font-weight: 700; - border-top: 1px solid rgba(34, 36, 38, 0.15); - -webkit-transition: background 0.1s ease, color 0.1s ease; - transition: background 0.1s ease, color 0.1s ease; -} -.ui.styled.accordion .accordion .title:first-child, -.ui.styled.accordion > .title:first-child { - border-top: none; -} -.ui.styled.accordion .accordion .content, -.ui.styled.accordion .content { - margin: 0; - padding: 0.5em 1em 1.5em; -} -.ui.styled.accordion .accordion .content { - padding: 0; - padding: 0.5em 1em 1.5em; -} -.ui.styled.accordion .accordion .active.title, -.ui.styled.accordion .accordion .title:hover, -.ui.styled.accordion .active.title, -.ui.styled.accordion .title:hover { - background: 0 0; - color: rgba(0, 0, 0, 0.87); -} -.ui.styled.accordion .accordion .active.title, -.ui.styled.accordion .accordion .title:hover { - background: 0 0; - color: rgba(0, 0, 0, 0.87); -} -.ui.styled.accordion .active.title { - background: 0 0; - color: rgba(0, 0, 0, 0.95); -} -.ui.styled.accordion .accordion .active.title { - background: 0 0; - color: rgba(0, 0, 0, 0.95); -} -.ui.accordion .accordion .active.content, -.ui.accordion .active.content { - display: block; -} -.ui.fluid.accordion, -.ui.fluid.accordion .accordion { - width: 100%; -} -.ui.inverted.accordion .title:not(.ui) { - color: rgba(255, 255, 255, 0.9); -} -@font-face { - font-family: Accordion; - src: url(data:application/x-font-ttf;charset=utf-8;base64,AAEAAAALAIAAAwAwT1MvMggjB5AAAAC8AAAAYGNtYXAPfOIKAAABHAAAAExnYXNwAAAAEAAAAWgAAAAIZ2x5Zryj6HgAAAFwAAAAyGhlYWT/0IhHAAACOAAAADZoaGVhApkB5wAAAnAAAAAkaG10eAJuABIAAAKUAAAAGGxvY2EAjABWAAACrAAAAA5tYXhwAAgAFgAAArwAAAAgbmFtZfC1n04AAALcAAABPHBvc3QAAwAAAAAEGAAAACAAAwIAAZAABQAAAUwBZgAAAEcBTAFmAAAA9QAZAIQAAAAAAAAAAAAAAAAAAAABEAAAAAAAAAAAAAAAAAAAAABAAADw2gHg/+D/4AHgACAAAAABAAAAAAAAAAAAAAAgAAAAAAACAAAAAwAAABQAAwABAAAAFAAEADgAAAAKAAgAAgACAAEAIPDa//3//wAAAAAAIPDZ//3//wAB/+MPKwADAAEAAAAAAAAAAAAAAAEAAf//AA8AAQAAAAAAAAAAAAIAADc5AQAAAAABAAAAAAAAAAAAAgAANzkBAAAAAAEAAAAAAAAAAAACAAA3OQEAAAAAAQASAEkAtwFuABMAADc0PwE2FzYXFh0BFAcGJwYvASY1EgaABQgHBQYGBQcIBYAG2wcGfwcBAQcECf8IBAcBAQd/BgYAAAAAAQAAAEkApQFuABMAADcRNDc2MzIfARYVFA8BBiMiJyY1AAUGBwgFgAYGgAUIBwYFWwEACAUGBoAFCAcFgAYGBQcAAAABAAAAAQAAqWYls18PPPUACwIAAAAAAM/9o+4AAAAAz/2j7gAAAAAAtwFuAAAACAACAAAAAAAAAAEAAAHg/+AAAAIAAAAAAAC3AAEAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAAQAAAAC3ABIAtwAAAAAAAAAKABQAHgBCAGQAAAABAAAABgAUAAEAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAADgCuAAEAAAAAAAEADAAAAAEAAAAAAAIADgBAAAEAAAAAAAMADAAiAAEAAAAAAAQADABOAAEAAAAAAAUAFgAMAAEAAAAAAAYABgAuAAEAAAAAAAoANABaAAMAAQQJAAEADAAAAAMAAQQJAAIADgBAAAMAAQQJAAMADAAiAAMAAQQJAAQADABOAAMAAQQJAAUAFgAMAAMAAQQJAAYADAA0AAMAAQQJAAoANABaAHIAYQB0AGkAbgBnAFYAZQByAHMAaQBvAG4AIAAxAC4AMAByAGEAdABpAG4AZ3JhdGluZwByAGEAdABpAG4AZwBSAGUAZwB1AGwAYQByAHIAYQB0AGkAbgBnAEYAbwBuAHQAIABnAGUAbgBlAHIAYQB0AGUAZAAgAGIAeQAgAEkAYwBvAE0AbwBvAG4ALgADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA) - format("truetype"), - url(data:application/font-woff;charset=utf-8;base64,d09GRk9UVE8AAASwAAoAAAAABGgAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAABDRkYgAAAA9AAAAS0AAAEtFpovuE9TLzIAAAIkAAAAYAAAAGAIIweQY21hcAAAAoQAAABMAAAATA984gpnYXNwAAAC0AAAAAgAAAAIAAAAEGhlYWQAAALYAAAANgAAADb/0IhHaGhlYQAAAxAAAAAkAAAAJAKZAedobXR4AAADNAAAABgAAAAYAm4AEm1heHAAAANMAAAABgAAAAYABlAAbmFtZQAAA1QAAAE8AAABPPC1n05wb3N0AAAEkAAAACAAAAAgAAMAAAEABAQAAQEBB3JhdGluZwABAgABADr4HAL4GwP4GAQeCgAZU/+Lix4KABlT/4uLDAeLa/iU+HQFHQAAAHkPHQAAAH4RHQAAAAkdAAABJBIABwEBBw0PERQZHnJhdGluZ3JhdGluZ3UwdTF1MjB1RjBEOXVGMERBAAACAYkABAAGAQEEBwoNVp38lA78lA78lA77lA773Z33bxWLkI2Qj44I9xT3FAWOj5CNkIuQi4+JjoePiI2Gi4YIi/uUBYuGiYeHiIiHh4mGi4aLho2Ijwj7FPcUBYeOiY+LkAgO+92L5hWL95QFi5CNkI6Oj4+PjZCLkIuQiY6HCPcU+xQFj4iNhouGi4aJh4eICPsU+xQFiIeGiYaLhouHjYePiI6Jj4uQCA74lBT4lBWLDAoAAAAAAwIAAZAABQAAAUwBZgAAAEcBTAFmAAAA9QAZAIQAAAAAAAAAAAAAAAAAAAABEAAAAAAAAAAAAAAAAAAAAABAAADw2gHg/+D/4AHgACAAAAABAAAAAAAAAAAAAAAgAAAAAAACAAAAAwAAABQAAwABAAAAFAAEADgAAAAKAAgAAgACAAEAIPDa//3//wAAAAAAIPDZ//3//wAB/+MPKwADAAEAAAAAAAAAAAAAAAEAAf//AA8AAQAAAAEAADfYOJZfDzz1AAsCAAAAAADP/aPuAAAAAM/9o+4AAAAAALcBbgAAAAgAAgAAAAAAAAABAAAB4P/gAAACAAAAAAAAtwABAAAAAAAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAEAAAAAtwASALcAAAAAUAAABgAAAAAADgCuAAEAAAAAAAEADAAAAAEAAAAAAAIADgBAAAEAAAAAAAMADAAiAAEAAAAAAAQADABOAAEAAAAAAAUAFgAMAAEAAAAAAAYABgAuAAEAAAAAAAoANABaAAMAAQQJAAEADAAAAAMAAQQJAAIADgBAAAMAAQQJAAMADAAiAAMAAQQJAAQADABOAAMAAQQJAAUAFgAMAAMAAQQJAAYADAA0AAMAAQQJAAoANABaAHIAYQB0AGkAbgBnAFYAZQByAHMAaQBvAG4AIAAxAC4AMAByAGEAdABpAG4AZ3JhdGluZwByAGEAdABpAG4AZwBSAGUAZwB1AGwAYQByAHIAYQB0AGkAbgBnAEYAbwBuAHQAIABnAGUAbgBlAHIAYQB0AGUAZAAgAGIAeQAgAEkAYwBvAE0AbwBvAG4ALgADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA) - format("woff"); - font-weight: 400; - font-style: normal; -} -.ui.accordion .accordion .title .dropdown.icon, -.ui.accordion .title .dropdown.icon { - font-family: Accordion; - line-height: 1; - -webkit-backface-visibility: hidden; - backface-visibility: hidden; - font-weight: 400; - font-style: normal; - text-align: center; -} -.ui.accordion .accordion .title .dropdown.icon:before, -.ui.accordion .title .dropdown.icon:before { - content: "\f0da"; -} diff --git a/docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/accordion.min.js b/docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/accordion.min.js deleted file mode 100755 index 662c70eaa..000000000 --- a/docs/source/_ext/sphinx_accordion/semantic-ui-2.4.2/accordion.min.js +++ /dev/null @@ -1,391 +0,0 @@ -!(function (F, A, e, q) { - "use strict"; - (A = - void 0 !== A && A.Math == Math - ? A - : "undefined" != typeof self && self.Math == Math - ? self - : Function("return this")()), - (F.fn.accordion = function (a) { - var v, - s = F(this), - b = new Date().getTime(), - y = [], - C = a, - O = "string" == typeof C, - x = [].slice.call(arguments, 1); - A.requestAnimationFrame || - A.mozRequestAnimationFrame || - A.webkitRequestAnimationFrame || - A.msRequestAnimationFrame; - return ( - s.each(function () { - var e, - c, - u = F.isPlainObject(a) - ? F.extend(!0, {}, F.fn.accordion.settings, a) - : F.extend({}, F.fn.accordion.settings), - d = u.className, - n = u.namespace, - g = u.selector, - l = u.error, - t = "." + n, - i = "module-" + n, - o = s.selector || "", - f = F(this), - m = f.find(g.title), - p = f.find(g.content), - r = this, - h = f.data(i); - (c = { - initialize: function () { - c.debug("Initializing", f), - c.bind.events(), - u.observeChanges && c.observeChanges(), - c.instantiate(); - }, - instantiate: function () { - (h = c), f.data(i, c); - }, - destroy: function () { - c.debug("Destroying previous instance", f), - f.off(t).removeData(i); - }, - refresh: function () { - (m = f.find(g.title)), (p = f.find(g.content)); - }, - observeChanges: function () { - "MutationObserver" in A && - ((e = new MutationObserver(function (e) { - c.debug("DOM tree modified, updating selector cache"), - c.refresh(); - })).observe(r, { childList: !0, subtree: !0 }), - c.debug("Setting up mutation observer", e)); - }, - bind: { - events: function () { - c.debug("Binding delegated events"), - f.on(u.on + t, g.trigger, c.event.click); - }, - }, - event: { - click: function () { - c.toggle.call(this); - }, - }, - toggle: function (e) { - var n = - e !== q - ? "number" == typeof e - ? m.eq(e) - : F(e).closest(g.title) - : F(this).closest(g.title), - t = n.next(p), - i = t.hasClass(d.animating), - o = t.hasClass(d.active), - a = o && !i, - s = !o && i; - c.debug("Toggling visibility of content", n), - a || s - ? u.collapsible - ? c.close.call(n) - : c.debug( - "Cannot close accordion content collapsing is disabled" - ) - : c.open.call(n); - }, - open: function (e) { - var n = - e !== q - ? "number" == typeof e - ? m.eq(e) - : F(e).closest(g.title) - : F(this).closest(g.title), - t = n.next(p), - i = t.hasClass(d.animating); - t.hasClass(d.active) || i - ? c.debug("Accordion already open, skipping", t) - : (c.debug("Opening accordion content", n), - u.onOpening.call(t), - u.onChanging.call(t), - u.exclusive && c.closeOthers.call(n), - n.addClass(d.active), - t.stop(!0, !0).addClass(d.animating), - u.animateChildren && - (F.fn.transition !== q && f.transition("is supported") - ? t.children().transition({ - animation: "fade in", - queue: !1, - useFailSafe: !0, - debug: u.debug, - verbose: u.verbose, - duration: u.duration, - }) - : t - .children() - .stop(!0, !0) - .animate({ opacity: 1 }, u.duration, c.resetOpacity)), - t.slideDown(u.duration, u.easing, function () { - t.removeClass(d.animating).addClass(d.active), - c.reset.display.call(this), - u.onOpen.call(this), - u.onChange.call(this); - })); - }, - close: function (e) { - var n = - e !== q - ? "number" == typeof e - ? m.eq(e) - : F(e).closest(g.title) - : F(this).closest(g.title), - t = n.next(p), - i = t.hasClass(d.animating), - o = t.hasClass(d.active); - (!o && !(!o && i)) || - (o && i) || - (c.debug("Closing accordion content", t), - u.onClosing.call(t), - u.onChanging.call(t), - n.removeClass(d.active), - t.stop(!0, !0).addClass(d.animating), - u.animateChildren && - (F.fn.transition !== q && f.transition("is supported") - ? t.children().transition({ - animation: "fade out", - queue: !1, - useFailSafe: !0, - debug: u.debug, - verbose: u.verbose, - duration: u.duration, - }) - : t - .children() - .stop(!0, !0) - .animate({ opacity: 0 }, u.duration, c.resetOpacity)), - t.slideUp(u.duration, u.easing, function () { - t.removeClass(d.animating).removeClass(d.active), - c.reset.display.call(this), - u.onClose.call(this), - u.onChange.call(this); - })); - }, - closeOthers: function (e) { - var n, - t, - i, - o = e !== q ? m.eq(e) : F(this).closest(g.title), - a = o.parents(g.content).prev(g.title), - s = o.closest(g.accordion), - l = g.title + "." + d.active + ":visible", - r = g.content + "." + d.active + ":visible"; - (i = u.closeNested - ? (n = s.find(l).not(a)).next(p) - : ((n = s.find(l).not(a)), - (t = s.find(r).find(l).not(a)), - (n = n.not(t)).next(p))), - 0 < n.length && - (c.debug("Exclusive enabled, closing other content", n), - n.removeClass(d.active), - i.removeClass(d.animating).stop(!0, !0), - u.animateChildren && - (F.fn.transition !== q && f.transition("is supported") - ? i.children().transition({ - animation: "fade out", - useFailSafe: !0, - debug: u.debug, - verbose: u.verbose, - duration: u.duration, - }) - : i - .children() - .stop(!0, !0) - .animate({ opacity: 0 }, u.duration, c.resetOpacity)), - i.slideUp(u.duration, u.easing, function () { - F(this).removeClass(d.active), c.reset.display.call(this); - })); - }, - reset: { - display: function () { - c.verbose("Removing inline display from element", this), - F(this).css("display", ""), - "" === F(this).attr("style") && - F(this).attr("style", "").removeAttr("style"); - }, - opacity: function () { - c.verbose("Removing inline opacity from element", this), - F(this).css("opacity", ""), - "" === F(this).attr("style") && - F(this).attr("style", "").removeAttr("style"); - }, - }, - setting: function (e, n) { - if ((c.debug("Changing setting", e, n), F.isPlainObject(e))) - F.extend(!0, u, e); - else { - if (n === q) return u[e]; - F.isPlainObject(u[e]) ? F.extend(!0, u[e], n) : (u[e] = n); - } - }, - internal: function (e, n) { - if ((c.debug("Changing internal", e, n), n === q)) return c[e]; - F.isPlainObject(e) ? F.extend(!0, c, e) : (c[e] = n); - }, - debug: function () { - !u.silent && - u.debug && - (u.performance - ? c.performance.log(arguments) - : ((c.debug = Function.prototype.bind.call( - console.info, - console, - u.name + ":" - )), - c.debug.apply(console, arguments))); - }, - verbose: function () { - !u.silent && - u.verbose && - u.debug && - (u.performance - ? c.performance.log(arguments) - : ((c.verbose = Function.prototype.bind.call( - console.info, - console, - u.name + ":" - )), - c.verbose.apply(console, arguments))); - }, - error: function () { - u.silent || - ((c.error = Function.prototype.bind.call( - console.error, - console, - u.name + ":" - )), - c.error.apply(console, arguments)); - }, - performance: { - log: function (e) { - var n, t; - u.performance && - ((t = (n = new Date().getTime()) - (b || n)), - (b = n), - y.push({ - Name: e[0], - Arguments: [].slice.call(e, 1) || "", - Element: r, - "Execution Time": t, - })), - clearTimeout(c.performance.timer), - (c.performance.timer = setTimeout( - c.performance.display, - 500 - )); - }, - display: function () { - var e = u.name + ":", - t = 0; - (b = !1), - clearTimeout(c.performance.timer), - F.each(y, function (e, n) { - t += n["Execution Time"]; - }), - (e += " " + t + "ms"), - o && (e += " '" + o + "'"), - (console.group !== q || console.table !== q) && - 0 < y.length && - (console.groupCollapsed(e), - console.table - ? console.table(y) - : F.each(y, function (e, n) { - console.log( - n.Name + ": " + n["Execution Time"] + "ms" - ); - }), - console.groupEnd()), - (y = []); - }, - }, - invoke: function (i, e, n) { - var o, - a, - t, - s = h; - return ( - (e = e || x), - (n = r || n), - "string" == typeof i && - s !== q && - ((i = i.split(/[\. ]/)), - (o = i.length - 1), - F.each(i, function (e, n) { - var t = - e != o - ? n + - i[e + 1].charAt(0).toUpperCase() + - i[e + 1].slice(1) - : i; - if (F.isPlainObject(s[t]) && e != o) s = s[t]; - else { - if (s[t] !== q) return (a = s[t]), !1; - if (!F.isPlainObject(s[n]) || e == o) - return ( - s[n] !== q ? (a = s[n]) : c.error(l.method, i), !1 - ); - s = s[n]; - } - })), - F.isFunction(a) ? (t = a.apply(n, e)) : a !== q && (t = a), - F.isArray(v) - ? v.push(t) - : v !== q - ? (v = [v, t]) - : t !== q && (v = t), - a - ); - }, - }), - O - ? (h === q && c.initialize(), c.invoke(C)) - : (h !== q && h.invoke("destroy"), c.initialize()); - }), - v !== q ? v : this - ); - }), - (F.fn.accordion.settings = { - name: "Accordion", - namespace: "accordion", - silent: !1, - debug: !1, - verbose: !1, - performance: !0, - on: "click", - observeChanges: !0, - exclusive: !0, - collapsible: !0, - closeNested: !1, - animateChildren: !0, - duration: 350, - easing: "easeOutQuad", - onOpening: function () {}, - onClosing: function () {}, - onChanging: function () {}, - onOpen: function () {}, - onClose: function () {}, - onChange: function () {}, - error: { method: "The method you called is not defined" }, - className: { active: "active", animating: "animating" }, - selector: { - accordion: ".accordion", - title: ".title", - trigger: ".title", - content: ".content", - }, - }), - F.extend(F.easing, { - easeOutQuad: function (e, n, t, i, o) { - return -i * (n /= o) * (n - 2) + t; - }, - }); -})(jQuery, window, document); diff --git a/docs/source/api.rst b/docs/source/api.rst deleted file mode 100644 index 648194eb6..000000000 --- a/docs/source/api.rst +++ /dev/null @@ -1,79 +0,0 @@ -=== -API -=== - -Datafile -======== -.. autoclass:: octue.resources.datafile.Datafile - :inherited-members: Filterable - :exclude-members: hash_non_class_object - - -Dataset -======= -.. autoclass:: octue.resources.dataset.Dataset - :exclude-members: hash_non_class_object - - -Manifest -======== -.. autoclass:: octue.resources.manifest.Manifest - :inherited-members: hash_non_class_object - - -Analysis -======== -.. autoclass:: octue.twined.resources.analysis.Analysis - :inherited-members: - - -Child -===== -.. autoclass:: octue.twined.resources.child.Child - - -Child emulator -============== -.. autoclass:: octue.twined.cloud.emulators.child.ChildEmulator - - -Filter containers -================= - -FilterSet ---------- -.. autoclass:: octue.resources.filter_containers.FilterSet - :inherited-members: set - -FilterList ----------- -.. autoclass:: octue.resources.filter_containers.FilterList - :inherited-members: list - -FilterDict ----------- -.. autoclass:: octue.resources.filter_containers.FilterDict - :inherited-members: UserDict - - -Configuration -============= - -Service configuration ---------------------- -.. autoclass:: octue.twined.configuration.ServiceConfiguration - - -Runner -====== -.. autoclass:: octue.twined.runner.Runner - - -Octue essential monitor messages -================================ -.. automodule:: octue.twined.essentials.monitor_messages - - -Octue log handler -================= -.. autofunction:: octue.log_handlers.apply_log_handler diff --git a/docs/source/asking_questions.rst b/docs/source/asking_questions.rst deleted file mode 100644 index 6c20deb71..000000000 --- a/docs/source/asking_questions.rst +++ /dev/null @@ -1,393 +0,0 @@ -.. _asking_questions: - -========================= -Asking services questions -========================= - -What is a question? -=================== -A question is a set of data (input values and/or an input manifest) sent to a child for processing/analysis. Questions -can be: - -- **Synchronous ("ask-and-wait"):** A question whose answer is waited for in real time - -- **Asynchronous ("fire-and-forget"):** A question whose answer is not waited for and is instead retrieved later. There - are two types: - - - **Regular:** Responses to these questions are automatically stored in an event store where they can be :ref:`retrieved using the Octue SDK ` - - - **Push endpoint:** Responses to these questions are pushed to an HTTP endpoint for asynchronous handling using Octue's - `django-twined `_ or custom logic in your own webserver. - -Questions are always asked to a *revision* of a service. You can ask a service a question if you have its -:ref:`SRUID `, project ID, and the necessary permissions. - - -Asking a question -================= - -.. important:: - If you're using an environment other than the ``main`` environment, then before asking any questions to your Twined - services, set the ``TWINED_SERVICES_TOPIC_NAME`` environment variable to the name of the Twined services Pub/Sub - topic (this is set when :ref:`deploying a service network `). It will be in the form - ``.octue.twined.services`` - -.. code-block:: python - - from octue.twined.resources import Child - - child = Child( - id="my-organisation/my-service:2.1.7", - backend={"name": "GCPPubSubBackend", "project_id": "my-project"}, - ) - - answer, question_uuid = child.ask( - input_values={"height": 32, "width": 3}, - input_manifest=manifest, - ) - - answer["output_values"] - >>> {"some": "data"} - - answer["output_manifest"]["my_dataset"].files - >>> , })> - - -.. _using_default_revision_tag: - -.. note:: - - Not including a service revision tag will cause your question to be sent to the default revision of the service - (usually the latest deployed version). This is determined by making a request to a `service registry - `_ if one or more - :ref:`registries are defined `. If none of the service registries contain an entry for - this service, a specific service revision tag must be used. - -You can also set the following options when you call :mod:`Child.ask `: - -- ``children`` - If the child has children of its own (i.e. grandchildren of the parent), this optional argument can be used to override the child's "default" children. This allows you to specify particular versions of grandchildren to use (see :ref:`this subsection below `). -- ``subscribe_to_logs`` - if true, the child will forward its logs to you -- ``allow_local_files`` - if true, local files/datasets are allowed in any input manifest you supply -- ``handle_monitor_message`` - if provided a function, it will be called on any monitor messages from the child -- ``record_events`` – if true, events received from the parent while it processes the question are saved to the ``Child.received_events`` property -- ``save_diagnostics`` – must be one of {"SAVE_DIAGNOSTICS_OFF", "SAVE_DIAGNOSTICS_ON_CRASH", "SAVE_DIAGNOSTICS_ON"}; if turned on, allow the input values and manifest (and its datasets) to be saved by the child either all the time or just if the analysis fails -- ``question_uuid`` - if provided, the question will use this UUID instead of a generated one -- ``push_endpoint`` - if provided, the result and other events produced during the processing of the question will be pushed to this HTTP endpoint (a URL) -- ``asynchronous`` - if true, don't wait for an answer to the question (the result and other events can be :ref:`retrieved from the event store later `) -- ``cpus`` - the number of CPUs to request for the question; defaults to the number set by the child service -- ``memory`` - the amount of memory to request for the question e.g. "256Mi" or "1Gi"; defaults to the amount set by the child service -- ``ephemeral_storage`` - the amount of ephemeral storage to request for the question e.g. "256Mi" or "1Gi"; defaults to the amount set by the child service -- ``timeout`` - how long in seconds to wait for an answer (``None`` by default - i.e. don't time out) - -If the question fails: - -- If ``raise_errors=False``, the unraised error is returned -- If ``raise_errors=False`` and ``max_retries > 0``, the question is retried up to this number of times -- If ``raise_errors=False``, ``max_retries > 0``, and ``prevent_retries_when`` is a list of exception types, the question is retried unless the error type is in the list -- If ``raise_errors=False``, ``log_errors=True``, and the question fails after its final retry, the error is logged - - -Exceptions raised by a child ----------------------------- -If a child raises an exception while processing your question, the exception will always be forwarded and re-raised in -your local service or python session. You can handle exceptions in whatever way you like. - -Timeouts --------- -If setting a timeout, bear in mind that the question has to reach the child, the child has to run its analysis on the -inputs sent to it (this will most likely make up the dominant part of the wait time), and the answer has to be sent back -to the parent. If you're not sure how long a particular analysis might take, it's best to set the timeout to ``None`` -initially or ask the owner/maintainer of the child for an estimate. - - -.. _retrieving_asynchronous_answers: - -Retrieving answers to asynchronous questions -============================================ -To retrieve results and other events from the processing of a question later, make sure you have the permissions to -access the event store and run: - -.. code-block:: python - - from octue.twined.cloud.pub_sub.bigquery import get_events - - events = get_events(question_uuid="53353901-0b47-44e7-9da3-a3ed59990a71") - - -**Options** - -- ``table_id`` - If you're not using the standard deployment, you can specify a different table here -- ``question_uuid`` - Retrieve events from this specific question -- ``parent_question_uuid`` - Retrieve events from questions triggered by the same parent question (this doesn't include the parent question's events) -- ``originator_question_uuid`` - Retrieve events for the entire tree of questions triggered by an originator question (a question asked manually through ``Child.ask``; this does include the originator question's events) -- ``kind`` - Only retrieve this kind of event if present (e.g. "result") -- ``include_backend_metadata`` - If ``True``, retrieve information about the service backend that produced the event -- ``limit`` - If set to a positive integer, limit the number of events returned to this - -.. note:: - - Only one of ``question_uuid``, ``parent_question_uuid``, and ``originator_question_uuid`` can be provided at one time. - - -.. collapse:: See an example output here... - - .. code-block:: python - - >>> events - [ - { - "event": { - "kind": "delivery_acknowledgement", - }, - }, - { - "event": { - "kind": "log_record", - "log_record": { - "args": null, - "created": 1709739861.5949728, - "exc_info": null, - "exc_text": null, - "filename": "app.py", - "funcName": "run", - "levelname": "INFO", - "levelno": 20, - "lineno": 28, - "module": "app", - "msecs": 594.9728488922119, - "msg": "Finished example analysis.", - "name": "app", - "pathname": "/workspace/example_service_cloud_run/app.py", - "process": 2, - "processName": "MainProcess", - "relativeCreated": 8560.13798713684, - "stack_info": null, - "thread": 68328473233152, - "threadName": "ThreadPoolExecutor-0_2" - } - }, - }, - { - "event": { - "kind": "heartbeat", - }, - }, - { - "event": { - "kind": "result", - "output_manifest": { - "datasets": { - "example_dataset": { - "files": [ - "gs://octue-sdk-python-test-bucket/example_output_datasets/example_dataset/output.dat" - ], - "id": "419bff6b-08c3-4c16-9eb1-5d1709168003", - "labels": [], - "name": "divergent-strange-gharial-of-pizza", - "path": "https://storage.googleapis.com/octue-sdk-python-test-bucket/example_output_datasets/example_dataset/.signed_metadata_files/divergent-strange-gharial-of-pizza", - "tags": {} - } - }, - "id": "a13713ae-f207-41c6-9e29-0a848ced6039", - "name": null - }, - "output_values": [1, 2, 3, 4, 5] - }, - }, - ] - ----- - -Asking multiple questions in parallel -===================================== -You can also ask multiple questions to a service in parallel - just provide questions as dictionaries of `Child.ask` -arguments: - -.. code-block:: python - - child.ask_multiple( - {"input_values": {"height": 32, "width": 3}}, - {"input_values": {"height": 12, "width": 10}}, - {"input_values": {"height": 7, "width": 32}}, - ) - >>> [ - ({"output_values": {"some": "output"}, "output_manifest": None}, '2681ef4e-4ab7-4cf9-8783-aad982d5e324'), - ({"output_values": {"another": "result"}, "output_manifest": None}, '474923bd-14b6-4f4c-9bfe-8148358f35cd'), - ({"output_values": {"different": "result"}, "output_manifest": None}, '9a50daae-2328-4728-9ddd-b2252474f118'), - ] - -This method uses multithreading, allowing all the questions to be asked at once instead of one after another. - -.. hint:: - - The maximum number of threads that can be used to ask questions in parallel can be set via the ``max_workers`` - argument. It has no effect on the total number of questions that can be asked, just how many can be in progress at - once. - - -Asking a question within a service -================================== -If you have :doc:`created your own Twined service ` and want to ask children questions, you can do -this more easily than above. Children are accessible from the ``analysis`` object by the keys you give them in the -:ref:`service configuration ` file. For example, you can ask an ``elevation`` service a question like -this: - -.. code-block:: python - - answer, question_uuid = analysis.children["elevation"].ask(input_values={"longitude": 0, "latitude": 1}) - -if these values are in your service configuration file: - -.. code-block:: json - - { - "children": [ - { - "key": "wind_speed", - "id": "template-child-services/wind-speed-service:2.1.1", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "my-project" - } - }, - { - "key": "elevation", - "id": "template-child-services/elevation-service:3.1.9", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "my-project" - } - } - ] - } - -and your ``twine.json`` file includes the child keys in its ``children`` field: - -.. code-block:: json - - { - "children": [ - { - "key": "wind_speed", - "purpose": "A service that returns the average wind speed for a given latitude and longitude.", - }, - { - "key": "elevation", - "purpose": "A service that returns the elevation for a given latitude and longitude.", - } - ] - } - -See the parent service's `service configuration `_ -and `app.py file `_ -in the `child-services app template `_ -to see this in action. - -.. _overriding_children: - -Overriding a child's children -============================= -If the child you're asking a question to has its own children (static children), you can override these by providing the -IDs of the children you want it to use (dynamic children) to the :mod:`Child.ask ` -method. Questions that would have gone to the static children will instead go to the dynamic children. Note that: - -- You must provide the children in the same format as they're provided in the :ref:`service configuration ` -- If you override one static child, you must override others, too -- The dynamic children must have the same keys as the static children (so the child knows which service to ask which - questions) -- You should ensure the dynamic children you provide are compatible with and appropriate for questions from the child - service - -For example, if the child requires these children in its service configuration: - -.. code-block:: json - - [ - { - "key": "wind_speed", - "id": "template-child-services/wind-speed-service:2.1.1", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "octue-sdk-python" - }, - }, - { - "key": "elevation", - "id": "template-child-services/elevation-service:3.1.9", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "octue-sdk-python" - }, - } - ] - -then you can override them like this: - -.. code-block:: python - - answer, question_uuid = child.ask( - input_values={"height": 32, "width": 3}, - children=[ - { - "key": "wind_speed", - "id": "my/own-service:1.0.0", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "octue-sdk-python" - }, - }, - { - "key": "elevation", - "id": "organisation/another-service:0.1.0", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "octue-sdk-python" - }, - }, - ], - ) - -Overriding beyond the first generation --------------------------------------- -It's an intentional choice to only go one generation deep with overriding children. If you need to be able to specify a -whole tree of children, grandchildren, and so on, please `upvote this issue. -`_ - - -.. _using_service_registries: - -Using a service registry -======================== -When asking a question, you can optionally specify one or more `service registries -`_ to resolve SRUIDs against. This checks if the service revision -exists (good for catching typos in SRUIDs) and raises an error if it doesn't. Service registries can also get the -default revision of a service if you don't provide a revision tag. Asking a question if without specifying a registry -will bypass these checks. - - -Specifying service registries ------------------------------ -You can specify service registries in two ways: - -1. For all questions asked inside a service. In the service configuration (``octue.yaml`` file): - - .. code-block:: yaml - - services: - - namespace: my-organisation - name: my-app - service_registries: - - name: my-registry - endpoint: blah.com/services - -2. For questions to a specific child, inside or outside a service: - - .. code-block:: python - - child = Child( - id="my-organisation/my-service:1.1.0", - backend={"name": "GCPPubSubBackend", "project_id": "my-project"}, - service_registries=[ - {"name": "my-registry", "endpoint": "blah.com/services"}, - ] - ) diff --git a/docs/source/authentication.rst b/docs/source/authentication.rst deleted file mode 100644 index 8c12f1450..000000000 --- a/docs/source/authentication.rst +++ /dev/null @@ -1,48 +0,0 @@ -============== -Authentication -============== -You need authentication while using ``octue`` to: - -- Access data from Google Cloud Storage -- Use, run, or deploy Twined services - -Authentication is provided by a GCP service account. - -Creating a service account -========================== -By setting up your Twined service network with the :doc:`Twined Terraform modules `, a set of -maintainer service accounts have already been created with the required permissions. - -Using a service account -======================= - -Locally -------- -1. Access your service accounts `here `_, making sure the - correct project is selected - -2. Click on the relevant service account, go to the "Keys" tab, and create (download) a JSON key for it - it will be - called ``-XXXXX.json``. - -.. DANGER:: - - It's best not to store this in your project to prevent accidentally committing it or building it into a docker - image layer. Instead, keep it somewhere else on your local system with any other service account keys you already - have. - - If you must keep within your project, it's good practice to name the file ``gcp-credentials.json`` and make - sure that ``gcp-cred*`` is in your ``.gitignore`` and ``.dockerignore`` files. - -2. If you're developing in a container (like a VSCode ``devcontainer``), mount the file into the container. You can - make gcloud available too - check out `this tutorial - `_. - -3. Set the ``GOOGLE_APPLICATION_CREDENTIALS`` environment variable to the absolute path of the key file. If using a - ``devcontainer``, make sure this is the path inside the container and not the path on your local machine. - -On GCP infrastructure / deployed services ------------------------------------------ -- Credentials are automatically provided when running code or services on GCP infrastructure, including the Kubernetes - cluster -- ``octue`` uses these when when running on these platforms, so there's no need to upload a service account key or - include one in service docker images diff --git a/docs/source/available_filters.rst b/docs/source/available_filters.rst deleted file mode 100644 index 82239a28b..000000000 --- a/docs/source/available_filters.rst +++ /dev/null @@ -1,103 +0,0 @@ -================= -Available filters -================= -Lots of filters are available when using the ``Dataset.files.filter`` method. We've broken them down by the type of -attribute the datafiles are being filtered by: - -- Numbers (e.g. ``int``, ``float``): - * ``is`` - * ``is_not`` - * ``equals`` - * ``not_equals`` - * ``lt`` - * ``lte`` - * ``gt`` - * ``gte`` - * ``in_range`` - * ``not_in_range`` - -- Iterables (e.g. ``list``, ``set``, ``tuple``, ``dictionary``): - * ``is`` - * ``is_not`` - * ``equals`` - * ``not_equals`` - * ``contains`` - * ``not_contains`` - * ``icontains`` - * ``not_icontains`` - -- ``bool`` - * ``is`` - * ``is_not`` - -- ``str`` - * ``is`` - * ``is_not`` - * ``equals`` - * ``not_equals`` - * ``iequals`` - * ``not_iequals`` - * ``lt`` (less than) - * ``lte`` (less than or equal) - * ``gt`` (greater than) - * ``gte`` (greater than or equal) - * ``contains`` - * ``not_contains`` - * ``icontains`` (case-insensitive contains) - * ``not_icontains`` - * ``starts_with`` - * ``not_starts_with`` - * ``ends_with`` - * ``not_ends_with`` - * ``in_range`` - * ``not_in_range`` - -- ``NoneType`` - * ``is`` - * ``is_not`` - -- ``LabelSet`` - * ``is`` - * ``is_not`` - * ``equals`` - * ``not_equals`` - * ``contains`` - * ``not_contains`` - * ``any_label_contains`` - * ``not_any_label_contains`` - * ``any_label_starts_with`` - * ``not_any_label_starts_with`` - * ``any_label_ends_with`` - * ``not_any_label_ends_with`` - -- ``datetime.datetime`` - * ``is`` - * ``is_not`` - * ``equals`` - * ``not_equals`` - * ``lt`` (less than) - * ``lte`` (less than or equal) - * ``gt`` (greater than) - * ``gte`` (greater than or equal) - * ``in_range`` - * ``not_in_range`` - * ``year_equals`` - * ``year_in`` - * ``month_equals`` - * ``month_in`` - * ``day_equals`` - * ``day_in`` - * ``weekday_equals`` - * ``weekday_in`` - * ``iso_weekday_equals`` - * ``iso_weekday_in`` - * ``time_equals`` - * ``time_in`` - * ``hour_equals`` - * ``hour_in`` - * ``minute_equals`` - * ``minute_in`` - * ``second_equals`` - * ``second_in`` - * ``in_date_range`` - * ``in_time_range`` diff --git a/docs/source/conf.py b/docs/source/conf.py deleted file mode 100644 index 023694a62..000000000 --- a/docs/source/conf.py +++ /dev/null @@ -1,196 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Read the Docs Template documentation build configuration file, created by -# sphinx-quickstart on Tue Aug 26 14:19:49 2014. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import subprocess - - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# import sys -# sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = ["sphinx_rtd_theme", "sphinx_tabs", "sphinx_toolbox.collapse", "sphinx.ext.autodoc"] - -autodoc_default_options = { - "members": True, - "member-order": "bysource", -} - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" - -# General information about the project. -project = "Octue SDK (Python)" -author = "Octue Ltd" -copyright = "2024, Octue Ltd" - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. - -# The full version, including alpha/beta/rc tags. -release = subprocess.check_output(["poetry", "version", "-s"]).decode().strip() - -# The short X.Y version. -version = ".".join(release.split(".")[0:2]) - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ["_build"] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -# keep_warnings = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = "sphinx_rtd_theme" - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -# html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -# html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_domain_indices = True - -# If false, no index is generated. -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Output file base name for HTML help builder. -# htmlhelp_basename = "ReadtheDocsTemplatedoc" - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -# man_pages = [("index", "readthedocstemplate", "Read the Docs Template Documentation", ["Read the Docs"], 1)] - -# If true, show URL addresses after external links. -# man_show_urls = False diff --git a/docs/source/creating_apps.rst b/docs/source/creating_apps.rst deleted file mode 100644 index ec8744e64..000000000 --- a/docs/source/creating_apps.rst +++ /dev/null @@ -1,162 +0,0 @@ -.. _creating_apps: - -app.py file -=========== -The ``app.py`` file is, as you might expect, the entrypoint to your app. It can contain any valid python including -imports and use of any number of external packages or internal/local packages. - -Structure ---------- -The ``app.py`` file must contain exactly one of the ``octue`` python app interfaces to serve as the entrypoint to your -code. These take a single :mod:`Analysis ` instance as a parameter or attribute: - -- **Option 1:** A function named ``run`` with the following signature: - - .. code-block:: python - - def run(analysis): - """A function that uses input and configuration from an ``Analysis`` - instance and stores any output values and output manifests on it. - It shouldn't return anything. - - :param octue.resources.Analysis analysis: - :return None: - """ - ... - -- **Option 2:** A class named ``App`` with the following signature: - - .. code-block:: python - - class App: - """A class that takes an ``Analysis`` instance and any number of - other parameters in its constructor. It can have any number of - methods but must always have a ``run`` method with the signature - shown below. - - :param octue.resources.Analysis analysis: - :return None: - """ - - def __init__(self, analysis): - self.analysis = analysis - ... - - def run(self): - """A method that that uses input and configuration from an - ``Analysis`` instance and stores any output values and - output manifests on it. It shouldn't return anything. - - :return None: - """ - ... - - ... - - -Accessing inputs and storing outputs ------------------------------------- -Your app must access configuration and input data from and store output data on the :mod:`analysis ` -parameter (for function-based apps) or attribute (for class-based apps). This allows standardised -configuration/input/output validation against the twine and interoperability of all Twined services while leaving you -freedom to do any kind of computation. To access the data, use the following attributes on the ``analysis`` -parameter/attribute: - -- Configuration values: ``analysis.configuration_values`` -- Configuration manifest: ``analysis.configuration_manifest`` -- Input values: ``analysis.input_values`` -- Input manifest: ``analysis.input_manifest`` -- Output values: ``analysis.output_values`` -- Output manifest: ``analysis.output_manifest`` - - -Sending monitor messages ------------------------- -As well as sending the final result of the analysis your app produces to the parent, you can send monitor messages as -computation progresses. This functionality can be used to update a plot or database in real time as the analysis is in -progress. - -.. code-block:: python - - def run(analysis): - some_data = {"x": 0, "y", 0.1, "z": 2} - analysis.send_monitor_message(data=some_data) - -Before sending monitor messages, the ``monitor_message_schema`` field must be provided in ``twine.json``. For example: - -.. code-block:: json - - { - ... - "monitor_message_schema": { - "x": { - "description": "Real component", - "type": "number" - }, - "y": { - "description": "Imaginary component", - "type": "number" - }, - "z": { - "description": "Number of iterations before divergence", - "type": "number", - "minimum": 0 - } - }, - ... - } - -Monitor messages can also be set up to send periodically in time. - -.. code-block:: python - - def run(analysis): - - # Define a data structure whose attributes can be accessed in real - # time as they're updated during the analysis. - class DataStructure: - def __init__(self): - self.x = 0 - self.y = 0 - self.z = 0 - - def as_dict(self): - """Add a method that provides the data in the format - required for the monitor messages. - - :return dict: - """ - return {"x": self.x, "y": self.y, "z": self.z} - - # Create an instance of the data structure. - my_updating_data = DataStructure() - - # Use the `as_dict` method to provide up-to-date data from the data - # structure to send as monitor messages every 60s. - analysis.set_up_periodic_monitor_message( - create_monitor_message=my_updating_data.as_dict, - period=60, - ) - - # Run long-running computations on the data structure that update its - # "x", "y", and "z" attributes in real time. The periodic monitor - # message will always send the current values of x, y, and z. - some_function(my_updating_data) - - -Finalising the analysis ------------------------ -When the analysis has finished, it is automatically finalised. This means: - -- The output values and manifest are validated against ``twine.json`` to ensure they're in the format promised by the - service. -- If the app produced an output manifest and the ``output_location`` field is set to a cloud directory path in the app - configuration, the output datasets are uploaded to this location. - -.. note:: - - You can manually call :mod:`analysis.finalise ` if you want to upload - any output datasets to a different location to the one specified in the service configuration. If you do this, the - analysis will not be finalised again - make sure you only call it when your output data is ready. Note that the - ``output_location`` and ``use_signed_urls_for_output_datasets`` settings in the service configuration are ignored if - you call it manually. diff --git a/docs/source/creating_services.rst b/docs/source/creating_services.rst deleted file mode 100644 index aac37d98a..000000000 --- a/docs/source/creating_services.rst +++ /dev/null @@ -1,155 +0,0 @@ -================= -Creating services -================= -One of the main features of the Octue SDK is to allow you to easily create services that can accept questions and -return answers. They can run locally on any machine or be deployed to the cloud. Currently: - -- The backend communication between twins uses Google Pub/Sub whether they're local or deployed -- Services are deployed to Google Cloud Run -- The language of the entrypoint must by ``python3`` (you can call processes using other languages within this though) - - -Anatomy of a Twined service -=========================== -A Twined service is defined by the following files (located in the repository root by default). - -app.py ------- -This is the entrypoint into your code - :doc:`read more here `. - - -twine.json ----------- -This file defines the schema for the service's configuration, input, and output data. Read more -`here `_ and see an example -`here `_. - -Dependencies file ------------------ -A file specifying your app's dependencies. This is a `setup.py file `_, -a `requirements.txt file `_, or a -`pyproject.toml file `_ listing all the python packages your app depends on -and the version ranges that are supported. - -.. _service_configuration: - -octue.yaml ----------- - - .. collapse:: This describes the service configuration - read more... - - ---- - - This file defines the basic structure of your service. It must contain at least: - - .. code-block:: yaml - - services: - - namespace: my-organisation - name: my-app - - It may also need the following key-value pairs: - - - ``app_source_path: `` - if your ``app.py`` file is not in the repository root - - All paths should be relative to the repository root. Other valid entries can be found in the - :mod:`ServiceConfiguration ` constructor. - - .. warning:: - - Currently, only one service can be defined per repository, but it must still appear as a list item of the - "services" key. At some point, it will be possible to define multiple services in one repository. - - If a service's app needs any configuration, asks questions to any other Twined services, or produces output - datafiles/datasets, you will need to provide some or all of the following values for that service: - - - ``configuration_values`` - - ``configuration_manifest`` - - ``children`` - - ``output_location`` - - ``use_signed_urls_for_output_datasets`` - - -Dockerfile (optional) ---------------------- - - .. collapse:: Provide this if your needs exceed the default Octue Dockerfile - read more... - - ---- - - Twined services run in a Docker container if they are deployed. They can also run this way locally. The SDK - provides a default ``Dockerfile`` for these purposes that will work for most cases: - - - For deploying to `Kubernetes `_ - - However, you may need to write and provide your own ``Dockerfile`` if your app requires: - - - Non-python or system dependencies (e.g. ``openfast``, ``wget``) - - Python dependencies that aren't installable via ``pip`` - - Private python packages - - Here are two examples of a custom ``Dockerfile`` that use different base images: - - - `A TurbSim service `_ - - `An OpenFAST service `_ - - If you do provide one, you must provide its path relative to your repository to the `build-twined-services` - GitHub Actions `workflow `_. - - As always, if you need help with this, feel free to drop us a message or raise an issue! - - -Where to specify the namespace, name, and revision tag ------------------------------------------------------- -See :ref:`here ` for service naming requirements. - -**Namespace** - -- Required: yes -- Set in: - - - ``octue.yaml`` - - ``OCTUE_SERVICE_NAMESPACE`` environment variable (takes priority) - -**Name** - -- Required: yes -- Set in: - - - ``octue.yaml`` - - ``OCTUE_SERVICE_NAME`` environment variable (takes priority) - -**Revision tag** - -- Required: no -- Default: a random "coolname" (e.g. ``hungry-hippo``) -- Set in: - - - ``OCTUE_SERVICE_REVISION_TAG`` environment variable - - If using ``octue twined service start`` command, the ``--revision-tag`` option (takes priority) - - -Template apps -============= -We've created some template apps for you to look at and play around with. We recommend going through them in this order: - -1. The `fractal app template `_ - - introduces a basic Twined service that returns output values to its parent. -2. The `using-manifests app template `_ - - introduces using a manifest of output datasets to return output files to its parent. -3. The `child-services app template `_ - - introduces asking questions to child services and using their answers to form an output to return to its parent. - - -Deploying services automatically -================================ -Automated deployment with Octue means: - -- Your service runs in Google Kubernetes Engine (GKE), ready to accept questions from and return answers to other services. -- You don't need to do anything to update your deployed service with new code changes - the service simply gets rebuilt - and re-deployed each time you push a commit to your ``main`` branch, or merge a pull request into it (other branches - and deployment strategies are available, but this is the default). -- Serverless is the default - your service only runs when questions from other services are sent to it, meaning there - are minimal costs to having it deployed but not in use. - -If you'd like help deploying services, contact us. To do it yourself, see :doc:`here `. diff --git a/docs/source/data_containers.rst b/docs/source/data_containers.rst deleted file mode 100644 index 6f670b328..000000000 --- a/docs/source/data_containers.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. _data_containers: - -================================== -Datafiles, datasets, and manifests -================================== -One of the main features of ``octue`` is making using, creating, and sharing scientific datasets easy. There are three -main data classes in the SDK that do this. - -- **Datafile** - :doc:`a single local or cloud file ` and its metadata. - -- **Dataset** - :doc:`a set of related datafiles ` that exist in the same location, plus metadata. - -- **Manifest** - :doc:`a set of related datasets ` that exist anywhere, plus metadata. Typically produced by - or for one analysis. diff --git a/docs/source/datafile.rst b/docs/source/datafile.rst deleted file mode 100644 index 32c405987..000000000 --- a/docs/source/datafile.rst +++ /dev/null @@ -1,308 +0,0 @@ -.. _datafile: - -======== -Datafile -======== - -.. admonition:: Definitions - - :mod:`Datafile ` - A single local or cloud file, its metadata, and helper methods. - - Locality - A datafile has one of these localities: - - - **Cloud-based:** it exists only in the cloud - - **Local:** it exists only on your local filesystem - - **Cloud-based and local:** it's cloud-based but has been downloaded for low-latency reading/writing - -.. tip:: - - Use a datafile to work with a file if you want to: - - - Read/write to local and cloud files in the same way - - Include it in a :doc:`dataset ` that can be sent to a Twined service for processing - - Add metadata to it for future sorting and filtering - -Key features -============ - -Work with local and cloud data ------------------------------- -Working with a datafile is the same whether it's local or cloud-based. It's also almost identical to using `python -built-in open function `_. For example, to write to a datafile: - -.. code-block:: python - - from octue.resources import Datafile - - datafile = Datafile("path/to/file.dat") - - # Or: - - datafile = Datafile("gs://my-bucket/path/to/file.dat") - - with datafile.open("w") as f: - f.write("Some data") - datafile.labels.add("processed") - -All the same file modes you'd use with the `python built-in open context manager -`_ are available for datafiles e.g. ``"r"`` and ``"a"``. - - -Automatic lazy downloading --------------------------- -Save time and bandwidth by only downloading when necessary. - -Downloading data from cloud datafiles is automatic and lazy so you get both low-latency content read/write and quick -metadata reads. This makes viewing and filtering by the metadata of cloud datasets and datafiles quick and avoids -unnecessary data transfer, energy usage, and costs. - -Datafile content isn't downloaded until you: - -- Try to read or write its contents using the :mod:`Datafile.open ` context manager -- Call its :mod:`download ` method -- Use its :mod:`local_path ` property - -Read more about downloading files :doc:`here `. - - -CLI command friendly --------------------- -Use any command line tool on your datafiles. Datafiles are python objects, but they represent real files that can be -fed to any CLI command you like - -.. code-block:: python - - import subprocess - output = subprocess.check_output(["openfast", datafile.local_path]) - - -Easy and expandable custom metadata ------------------------------------ -Find the needle in the haystack by making your data searchable. You can set the following metadata on a datafile: - -- Timestamp -- Labels (a set of lowercase strings) -- Tags (a dictionary of key-value pairs) - -This metadata is stored locally in a ``.octue`` file for local datafiles or on the cloud objects for cloud datafiles and -is used during ``Datafile`` instantiation. It can be accessed like this: - -.. code-block:: python - - datafile.timestamp - >>> datetime.datetime(2022, 5, 4, 17, 57, 57, 136739) - - datafile.labels - >>> {"processed"} - - datafile.tags - >>> {"organisation": "octue", "energy": "renewable"} - -You can update the metadata by setting it on the instance while inside the :mod:`Datafile.open ` context manager. - -.. code-block:: python - - with datafile.open("a"): - datafile.labels.add("updated") - -You can do this outside the context manager too, but you then need to call the update method: - -.. code-block:: python - - datafile.labels.add("updated") - datafile.update_metadata() - - -Upload an existing local datafile ---------------------------------- -Back up and share your datafiles for collaboration. You can upload an existing local datafile to the cloud without -using the :mod:`Datafile.open ` context manager if you don't need to modify its -contents: - -.. code-block:: python - - datafile.upload("gs://my-bucket/my_datafile.dat", update_metadata=True) - - -Get file and metadata hashes ----------------------------- -Make your analysis reproducible: guarantee a datafile contains exactly the same data as before by checking its hash. - -.. code-block:: python - - datafile.hash_value - >>> 'mnG7TA==' - -You can also check that any metadata is the same. - -.. code-block:: python - - datafile.metadata_hash_value - >>> 'DIgCHg==' - - -Immutable ID ------------- -Each datafile has an immutable UUID: - -.. code-block:: python - - datafile.id - >>> '9a1f9b26-6a48-4f2d-be80-468d3270d79b' - - -Check a datafile's locality ---------------------------- -Is this datafile local or in the cloud? - -.. code-block:: python - - datafile.exists_locally - >>> True - - datafile.exists_in_cloud - >>> False - -A cloud datafile that has been downloaded will return ``True`` for both of these properties. - - -Represent HDF5 files --------------------- -Support fast I/O processing and storage. - -.. warning:: - If you want to represent HDF5 files with a ``Datafile``, you must include the extra requirements provided by the - ``hdf5`` key at installation i.e. - - .. code-block:: shell - - pip install octue[hdf5] - - or - - .. code-block:: shell - - poetry add octue -E hdf5 - - -Usage examples -============== - -The ``Datafile`` class can be used functionally or as a context manager. When used as a context manager, it is analogous -with the `python built-in open function `_. On exiting the context -(the ``with`` block), it closes the datafile locally and, if the datafile also exists in the cloud, updates the cloud -object with any data or metadata changes. - - -.. image:: images/datafile_use_cases.png - - -Example A ---------- -**Scenario:** Download a cloud object, calculate Octue metadata from its contents, and add the new metadata to the cloud object - -**Starting point:** Object in cloud with or without Octue metadata - -**Goal:** Object in cloud with updated metadata - -.. code-block:: python - - from octue.resources import Datafile - - - datafile = Datafile("gs://my-bucket/path/to/data.csv") - - with datafile.open() as f: - data = f.read() - new_metadata = metadata_calculating_function(data) - - datafile.timestamp = new_metadata["timestamp"] - datafile.tags = new_metadata["tags"] - datafile.labels = new_metadata["labels"] - - -Example B ---------- -**Scenario:** Add or update Octue metadata on an existing cloud object *without downloading its content* - -**Starting point:** A cloud object with or without Octue metadata - -**Goal:** Object in cloud with updated metadata - -.. code-block:: python - - from datetime import datetime - from octue.resources import Datafile - - - datafile = Datafile("gs://my-bucket/path/to/data.csv") - - datafile.timestamp = datetime.now() - datafile.tags = {"manufacturer": "Vestas", "output": "1MW"} - datafile.labels = {"new"} - - datafile.upload(update_metadata=True) # Or, datafile.update_metadata() - - -Example C ---------- -**Scenario:** Read in the data and Octue metadata of an existing cloud object without intent to update it in the cloud - -**Starting point:** A cloud object with Octue metadata - -**Goal:** Cloud object data (contents) and metadata held locally in local variables - -.. code-block:: python - - from octue.resources import Datafile - - - datafile = Datafile("gs://my-bucket/path/to/data.csv") - - with datafile.open() as f: - data = f.read() - - metadata = datafile.metadata() - - -Example D ---------- -**Scenario:** Create a new cloud object from local data, adding Octue metadata - -**Starting point:** A file-like locally (or content data in local variable) with Octue metadata stored in local variables - -**Goal:** A new object in the cloud with data and Octue metadata - -For creating new data in a new local file: - -.. code-block:: python - - from octue.resources import Datafile - - - datafile = Datafile( - "path/to/local/file.dat", - tags={"cleaned": True, "type": "linear"}, - labels={"Vestas"} - ) - - with datafile.open("w") as f: - f.write("This is some cleaned data.") - - datafile.upload("gs://my-bucket/path/to/data.dat") - - -For existing data in an existing local file: - -.. code-block:: python - - from octue.resources import Datafile - - - tags = {"cleaned": True, "type": "linear"} - labels = {"Vestas"} - - datafile = Datafile(path="path/to/local/file.dat", tags=tags, labels=labels) - datafile.upload("gs://my-bucket/path/to/data.dat") diff --git a/docs/source/dataset.rst b/docs/source/dataset.rst deleted file mode 100644 index f7f4a686b..000000000 --- a/docs/source/dataset.rst +++ /dev/null @@ -1,232 +0,0 @@ -.. _dataset: - -======= -Dataset -======= - -.. admonition:: Definitions - - :mod:`Dataset ` - A set of related :doc:`datafiles ` that exist in the same location, dataset metadata, and helper - methods. - - Locality - A dataset has one of these localities: - - - **Cloud-based:** it exists only in the cloud - - **Local:** it exists only on your local filesystem - -.. tip:: - - Use a dataset if you want to: - - - Group together a set of files that naturally relate to each other e.g. a timeseries that's been split into - multiple files. - - Add metadata to it for future sorting and filtering - - Include it in a :doc:`manifest ` with other datasets and send them to a Twined service for processing - - -Key features -============ - -Work with local and cloud datasets ----------------------------------- -Working with a dataset is the same whether it's local or cloud-based. - -.. code-block:: python - - from octue.resources import Dataset - - dataset = Dataset(path="path/to/dataset") - - dataset = Dataset(path="gs://my-bucket/path/to/dataset") - - -.. warning:: - - Datasets recurse all subdirectories by default unless ``recursive=False`` is set. - - -Upload a dataset ----------------- -Back up and share your datasets for collaboration. - -.. code-block:: python - - dataset.upload("gs://my-bucket/path/to/upload") - - -Download a dataset ------------------- -Use a shared or public dataset or retrieve a backup. - -.. code-block:: python - - dataset.download("path/to/download") - - -Easy and expandable custom metadata ------------------------------------ -Find the needle in the haystack by making your data searchable. You can set the following metadata on a dataset: - -- Name -- Labels (a set of lowercase strings) -- Tags (a dictionary of key-value pairs) - -This metadata is stored locally in a ``.octue`` file in the same directory as the dataset and is used during -``Dataset`` instantiation. It can be accessed like this: - -.. code-block:: python - - dataset.name - >>> "my-dataset" - - dataset.labels - >>> {"processed"} - - dataset.tags - >>> {"organisation": "octue", "energy": "renewable"} - -You can update the metadata by setting it on the instance while inside the ``Dataset`` context manager. - -.. code-block:: python - - with dataset: - datafile.labels.add("updated") - -You can do this outside the context manager too, but you then need to call the update method: - -.. code-block:: python - - dataset.labels.add("updated") - dataset.update_metadata() - - -Get dataset and metadata hashes -------------------------------- -Make your analysis reproducible: guarantee a dataset contains exactly the same data as before by checking its hash. - -.. code-block:: python - - dataset.hash_value - >>> 'uvG7TA==' - -.. note:: - - A dataset's hash is a function of its datafiles' hashes. Datafile and dataset metadata do not affect it. - -You can also check that dataset metadata is the same. - -.. code-block:: python - - dataset.metadata_hash_value - >>> 'DIgCHg==' - - -Immutable ID ------------- -Each dataset has an immutable UUID: - -.. code-block:: python - - dataset.id - >>> '9a1f9b26-6a48-4f2d-be80-468d3270d79c' - - -Check a dataset's locality ---------------------------- -Is this dataset local or in the cloud? - -.. code-block:: python - - dataset.exists_locally - >>> True - - dataset.exists_in_cloud - >>> False - -A dataset can only return ``True`` for one of these at a time. - - -Filter datasets ---------------- -Narrow down a dataset to just the files you want to avoiding extra downloading and processing. - -Datafiles in a dataset are stored in a :mod:`FilterSet `, meaning they -can be easily filtered by any attribute of the datafiles contained e.g. name, extension, ID, timestamp, tags, labels, -size. The filtering syntax is similar to Django's i.e. - -.. code-block:: shell - - # Get datafiles that have an attribute that satisfies the filter. - dataset.files.filter(__=) - - # Or, if your filter is a simple equality filter: - dataset.files.filter(=) - -Here's an example: - -.. code-block:: python - - # Make a dataset. - dataset = Dataset( - path="blah", - files=[ - Datafile(path="my_file.csv", labels=["one", "a", "b" "all"]), - Datafile(path="your_file.txt", labels=["two", "a", "b", "all"), - Datafile(path="another_file.csv", labels=["three", "all"]), - ] - ) - - # Filter it! - dataset.files.filter(name__starts_with="my") - >>> })> - - dataset.files.filter(extension="csv") - >>> , })> - - dataset.files.filter(labels__contains="a") - >>> , })> - -You can iterate through the filtered files: - -.. code-block:: python - - for datafile in dataset.files.filter(labels__contains="a"): - print(datafile.name) - >>> 'my_file.csv' - 'your_file.txt' - -If there's just one result, get it via the :mod:`FilterSet.one ` method: - -.. code-block:: python - - dataset.files.filter(name__starts_with="my").one() - >>> - -You can also chain filters or specify them all at the same time - these two examples produce the same result: - -.. code-block:: python - - # Chaining multiple filters. - dataset.files.filter(extension="csv").filter(labels__contains="a") - >>> })> - - # Specifying multiple filters at once. - dataset.files.filter(extension="csv", labels__contains="a") - >>> })> - -For the full list of available filters, :doc:`click here `. - - -Order datasets --------------- -A dataset can also be ordered by any of the attributes of its datafiles: - -.. code-block:: python - - dataset.files.order_by("name") - >>> , , ])> - -The ordering can also be carried out in reverse (i.e. descending order) by passing ``reverse=True`` as a second argument -to the :mod:`FilterSet.order_by ` method. diff --git a/docs/source/deploying_services.rst b/docs/source/deploying_services.rst deleted file mode 100644 index bbf913d02..000000000 --- a/docs/source/deploying_services.rst +++ /dev/null @@ -1,67 +0,0 @@ -.. _deploying_services_advanced: - -====================================== -Deploying services (developer's guide) -====================================== -This is a guide for developers that want to deploy Twined services themselves - it is not needed if Octue manages your -services for you or if you are only asking questions to existing Twined services. - -What is deployment? -=================== -Deploying a Twined service means the service: - -* Is a docker image that is spun up and down in a Kubernetes cluster on demand -* Is ready at any time to answer questions from users and other Twined services in the service network -* Can ask questions to any other Twined service in the service network -* Will automatically spin down after it has finished answering a question -* Will automatically build and redeploy after a relevant code change (e.g. on push or merge into ``main``) - -We can split deployment into service deployment and infrastructure deployment. - -Deploying a service -=================== -Assuming the service network infrastructure already exists, a service can be deployed by building and pushing its docker -image to the service network's Artifact Registry repository. We recommend pushing a new image for each release of the -code e.g. on merge into the ``main`` branch. Each new image is the deployment of a new service revision. This can be -done automatically: - -- Follow the `instructions `_ - to add the `build-twined-service `_ - GitHub Actions workflow to your service's GitHub repository. Set its trigger to merge or push to ``main`` (see example - below) -- This needs to be done **once for every service** you want to deploy -- A live example can be `found here `_ - including automated pre-deployment testing and creation of a GitHub release - -You can now :doc:`ask your service some questions `! It will be available in the service network as -``/:`` (e.g. ``octue/example-service-kueue:0.1.1``). - - -Deploying the infrastructure -============================ - -Prerequisites -------------- -Twined services are currently deployable to Google Cloud Platform (GCP). You must have "owner" level access to the GCP -project you're deploying to and billing must be set up for it. - -Deploying step-by-step ----------------------- -There are two steps to deploying the infrastructure: - -1. Deploy the core infrastructure (storage bucket, event store, IAM service accounts and roles) -2. Deploy the Kubernetes cluster, event handler, service registry, and Pub/Sub topic - -1. Deploy core infrastructure -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- Follow `the instructions `_ to deploy the resources in the - ``terraform-octue-twined-core`` Terraform module -- This only needs to be done once per service network - -2. Deploy Kubernetes cluster -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- Follow the `instructions `_ to deploy the resources in the - ``terraform-octue-twined-cluster`` Terraform module -- This only needs to be done once per service network diff --git a/docs/source/downloading_datafiles.rst b/docs/source/downloading_datafiles.rst deleted file mode 100644 index e88eca4d4..000000000 --- a/docs/source/downloading_datafiles.rst +++ /dev/null @@ -1,29 +0,0 @@ -========================================= -More information on downloading datafiles -========================================= - -- To avoid unnecessary data transfer and costs, cloud datafiles are not downloaded locally - `until necessary `_. -- When downloaded, they are downloaded by default to a temporary local file that will exist at least as long as the - python session is running -- Calling ``Datafile.download`` or using ``Datafile.local_path`` again will not re-download the file -- Any changes made to the datafile via the ``Datafile.open`` method are made to the local copy and then synced with - the cloud object - -.. warning:: - - External changes to cloud files will not be synced locally unless the datafile is re-instantiated. - -- If you want a cloud datafile to be downloaded to a permanent location, you can do one of: - - .. code-block:: python - - datafile.download(local_path="my/local/path.csv") - - datafile.local_path = "my/local/path.csv" - -- To pre-set a permanent download location on instantiation, run: - - .. code-block:: python - - datafile = Datafile("gs://my-bucket/path/to/file.dat", local_path="my/local/path.csv") diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index c5dffb873..000000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,98 +0,0 @@ -================== -Octue SDK (Python) -================== - -The python SDK for `Octue `_ Twined scientific data services and digital twins - get faster data -groundwork so you have more time for the science! - -.. _service_definition: - -.. admonition:: Definition - - Octue Twined service - A data service or digital twin built on the ``octue`` SDK that can be asked questions, process them, and return - answers. Twined services can communicate with each other with minimal extra setup. - - -Key features -============ - -**Unified cloud/local file, dataset, and manifest operations** - -- Create and build datasets easily -- Organise them with timestamps, labels, and tags -- Filter and combine them using this metadata -- Store them locally or in the cloud (or both for low-latency reading/writing with cloud-guaranteed data availability) -- Use internet/cloud-based datasets as if they were local e.g. - - - ``https://example.com/important_dataset.dat`` - - ``gs://example-bucket/important_dataset.dat`` - -- Create manifests (a set of datasets needed for a particular analysis) to modularise your dataset input/output - -**Ask existing services questions from anywhere** - -- Send them data to process from anywhere -- Automatically have their logs, monitor messages, and any errors forwarded to you and displayed as if they were local -- Receive their output data as JSON -- Receive a manifest of any output datasets they produce for you to download or access as you wish - -**Create, run, and deploy your apps as services** - -- No need to change your app - just wrap it -- Use the ``octue`` CLI to run your service locally or deploy it to Google Kubernetes Engine (GKE) -- Create JSON-schema interfaces to explicitly define the form of configuration, input, and output data -- Ask other services questions as part of your app (i.e. build trees of services) -- Automatically display readable, colourised logs, or use your own log handler -- Avoid time-consuming and confusing devops, cloud configuration, and backend maintenance - -**High standards, quick responses, and good intentions** - -- Open-source and transparent on GitHub - anyone can see the code and raise an issue -- Automated testing, standards, releases, and deployment -- High test coverage -- Works on MacOS, Linux, and Windows -- Developed not-for-profit for the renewable energy industry - - -Need help, found a bug, or want to request a new feature? -========================================================= -We use `GitHub Issues `_ [#]_ to manage: - -- Bug reports -- Feature requests -- Support requests - - -.. rubric:: Footnotes - -.. [#] Bug reports, feature requests and support requests, may also be made directly to your Octue support contact, or - via the `support pages `_. - - -.. - The table of contents tree is hidden in the page content but will show up as a navigation bar for all pages. - -.. toctree:: - :maxdepth: 1 - :hidden: - - installation - data_containers - datafile - dataset - manifest - services - asking_questions - creating_services - updating_services - running_services_locally - deploying_services - testing_services - troubleshooting_services - logging - authentication - inter_service_compatibility - api - license - version_history diff --git a/docs/source/installation.rst b/docs/source/installation.rst deleted file mode 100644 index 64ed4f0a8..000000000 --- a/docs/source/installation.rst +++ /dev/null @@ -1,34 +0,0 @@ -.. _chapter-installation: - -============ -Installation -============ - - -Pip -=== - -.. code-block:: shell - - pip install octue==x.y.z - - -Poetry -====== -Read more about Poetry `here `_. - -.. code-block:: shell - - poetry add octue=x.y.z - - -Add to your dependencies -======================== -To use a specific version of the Octue SDK in your python application, simply add: - -.. code-block:: shell - - octue==x.y.z - -to your ``requirements.txt`` or ``setup.py`` file, where ``x.y.z`` is your preferred version of the SDK (we recommend -the latest stable version). diff --git a/docs/source/inter_service_compatibility.rst b/docs/source/inter_service_compatibility.rst deleted file mode 100644 index d49691ad3..000000000 --- a/docs/source/inter_service_compatibility.rst +++ /dev/null @@ -1,133 +0,0 @@ -=========================== -Inter-service compatibility -=========================== - -Twined services acting as parents and children communicate with each other according to the `services communication -schema `_. The table below shows which ``octue`` versions parents -can run (rows) to send questions compatible with versions children are running (columns). Note that this table does not -display whether children's responses are compatible with the parent, just that a child is able to accept a question. - -**Key** - -- ``0`` = incompatible -- ``1`` = compatible - -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| | 0.67.0 | 0.66.1 | 0.66.0 | 0.65.0 | 0.64.0 | 0.63.0 | 0.62.1 | 0.62.0 | 0.61.2 | 0.61.1 | 0.61.0 | 0.60.2 | 0.60.1 | 0.60.0 | 0.59.1 | 0.59.0 | 0.58.0 | 0.57.2 | 0.57.1 | 0.57.0 | 0.56.0 | 0.55.0 | 0.54.0 | 0.53.0 | 0.52.2 | 0.52.1 | 0.52.0 | 0.51.0 | 0.50.1 | 0.50.0 | 0.49.2 | 0.49.1 | 0.49.0 | 0.48.0 | 0.47.2 | 0.47.1 | 0.47.0 | 0.46.3 | 0.46.2 | 0.46.1 | 0.46.0 | 0.45.0 | 0.44.0 | 0.43.7 | 0.43.6 | 0.43.5 | 0.43.4 | 0.43.3 | 0.43.2 | 0.43.1 | 0.43.0 | 0.42.1 | 0.42.0 | 0.41.1 | 0.41.0 | 0.40.2 | 0.40.1 | 0.40.0 | -+========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+==========+ -| 0.67.0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.66.1 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.66.0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.65.0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.64.0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.63.0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.62.1 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.62.0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.61.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.61.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.61.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.60.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.60.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.60.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.59.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.59.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.58.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.57.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.57.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.57.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.56.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.55.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.54.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.53.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.52.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.52.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.52.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.51.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.50.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.50.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.49.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.49.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.49.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.48.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.47.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.47.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.47.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.46.3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.46.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.46.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.46.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.45.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.44.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.7 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.6 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.5 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.43.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.42.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.42.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.41.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.41.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.40.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.40.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ -| 0.40.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -+--------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+----------+ diff --git a/docs/source/license.rst b/docs/source/license.rst deleted file mode 100644 index 3ac973176..000000000 --- a/docs/source/license.rst +++ /dev/null @@ -1,13 +0,0 @@ -======= -License -======= - -The Boring Bit -============== - -See `the octue-sdk-python license `_. - -Third Party Libraries -===================== - -**octue-sdk-python** includes or is linked against code from third party libraries - see `our attributions page `_. diff --git a/docs/source/logging.rst b/docs/source/logging.rst deleted file mode 100644 index 6a717a489..000000000 --- a/docs/source/logging.rst +++ /dev/null @@ -1,53 +0,0 @@ -======= -Logging -======= -By default, ``octue`` streams your logs to ``stderr`` in a nice, readable format so your log messages are immediately -visible when you start developing without any extra configuration. If you prefer to use your own handlers or formatters, -simply set ``USE_OCTUE_LOG_HANDLER=0`` in the environment running your app. - - -Readable logs -------------- -Some advantages of the Octue log handler are: - -- Its readable format -- Its clear separation of log **context** from log **message**. - -Below, the context is on the left and includes: - -- The time -- Log level -- Module producing the log -- Octue analysis ID - -This is followed by the actual log message on the right: - -.. code-block:: - - [2021-07-10 20:03:12,713 | INFO | octue.runner | analysis-102ee7d5-4b94-4f8a-9dcd-36dbd00662ec] Hello! The child services template app is running! - - -Colourised services -------------------- -Another advantage to using the Octue log handler is that each Twined service is coloured according to its position in the -tree, making it much easier to read log messages from multiple levels of children. - -.. image:: images/coloured_logs.png - -In this example: - -- The log context is in blue -- Anything running in the root parent service's app is labeled with the analysis ID in green -- Anything running in the immediate child services (``elevation`` and ``wind_speed``) are labelled with the analysis ID - in yellow -- Any children further down the tree (i.e. children of the child services and so on) will have their own labels in - other colours consistent to their level - - -Add extra information ---------------------- -You can add certain log record attributes to the logging context by also providing the following environment variables: - -- ``INCLUDE_LINE_NUMBER_IN_LOGS=1`` - include the line number -- ``INCLUDE_PROCESS_NAME_IN_LOGS=1`` - include the process name -- ``INCLUDE_THREAD_NAME_IN_LOGS=1`` - include the thread name diff --git a/docs/source/manifest.rst b/docs/source/manifest.rst deleted file mode 100644 index ea713bebe..000000000 --- a/docs/source/manifest.rst +++ /dev/null @@ -1,119 +0,0 @@ -.. _manifest: - -======== -Manifest -======== - -.. admonition:: Definitions - - :mod:`Manifest ` - A set of related cloud and/or local :doc:`datasets `, metadata, and helper methods. Typically produced - by or needed for processing by a Twined service. - -.. tip:: - - Use a manifest to send :doc:`datasets ` to a Twined service as a question (for processing) - the service - will send an output manifest back with its answer if the answer includes output datasets. - - -Key features -============ - -Group related datasets together -------------------------------- -Make a clear grouping of datasets needed for a particular analysis. - -.. code-block:: python - - from octue.resources import Manifest - - manifest = Manifest( - datasets={ - "my_dataset_0": "gs://my-bucket/my_dataset_0", - "my_dataset_1": "gs://my-bucket/my_dataset_1", - "my_dataset_2": "gs://another-bucket/my_dataset_2", - } - ) - - -Send datasets to a service --------------------------- -Get a Twined service to analyse data for you as part of a larger analysis. - -.. code-block:: python - - from octue.twined.resources import Child - - child = Child( - id="octue/wind-speed:2.1.0", - backend={"name": "GCPPubSubBackend", "project_id": "my-project"}, - ) - - answer, question_uuid = child.ask(input_manifest=manifest) - -See :doc:`here ` for more information. - - -Receive datasets from a service -------------------------------- -Access output datasets from a Twined service from the cloud when you're ready. - -.. code-block:: python - - manifest = answer["output_manifest"] - manifest["an_output_dataset"].files - >>> , })> - -.. hint:: - - Datasets in an output manifest are stored in the cloud. You’ll need to keep a reference to where they are to access - them - the output manifest is this reference. You’ll need to use it straight away or save it to make use of it. - - -Download all datasets from a manifest -------------------------------------- -Download all or a subset of datasets from a manifest. - -.. code-block:: python - - manifest.download() - >>> { - "my_dataset": "/path/to/dataset" - } - -.. note:: - - Datasets are downloaded to a temporary directory if no paths are given. - - -Further information -=================== - -Manifests of local datasets ---------------------------- - -You can include local datasets in your manifest if you can guarantee all services that need them can access them. A use -case for this is, for example, a supercomputer cluster running several ``octue`` services locally that process and -transfer large amounts of data. It is much faster to store and access the required datasets locally than upload them to -the cloud and then download them again for each service (as would happen with cloud datasets). - -.. warning:: - - If you want to ask a child a question that includes a manifest containing one or more local datasets, you must - include the :mod:`allow_local_files ` parameter. For example, if you have an - analysis object with a child called "wind_speed": - - .. code-block:: python - - input_manifest = Manifest( - datasets={ - "my_dataset_0": "gs://my-bucket/my_dataset_0", - "my_dataset_1": "local/path/to/my_dataset_1", - } - ) - - answer, question_uuid = analysis.children["wind_speed"].ask( - input_values=analysis.input_values, - input_manifest=analysis.input_manifest, - allow_local_files=True, - ) diff --git a/docs/source/running_services_locally.rst b/docs/source/running_services_locally.rst deleted file mode 100644 index 1c037ba94..000000000 --- a/docs/source/running_services_locally.rst +++ /dev/null @@ -1,72 +0,0 @@ -======================== -Running services locally -======================== -Services can be operated locally (e.g. for testing or ad-hoc data processing). You can: - -- Run your service once (i.e. run one analysis): - - - Via the ``octue`` CLI - - By using the ``octue`` library in a python script - -- Start your service as a child, allowing it to answer any number of questions from any other Twined service: - - - Via the CLI - - -Running a service once -====================== - -Via the CLI ------------ -1. Ensure you've created a valid :ref:`octue.yaml ` file for your service - -2. Run: - - .. code-block:: shell - - octue twined question ask local --input-values='{"some": "input"}' - -The output values and/or manifest will be printed to ``stdout`` but are also stored in the event store. - -Via a python script -------------------- -Imagine we have a simple app that calculates the area of a square. It could be run locally on a given height and width -like this: - -.. code-block:: python - - from octue.twined.runner import Runner - - runner = Runner(app_src="path/to/app.py", twine="path/to/twine.json") - analysis = runner.run(input_values={"height": 5, "width": 10}) - - analysis.output_values - >>> {"area": 50} - - analysis.output_manifest - >>> None - -See the :mod:`Runner ` API documentation for more advanced usage including providing configuration, -children, and an input manifest. - - -Starting a service as a child -============================= - -Via the CLI ------------ - -1. Ensure you've created a valid :ref:`octue.yaml ` file for your service -2. Run: - - .. code-block:: shell - - octue twined service start - -This will run the service as a child waiting for questions until you press ``Ctrl + C`` or an error is encountered. The -service will be available to be questioned by other services at the service ID ``organisation/name`` as specified in -the ``octue.yaml`` file. - -.. tip:: - - You can use the ``--timeout`` option to stop the service after a given number of seconds. diff --git a/docs/source/services.rst b/docs/source/services.rst deleted file mode 100644 index 54fecd3d2..000000000 --- a/docs/source/services.rst +++ /dev/null @@ -1,98 +0,0 @@ -.. _services: - -===================== -Octue Twined services -===================== - -There's a growing range of live :ref:`services ` in the Octue ecosystem that you can query, mostly -related to wind energy and other renewables. Here's a quick glossary of terms before we tell you more: - -.. admonition:: Definitions - - Twined service - See :ref:`here `. - - Child - A Twined service that can be asked a question. This name reflects the tree structure of services (specifically, - `a DAG `_) formed by the service asking the question (the - parent), the child it asks the question to, any children that the child asks questions to as part of forming - its answer, and so on. - - Parent - A Twined service that asks a question to another Twined service (a child). - - Asking a question - Sending data (input values and/or an input manifest) to a child for processing/analysis. - - Receiving an answer - Receiving data (output values and/or an output manifest) from a child you asked a question to. - - Twined ecosystem - The set of services running the Octue SDK as their backend. These services guarantee: - - - Defined input/output JSON schemas and validation - - An easy and consistent interface for asking them questions and receiving their answers - - Logs, exceptions, and monitor messages forwarded to you - - High availability (if deployed in the cloud) - - -.. _service_naming: - -Service names -============= - -Questions are always asked to a *revision* of a service. Services revisions are named in a similar way to docker images. -They look like ``namespace/name:tag`` where the tag is often a semantic version (but doesn't have to be). - -.. admonition:: Definitions - - Service revision - A specific instance of a Twined service that can be individually addressed. The revision could correspond to a - version of the service, a dynamic development branch for it, or a deliberate duplication or variation of it. - - .. _sruid_definition: - - Service revision unique identifier (SRUID) - The combination of a service revision's namespace, name, and revision tag that uniquely identifies it. For - example, ``octue/my-service:1.3.0`` where the namespace is ``octue``, the name is ``my-service``, and the - revision tag is ``1.3.0``. - - Service namespace - The group to which the service belongs e.g. your name or your organisation's name. If in doubt, use the GitHub - handle of the user or organisation publishing the services. - - Namespaces must be lower kebab case (i.e. they may contain the letters [a-z], numbers [0-9], and hyphens [-]). - They may not begin or end with hyphens. - - Service name - A name to uniquely identify the service within its namespace. This usually corresponds to the name of the GitHub - repository for the service. Names must be lower kebab case (i.e. they may contain the letters [a-z], numbers - [0-9] and hyphens [-]). They may not begin or end with hyphens. - - Service revision tag - A tag that uniquely identifies a particular revision of a service. The revision tag could be a: - - - Commit hash (e.g. ``a3eb45``) - - Semantic version (e.g. ``0.12.4``) - - Branch name (e.g. ``development``) - - Particular environment the service is deployed in (e.g. ``production``) - - Combination of these (e.g. ``0.12.4-production``) - - Tags may contain lowercase and uppercase letters, numbers, underscores, periods, and hyphens, but can't start - with a period or a dash. They can contain a maximum of 128 characters. These requirements are the same as the - `Docker tag format `_. - - Service ID - The SRUID is a special case of a service ID. A service ID can be an SRUID or just the service namespace and - name. It can be used to ask a question to a service without specifying a specific revision of it. This enables - asking questions to, for example, the service ``octue/my-service`` and automatically having them routed to its - default (usually latest) revision. :ref:`See here for more info`. - - -Service communication standard -============================== - -Twined services communicate according to the service communication standard. The JSON schema defining this can be found -`here `_. Messages received by services are validated against it -and invalid messages are rejected. The schema is in beta, so (rare) breaking changes are reflected in the minor version -number. diff --git a/docs/source/testing_services.rst b/docs/source/testing_services.rst deleted file mode 100644 index d7f8e6673..000000000 --- a/docs/source/testing_services.rst +++ /dev/null @@ -1,288 +0,0 @@ -.. _testing_services: - -================ -Testing services -================ -We recommend writing automated tests for your service so anyone who wants to use it can have confidence in its quality -and reliability at a glance. `Here's an example test `_ -for our example service. - - -Emulating children -================== -If your app has children, you should emulate them in your tests instead of communicating with the real ones. This makes -your tests: - -- **Independent** of anything external to your app code - i.e. independent of the remote child, your internet connection, - and communication between your app and the child (Google Pub/Sub). -- **Much faster** - the emulation will complete in a few milliseconds as opposed to the time it takes the real child to - actually run an analysis, which could be minutes, hours, or days. Tests for our `child services template app - `_ run - around **900 times faster** when the children are emulated. - -The Child Emulator ------------------- -We've written a child emulator that takes a list of events and returns them to the parent for handling in the order -given - without contacting the real child or using Pub/Sub. Any events a real child can produce are supported. -:mod:`Child ` instances can be mocked like-for-like by -:mod:`ChildEmulator ` instances without the parent knowing. - -Event kinds -------------- -You can emulate any event that your app (the parent) can handle. The table below shows what these are. - -+------------------------------+----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Event kind | Number of events supported | Example | -+==============================+============================+=================================================================================================================================================================================================+ -| ``delivery_acknowledgement`` | One | {"event": {"kind": "delivery_acknowledgement"}, "attributes": {"question_uuid": 79192e90-9022-4797-b6c7-82dc097dacdb, ...}} | -+------------------------------+----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| ``heartbeat`` | Any number | {"event": {"kind": "heartbeat"}, "attributes": {"question_uuid": 79192e90-9022-4797-b6c7-82dc097dacdb, ...} | -+------------------------------+----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| ``log_record`` | Any number | {"event": {"kind": "log_record": "log_record": {"msg": "Starting analysis."}}, "attributes": {"question_uuid": 79192e90-9022-4797-b6c7-82dc097dacdb, ...} | -+------------------------------+----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| ``monitor_message`` | Any number | {"event": {"kind": "monitor_message": "data": '{"progress": "35%"}'}, "attributes": {"question_uuid": 79192e90-9022-4797-b6c7-82dc097dacdb, ...} | -+------------------------------+----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| ``exception`` | One | {"event": {"kind": "exception", "exception_type": "ValueError", "exception_message": "x cannot be less than 10."}, "attributes": {"question_uuid": 79192e90-9022-4797-b6c7-82dc097dacdb, ...} | -+------------------------------+----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| ``result`` | One | {"event": {"kind": "result", "output_values": {"my": "results"}, "output_manifest": None}, "attributes": {"question_uuid": 79192e90-9022-4797-b6c7-82dc097dacdb, ...} | -+------------------------------+----------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - -**Notes** - -- Event formats and contents must conform with the `service communication schema `_. -- Every event must be accompanied with the required event attributes -- The ``log_record`` key of a ``log_record`` event is any dictionary that the ``logging.makeLogRecord`` function can - convert into a log record. -- The ``data`` key of a ``monitor_message`` event must be a JSON-serialised string -- Any events after a ``result`` or ``exception`` event won't be passed to the parent because execution of the child - emulator will have ended. - - -Instantiating a child emulator ------------------------------- - -.. code-block:: python - - events = [ - { - { - "event": { - "kind": "log_record", - "log_record": {"msg": "Starting analysis."}, - ... # Left out for brevity. - }, - "attributes": { - "datetime": "2024-04-11T10:46:48.236064", - "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", - "retry_count": 0, - "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", - "parent_question_uuid": "5776ad74-52a6-46f7-a526-90421d91b8b2", - "originator_question_uuid": "86dc55b2-4282-42bd-92d0-bd4991ae7356", - "parent": "octue/test-service:1.0.0", - "originator": "octue/test-service:1.0.0", - "sender": "octue/test-service:1.0.0", - "sender_type": "CHILD", - "sender_sdk_version": "0.51.0", - "recipient": "octue/another-service:3.2.1" - }, - }, - }, - { - "event": { - "kind": "monitor_message", - "data": '{"progress": "35%"}', - }, - "attributes": { - ... # Left out for brevity. - }, - }, - { - "event": { - "kind": "log_record", - "log_record": {"msg": "Finished analysis."}, - ... # Left out for brevity. - }, - "attributes": { - ... # Left out for brevity. - }, - }, - { - "event": { - "kind": "result", - "output_values": [1, 2, 3, 4, 5], - }, - "attributes": { - ... # Left out for brevity. - }, - }, - ] - - child_emulator = ChildEmulator(events) - - def handle_monitor_message(message): - ... - - result, question_uuid = child_emulator.ask( - input_values={"hello": "world"}, - handle_monitor_message=handle_monitor_message, - ) - >>> {"output_values": [1, 2, 3, 4, 5], "output_manifest": None} - - -Using the child emulator ------------------------- -To emulate your children in tests, patch the :mod:`Child ` class with the -:mod:`ChildEmulator ` class. - -.. code-block:: python - - from unittest.mock import patch - - from octue.twined.runner import Runner - from octue.twined.cloud.emulators import ChildEmulator - - - app_directory_path = "path/to/directory_containing_app" - - # You can explicitly specify your children here as shown or - # read the same information in from your service configuration file. - children = [ - { - "key": "my_child", - "id": "octue/my-child-service:2.1.0", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "my-project" - } - }, - ] - - runner = Runner( - app_src=app_directory_path, - twine=os.path.join(app_directory_path, "twine.json"), - children=children, - service_id="your-org/your-service:2.1.0", - ) - - emulated_children = [ - ChildEmulator( - events=[ - { - "event": { - "kind": "result", - "output_values": [300], - }, - "attributes": { - "datetime": "2024-04-11T10:46:48.236064", - "uuid": "a9de11b1-e88f-43fa-b3a4-40a590c3443f", - "retry_count": 0, - "question_uuid": "d45c7e99-d610-413b-8130-dd6eef46dda6", - "parent_question_uuid": "5776ad74-52a6-46f7-a526-90421d91b8b2", - "originator_question_uuid": "86dc55b2-4282-42bd-92d0-bd4991ae7356", - "parent": "you/your-service:2.1.0", - "originator": "you/your-service:2.1.0", - "sender": "octue/my-child-service:2.1.0", - "sender_type": "CHILD", - "sender_sdk_version": "0.56.0", - "recipient": "you/your-service:2.1.0" - }, - }, - ], - ) - ] - - with patch("octue.runner.Child", side_effect=emulated_children): - analysis = runner.run(input_values={"some": "input"}) - - analysis.output_values - >>> [300] - - analysis.output_manifest - >>> None - - -**Notes** - -- If your app uses more than one child, provide more child emulators in the ``emulated_children`` list in the order - they're asked questions in your app. -- If a given child is asked more than one question, provide a child emulator for each question asked in the same order - the questions are asked. - - -Creating a test fixture -======================= -Since the child is *emulated*, it doesn't actually do any calculation - if you change the inputs, the outputs won't -change correspondingly (or at all). So, it's up to you to define a set of realistic inputs and corresponding outputs -(the list of emulated events) to test your service. These are called **test fixtures**. - -.. note:: - Unlike a real child, the **inputs** given to the emulator aren't validated against the schema in the child's twine - - this is because the twine is only available to the real child. This is ok - you're testing your service, not the - child your service contacts. The events given to the emulator are still validated against the service communication - schema, though. - -You can create test fixtures manually or by using the ``Child.received_events`` property after questioning a real -child. - -.. code-block:: python - - import json - from octue.twined.resources import Child - - - child = Child( - id="octue/my-child:2.1.0", - backend={"name": "GCPPubSubBackend", "project_id": "my-project"}, - ) - - result, question_uuid = child.ask(input_values=[1, 2, 3, 4]) - - child.received_events - >>> [ - { - "event": { - 'kind': 'delivery_acknowledgement', - }, - "attributes": { - ... # Left out for brevity. - }, - }, - { - "event": { - 'kind': 'log_record', - 'log_record': { - 'msg': 'Finished analysis.', - 'args': None, - 'levelname': 'INFO', - ... # Left out for brevity. - }, - }, - "attributes": { - ... # Left out for brevity. - }, - }, - { - "event": { - 'kind': 'result', - 'output_values': {"some": "results"}, - }, - "attributes": { - ... # Left out for brevity. - }, - }, - ] - -You can then feed these into a child emulator to emulate one possible response of the child: - -.. code-block:: python - - from octue.twined.cloud.emulators import ChildEmulator - - - child_emulator = ChildEmulator(events=child.received_events) - result, question_uuid = child_emulator.ask(input_values=[1, 2, 3, 4]) - - result - >>> {"some": "results"} - -You can also create test fixtures from :ref:`downloaded service crash diagnostics `. diff --git a/docs/source/troubleshooting_services.rst b/docs/source/troubleshooting_services.rst deleted file mode 100644 index a0f92f255..000000000 --- a/docs/source/troubleshooting_services.rst +++ /dev/null @@ -1,128 +0,0 @@ -======================== -Troubleshooting services -======================== - -Diagnostics -=========== -Services save the following data to the cloud if they crash while processing a question (the default), or when they -finish processing a question successfully if diagnostics are permanently turned on (not the default): - -- Input values -- Input manifest and datasets -- Child configuration values -- Child configuration manifest and datasets -- Inputs to and events received in response to each question the service asked its children (if it has any). These are - stored in the order the questions were asked. - -.. important:: - - For this feature to be enabled, the child must have the ``diagnostics_cloud_path`` field in its service - configuration (:ref:`octue.yaml ` file) set to a Google Cloud Storage path. - - -Accessing diagnostics -===================== -If diagnostics are enabled, a service will upload the diagnostics and send the upload path to the parent as a log -message. A user with credentials to access this path can use the ``octue`` CLI to retrieve the diagnostics data: - -.. code-block:: shell - - octue twined question diagnostics - -More information on the command: - -.. code-block:: - - >>> octue twined question diagnostics -h - - Usage: octue twined question diagnostics [OPTIONS] CLOUD_PATH - - Download diagnostics for an analysis from the given directory in - Google Cloud Storage. The cloud path should end in the analysis ID. - - CLOUD_PATH: The path to the directory in Google Cloud Storage containing the - diagnostics data. - - Options: - --local-path DIRECTORY The path to a directory to store the directory of - diagnostics data in. Defaults to the current working - directory. - --download-datasets If provided, download any datasets from the - diagnostics and update their paths in their - manifests to the new local paths. - -h, --help Show this message and exit. - -.. _test_fixtures_from_diagnostics: - -Creating test fixtures from diagnostics -======================================= -You can create test fixtures directly from diagnostics, allowing you to recreate the exact conditions that caused -your service to fail. - -.. code-block:: python - - from unittest.mock import patch - - from octue.twined.runner import Runner - from octue.twined.utils.testing import load_test_fixture_from_diagnostics - - - ( - configuration_values, - configuration_manifest, - input_values, - input_manifest, - child_emulators, - ) = load_test_fixture_from_diagnostics(path="path/to/downloaded/diagnostics") - - # You can explicitly specify your children here as shown or - # read the same information in from your service configuration file. - children = [ - { - "key": "my_child", - "id": "octue/my-child-service:2.1.0", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "my-project", - } - }, - { - "key": "another_child", - "id": "octue/another-child-service:2.1.0", - "backend": { - "name": "GCPPubSubBackend", - "project_id": "my-project", - } - } - ] - - runner = Runner( - app_src="path/to/directory_containing_app", - twine=os.path.join(app_directory_path, "twine.json"), - children=children, - configuration_values=configuration_values, - configuration_manifest=configuration_manifest, - service_id="your-org/your-service:2.1.0", - ) - - with patch("octue.runner.Child", side_effect=child_emulators): - analysis = runner.run(input_values=input_values, input_manifest=input_manifest) - - -Disabling diagnostics -===================== -When asking a question to a child, parents can disable diagnostics upload in the child on a question-by-question -basis by setting ``save_diagnostics`` to ``"SAVE_DIAGNOSTICS_OFF"`` in :mod:`Child.ask `. -For example: - -.. code-block:: python - - child = Child( - id="my-organisation/my-service:2.1.0", - backend={"name": "GCPPubSubBackend", "project_id": "my-project"}, - ) - - answer, question_uuid = child.ask( - input_values={"height": 32, "width": 3}, - save_diagnostics="SAVE_DIAGNOSTICS_OFF", - ) diff --git a/docs/source/twined/about.rst b/docs/source/twined/about.rst deleted file mode 100644 index a39cb2cd4..000000000 --- a/docs/source/twined/about.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _about: - -============ -About Twines -============ - -**Twined** is a framework for describing a digital twin or data service. - -We call these descriptions "twines". To just get started building a *twine*, check out the :ref:`quick_start`. To -get into the detail of what's in a *twine*, see :ref:`anatomy`. - -Here, we look at requirements for the framework, our motivations and background, and some of the decisions made while -developing **twined**. - -.. toctree:: - :maxdepth: 1 - - about_digital_twins - about_requirements - about_introducing_json_schema - about_other_considerations diff --git a/docs/source/twined/about_digital_twins.rst b/docs/source/twined/about_digital_twins.rst deleted file mode 100644 index b1ff3fda2..000000000 --- a/docs/source/twined/about_digital_twins.rst +++ /dev/null @@ -1,55 +0,0 @@ -.. _digital_twins: - -============= -Digital Twins -============= - -A digital twin is a virtual representation of a real life being - a physical asset like a wind turbine or car - or even -a human. - -There are three reasons why you might want to create a digital twin: - - Monitoring - - Prediction - - Optimisation - -On its own, a digital twin can be quite useful. For example, a twin might embody an AI-based analysis to predict power -output of a turbine. - -.. figure:: ../images/digital_twin_component_basic.svg - :width: 400px - :align: center - :figclass: align-center - :alt: A digital twin component - - A digital twin consists of some kind of analysis or processing task, which could be run many times per second, or - daily, down to occasionally or sometimes only once (the same as a "normal" analysis). - -Coupling digital twins is generally even more useful. You might wish to couple your turbine twin with a representation -of the local power grid, and a representation of a factory building to determine power demand... enabling you to -optimise your factory plant for lowest energy cost whilst intelligently selling surplus power to the grid. - -.. figure:: ../images/digital_twin_hierarchy.svg - :width: 350px - :align: center - :figclass: align-center - :alt: Hierarchy of digital twins - - A hierarchy of digital twins. Each blue circle represents a twin, coupled to its neighbours. Yellow nodes are where - schema are used to connect twins. - - -.. _gemini_principles: - -Gemini Principles -================= - -The Gemini Principles have been derived by the -`Centre for Digital Built Britain (CDBB) `_. -We strongly recommend you give them a read if embarking on a digital twins project. - -The aim of **twined** is to enable the following principles. In particular: - -#. Openness (open-source project to create schema for twins that can be run anywhere, anywhen) -#. Federation (encouraging a standardised way of connecting twins together) -#. Security (making sure schemas and data can be read safely) -#. Public Good (see our nano-rant about climate change in :ref:`reason_for_being`) diff --git a/docs/source/twined/about_introducing_json_schema.rst b/docs/source/twined/about_introducing_json_schema.rst deleted file mode 100644 index 09f00f906..000000000 --- a/docs/source/twined/about_introducing_json_schema.rst +++ /dev/null @@ -1,132 +0,0 @@ -.. _introducing_json_schema: - -======================= -Introducing JSON Schema -======================= - -``JSON`` is a data interchange format that has rapidly taken over as the defacto web-based data communication standard -in recent years. - -``JSONSchema`` is a way of specifying what a ``JSON`` document should contain. The Schema are, themselves, written in -``JSON``! - -Whilst schema can become extremely complicated in some scenarios, they are best designed to be quite succinct. See below -for the schema (and matching ``JSON``) for an integer and a string variable. - -**JSON:** - -.. code-block:: json - - { - "id": 1, - "name": "Tom" - } - - -**Schema:** - -.. code-block:: json - - { - "type": "object", - "title": "An id number and a name", - "properties": { - "id": { - "type": "integer", - "title": "An integer number", - "default": 0 - }, - "name": { - "type": "string", - "title": "A string name", - "default": "" - } - } - } - - -.. _useful_resources: - -Useful resources -================ -.. list-table:: - :widths: auto - :header-rows: 1 - - * - Link - - Resource - * - https://jsonschema.net/ - - Useful web tool for inferring schema from existing json - * - https://jsoneditoronline.org - - A powerful online editor for json, allowing manipulation of large documents better than most text editors - * - https://www.json.org/ - - The JSON standard spec - * - https://json-schema.org/ - - The (draft standard) JSONSchema spec - * - https://rjsf-team.github.io/react-jsonschema-form/ - - A front end library for generating webforms directly from a schema - - -.. _human_readbility: - -Human readability -================= - -Back in our :ref:`requirements` section, we noted it was important for humans to read and understand schema. - -The actual documents themselves are pretty easy to read by technical users. But, for non technical users, readability can be -enhanced even further by the ability to turn ``JSONSchema`` into web forms automatically. For our example above, we can -autogenerate a web form straight from the schema: - -.. figure:: ../images/schema_form_example.png - :width: 500px - :align: center - :figclass: align-center - :alt: Web form from an example schema - - Web form generated from the example schema above. - -Thus, we can take a schema (or a part of a schema) and use it to generate a control form for a digital twin in a web -interface without writing a separate form component - great for ease and maintainability. - - -.. _why_not_xml: - -Why not XML? -============ - -In a truly excellent `three-part blog `_, writer Seva Savris takes us -through the ups and downs of ``JSON`` versus ``XML``; well worth a read if wishing to understand the respective technologies -better. - -In short, both ``JSON`` and ``XML`` are generalised data interchange specifications and can both can do what we want here. -We choose ``JSON`` because: - -#. Textual representation is much more concise and easy to understand (very important where non-developers like - engineers and scientists must be expected to interpret schema) - -#. `Attack vectors `_. Because entities in ``XML`` - are not necessarily primitives (unlike in ``JSON``), an ``XML`` document parser in its default state may leave a system - open to XXE injection attacks and DTD validation attacks, and therefore requires hardening. ``JSON`` documents are - similarly afflicated (just like any kind of serialized data) but default parsers, operating on the premise of only - deserializing to primitive types, are safe by default - it is only when nondefault parsering or deserialization - techniques (such as ``JSONP``) are used that the application becomes vulnerable. By utilising a default ``JSON`` parser - we can therefore significantly shrink the attack surface of the system. See - `this blog post `_ for further discussion. - -#. ``XML`` is powerful... perhaps too powerful. The standard can be adapted greatly, resulting in high encapsulation - and a high resilience to future unknowns. Both beneficial. However, this requires developers of twins to maintain - interfaces of very high complexity, adaptable to a much wider variety of input. To enable developers to progress, we - suggest handling changes and future unknowns through well-considered versioning, whilst keeping their API simple. - -#. ``XML`` allows baked-in validation of data and attributes. Whilst advantageous in some situations, this is not a - benefit here. We wish validation to be one-sided: validation of data accepted/generated by a digital twin should be - occur within (at) the boundaries of that twin. - -#. Required validation capabilities, built into ``XML`` are achievable with ``JSONSchema`` (otherwise missing from the - pure ``JSON`` standard) - -#. ``JSON`` is a more compact expression than XML, significantly reducing memory and bandwidth requirements. Whilst - not a major issue for most modern PCS, sensors on the edge may have limited memory, and both memory and bandwidth at - scale are extremely expensive. Thus for extremely large networks of interconnected systems there could be significant - speed and cost savings. diff --git a/docs/source/twined/about_other_considerations.rst b/docs/source/twined/about_other_considerations.rst deleted file mode 100644 index 639942a51..000000000 --- a/docs/source/twined/about_other_considerations.rst +++ /dev/null @@ -1,105 +0,0 @@ -.. _other_considerations: - -==================== -Other Considerations -==================== - -A variety of thoughts that arose whilst architecting **twined**. - -.. _bash_style_stdio: - -Bash-style stdio -================ - -Some thought was given to using a very old-school-unix approach to piping data between twins, via stdout. - -Whilst attractive (as being a wildly fast way of piping data between twins on the same machine) it was felt this -was insufficiently general, eg: - - - where twins don't exist on the same machine or container, making it cumbersome to engineer common iostreams - - where slight differences between different shells might lead to incompatibilities or changes in behaviour - -And also unfriendly, eg: - - - engineers or scientists unfamiliar with subtleties of bash shell scripting encounter difficulty piping data around - - difficult to build friendly web based tools to introspect the data and configuration - - bound to be headaches on windows platforms, even though windows now supports bash - - easy to corrupt using third party libraries (e.g. which print to stdout) - - -.. _Units: - -Units -===== - -Being used (mostly) for engineering and scientific analysis, it was tempting to add in a specified sub-schema for units. -For example, mandating that where values can be given in units, they be specified in a certain way, like: - -.. code-block:: javascript - - { - "wind_speed": { - "value": 10.2, - "units": "mph" - } - } - -or (more succinct): - -.. code-block:: javascript - - { - "wind_speed": 10.2, - "wind_speed_units": "mph" - } - -It's still extremely tempting to provide this facility; or at least provide some way of specifying in the schema -what units a value should be provided in. Thinking about it but don't have time right now. -If anybody wants to start crafting a PR with an extension or update to **twined** that facilitates this; please raise an -issue to start progressing it. - - -.. _variable_style: - -Variable Style -============== - -A premptive stamp on the whinging... - -Note that in the ``JSON`` descriptions above, all variables are named in ``snake_case`` rather than ``camelCase``. This -decision, more likely than even Brexit to divide opinions, is based on: - -- The languages we anticipate being most popular for building twins seem to trend toward snake case (eg - `python `_, `c++ `_) - although to be fair we might've woefully misjudged which languages start emerging. - -- The reservation of snake case for the schema spec has the subtle advantage that in future, we might be able to use - camelCase within the spec to denote class types in some useful way, just like in python. Not sure yet; just mulling. - -- The :ref:`requirements` mention human-readability as a must; - `this paper `_ - suggests a 20% slower comprehension of camel case than snake, although to be fair that's probably arguable. - -- We're starting in Python so are taking a lead from PEP8, which is bar none the most successful style guide on the - planet, because it got everybody on the same page really early on. - -If existing code that you're dropping in uses camelCase, please don't file that as an issue... converting property -names automatically after schema validation generation is trivial, there are tons of libraries (like -`humps `_) to do it. - -We'd also consider a pull request for a built-in utility converting `to `_ and -`from `_ that does this following validation and prior to returning results. -Suggest your proposed approach on the `issues board `_. - - -.. _language_choice: - -Language Choice -=============== - -**twined** is presently released in python only. It won't be too hard to replicate functionality in other languages, and -we're considering other languages at present, so might be easily persuadable ;) - -If you require implementation of **twined** in a different language, -and are willing to consider sponsorship of development and maintenance of that library, please -`file an issue `_. diff --git a/docs/source/twined/about_requirements.rst b/docs/source/twined/about_requirements.rst deleted file mode 100644 index d2e828ef0..000000000 --- a/docs/source/twined/about_requirements.rst +++ /dev/null @@ -1,29 +0,0 @@ -.. _requirements: - -Requirements of the framework -=================================== - -A *twine* must describe a digital twin, and have multiple roles. It must: - -#. Define what data is required by a digital twin, in order to run -#. Define what data will be returned by the twin following a successful run -#. Define the formats of these data, in such a way that incoming data can be validated -#. Define what other (1st or 3rd party) twins / services are required by this one in order for it to run. - -If this weren't enough, the description: - -#. Must be trustable (i.e. a *twine* from an untrusted, corrupt or malicious third party should be safe to at least read) -#. Must be machine-readable *and machine-understandable* [1]_ -#. Must be human-readable *and human-understandable* [1]_ -#. Must be discoverable (that is, searchable/indexable) otherwise people won't know it's there in orer to use it. - -Fortunately for digital twin developers, several of these requirements have already been seen for data interchange -formats developed for the web. **twined** uses ``JSON`` and ``JSONSchema`` to help interchange data. - -If you're not already familiar with ``JSONSchema`` (or wish to know why **twined** uses ``JSON`` over the seemingly more -appropriate ``XML`` standard), see :ref:`introducing_json_schema`. - - -.. Footnotes: - -.. [1] *Understandable* essentially means that, once read, the machine or human knows what it actually means and what to do with it. diff --git a/docs/source/twined/anatomy.rst b/docs/source/twined/anatomy.rst deleted file mode 100644 index dfc4e40e8..000000000 --- a/docs/source/twined/anatomy.rst +++ /dev/null @@ -1,101 +0,0 @@ -.. _anatomy: - -========================= -Anatomy Of The Twine File -========================= - -The main point of **twined** is to enable engineers and scientists to easily (and rigorously) define a digital twin -or data service. - -This is done by adding a ``twine.json`` file to the repository containing your code. Adding a *twine* means you can: - -- communicate (to you or a colleague) what data is required by this service -- communicate (to another service / machine) what data is required -- deploy services automatically with a provider like `Octue `_. - -To just get started building a *twine*, check out the :ref:`quick_start`. To learn more about twines in general, -see :ref:`about`. Here, we describe the parts of a *twine* ("strands") and what they mean. - -.. _strands: - -Strands -======= - -A *twine* has several sections, called *strands*. Each defines a different kind of data required (or produced) by the -twin. - -.. list-table:: - :widths: 30 70 - :header-rows: 1 - - * - Strand - - Describes the twin's requirements for... - * - :ref:`Configuration Values ` - - Data, in JSON form, used for configuration of the twin/service. - * - :ref:`Configuration Manifest ` - - Files/datasets required by the twin at configuration/startup - * - :ref:`Input Values ` - - Data, in JSON form, passed to the twin in order to trigger an analysis - * - :ref:`Input Manifest ` - - Files/datasets passed with Input Values to trigger an analysis - * - :ref:`Output Values ` - - Data, in JSON form, that will be produced by the twin (in response to inputs) - * - :ref:`Output Manifest ` - - Files/datasets that will be produced by the twin (in response to inputs) - * - :ref:`Credentials ` - - Credentials that are required by the twin in order to access third party services - * - :ref:`Children ` - - Other twins, access to which are required for this twin to function - * - :ref:`Monitors ` - - Visual and progress outputs from an analysis - - -.. toctree:: - :maxdepth: 1 - :hidden: - - anatomy_values - anatomy_manifest - anatomy_credentials - anatomy_monitors - anatomy_children - - -.. _twine_file_schema: - -Twine File Schema -================= - -Because the ``twine.json`` file itself is in ``JSON`` format with a strict structure, **twined** uses a schema to make -that twine files are correctly written (a "schema-schema", if you will, since a twine already contains schema). Try not -to think about it. But if you must, the *twine* schema is -`here `_. - -The first thing **twined** always does is check that the ``twine.json`` file itself is valid, and give you a -descriptive error if it isn't. - - -.. _other_external_io: - -Other External I/O -================== - -A twin might: - -- GET/POST data from/to an external API, -- query/update a database, -- upload files to an object store, -- trigger events in another network, or -- perform pretty much any interaction you can think of with other applications over the web. - -However, such data exchange may not be controllable by **twined** (which is intended to operate at the boundaries of the -twin) unless the resulting data is returned from the twin (and must therefore be compliant with the schema). - -So, there's nothing for **twined** to do here, and no need for a strand in the *twine* file. However, interacting with -third party APIs or databases might require some credentials. See :ref:`credentials_strand` for help with that. - -.. NOTE:: - This is actually a very common scenario. For example, the purpose of the twin might be to fetch data (like a weather - forecast) from some external API then return it in the ``output_values`` for use in a network of digital twins. - But its the twin developer's job to do the fetchin' and make sure the resulting data is compliant with the - ``output_values_schema`` (see :ref:`values_based_strands`). diff --git a/docs/source/twined/anatomy_children.rst b/docs/source/twined/anatomy_children.rst deleted file mode 100644 index 8f1723746..000000000 --- a/docs/source/twined/anatomy_children.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _children_strand: - -=============== -Children Strand -=============== - -.. ATTENTION:: - - Coming Soon! diff --git a/docs/source/twined/anatomy_credentials.rst b/docs/source/twined/anatomy_credentials.rst deleted file mode 100644 index cbed9f059..000000000 --- a/docs/source/twined/anatomy_credentials.rst +++ /dev/null @@ -1,78 +0,0 @@ -.. _credentials_strand: - -================== -Credentials Strand -================== - -In order to: - -- GET/POST data from/to an API, -- query a database, or -- connect to a socket (for receiving Values or emitting Values, Monitors or Logs), - -A digital twin must have *access* to it. API keys, database URIs, etc must be supplied to the digital twin but -treated with best practice with respect to security considerations. The purpose of the ``credentials`` strand is to -dictate what credentials the twin requires in order to function. - -.. _defining_the_credentials_strand: - -Defining the Credentials Strand -=============================== - -This is the simplest of the strands, containing a list of credentials (whose ``NAMES_SHOULD_BE_SHOUTY_SNAKE_CASE``) with -a reminder of the purpose. - -.. code-block:: javascript - - { - "credentials": [ - { - "name": "SECRET_THE_FIRST", - "purpose": "Token for accessing a 3rd party API service" - }, - { - "name": "SECRET_THE_SECOND", - "purpose": "Token for accessing a 3rd party API service" - }, - { - "name": "SECRET_THE_THIRD", - "purpose": "Another secret, like a password for a sandbox or local database" - } - ] - } - -.. _supplying_credentials: - -Supplying Credentials -===================== - -.. ATTENTION:: - - *Credentials should never be hard-coded into application code* - - Do you trust the twin code? If you insert credentials to your own database into a digital twin - provided by a third party, you better be very sure that twin isn't going to scrape all that data out then send - it elsewhere! - - Alternatively, if you're building a twin requiring such credentials, it's your responsibility to give the end - users confidence that you're not abusing their access. - - There'll be a lot more discussion on these issues, but it's outside the scope of **twined** - all we do here is - make sure a twin has the credentials it requires. - -Credentials should be securely managed by whatever system is managing the twin, then made accessible to the twin -in the form of environment variables: - -.. code-block:: javascript - - SERVICE_API_KEY=someLongTokenTHatYouProbablyHaveToPayTheThirdPartyProviderLoadsOfMoneyFor - -Credentials may also reside in a ``.env`` file in the current directory, either in the format above -(with a new line for each variable) or, for convenience, as bash exports like: - -.. code-block:: javascript - - export SERVICE_API_KEY=someLongTokenTHatYouProbablyHaveToPayTheThirdPartyProviderLoadsOfMoneyFor - -The ``validate_credentials()`` method of the ``Twine`` class checks for their presence and, where contained in a -``.env`` file, ensures they are loaded into the environment. diff --git a/docs/source/twined/anatomy_manifest.rst b/docs/source/twined/anatomy_manifest.rst deleted file mode 100644 index 1f342b57a..000000000 --- a/docs/source/twined/anatomy_manifest.rst +++ /dev/null @@ -1,369 +0,0 @@ -.. _manifest_strands: - -====================== -Manifest-based Strands -====================== - -Frequently, twins operate on files containing some kind of data. These files need to be made accessible to the code -running in the twin, in order that their contents can be read and processed. Conversely, a twin might produce an output -dataset which must be understood by users. - -The ``configuration_manifest``, ``input_manifest`` and ``output_manifest`` strands describe what kind of datasets (and -associated files) are required / produced. - -.. NOTE:: - - Files are always contained in datasets, even if there's only one file. It's so that we can keep nitty-gritty file - metadata separate from the more meaningful, higher level metadata like what a dataset is for. - -.. tabs:: - - .. group-tab:: Configuration Manifest Strand - - This describes datasets/files that are required at startup of the twin / service. They typically contain a - resource that the twin might use across many analyses. - - For example, a twin might predict failure for a particular component, given an image. It will require a trained - ML model (saved in a ``*.pickle`` or ``*.json``). While many thousands of predictions might be done over the - period that the twin is deployed, all predictions are done using this version of the model - so the model file is - supplied at startup. - - .. group-tab:: Input Manifest Strand - - These files are made available for the twin to run a particular analysis with. Each analysis will likely have - different input datasets. - - For example, a twin might be passed a dataset of LiDAR ``*.scn`` files and be expected to compute atmospheric flow - properties as a timeseries (which might be returned in the :ref:`output values ` for onward - processing and storage). - - .. group-tab:: Output Manifest Strand - - Files are created by the twin during an analysis, tagged and stored as datasets for some onward purpose. - This strand is not used for sourcing data; it enables users or other services to understand appropriate search - terms to retrieve datasets produced. - - -.. _describing_manifests: - -Describing Manifests -==================== - -Manifest-based strands are a **description of what files are needed**. The purpose of the manifest strands is to -provide a helper to a wider system providing datafiles to digital twins. - -.. tabs:: - - .. group-tab:: Configuration Manifest Strand - - .. accordion:: - - .. accordion-row:: Show twine containing this strand - - .. literalinclude:: ../../octue/twined/examples/damage_classifier_service/twine.json - :language: javascript - - .. accordion-row:: Show a matching file manifest - - .. literalinclude:: ../../octue/twined/examples/damage_classifier_service/data/configuration_manifest.json - :language: javascript - - - .. group-tab:: Input Manifest Strand - - Here we specify that two datasets (and all or some of the files associated with them) are - required, for a service that cross-checks meteorological mast data and power output data for a wind farm. - - .. accordion:: - - .. accordion-row:: Show twine containing this strand - - .. literalinclude:: ../../octue/twined/examples/met_mast_scada_service/strands/input_manifest.json - :language: javascript - - .. accordion-row:: Show a matching file manifest - - .. literalinclude:: ../../octue/twined/examples/met_mast_scada_service/data/input_manifest.json - :language: javascript - - .. group-tab:: Output Manifest Strand - - .. accordion:: - - .. accordion-row:: Show twine containing this strand - - .. literalinclude:: ../../octue/twined/examples/met_mast_scada_service/strands/output_manifest.json - :language: javascript - - .. accordion-row:: Show a matching file manifest - - .. literalinclude:: ../../octue/twined/examples/met_mast_scada_service/data/output_manifest.json - :language: javascript - - - -.. _file_tag_templates: - -File tag templates -================== - -Datafiles can be tagged with key-value pairs of relevant metadata that can be used in analyses. Certain datasets might -need one set of metadata on each file, while others might need a different set. The required (or optional) file tags can be -specified in the twine in the ``file_tags_template`` property of each dataset of any ``manifest`` strand. Each file in -the corresponding manifest strand is then validated against its dataset's file tag template to ensure the required tags -are present. - -.. tabs:: - - .. group-tab:: Manifest strand with file tag template - - The example below is for an input manifest, but the format is the same for configuration and output manifests. - - .. accordion:: - - .. accordion-row:: Show twine containing a manifest strand with a file tag template - - .. code-block:: javascript - - { - "input_manifest": { - "datasets": [ - { - "key": "met_mast_data", - "purpose": "A dataset containing meteorological mast data", - "file_tags_template": { - "type": "object", - "properties": { - "manufacturer": {"type": "string"}, - "height": {"type": "number"}, - "is_recycled": {"type": "boolean"} - }, - "required": ["manufacturer", "height", "is_recycled"] - } - } - ] - } - } - - .. accordion-row:: Show a matching file manifest - - .. code-block:: javascript - - { - "id": "8ead7669-8162-4f64-8cd5-4abe92509e17", - "datasets": [ - { - "id": "7ead7669-8162-4f64-8cd5-4abe92509e17", - "name": "met_mast_data", - "tags": {}, - "labels": ["met", "mast", "wind"], - "files": [ - { - "path": "input/datasets/7ead7669/file_1.csv", - "cluster": 0, - "sequence": 0, - "extension": "csv", - "labels": ["mykeyword1", "mykeyword2"], - "tags": { - "manufacturer": "vestas", - "height": 500, - "is_recycled": true - }, - "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", - "name": "file_1.csv" - }, - { - "path": "input/datasets/7ead7669/file_1.csv", - "cluster": 0, - "sequence": 1, - "extension": "csv", - "labels": [], - "tags": { - "manufacturer": "vestas", - "height": 500, - "is_recycled": true - }, - "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", - "name": "file_1.csv" - } - ] - } - ] - } - - .. group-tab:: Manifest strand with a remote file tag template - - A remote reference can also be given for a file tag template. If the tag template somewhere public, this is - useful for sharing the template between one or more teams working on the same type of data. - - The example below is for an input manifest, but the format is the same for configuration and output manifests. - It also shows two different tag templates being specified for two different types of dataset required by the - manifest. - - .. accordion:: - - .. accordion-row:: Show twine using a remote tag template - - .. code-block:: javascript - - { - "input_manifest": { - "datasets": [ - { - "key": "met_mast_data", - "purpose": "A dataset containing meteorological mast data", - "file_tags_template": { - "$ref": "https://refs.schema.octue.com/octue/my-file-type-tag-template/0.0.0.json" - } - }, - { - "key": "some_other_kind_of_dataset", - "purpose": "A dataset containing something else", - "file_tags_template": { - "$ref": "https://refs.schema.octue.com/octue/another-file-type-tag-template/0.0.0.json" - } - } - ] - } - } - - .. accordion-row:: Show a matching file manifest - - .. code-block:: javascript - - { - "id": "8ead7669-8162-4f64-8cd5-4abe92509e17", - "datasets": [ - { - "id": "7ead7669-8162-4f64-8cd5-4abe92509e17", - "name": "met_mast_data", - "tags": {}, - "labels": ["met", "mast", "wind"], - "files": [ - { - "path": "input/datasets/7ead7669/file_1.csv", - "cluster": 0, - "sequence": 0, - "extension": "csv", - "labels": ["mykeyword1", "mykeyword2"], - "tags": { - "manufacturer": "vestas", - "height": 500, - "is_recycled": true - }, - "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", - "name": "file_1.csv" - }, - { - "path": "input/datasets/7ead7669/file_1.csv", - "cluster": 0, - "sequence": 1, - "extension": "csv", - "labels": [], - "tags": { - "manufacturer": "vestas", - "height": 500, - "is_recycled": true - }, - "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", - "name": "file_1.csv" - } - ] - }, - { - "id": "7ead7669-8162-4f64-8cd5-4abe92509e29", - "name": "some_other_kind_of_dataset", - "tags": {}, - "labels": ["my-label"], - "files": [ - { - "path": "input/datasets/7eadpp9/interesting_file.dat", - "cluster": 0, - "sequence": 0, - "extension": "dat", - "labels": [], - "tags": { - "length": 864, - "orientation_angle": 85 - }, - "id": "abff07bc-7c19-4ed5-be6d-a6546eae9071", - "name": "interesting_file.csv" - }, - } - ] - } - - -TODO - clean up or remove this section - -.. _how_filtering_works: - -How Filtering Works -=================== - -It's the job of **twined** to make sure of two things: - -1. make sure the *twine* file itself is valid, - - - **File data (input, output)** - - Files are not streamed directly to the digital twin (this would require extreme bandwidth in whatever system is - orchestrating all the twins). Instead, files should be made available on the local storage system; i.e. a volume - mounted to whatever container or VM the digital twin runs in. - - Groups of files are described by a ``manifest``, where a manifest is (in essence) a catalogue of files in a - dataset. - - A digital twin might receive multiple manifests, if it uses multiple datasets. For example, it could use a 3D - point cloud LiDAR dataset, and a meteorological dataset. - - .. code-block:: javascript - - { - "manifests": [ - { - "type": "dataset", - "id": "3c15c2ba-6a32-87e0-11e9-3baa66a632fe", // UUID of the manifest - "files": [ - { - "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", // UUID of that file - "sha1": "askjnkdfoisdnfkjnkjsnd" // for quality control to check correctness of file contents - "name": "Lidar - 4 to 10 Dec.csv", - "path": "local/file/path/to/folder/containing/it/", - "type": "csv", - "metadata": { - }, - "size_bytes": 59684813, - "tags": {"special_number": 1}, - "labels": ["lidar", "helpful", "information", "like"], // Searchable, parsable and filterable - }, - { - "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", - "name": "Lidar - 11 to 18 Dec.csv", - "path": "local/file/path/to/folder/containing/it/", - "type": "csv", - "metadata": { - }, - "size_bytes": 59684813, - "tags": {"special_number": 2}, - "labels": ["lidar", "helpful", "information", "like"] // Searchable, parsable and filterable - }, - { - "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", - "name": "Lidar report.pdf", - "path": "local/file/path/to/folder/containing/it/", - "type": "pdf", - "metadata": { - }, - "size_bytes": 484813, - "tags": {}, - "labels": ["report"] // Searchable, parsable and filterable - } - ] - }, - { - // ... another dataset manifest ... - } - ] - } diff --git a/docs/source/twined/anatomy_monitors.rst b/docs/source/twined/anatomy_monitors.rst deleted file mode 100644 index 7d034b3e0..000000000 --- a/docs/source/twined/anatomy_monitors.rst +++ /dev/null @@ -1,55 +0,0 @@ -.. _monitors_strand: - -====================== -Monitor Message Strand -====================== - -The ``monitor_message_schema`` strand is *values-based* meaning the data that matches the strand is in JSON form. It is -a *json schema* which describes a monitor message. - -.. tabs:: - - .. group-tab:: Monitors Strand - - There are two kinds of monitoring data required from a digital twin. - - **Monitor data (output)** - - Values for health and progress monitoring of the twin, for example percentage progress, iteration number and - status - perhaps even residuals graphs for a converging calculation. Broadly speaking, this should be user-facing - information. - - *This kind of monitoring data can be in a suitable form for display on a dashboard* - - **Log data (output)** - - Logged statements, typically in iostream form, produced by the twin (e.g. via python's ``logging`` module) must be - capturable as an output for debugging and monitoring purposes. Broadly speaking, this should be developer-facing - information. - - - -Let's look at basic examples for twines containing each of these strands: - - -.. tabs:: - - .. group-tab:: Monitors Strand - - **Monitor data (output)** - - .. code-block:: javascript - - { - "monitor_message_schema": { - "type": "object", - "properties": { - "my_property": { - "type": "number" - } - }, - "required": ["my_property"] - } - } - - **Log data (output)** diff --git a/docs/source/twined/anatomy_values.rst b/docs/source/twined/anatomy_values.rst deleted file mode 100644 index f624d30de..000000000 --- a/docs/source/twined/anatomy_values.rst +++ /dev/null @@ -1,128 +0,0 @@ -.. _values_based_strands: - -==================== -Values-based Strands -==================== - -The ``configuration_values_schema``, ``input_values_schema`` and ``output_values_schema`` strands are *values-based*, -meaning the data that matches these strands is in JSON form. - -Each of these strands is a *json schema* which describes that data. - -.. tabs:: - - .. group-tab:: Configuration Values Strand - - This strand is a ``configuration_values_schema``, that is used to check validity of any - ``configuration_values`` data supplied to the twin at startup. - - The Configuration Values Strand is generally used to define control parameters relating to what the twin should - do, or how it should operate. - - For example, should it produce output images as low resolution PNGs or as SVGs? How many iterations of a fluid - flow solver should be used? What is the acceptable error level on an classifier algorithm? - - .. group-tab:: Input Values Strand - - This strand is an ``input_values_schema``, that is used to check validity of ``input_values`` data supplied to the - twin at the beginning of an analysis task. - - The Input Values Strand is generally used to define actual data which will be processed by the twin. Sometimes, it - may be used to define control parameters specific to an analysis. - - For example, if a twin cleans and detects anomalies in a 10-minute timeseries of 1Hz data, the ``input_values`` - might contain an array of data and a list of corresponding timestamps. It may also contain a control parameter - specifying which algorithm is used to do the detection. - - .. NOTE:: - Depending on the way the twin is deployed (see :ref:`deployment`), the ``input_values`` might come in from a - web request, over a websocket or called directly from the command line or another library. - - However they come, if the new ``input_values`` validate against the ``input_values_schema`` strand, - then analysis can proceed. - - .. group-tab:: Output Values Strand - - This strand is an ``output_values_schema``, that is used to check results (``output_values``) computed during an - analysis. This ensures that the application wrapped up within the *twine* is operating correctly, and - enables other twins/services or the end users to see what outputs they will get. - - For example,if a twin cleans and detects anomalies in a 10-minute timeseries of 1Hz data, the ``output_values`` - might contain an array of data interpolated onto regular timestamps, with missing values filled in and a list of - warnings where anomalies were found. - - -Let's look at basic examples for twines containing each of these strands: - - -.. tabs:: - - .. group-tab:: Configuration Values Strand - - This *twine* contains an example ``configuration_values_schema`` with one control parameter. - - `Many more detailed and specialised examples are available in the GitHub repository `_ - - .. code-block:: javascript - - { - "configuration_values_schema": { - "title": "The example configuration form", - "description": "The Configuration Values Strand of an example twine", - "type": "object", - "properties": { - "n_iterations": { - "description": "An example of an integer configuration variable, called 'n_iterations'.", - "type": "integer", - "minimum": 1, - "maximum": 10, - "default": 5 - } - } - } - } - - Matching ``configuration_values`` data could look like this: - - .. code-block:: javascript - - { - "n_iterations": 8, - } - - - .. group-tab:: Input Values Strand - - This *twine* contains an example ``input_values_schema`` with one input value, which marked as required. - - Many more detailed and specialised examples are available in :ref:`examples`. - - .. code-block:: javascript - - { - "input_values_schema": { - "title": "Input Values", - "description": "The input values strand of an example twine, with a required height value", - "type": "object", - "properties": { - "height": { - "description": "An example of an integer value called 'height'", - "type": "integer", - "minimum": 2 - } - }, - "required": ["height"] - }, - - Matching ``input_values`` data could look like this: - - .. code-block:: javascript - - { - "height": 13, - } - - - .. group-tab:: Output Values Strand - - Stuff diff --git a/docs/source/twined/deployment.rst b/docs/source/twined/deployment.rst deleted file mode 100644 index 0cec07339..000000000 --- a/docs/source/twined/deployment.rst +++ /dev/null @@ -1,66 +0,0 @@ -.. _deployment: - -========== -Deployment -========== - - -.. _deploying_with_octue: - -Deploying with Octue -==================== - -`Octue `_ provides automated deployment to a cloud provider (like GCP or Azure), along with -permissions and user management, monitoring, logging and data storage management out of the box. - -There are also a whole bunch of collaborative helper tools, like the graphical -`twine builder `_ and manifesting tools, designed to speed up the process of building -and using twines. - -The full set of services is in early beta, `get in touch `_ and we can help you -architect systems - from small data services to large networks of :ref:`digital_twins`. - - -.. _deploying_with_doctue: - -Coming Soon - Deploying with doctue -=================================== - -Once we've bedded down our services internally at Octue, we'll be open-sourcing more parts of our build/deploy process, -including docker containers with pre-configured servers to run and monitor twine-based services and digital twins. - -This will allow services to be easily spun up on GCP, Azure Digital Ocean etc., and be a nice halfway house between -fully managed system on Octue and running your own webserver. Of course, -without all the collaborative and data management features that Octue provides ;) - -We're looking for commercial sponsors for this part of the process - if that could be you, please -`get in touch `_ - - -.. _deploying_as_a_cli: - -Deploying as a command-line application -======================================= - -Use the open-source `octue app template `_ as a guide. Write your new -python code (or call your existing tools/libraries) within it. It's set up to wrap and check configuration, inputs and -outputs using twined. Follow the instructions there to set up your inputs, and your files, and run an analysis. - - -.. _deployment_with_a_web_server: - -Deploying with your own web server -================================== - -You can use any python based web server (need another language? see :ref:`language_choice`): - -- Add ``configuration_values_data`` to your webserver config -- Set up an endpoint to allow. -- Set up an endpoint to handle incoming requests / socket messages - these will be ``input_values_data``. -- Treat these requests / messages as events which trigger a task. -- In your task framework (e.g. your celery task), either: - - Use **twined** directly to validate the ``input_values_data``/``output_values_data`` (and, on startup, the - ``configuration_values_data``) and handle running any required analysis yourself, or - - import your analysis app (as built in :ref:`deploying_as_a_cli`) and call it with the configuration and input - data in your task framework. -- Return the result to the client. diff --git a/docs/source/twined/examples.rst b/docs/source/twined/examples.rst deleted file mode 100644 index a44f680ac..000000000 --- a/docs/source/twined/examples.rst +++ /dev/null @@ -1,192 +0,0 @@ -.. _examples: - -======== -Examples -======== - -Here, we look at example use cases for the library, and show how to use it in python. - -It's also well worth looking at the unit test cases -copied straight from the unit test cases, so you can always check there to see how everything hooks up. - - -.. _example_equipment_installation_cost: - -[Simple] Equipment installation cost -==================================== - -.. tabs:: - - .. group-tab:: Scenario - - You need to provide your team with an estimate for installation cost of an equipment foundation. - - It's a straightforward calculation for you, but the Logistics Team keeps changing the installation position, to - try and optimise the overall project logistics. - - Each time the locations change, the GIS team gives you an updated embedment depth, which is what you use - (along with steel cost and foundation type), to calculate cost and report it back. - - This twine allows you to define to create a wrapper around your scripts that communicates to the GIS team what you - need as an input, communicate to the logistics team what they can expect as an output. - - When deployed as a digital twin, the calculation gets automatically updated, leaving you free to get on with - all the other work! - - .. group-tab:: Twine - - We specify the ``steel_cost`` and ``foundation_type`` as ``configuration`` values, which you can set on startup of the twin. - - Once the twin is running, it requires the ``embedment_depth`` as an ``input_value`` from the GIS team. A member - of the GIS team can use your twin to get ``foundation_cost`` directly. - - .. code-block:: javascript - - { - "title": "Foundation Cost Model", - "description": "This twine helps compute the cost of an installed foundation.", - "children": [ - ], - "configuration_values_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Foundation cost twin configuration", - "description": "Set config parameters and constants at startup of the twin.", - "type": "object", - "properties": { - "steel_cost": { - "description": "The cost of steel in GBP/m^3. To get a better predictive model, you could add an economic twin that forecasts the cost of steel using the project timetable.", - "type": "number", - "minimum": 0, - "default": 3000 - }, - "foundation_type": { - "description": "The type of foundation being used.", - "type": "string", - "pattern": "^(monopile|twisted-jacket)$", - "default": "monopile" - } - } - }, - "input_values_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Input Values schema for the foundation cost twin", - "description": "These values are supplied to the twin asynchronously over a web socket. So as these values change, the twin can reply with an update.", - "type": "object", - "properties": { - "embedment_depth": { - "description": "Embedment depth in metres", - "type": "number", - "minimum": 10, - "maximum": 500 - } - } - }, - "output_manifest": { - "datasets": [] - }, - "output_values_schema": { - "title": "Output Values schema for the foundation cost twin", - "description": "The response supplied to a change in input values will always conform to this schema.", - "type": "object", - "properties": { - "foundation_cost": { - "description": "The foundation cost.", - "type": "integer", - "minimum": 2 - } - } - } - } - - -.. _example_site_weather_conditions: - -[Simple] Site weather conditions -================================ - -.. tabs:: - - .. group-tab:: Scenario - - You need to be able to get characteristic weather conditions at a specific location, for a range of reasons - including assessing extreme design loads. The values you need are computed in a script, which calls a Weather - API (provided by a third party), but also needs a dataset of "Wind Resource" files. - - .. group-tab:: Twine - - .. code-block:: javascript - - { - "title": "Weather Service Digital Twin", - "description": "Provides a model for design extreme weather conditions given a location", - "notes": "Easily extendable with children to add forecast and historical data of different types.", - "credentials": [ - { - "name": "WEATHER_API_SECRET_KEY", - "purpose": "Token for accessing a 3rd party weather API service" - } - ], - "input_manifest": { - "datasets": [ - { - "key": "wind_resource_data", - "purpose": "A dataset containing Wind Resource Grid files" - } - ] - }, - "input_values_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Input Values for the weather service twin", - "description": "This is a simple example for getting metocean conditions at a single location", - "type": "object", - "properties": { - "location": { - "description": "Location", - "type": "object", - "properties": { - "latitude": { - "type": "number", - "minimum": -90, - "maximum": 90 - }, - "longitude": { - "type": "number", - "minimum": -180, - "maximum": 180 - }, - "srid": { - "description": "The Spatial Reference System ID for the coordinate. Default is 4326 (WGS84)", - "type": "integer", - "default": 4326 - } - } - } - } - }, - "output_manifest": { - "datasets": [ - { - "key": "production_data", - "purpose": "A dataset containing production data", - "tags": {"cleaned": true}, - "labels": ["production", "wind"] - } - ] - }, - "output_values_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Output Values for the metocean service twin", - "description": "The output values strand of an example twine", - "type": "object", - "properties": { - "water_depth": { - "description": "Design water depth for use in concept calculations", - "type": "number" - }, - "extreme_wind_speed": { - "description": "Extreme wind speed value for use in concept calculations", - "type": "number" - } - } - } - } diff --git a/docs/source/twined/favicon.ico b/docs/source/twined/favicon.ico deleted file mode 100644 index 97f2f385efe4238217885f7221afbc306f7d6627..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1150 zcmbu8JugF17{{N|YB97SEzx>Ox`?3`G1bdv@C68wGzJnMKx{ghh)5(B#AvXIq|!k& z5ki?vmeR#SB@B{=`aQSzB&F9t+x+gi=iLAQInQ%mL|ps>0pZyr^KOxli1Y$OjFiV| zMEDksm9hF2kN)VP*mTj(z`*$6t~~Db$=%k#FeIQ8v|r&A<7>b(1vl^s>2NU9^swFE zg8mXdAPbY=0V@VN^?P6m-k<~pSZQlLbLY2XKD6slfD*jIB3LodsbA+c-ecb5p!rz2 zqVkwlU-!l|p4*db> z+(+O8W2tJAhV8#VU#q{?{QtUsBjjGWe*fLSE8^(> z>6%T$U(b)8V{5(fI-TFuj$p={--$J&+XTn+t9xbV@bmthM?$B5&VM`JTX4KTX0PoS z8cWwB0dbL%;IG3w2D4CgfA9SwC-oxBE|FwX#G4cGzm(~I7g0=Kzo|yG>dE2#0zB|~ A-2eap diff --git a/docs/source/twined/index.rst b/docs/source/twined/index.rst deleted file mode 100644 index 509b55944..000000000 --- a/docs/source/twined/index.rst +++ /dev/null @@ -1,106 +0,0 @@ -.. ATTENTION:: - This library is in very early stages. Like the idea of it? Please - `star us on GitHub `_ and contribute via the - `issues board `_ and - `roadmap `_. - -====== -Twined -====== - -**twined** is a library to help create and connect :ref:`digital_twins` and data services. - -.. epigraph:: - *"Twined" [t-why-nd] ~ encircled, twisted together, interwoven* - -A digital twin is a virtual representation of a real life being - a physical asset like a wind turbine or car - or even -a human. Like real things, digital twins need to interact, so can be connected together, but need a common communication -framework to do so. - -**twined** helps you to define a single file, a "twine", that defines a digital twin / data service. It specifies -specifying its data interfaces, connections to other twins, and other requirements. - -Any person, or any computer, can read a twine and understand *what-goes-in* and *what-comes-out*. That makes it easy to -collaborate with other teams, since everybody is crystal clear about what's needed. - -.. figure:: ../images/digital_twin_hierarchy.svg - :width: 350px - :align: center - :figclass: align-center - :alt: Hierarchy of digital twins - - Digital twins / data services connected in a hierarchy. Each blue circle represents a twin, coupled to its neighbours. - Yellow nodes are where schema are used to connect twins. - - -.. _aims: - -Aims -==== - -**twined** provides a toolkit to help create and validate "twines" - descriptions of a digital twin, what data it -requires, what it does and how it works. - -The goals of this **twined** library are as follows: - - Provide a clear framework for what a *twine* can and/or must contain - - Provide functions to validate incoming data against a known *twine* - - Provide functions to check that a *twine* itself is valid - - Provide (or direct you to) tools to create *twines* describing what you require - -In :ref:`anatomy`, we describe the different parts of a twine (examining how digital twins connect and interact... -building them together in hierarchies and networks). But you may prefer to dive straight in with the :ref:`quick_start` -guide. - -The scope of **twined** is not large. Many other libraries will deal with hosting and deploying digital twins, still -more will deal with the actual analyses done within them. **twined** purely deals with parsing and checking the -information exchanged. - - -.. _reason_for_being: - -Raison d'etre -============= - -Octue believes that a lynchpin of solving climate change is the ability for all engineering, manufacturing, supply -chain and infrastructure plant to be connected together, enabling strong optimisation and efficient use of these -systems. - -To enable engineers and scientists to build, connect and run digital twins in large networks (or even in small teams!) -it is necessary for everyone to be on the same page - the :ref:`gemini_principles` are a great way to start with that, -which is why we've released this part of our technology stack as open source, to support those principles and help -develop a wider ecosystem. - -The main goal is to **help engineers and scientists focus on doing engineering and science** - instead of apis, data -cleaning/management, and all this cloud-pipeline-devops-test-ci-ml BS that takes up 90% of a scientist's time, when they -should be spending their valuable time researching migratory patterns of birds, or cell structures, or wind turbine -performance, or whatever excites them. - -.. _uses: - -Uses -===== - -At `Octue `_, **twined** is used as a core part of our application creation process: - - * As a format to communicate requirements to our partners in research projects - * As a tool to validate incoming data to digital twins - * As a framework to help establish schema when designing digital twins - * As a source of information on digital twins in our network, to help map and connect twins together - -We'd like to hear about your use case. Please get in touch! - -We use the `GitHub Issue Tracker `_ to manage bug reports and feature requests. -Please note, this is not a "general help" forum; we recommend Stack Overflow for such questions. For really gnarly -issues or for help designing digital twin schema, Octue is able to provide application support services for those -building digital twins using **twined**. - -.. toctree:: - :maxdepth: 2 - - self - quick_start - anatomy - about - deployment - license - version_history diff --git a/docs/source/twined/license.rst b/docs/source/twined/license.rst deleted file mode 100644 index 3f6097513..000000000 --- a/docs/source/twined/license.rst +++ /dev/null @@ -1,85 +0,0 @@ -.. _license: - -======= -License -======= - -Octue maintains **twined** as an open source project, under the MIT license. - -The boring bit -============== - -Copyright (c) 2013-2024 Octue Ltd, All Rights Reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -Third Party Libraries -===================== - -**twined** includes or is linked against the following third party libraries: - - -Plotly.js ---------- -The MIT License (MIT) - -Copyright (c) 2020 Plotly, Inc - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - -jsonschema ----------- - -Copyright (c) 2013 Julian Berman - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/docs/source/twined/lifecycle.rst b/docs/source/twined/lifecycle.rst deleted file mode 100644 index 404bd3c5c..000000000 --- a/docs/source/twined/lifecycle.rst +++ /dev/null @@ -1,30 +0,0 @@ - -.. - - Data matching the ``configuration_values_schema`` is supplied to the digital twin / data service at - startup. - - It's generally used to define control parameters relating to what the service should do, or how it should operate. - For example, should it produce output images as low resolution PNGs or as SVGs? How many iterations of a fluid - flow solver should be used? What is the acceptable error level on an classifier algorithm? - - Input Values - - Once configuration data supplied to a service has been validated, it can accept inputs and run analyses - using them. - - Depending on the way it's deployed (see :ref:`deployment`), the ``input_values`` might come in from a web request, - over a websocket or called directly from the command line or another library. - - However it comes, new ``input_values``, which are in ``JSON`` format, are checked against the - ``input_values_schema`` strand of the twine. If they match, then analysis can proceed. - - Output Values - - Once a service has Data matching the ``output_values_schema`` is supplied to the service while it's running. Depending on the way - it's deployed, the values might come in from a web request, over a websocket or called directly from - another library - - Input For example current rotor speed, or forecast wind direction. - - Values might be passed at instantiation of a twin (typical application-like process) or via a socket. diff --git a/docs/source/twined/quick_start.rst b/docs/source/twined/quick_start.rst deleted file mode 100644 index 3428b7b34..000000000 --- a/docs/source/twined/quick_start.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. _quick_start: - -============ -Quick Start -============ - -.. toctree:: - :maxdepth: 2 - - quick_start_installation - quick_start_create_your_first_twine diff --git a/docs/source/twined/quick_start_create_your_first_twine.rst b/docs/source/twined/quick_start_create_your_first_twine.rst deleted file mode 100644 index 768069b85..000000000 --- a/docs/source/twined/quick_start_create_your_first_twine.rst +++ /dev/null @@ -1,111 +0,0 @@ -.. _create_your_first_twine: - -Create your first twine -======================= - -Let's say we want a digital twin that accepts two values, uses them to make a calculation, then gives the result. Anyone connecting to the twin will need to know what values it requires, and what it responds with. - -First, create a blank text file, call it `twine.json`. We'll give the twin a title and description. -Paste in the following: - -.. code-block:: javascript - - { - "title": "My first digital twin... of an atomising discombobulator", - "description": "A simple example... estimates the `foz` value of an atomising discombobulator." - } - -Now, let's define an input values strand, to specify what values are required by the twin. For this we use a json schema -(you can read more about them in :ref:`introducing_json_schema`). Add the ``input_values`` field, so your twine looks like this: - -.. code-block:: javascript - - { - "title": "My first digital twin", - "description": "A simple example to build on..." - "input_values_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Input Values schema for my first digital twin", - "description": "These values are supplied to the twin by another program (often over a websocket, depending on your integration provider). So as these values change, the twin can reply with an update.", - "type": "object", - "properties": { - "foo": { - "description": "The foo value... speed of the discombobulator's input bobulation module, in m/s", - "type": "number", - "minimum": 10, - "maximum": 500 - }, - "baz": { - "description": "The baz value... period of the discombobulator's recombulation unit, in s", - "type": "number", - "minimum": 0, - "maximum": 1000 - } - } - } - } - -Finally, let's define an output values strand, to define what kind of data is returned by the twin: - -.. code-block:: javascript - - "output_values_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "Output Values schema for my first digital twin", - "description": "The twin will output data that matches this schema", - "type": "object", - "properties": { - "foz": { - "description": "Estimate of the foz value... efficiency of the discombobulator in %", - "type": "number", - "minimum": 10, - "maximum": 500 - } - } - } - - -.. _load_the_twine: - -Load the twine -============== - -**twined** provides a `Twine()` class to load a twine (from a file or a json string). -The loading process checks the twine itself is valid. It's as simple as: - -.. code-block:: py - - from octue.twined import Twine - - my_twine = Twine(source='twine.json') - - -.. _validate_some_inputs: - -Validate some inputs -==================== - -Say we have some json that we want to parse and validate, to make sure it matches what's required for input values. - -.. code-block:: py - - my_input_values = my_twine.validate_input_values(json='{"foo": 30, "baz": 500}') - -You can read the values from a file too. Paste the following into a file named ``input_values.json``: - -.. code-block:: javascript - - { - "foo": 30, - "baz": 500 - } - -Then parse and validate directly from the file: - -.. code-block:: py - - my_input_values = my_twine.validate_input_values(source="input_values.json") - - -.. ATTENTION:: - LIBRARY IS UNDER CONSTRUCTION! WATCH THIS SPACE FOR MORE! diff --git a/docs/source/twined/quick_start_installation.rst b/docs/source/twined/quick_start_installation.rst deleted file mode 100644 index 0cbc136cd..000000000 --- a/docs/source/twined/quick_start_installation.rst +++ /dev/null @@ -1,41 +0,0 @@ -.. _installation: - -============ -Installation -============ - -**twined** is available on `pypi `_, so installation into your python virtual environment is dead -simple: - -.. code-block:: py - - pip install twined - -Don't have a virtual environment with pip? You probably should! ``pyenv`` is your friend. Google it. - - -.. _compilation: - -Compilation -============ - -There is presently no need to compile **twined**, as it's written entirely in python. - - -.. _third_party_library_installation: - -Third party library installation -================================ - -**twined** is for python >= 3.6 so expects that. Other dependencies can be checked in ``setup.py``, and will -automatically installed during the installation above. - - -.. _third_party_build_requirements: - -Third party build requirements -============================== - -.. ATTENTION:: - Woohoo! There are no crazy dependencies that you have to compile and build for your particular system. - (you know the ones... they never *actually* compile, right?). We aim to keep it this way. diff --git a/docs/source/twined/version_history.rst b/docs/source/twined/version_history.rst deleted file mode 100644 index f233425e8..000000000 --- a/docs/source/twined/version_history.rst +++ /dev/null @@ -1,41 +0,0 @@ -.. _version_history: - -=============== -Version History -=============== - -Origins -======= - -**twined** began as an internal tool at Octue, enabling applications to be connected together in the Octue ecosystem. - -The twined library is presently being ported out of Octue's SDKs as it became clear that it would be most beneficial to -open-source the framework we developed to connect applications and digital twins together. - - -.. _version_0.0.x: - -0.0.x -===== - -Initial library framework - development version. Highly unstable! Let's see what happens... - -New Features ------------- -#. Documentation -#. Travis- and RTD- based test and documentation build with Codecov integration -#. Load and validation of twine itself against twine schema -#. Main ``Twine()`` class with strands set as attributes -#. Validation of input, config and output values against twine -#. Validation of manifest json -#. Credential parsing from the environment and validation -#. Hook allowing instantiation of inputs and config to a given class e.g. ``Manifest`` -#. Tests to cover the majority of functionality - -Backward Incompatible API Changes ---------------------------------- -#. n/a (Initial release) - -Bug Fixes & Minor Changes -------------------------- -#. n/a (Initial Release) diff --git a/docs/source/updating_services.rst b/docs/source/updating_services.rst deleted file mode 100644 index ead5eebdb..000000000 --- a/docs/source/updating_services.rst +++ /dev/null @@ -1,95 +0,0 @@ -.. _updating_services: - -Updating a Twined service -========================= - -This page describes how to update an existing, deployed Twined service - in other words, how to deploy a new Twined -service revision. - -We assume that: - -- Your service's repository is on GitHub and you have push access to it -- The `standard Twined service deployment GitHub Actions workflow `_ - is set up in the repository and being used to build and push the service image to the artifact registry on merge of a - pull request into the ``main`` branch (see an example `here `_) -- A release workflow is set up that will tag and release the new service revision on GitHub (see an example - `here `_) - -Instructions -------------- - -1. Check out and pull the ``main`` branch to make sure you're up to date with the latest changes - - .. code-block:: shell - - git checkout main - git pull - -2. Install your service locally so you can run the tests and your development environment can lint the code etc.: - - .. code-block:: shell - - poetry install - -3. Set up `pre-commit `_ to enforce code quality: - - .. code-block:: shell - - pre-commit install && pre-commit install -t commit-msg - -4. Check out a new branch so you can work independently of any other work on the code happening at the same time - - .. code-block:: shell - - git checkout -b my-new-feature - -5. Add and make changes to your app's code as needed, committing each self-contained change. Use the `Conventional - Commits `_ commit message format so the new version for your service - can be automatically calculated. - - .. code-block:: shell - - git add a-new-file another-new-file - git commit -m "Your commit message" - ...repeat... - - Push your commits frequently so your work is backed up on GitHub - - .. code-block:: shell - - git push - -6. Write any new tests you need to verify your code works and update any old tests as needed - -7. Run the tests locally using ``pytest`` and fix anything that makes them fail - - .. image:: images/updating_services/pytest.png - -8. Update the `semantic version `_ of your app. This communicates to anyone updating from a - previous version of the service whether they can use it as before or if there might be changes they need to make to - their own code or data first. - - - ``poetry version patch`` for a bug fix or small non-code change - - ``poetry version minor`` for a new feature - - ``poetry version major`` for a breaking change - - Don't forget to commit this change, too. - -9. When you're ready to review the changes, head to GitHub and open a pull request of your branch into ``main``. This - makes it easy for you and anyone else to see what's changed. Check the "Files Changed" tab to make sure everything's - there and consistent (it's easy to forget to push a commit). Ask your colleagues to review the code if required. - - .. image:: images/updating_services/diff.png - -10. When you're ready to release the new version of your service, check that the GitHub checks have passed. These ensure - code quality, that the tests pass, and that the new version number is correct. - - .. image:: images/updating_services/checks.png - -11. Merge the pull request into ``main``. This will run the deployment workflow (usually called ``cd`` - continuous - deployment), making the new version of the service available to everyone. - -12. Check that the deployment workflow has run successfully (this can take a few minutes). You can check the progress in - the "Actions" tab of the GitHub repository - - .. image:: images/updating_services/deployment.png diff --git a/docs/source/version_history.rst b/docs/source/version_history.rst deleted file mode 100644 index 537c800e9..000000000 --- a/docs/source/version_history.rst +++ /dev/null @@ -1,25 +0,0 @@ -.. _chapter-version-history: - -=============== -Version History -=============== -See our `releases on GitHub. `_ - -Semantic versioning -=================== -We use `semantic versioning `_ so you can see when new releases make breaking changes or just add -new features or bug fixes. Breaking changes are highlighted in our pull request descriptions and release notes. - -.. important:: - - Note that ``octue`` is still in beta, so its major version number remains at 0 (i.e. ``0.y.z``). This means that, - for now, both breaking changes and new features are denoted by an increase in the minor version number (``y`` in - ``x.y.z``). When we come out of beta, breaking changes will be denoted by an increase in the major version number - (``x`` in ``x.y.z``). - - -Deprecated code -=============== -When code is deprecated, it will still work but a deprecation warning will be issued with a suggestion on how to update -it. After an adjustment period, deprecations will be removed from the codebase according to the `code removal schedule `_. -This constitutes a breaking change. From 3c25472161efbaf06ec61a5dbdc4e40f5ad4f6ec Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 19:17:53 -0400 Subject: [PATCH 40/69] DOC: Update path to image in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 19af0d04b..8ef8a1358 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10961975.svg)](https://doi.org/10.5281/zenodo.10961975) -# Octue Python SDK Purple Fruit Snake +# Octue Python SDK Purple Fruit Snake The python SDK for running [Octue](https://octue.com) data services, digital twins, and applications - get faster data groundwork so you have more time for the science! From 3c3fd6ecddc87a9ea43304c2bdf8e5beb39a7939 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Sun, 17 Aug 2025 19:18:41 -0400 Subject: [PATCH 41/69] DOC: Remove old docs requirements file skipci --- docs/requirements.txt | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index df0e58bfb..000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Required by the python script for building documentation -poetry==1.2.0b2 -Sphinx>=5,<8 -sphinx-rtd-theme>=1,<2 -sphinx-tabs>=3.4.0,<4 -sphinx-toolbox>=3,<4 -git+https://github.com/octue/octue-sdk-python.git@main From 6d163947f71cfbb2040e75f4ab49b3bd8d16dd71 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 11:50:36 -0400 Subject: [PATCH 42/69] DOC: Enable search in docs skipci --- docs/mkdocs.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index f57dccc2b..b6d3f7d0a 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -11,6 +11,9 @@ theme: - navigation.instant.prefetch - navigation.tracking - navigation.path + - search.suggest + - search.highlight + - search.share palette: # Palette toggle for light mode @@ -38,6 +41,7 @@ markdown_extensions: plugins: - privacy - glightbox + - search nav: - index.md From 1ebad21fb103a48aed0233fc79d85d5a6dde514c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 13:09:07 -0400 Subject: [PATCH 43/69] DOC: Add copyright and links to footer --- docs/mkdocs.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index b6d3f7d0a..3318a5aae 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -11,6 +11,7 @@ theme: - navigation.instant.prefetch - navigation.tracking - navigation.path + - navigation.footer - search.suggest - search.highlight - search.share @@ -64,3 +65,14 @@ nav: # - api.md - license.md - version_history.md + +copyright: Copyright © 2025 Octue + +extra: + social: + - icon: material/github + link: https://github.com/octue + - icon: material/linkedin + link: https://www.linkedin.com/company/octue + - icon: material/web + link: https://octue.com From 3d9be3b8c39484a7af04f925f7c2d47fdeabeb02 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 13:56:56 -0400 Subject: [PATCH 44/69] DEP: Add `mike` for docs --- poetry.lock | 82 ++++++++++++++++++++++++++++++++++++++++++++++++-- pyproject.toml | 1 + 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index f14730f5b..e0486ba07 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1122,7 +1122,7 @@ version = "8.6.1" description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["main", "dev"] files = [ {file = "importlib_metadata-8.6.1-py3-none-any.whl", hash = "sha256:02a89390c1e15fdfdc0d7c6b25cb3e62650d0494005c97d6f148bf5b9787525e"}, {file = "importlib_metadata-8.6.1.tar.gz", hash = "sha256:310b41d755445d74569f993ccfc22838295d9fe005425094fad953d7f15c8580"}, @@ -1140,6 +1140,26 @@ perf = ["ipython"] test = ["flufl.flake8", "importlib_resources (>=1.3) ; python_version < \"3.9\"", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] type = ["pytest-mypy"] +[[package]] +name = "importlib-resources" +version = "6.5.2" +description = "Read resources from Python packages" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "importlib_resources-6.5.2-py3-none-any.whl", hash = "sha256:789cfdc3ed28c78b67a06acb8126751ced69a3d5f79c095a98298cd8a760ccec"}, + {file = "importlib_resources-6.5.2.tar.gz", hash = "sha256:185f87adef5bcc288449d98fb4fba07cea78bc036455dd44c5fc4a2fe78fed2c"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["jaraco.test (>=5.4)", "pytest (>=6,!=8.1.*)", "zipp (>=3.17)"] +type = ["pytest-mypy"] + [[package]] name = "iniconfig" version = "2.1.0" @@ -1306,6 +1326,32 @@ files = [ {file = "mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8"}, ] +[[package]] +name = "mike" +version = "2.1.3" +description = "Manage multiple versions of your MkDocs-powered documentation" +optional = false +python-versions = "*" +groups = ["dev"] +files = [ + {file = "mike-2.1.3-py3-none-any.whl", hash = "sha256:d90c64077e84f06272437b464735130d380703a76a5738b152932884c60c062a"}, + {file = "mike-2.1.3.tar.gz", hash = "sha256:abd79b8ea483fb0275b7972825d3082e5ae67a41820f8d8a0dc7a3f49944e810"}, +] + +[package.dependencies] +importlib-metadata = "*" +importlib-resources = "*" +jinja2 = ">=2.7" +mkdocs = ">=1.0" +pyparsing = ">=3.0" +pyyaml = ">=5.1" +pyyaml-env-tag = "*" +verspec = "*" + +[package.extras] +dev = ["coverage", "flake8 (>=3.0)", "flake8-quotes", "shtab"] +test = ["coverage", "flake8 (>=3.0)", "flake8-quotes", "shtab"] + [[package]] name = "mkdocs" version = "1.6.1" @@ -1934,6 +1980,21 @@ pyyaml = "*" [package.extras] extra = ["pygments (>=2.19.1)"] +[[package]] +name = "pyparsing" +version = "3.2.3" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "pyparsing-3.2.3-py3-none-any.whl", hash = "sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf"}, + {file = "pyparsing-3.2.3.tar.gz", hash = "sha256:b9c13f1ab8b3b542f72e28f634bad4de758ab3ce4546e4301970ad6fa77c38be"}, +] + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + [[package]] name = "pytest" version = "7.4.4" @@ -2918,6 +2979,21 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "verspec" +version = "0.1.0" +description = "Flexible version handling" +optional = false +python-versions = "*" +groups = ["dev"] +files = [ + {file = "verspec-0.1.0-py3-none-any.whl", hash = "sha256:741877d5633cc9464c45a469ae2a31e801e6dbbaa85b9675d481cda100f11c31"}, + {file = "verspec-0.1.0.tar.gz", hash = "sha256:c4504ca697b2056cdb4bfa7121461f5a0e81809255b41c03dda4ba823637c01e"}, +] + +[package.extras] +test = ["coverage", "flake8 (>=3.7)", "mypy", "pretend", "pytest"] + [[package]] name = "virtualenv" version = "20.31.2" @@ -3089,7 +3165,7 @@ version = "3.21.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["main", "dev"] files = [ {file = "zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931"}, {file = "zipp-3.21.0.tar.gz", hash = "sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4"}, @@ -3109,4 +3185,4 @@ hdf5 = ["h5py"] [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "1093362b1bfb65a7d8bcd257d7fea3d02d712d5b979e996953912880f44b873e" +content-hash = "2cc578caf8bdf9a2a86a39d2e62c30d6cd0929e34b29a664dcb89cbd391171ec" diff --git a/pyproject.toml b/pyproject.toml index 3d0f59d88..4f894ee30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ sphinx-toolbox = "^3" ruff = "^0.6.9" mkdocs-material = "^9.6.17" mkdocs-glightbox = "^0.4.0" +mike = "^2.1.3" [tool.ruff] line-length = 120 From 5b4518ff4bd99b87cfa25ee598eef9f87afa41bb Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 13:57:10 -0400 Subject: [PATCH 45/69] DOC: Add versioning to docs skipci --- docs/mkdocs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 3318a5aae..bf1d74e3d 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -76,3 +76,5 @@ extra: link: https://www.linkedin.com/company/octue - icon: material/web link: https://octue.com + version: + provider: mike From 65162208b1e10b5599a7f1df3b9fd2479cd8aab4 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 14:03:25 -0400 Subject: [PATCH 46/69] DOC: Use Octue colours in docs --- docs/docs/stylesheets/extra.css | 5 +++++ docs/mkdocs.yml | 3 +++ 2 files changed, 8 insertions(+) create mode 100644 docs/docs/stylesheets/extra.css diff --git a/docs/docs/stylesheets/extra.css b/docs/docs/stylesheets/extra.css new file mode 100644 index 000000000..3017dc185 --- /dev/null +++ b/docs/docs/stylesheets/extra.css @@ -0,0 +1,5 @@ +:root > * { + --md-primary-fg-color: #b4232f; + --md-primary-fg-color--light: #3498db; + --md-primary-fg-color--dark: #4d4d4f; +} diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index bf1d74e3d..6fb8a5eec 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -78,3 +78,6 @@ extra: link: https://octue.com version: provider: mike + +extra_css: + - stylesheets/extra.css From a5c4b2462f451df1e1724c41da90b26feda518ce Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 17:38:07 -0400 Subject: [PATCH 47/69] DOC: Add Octue logo to docs --- docs/docs/images/logos_square-dark.png | Bin 0 -> 1240 bytes docs/mkdocs.yml | 1 + 2 files changed, 1 insertion(+) create mode 100644 docs/docs/images/logos_square-dark.png diff --git a/docs/docs/images/logos_square-dark.png b/docs/docs/images/logos_square-dark.png new file mode 100644 index 0000000000000000000000000000000000000000..73d33f758789e8b1ef85704425aec17059488125 GIT binary patch literal 1240 zcmV;}1Sk86P)I88os5(KtL6iyVZs2aaAh&19yy&uEqt9x{{B2tM+KEib*B_|V+_1@ug zs0qDh0zNvRZ-N>aKgm3mohgp15xiJg#h00zKs0Qcl~mx?h9QYQd>2hb?>4M-V|F>gXV#K_1zSoSNvc))IVXw;2evZOvc4m0;TN9{)j9n`f6OH8?zEy? z{ic#!!XFfaX+C5$WD=}}d?F;2D!|{t`6p#Mv2WXBNXQM{^y%v}Vn_p1ASQ;O+^5T@ zd`T)+KN>G66?qMnPtTpmNe$V6dzt(U`vN*O)UdA(T6}_i%{bur-Kqdb%uh^{7{FEy z{t@HX$GwG4Q%y53s;{$pqp0NUZJ)O@fB&_n#V>1;6XX|8J!J)ZsaVX zC*}q7AO_%~`(5?0OHM&3VZ!y=r~vGgW50EO2q9+&U7>19VEirI2S-j0eO}Iu7=YVI zEZhf24pTy$BRMtnsdeBYgd9VRof3?4TrfE=j5ia5cCmVxUK}~vt&6FyhBaV(&j5aF zX1tDu@xhR@0Qj_;GdZ;HjvVdEc+P1>jjal zd%!%RCx$zir!?n&d9MhR|f`}EoqbSSJZ)H%V( zLb!G~Z>NY)aZV_8C@*;7=Om9M{u#ukrq9G$8U7FX7pwJHc zUZ01Iu^nCx-VhyG*g3|*4Cy^*bQ@sM8O;?Jl|t5mKDOt4)P!miVSLOeDkScp%Gsj^ zV}`6lBld_us;0`N#2-@WZLjT$)}b=e8HY;3(#;*N4UOVsA5?Z32U8Kmo_k7p&dU&X z>C?ff2x{PJBm- zQX_-S{~Gb#91FbtbD_~2sV+gzY;o)C8o-U50sIGE0KKyq%UKox0000 Date: Mon, 18 Aug 2025 17:40:54 -0400 Subject: [PATCH 48/69] DOC: Add Octue logo favicon to docs skipci --- docs/mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index ade8ae567..f3889e8ef 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -4,6 +4,7 @@ site_url: https://docs.twined.octue.com theme: name: material logo: images/logos_square-dark.png + favicon: images/logos_square-dark.png features: - navigation.top From d0a06d32c2970111c352f3a176cd5f627e21410a Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 17:51:43 -0400 Subject: [PATCH 49/69] DEP: Remove dependencies and config from old documentation --- .pre-commit-config.yaml | 13 - .readthedocs.yaml | 16 - poetry.lock | 732 +--------------------------------------- pyproject.toml | 6 +- 4 files changed, 3 insertions(+), 764 deletions(-) delete mode 100644 .readthedocs.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 62f0b4be1..cb490c791 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,19 +31,6 @@ repos: hooks: - id: prettier - # - repo: https://github.com/thclark/pre-commit-sphinx - # rev: 0.0.3 - # hooks: - # - id: build-docs - # language_version: python3 - # additional_dependencies: - # - "poetry==1.2.0b2" - # - "Sphinx>=5,<8" - # - "sphinx-rtd-theme>=1,<2" - # - "sphinx-tabs>=3,<4" - # - "sphinx-toolbox>=3" - # - "git+https://github.com/octue/octue-sdk-python.git@main" - - repo: https://github.com/windpioneers/pre-commit-hooks rev: 0.0.5 hooks: diff --git a/.readthedocs.yaml b/.readthedocs.yaml deleted file mode 100644 index 71a0bcbdb..000000000 --- a/.readthedocs.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Read the Docs configuration file for Sphinx projects -# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details - -version: 2 - -build: - os: ubuntu-22.04 - tools: - python: "3.10" - -sphinx: - configuration: docs/source/conf.py - -python: - install: - - requirements: docs/requirements.txt diff --git a/poetry.lock b/poetry.lock index e0486ba07..148c2feb2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,55 +1,5 @@ # This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. -[[package]] -name = "alabaster" -version = "0.7.16" -description = "A light, configurable Sphinx theme" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "alabaster-0.7.16-py3-none-any.whl", hash = "sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92"}, - {file = "alabaster-0.7.16.tar.gz", hash = "sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65"}, -] - -[[package]] -name = "apeye" -version = "1.4.1" -description = "Handy tools for working with URLs and APIs." -optional = false -python-versions = ">=3.6.1" -groups = ["dev"] -files = [ - {file = "apeye-1.4.1-py3-none-any.whl", hash = "sha256:44e58a9104ec189bf42e76b3a7fe91e2b2879d96d48e9a77e5e32ff699c9204e"}, - {file = "apeye-1.4.1.tar.gz", hash = "sha256:14ea542fad689e3bfdbda2189a354a4908e90aee4bf84c15ab75d68453d76a36"}, -] - -[package.dependencies] -apeye-core = ">=1.0.0b2" -domdf-python-tools = ">=2.6.0" -platformdirs = ">=2.3.0" -requests = ">=2.24.0" - -[package.extras] -all = ["cachecontrol[filecache] (>=0.12.6)", "lockfile (>=0.12.2)"] -limiter = ["cachecontrol[filecache] (>=0.12.6)", "lockfile (>=0.12.2)"] - -[[package]] -name = "apeye-core" -version = "1.1.5" -description = "Core (offline) functionality for the apeye library." -optional = false -python-versions = ">=3.6.1" -groups = ["dev"] -files = [ - {file = "apeye_core-1.1.5-py3-none-any.whl", hash = "sha256:dc27a93f8c9e246b3b238c5ea51edf6115ab2618ef029b9f2d9a190ec8228fbf"}, - {file = "apeye_core-1.1.5.tar.gz", hash = "sha256:5de72ed3d00cc9b20fea55e54b7ab8f5ef8500eb33a5368bc162a5585e238a55"}, -] - -[package.dependencies] -domdf-python-tools = ">=2.6.0" -idna = ">=2.5" - [[package]] name = "appdirs" version = "1.4.4" @@ -82,21 +32,6 @@ docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphi tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\""] -[[package]] -name = "autodocsumm" -version = "0.2.14" -description = "Extended sphinx autodoc including automatic autosummaries" -optional = false -python-versions = ">=3.7" -groups = ["dev"] -files = [ - {file = "autodocsumm-0.2.14-py3-none-any.whl", hash = "sha256:3bad8717fc5190802c60392a7ab04b9f3c97aa9efa8b3780b3d81d615bfe5dc0"}, - {file = "autodocsumm-0.2.14.tar.gz", hash = "sha256:2839a9d4facc3c4eccd306c08695540911042b46eeafcdc3203e6d0bab40bc77"}, -] - -[package.dependencies] -Sphinx = ">=4.0,<9.0" - [[package]] name = "babel" version = "2.17.0" @@ -132,51 +67,6 @@ files = [ [package.extras] extras = ["regex"] -[[package]] -name = "beautifulsoup4" -version = "4.13.4" -description = "Screen-scraping library" -optional = false -python-versions = ">=3.7.0" -groups = ["dev"] -files = [ - {file = "beautifulsoup4-4.13.4-py3-none-any.whl", hash = "sha256:9bbbb14bfde9d79f38b8cd5f8c7c85f4b8f2523190ebed90e950a8dea4cb1c4b"}, - {file = "beautifulsoup4-4.13.4.tar.gz", hash = "sha256:dbb3c4e1ceae6aefebdaf2423247260cd062430a410e38c66f2baa50a8437195"}, -] - -[package.dependencies] -soupsieve = ">1.2" -typing-extensions = ">=4.0.0" - -[package.extras] -cchardet = ["cchardet"] -chardet = ["chardet"] -charset-normalizer = ["charset-normalizer"] -html5lib = ["html5lib"] -lxml = ["lxml"] - -[[package]] -name = "cachecontrol" -version = "0.14.3" -description = "httplib2 caching for requests" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "cachecontrol-0.14.3-py3-none-any.whl", hash = "sha256:b35e44a3113f17d2a31c1e6b27b9de6d4405f84ae51baa8c1d3cc5b633010cae"}, - {file = "cachecontrol-0.14.3.tar.gz", hash = "sha256:73e7efec4b06b20d9267b441c1f733664f989fb8688391b670ca812d70795d11"}, -] - -[package.dependencies] -filelock = {version = ">=3.8.0", optional = true, markers = "extra == \"filecache\""} -msgpack = ">=0.5.2,<2.0.0" -requests = ">=2.16.0" - -[package.extras] -dev = ["CacheControl[filecache,redis]", "build", "cherrypy", "codespell[tomli]", "furo", "mypy", "pytest", "pytest-cov", "ruff", "sphinx", "sphinx-copybutton", "tox", "types-redis", "types-requests"] -filecache = ["filelock (>=3.8.0)"] -redis = ["redis (>=2.10.5)"] - [[package]] name = "cachetools" version = "5.5.2" @@ -420,25 +310,6 @@ files = [ [package.extras] toml = ["toml"] -[[package]] -name = "cssutils" -version = "2.11.1" -description = "A CSS Cascading Style Sheets library for Python" -optional = false -python-versions = ">=3.8" -groups = ["dev"] -files = [ - {file = "cssutils-2.11.1-py3-none-any.whl", hash = "sha256:a67bfdfdff4f3867fab43698ec4897c1a828eca5973f4073321b3bccaf1199b1"}, - {file = "cssutils-2.11.1.tar.gz", hash = "sha256:0563a76513b6af6eebbe788c3bf3d01c920e46b3f90c8416738c5cfc773ff8e2"}, -] - -[package.dependencies] -more-itertools = "*" - -[package.extras] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -test = ["cssselect", "importlib-resources ; python_version < \"3.9\"", "jaraco.test (>=5.1)", "lxml ; python_version < \"3.11\"", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)"] - [[package]] name = "dateparser" version = "1.1.1" @@ -480,22 +351,6 @@ wrapt = ">=1.10,<2" [package.extras] dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools ; python_version >= \"3.12\"", "tox"] -[[package]] -name = "dict2css" -version = "0.3.0.post1" -description = "A μ-library for constructing cascading style sheets from Python dictionaries." -optional = false -python-versions = ">=3.6" -groups = ["dev"] -files = [ - {file = "dict2css-0.3.0.post1-py3-none-any.whl", hash = "sha256:f006a6b774c3e31869015122ae82c491fd25e7de4a75607a62aa3e798f837e0d"}, - {file = "dict2css-0.3.0.post1.tar.gz", hash = "sha256:89c544c21c4ca7472c3fffb9d37d3d926f606329afdb751dc1de67a411b70719"}, -] - -[package.dependencies] -cssutils = ">=2.2.0" -domdf-python-tools = ">=2.2.0" - [[package]] name = "distlib" version = "0.3.9" @@ -508,38 +363,6 @@ files = [ {file = "distlib-0.3.9.tar.gz", hash = "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403"}, ] -[[package]] -name = "docutils" -version = "0.18.1" -description = "Docutils -- Python Documentation Utilities" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -groups = ["dev"] -files = [ - {file = "docutils-0.18.1-py2.py3-none-any.whl", hash = "sha256:23010f129180089fbcd3bc08cfefccb3b890b0050e1ca00c867036e9d161b98c"}, - {file = "docutils-0.18.1.tar.gz", hash = "sha256:679987caf361a7539d76e584cbeddc311e3aee937877c87346f31debc63e9d06"}, -] - -[[package]] -name = "domdf-python-tools" -version = "3.10.0" -description = "Helpful functions for Python 🐍 🛠️" -optional = false -python-versions = ">=3.6" -groups = ["dev"] -files = [ - {file = "domdf_python_tools-3.10.0-py3-none-any.whl", hash = "sha256:5e71c1be71bbcc1f881d690c8984b60e64298ec256903b3147f068bc33090c36"}, - {file = "domdf_python_tools-3.10.0.tar.gz", hash = "sha256:2ae308d2f4f1e9145f5f4ba57f840fbfd1c2983ee26e4824347789649d3ae298"}, -] - -[package.dependencies] -natsort = ">=7.0.1" -typing-extensions = ">=3.7.4.1" - -[package.extras] -all = ["pytz (>=2019.1)"] -dates = ["pytz (>=2019.1)"] - [[package]] name = "exceptiongroup" version = "1.3.0" @@ -1052,28 +875,6 @@ files = [ [package.dependencies] numpy = ">=1.19.3" -[[package]] -name = "html5lib" -version = "1.1" -description = "HTML parser based on the WHATWG HTML specification" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -groups = ["dev"] -files = [ - {file = "html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d"}, - {file = "html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f"}, -] - -[package.dependencies] -six = ">=1.9" -webencodings = "*" - -[package.extras] -all = ["chardet (>=2.2)", "genshi", "lxml ; platform_python_implementation == \"CPython\""] -chardet = ["chardet (>=2.2)"] -genshi = ["genshi"] -lxml = ["lxml ; platform_python_implementation == \"CPython\""] - [[package]] name = "identify" version = "2.6.10" @@ -1104,18 +905,6 @@ files = [ [package.extras] all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] -[[package]] -name = "imagesize" -version = "1.4.1" -description = "Getting image size from png/jpeg/jpeg2000/gif file" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -groups = ["dev"] -files = [ - {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"}, - {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, -] - [[package]] name = "importlib-metadata" version = "8.6.1" @@ -1455,108 +1244,6 @@ files = [ {file = "mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443"}, ] -[[package]] -name = "more-itertools" -version = "10.7.0" -description = "More routines for operating on iterables, beyond itertools" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "more_itertools-10.7.0-py3-none-any.whl", hash = "sha256:d43980384673cb07d2f7d2d918c616b30c659c089ee23953f601d6609c67510e"}, - {file = "more_itertools-10.7.0.tar.gz", hash = "sha256:9fddd5403be01a94b204faadcff459ec3568cf110265d3c54323e1e866ad29d3"}, -] - -[[package]] -name = "msgpack" -version = "1.1.0" -description = "MessagePack serializer" -optional = false -python-versions = ">=3.8" -groups = ["dev"] -files = [ - {file = "msgpack-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7ad442d527a7e358a469faf43fda45aaf4ac3249c8310a82f0ccff9164e5dccd"}, - {file = "msgpack-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:74bed8f63f8f14d75eec75cf3d04ad581da6b914001b474a5d3cd3372c8cc27d"}, - {file = "msgpack-1.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:914571a2a5b4e7606997e169f64ce53a8b1e06f2cf2c3a7273aa106236d43dd5"}, - {file = "msgpack-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c921af52214dcbb75e6bdf6a661b23c3e6417f00c603dd2070bccb5c3ef499f5"}, - {file = "msgpack-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8ce0b22b890be5d252de90d0e0d119f363012027cf256185fc3d474c44b1b9e"}, - {file = "msgpack-1.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:73322a6cc57fcee3c0c57c4463d828e9428275fb85a27aa2aa1a92fdc42afd7b"}, - {file = "msgpack-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e1f3c3d21f7cf67bcf2da8e494d30a75e4cf60041d98b3f79875afb5b96f3a3f"}, - {file = "msgpack-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:64fc9068d701233effd61b19efb1485587560b66fe57b3e50d29c5d78e7fef68"}, - {file = "msgpack-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:42f754515e0f683f9c79210a5d1cad631ec3d06cea5172214d2176a42e67e19b"}, - {file = "msgpack-1.1.0-cp310-cp310-win32.whl", hash = "sha256:3df7e6b05571b3814361e8464f9304c42d2196808e0119f55d0d3e62cd5ea044"}, - {file = "msgpack-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:685ec345eefc757a7c8af44a3032734a739f8c45d1b0ac45efc5d8977aa4720f"}, - {file = "msgpack-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3d364a55082fb2a7416f6c63ae383fbd903adb5a6cf78c5b96cc6316dc1cedc7"}, - {file = "msgpack-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:79ec007767b9b56860e0372085f8504db5d06bd6a327a335449508bbee9648fa"}, - {file = "msgpack-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6ad622bf7756d5a497d5b6836e7fc3752e2dd6f4c648e24b1803f6048596f701"}, - {file = "msgpack-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e59bca908d9ca0de3dc8684f21ebf9a690fe47b6be93236eb40b99af28b6ea6"}, - {file = "msgpack-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e1da8f11a3dd397f0a32c76165cf0c4eb95b31013a94f6ecc0b280c05c91b59"}, - {file = "msgpack-1.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:452aff037287acb1d70a804ffd022b21fa2bb7c46bee884dbc864cc9024128a0"}, - {file = "msgpack-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8da4bf6d54ceed70e8861f833f83ce0814a2b72102e890cbdfe4b34764cdd66e"}, - {file = "msgpack-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:41c991beebf175faf352fb940bf2af9ad1fb77fd25f38d9142053914947cdbf6"}, - {file = "msgpack-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a52a1f3a5af7ba1c9ace055b659189f6c669cf3657095b50f9602af3a3ba0fe5"}, - {file = "msgpack-1.1.0-cp311-cp311-win32.whl", hash = "sha256:58638690ebd0a06427c5fe1a227bb6b8b9fdc2bd07701bec13c2335c82131a88"}, - {file = "msgpack-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fd2906780f25c8ed5d7b323379f6138524ba793428db5d0e9d226d3fa6aa1788"}, - {file = "msgpack-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:d46cf9e3705ea9485687aa4001a76e44748b609d260af21c4ceea7f2212a501d"}, - {file = "msgpack-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5dbad74103df937e1325cc4bfeaf57713be0b4f15e1c2da43ccdd836393e2ea2"}, - {file = "msgpack-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:58dfc47f8b102da61e8949708b3eafc3504509a5728f8b4ddef84bd9e16ad420"}, - {file = "msgpack-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4676e5be1b472909b2ee6356ff425ebedf5142427842aa06b4dfd5117d1ca8a2"}, - {file = "msgpack-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17fb65dd0bec285907f68b15734a993ad3fc94332b5bb21b0435846228de1f39"}, - {file = "msgpack-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a51abd48c6d8ac89e0cfd4fe177c61481aca2d5e7ba42044fd218cfd8ea9899f"}, - {file = "msgpack-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2137773500afa5494a61b1208619e3871f75f27b03bcfca7b3a7023284140247"}, - {file = "msgpack-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:398b713459fea610861c8a7b62a6fec1882759f308ae0795b5413ff6a160cf3c"}, - {file = "msgpack-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:06f5fd2f6bb2a7914922d935d3b8bb4a7fff3a9a91cfce6d06c13bc42bec975b"}, - {file = "msgpack-1.1.0-cp312-cp312-win32.whl", hash = "sha256:ad33e8400e4ec17ba782f7b9cf868977d867ed784a1f5f2ab46e7ba53b6e1e1b"}, - {file = "msgpack-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:115a7af8ee9e8cddc10f87636767857e7e3717b7a2e97379dc2054712693e90f"}, - {file = "msgpack-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:071603e2f0771c45ad9bc65719291c568d4edf120b44eb36324dcb02a13bfddf"}, - {file = "msgpack-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0f92a83b84e7c0749e3f12821949d79485971f087604178026085f60ce109330"}, - {file = "msgpack-1.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4a1964df7b81285d00a84da4e70cb1383f2e665e0f1f2a7027e683956d04b734"}, - {file = "msgpack-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59caf6a4ed0d164055ccff8fe31eddc0ebc07cf7326a2aaa0dbf7a4001cd823e"}, - {file = "msgpack-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0907e1a7119b337971a689153665764adc34e89175f9a34793307d9def08e6ca"}, - {file = "msgpack-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65553c9b6da8166e819a6aa90ad15288599b340f91d18f60b2061f402b9a4915"}, - {file = "msgpack-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7a946a8992941fea80ed4beae6bff74ffd7ee129a90b4dd5cf9c476a30e9708d"}, - {file = "msgpack-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4b51405e36e075193bc051315dbf29168d6141ae2500ba8cd80a522964e31434"}, - {file = "msgpack-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4c01941fd2ff87c2a934ee6055bda4ed353a7846b8d4f341c428109e9fcde8c"}, - {file = "msgpack-1.1.0-cp313-cp313-win32.whl", hash = "sha256:7c9a35ce2c2573bada929e0b7b3576de647b0defbd25f5139dcdaba0ae35a4cc"}, - {file = "msgpack-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:bce7d9e614a04d0883af0b3d4d501171fbfca038f12c77fa838d9f198147a23f"}, - {file = "msgpack-1.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c40ffa9a15d74e05ba1fe2681ea33b9caffd886675412612d93ab17b58ea2fec"}, - {file = "msgpack-1.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1ba6136e650898082d9d5a5217d5906d1e138024f836ff48691784bbe1adf96"}, - {file = "msgpack-1.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0856a2b7e8dcb874be44fea031d22e5b3a19121be92a1e098f46068a11b0870"}, - {file = "msgpack-1.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:471e27a5787a2e3f974ba023f9e265a8c7cfd373632247deb225617e3100a3c7"}, - {file = "msgpack-1.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:646afc8102935a388ffc3914b336d22d1c2d6209c773f3eb5dd4d6d3b6f8c1cb"}, - {file = "msgpack-1.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:13599f8829cfbe0158f6456374e9eea9f44eee08076291771d8ae93eda56607f"}, - {file = "msgpack-1.1.0-cp38-cp38-win32.whl", hash = "sha256:8a84efb768fb968381e525eeeb3d92857e4985aacc39f3c47ffd00eb4509315b"}, - {file = "msgpack-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:879a7b7b0ad82481c52d3c7eb99bf6f0645dbdec5134a4bddbd16f3506947feb"}, - {file = "msgpack-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:53258eeb7a80fc46f62fd59c876957a2d0e15e6449a9e71842b6d24419d88ca1"}, - {file = "msgpack-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7e7b853bbc44fb03fbdba34feb4bd414322180135e2cb5164f20ce1c9795ee48"}, - {file = "msgpack-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3e9b4936df53b970513eac1758f3882c88658a220b58dcc1e39606dccaaf01c"}, - {file = "msgpack-1.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46c34e99110762a76e3911fc923222472c9d681f1094096ac4102c18319e6468"}, - {file = "msgpack-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a706d1e74dd3dea05cb54580d9bd8b2880e9264856ce5068027eed09680aa74"}, - {file = "msgpack-1.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:534480ee5690ab3cbed89d4c8971a5c631b69a8c0883ecfea96c19118510c846"}, - {file = "msgpack-1.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8cf9e8c3a2153934a23ac160cc4cba0ec035f6867c8013cc6077a79823370346"}, - {file = "msgpack-1.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3180065ec2abbe13a4ad37688b61b99d7f9e012a535b930e0e683ad6bc30155b"}, - {file = "msgpack-1.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c5a91481a3cc573ac8c0d9aace09345d989dc4a0202b7fcb312c88c26d4e71a8"}, - {file = "msgpack-1.1.0-cp39-cp39-win32.whl", hash = "sha256:f80bc7d47f76089633763f952e67f8214cb7b3ee6bfa489b3cb6a84cfac114cd"}, - {file = "msgpack-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:4d1b7ff2d6146e16e8bd665ac726a89c74163ef8cd39fa8c1087d4e52d3a2325"}, - {file = "msgpack-1.1.0.tar.gz", hash = "sha256:dd432ccc2c72b914e4cb77afce64aab761c1137cc698be3984eee260bcb2896e"}, -] - -[[package]] -name = "natsort" -version = "8.4.0" -description = "Simple yet flexible natural sorting in Python." -optional = false -python-versions = ">=3.7" -groups = ["dev"] -files = [ - {file = "natsort-8.4.0-py3-none-any.whl", hash = "sha256:4732914fb471f56b5cce04d7bae6f164a592c7712e1c85f9ef585e197299521c"}, - {file = "natsort-8.4.0.tar.gz", hash = "sha256:45312c4a0e5507593da193dedd04abb1469253b601ecaf63445ad80f0a1ea581"}, -] - -[package.extras] -fast = ["fastnumbers (>=2.0.0)"] -icu = ["PyICU (>=1.0.0)"] - [[package]] name = "nodeenv" version = "1.9.1" @@ -2400,77 +2087,6 @@ files = [ [package.dependencies] pyasn1 = ">=0.1.3" -[[package]] -name = "ruamel-yaml" -version = "0.18.10" -description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" -optional = false -python-versions = ">=3.7" -groups = ["dev"] -files = [ - {file = "ruamel.yaml-0.18.10-py3-none-any.whl", hash = "sha256:30f22513ab2301b3d2b577adc121c6471f28734d3d9728581245f1e76468b4f1"}, - {file = "ruamel.yaml-0.18.10.tar.gz", hash = "sha256:20c86ab29ac2153f80a428e1254a8adf686d3383df04490514ca3b79a362db58"}, -] - -[package.dependencies] -"ruamel.yaml.clib" = {version = ">=0.2.7", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.13\""} - -[package.extras] -docs = ["mercurial (>5.7)", "ryd"] -jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] - -[[package]] -name = "ruamel-yaml-clib" -version = "0.2.12" -description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -markers = "platform_python_implementation == \"CPython\" and python_version < \"3.13\"" -files = [ - {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:11f891336688faf5156a36293a9c362bdc7c88f03a8a027c2c1d8e0bcde998e5"}, - {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:a606ef75a60ecf3d924613892cc603b154178ee25abb3055db5062da811fd969"}, - {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd5415dded15c3822597455bc02bcd66e81ef8b7a48cb71a33628fc9fdde39df"}, - {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f66efbc1caa63c088dead1c4170d148eabc9b80d95fb75b6c92ac0aad2437d76"}, - {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:22353049ba4181685023b25b5b51a574bce33e7f51c759371a7422dcae5402a6"}, - {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:932205970b9f9991b34f55136be327501903f7c66830e9760a8ffb15b07f05cd"}, - {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win32.whl", hash = "sha256:3eac5a91891ceb88138c113f9db04f3cebdae277f5d44eaa3651a4f573e6a5da"}, - {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win_amd64.whl", hash = "sha256:ab007f2f5a87bd08ab1499bdf96f3d5c6ad4dcfa364884cb4549aa0154b13a28"}, - {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:4a6679521a58256a90b0d89e03992c15144c5f3858f40d7c18886023d7943db6"}, - {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:d84318609196d6bd6da0edfa25cedfbabd8dbde5140a0a23af29ad4b8f91fb1e"}, - {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb43a269eb827806502c7c8efb7ae7e9e9d0573257a46e8e952f4d4caba4f31e"}, - {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:811ea1594b8a0fb466172c384267a4e5e367298af6b228931f273b111f17ef52"}, - {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cf12567a7b565cbf65d438dec6cfbe2917d3c1bdddfce84a9930b7d35ea59642"}, - {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7dd5adc8b930b12c8fc5b99e2d535a09889941aa0d0bd06f4749e9a9397c71d2"}, - {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win32.whl", hash = "sha256:bd0a08f0bab19093c54e18a14a10b4322e1eacc5217056f3c063bd2f59853ce4"}, - {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win_amd64.whl", hash = "sha256:a274fb2cb086c7a3dea4322ec27f4cb5cc4b6298adb583ab0e211a4682f241eb"}, - {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:20b0f8dc160ba83b6dcc0e256846e1a02d044e13f7ea74a3d1d56ede4e48c632"}, - {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:943f32bc9dedb3abff9879edc134901df92cfce2c3d5c9348f172f62eb2d771d"}, - {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95c3829bb364fdb8e0332c9931ecf57d9be3519241323c5274bd82f709cebc0c"}, - {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:749c16fcc4a2b09f28843cda5a193e0283e47454b63ec4b81eaa2242f50e4ccd"}, - {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bf165fef1f223beae7333275156ab2022cffe255dcc51c27f066b4370da81e31"}, - {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:32621c177bbf782ca5a18ba4d7af0f1082a3f6e517ac2a18b3974d4edf349680"}, - {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win32.whl", hash = "sha256:e8c4ebfcfd57177b572e2040777b8abc537cdef58a2120e830124946aa9b42c5"}, - {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win_amd64.whl", hash = "sha256:0467c5965282c62203273b838ae77c0d29d7638c8a4e3a1c8bdd3602c10904e4"}, - {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:4c8c5d82f50bb53986a5e02d1b3092b03622c02c2eb78e29bec33fd9593bae1a"}, - {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:e7e3736715fbf53e9be2a79eb4db68e4ed857017344d697e8b9749444ae57475"}, - {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b7e75b4965e1d4690e93021adfcecccbca7d61c7bddd8e22406ef2ff20d74ef"}, - {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96777d473c05ee3e5e3c3e999f5d23c6f4ec5b0c38c098b3a5229085f74236c6"}, - {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:3bc2a80e6420ca8b7d3590791e2dfc709c88ab9152c00eeb511c9875ce5778bf"}, - {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e188d2699864c11c36cdfdada94d781fd5d6b0071cd9c427bceb08ad3d7c70e1"}, - {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win32.whl", hash = "sha256:6442cb36270b3afb1b4951f060eccca1ce49f3d087ca1ca4563a6eb479cb3de6"}, - {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win_amd64.whl", hash = "sha256:e5b8daf27af0b90da7bb903a876477a9e6d7270be6146906b276605997c7e9a3"}, - {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:fc4b630cd3fa2cf7fce38afa91d7cfe844a9f75d7f0f36393fa98815e911d987"}, - {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bc5f1e1c28e966d61d2519f2a3d451ba989f9ea0f2307de7bc45baa526de9e45"}, - {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a0e060aace4c24dcaf71023bbd7d42674e3b230f7e7b97317baf1e953e5b519"}, - {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2f1c3765db32be59d18ab3953f43ab62a761327aafc1594a2a1fbe038b8b8a7"}, - {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d85252669dc32f98ebcd5d36768f5d4faeaeaa2d655ac0473be490ecdae3c285"}, - {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e143ada795c341b56de9418c58d028989093ee611aa27ffb9b7f609c00d813ed"}, - {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win32.whl", hash = "sha256:beffaed67936fbbeffd10966a4eb53c402fafd3d6833770516bf7314bc6ffa12"}, - {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win_amd64.whl", hash = "sha256:040ae85536960525ea62868b642bdb0c2cc6021c9f9d507810c0c604e66f5a7b"}, - {file = "ruamel.yaml.clib-0.2.12.tar.gz", hash = "sha256:6c8fbb13ec503f99a91901ab46e0b07ae7941cd527393187039aec586fdfd36f"}, -] - [[package]] name = "ruff" version = "0.6.9" @@ -2532,324 +2148,6 @@ files = [ {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] -[[package]] -name = "snowballstemmer" -version = "3.0.1" -description = "This package provides 32 stemmers for 30 languages generated from Snowball algorithms." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*" -groups = ["dev"] -files = [ - {file = "snowballstemmer-3.0.1-py3-none-any.whl", hash = "sha256:6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064"}, - {file = "snowballstemmer-3.0.1.tar.gz", hash = "sha256:6d5eeeec8e9f84d4d56b847692bacf79bc2c8e90c7f80ca4444ff8b6f2e52895"}, -] - -[[package]] -name = "soupsieve" -version = "2.7" -description = "A modern CSS selector implementation for Beautiful Soup." -optional = false -python-versions = ">=3.8" -groups = ["dev"] -files = [ - {file = "soupsieve-2.7-py3-none-any.whl", hash = "sha256:6e60cc5c1ffaf1cebcc12e8188320b72071e922c2e897f737cadce79ad5d30c4"}, - {file = "soupsieve-2.7.tar.gz", hash = "sha256:ad282f9b6926286d2ead4750552c8a6142bc4c783fd66b0293547c8fe6ae126a"}, -] - -[[package]] -name = "sphinx" -version = "7.3.7" -description = "Python documentation generator" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "sphinx-7.3.7-py3-none-any.whl", hash = "sha256:413f75440be4cacf328f580b4274ada4565fb2187d696a84970c23f77b64d8c3"}, - {file = "sphinx-7.3.7.tar.gz", hash = "sha256:a4a7db75ed37531c05002d56ed6948d4c42f473a36f46e1382b0bd76ca9627bc"}, -] - -[package.dependencies] -alabaster = ">=0.7.14,<0.8.0" -babel = ">=2.9" -colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} -docutils = ">=0.18.1,<0.22" -imagesize = ">=1.3" -Jinja2 = ">=3.0" -packaging = ">=21.0" -Pygments = ">=2.14" -requests = ">=2.25.0" -snowballstemmer = ">=2.0" -sphinxcontrib-applehelp = "*" -sphinxcontrib-devhelp = "*" -sphinxcontrib-htmlhelp = ">=2.0.0" -sphinxcontrib-jsmath = "*" -sphinxcontrib-qthelp = "*" -sphinxcontrib-serializinghtml = ">=1.1.9" -tomli = {version = ">=2", markers = "python_version < \"3.11\""} - -[package.extras] -docs = ["sphinxcontrib-websupport"] -lint = ["flake8 (>=3.5.0)", "importlib_metadata", "mypy (==1.9.0)", "pytest (>=6.0)", "ruff (==0.3.7)", "sphinx-lint", "tomli", "types-docutils", "types-requests"] -test = ["cython (>=3.0)", "defusedxml (>=0.7.1)", "pytest (>=6.0)", "setuptools (>=67.0)"] - -[[package]] -name = "sphinx-autodoc-typehints" -version = "2.3.0" -description = "Type hints (PEP 484) support for the Sphinx autodoc extension" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "sphinx_autodoc_typehints-2.3.0-py3-none-any.whl", hash = "sha256:3098e2c6d0ba99eacd013eb06861acc9b51c6e595be86ab05c08ee5506ac0c67"}, - {file = "sphinx_autodoc_typehints-2.3.0.tar.gz", hash = "sha256:535c78ed2d6a1bad393ba9f3dfa2602cf424e2631ee207263e07874c38fde084"}, -] - -[package.dependencies] -sphinx = ">=7.3.5" - -[package.extras] -docs = ["furo (>=2024.1.29)"] -numpy = ["nptyping (>=2.5)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.4.4)", "defusedxml (>=0.7.1)", "diff-cover (>=9)", "pytest (>=8.1.1)", "pytest-cov (>=5)", "sphobjinv (>=2.3.1)", "typing-extensions (>=4.11)"] - -[[package]] -name = "sphinx-jinja2-compat" -version = "0.3.0" -description = "Patches Jinja2 v3 to restore compatibility with earlier Sphinx versions." -optional = false -python-versions = ">=3.6" -groups = ["dev"] -files = [ - {file = "sphinx_jinja2_compat-0.3.0-py3-none-any.whl", hash = "sha256:b1e4006d8e1ea31013fa9946d1b075b0c8d2a42c6e3425e63542c1e9f8be9084"}, - {file = "sphinx_jinja2_compat-0.3.0.tar.gz", hash = "sha256:f3c1590b275f42e7a654e081db5e3e5fb97f515608422bde94015ddf795dfe7c"}, -] - -[package.dependencies] -jinja2 = ">=2.10" -markupsafe = ">=1" -standard-imghdr = {version = "3.10.14", markers = "python_version >= \"3.13\""} - -[[package]] -name = "sphinx-prompt" -version = "1.8.0" -description = "Sphinx directive to add unselectable prompt" -optional = false -python-versions = ">=3.9,<4.0" -groups = ["dev"] -files = [ - {file = "sphinx_prompt-1.8.0-py3-none-any.whl", hash = "sha256:369ecc633f0711886f9b3a078c83264245be1adf46abeeb9b88b5519e4b51007"}, - {file = "sphinx_prompt-1.8.0.tar.gz", hash = "sha256:47482f86fcec29662fdfd23e7c04ef03582714195d01f5d565403320084372ed"}, -] - -[package.dependencies] -docutils = "*" -pygments = "*" -Sphinx = ">=7.0.0,<8.0.0" - -[[package]] -name = "sphinx-rtd-theme" -version = "1.3.0" -description = "Read the Docs theme for Sphinx" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" -groups = ["dev"] -files = [ - {file = "sphinx_rtd_theme-1.3.0-py2.py3-none-any.whl", hash = "sha256:46ddef89cc2416a81ecfbeaceab1881948c014b1b6e4450b815311a89fb977b0"}, - {file = "sphinx_rtd_theme-1.3.0.tar.gz", hash = "sha256:590b030c7abb9cf038ec053b95e5380b5c70d61591eb0b552063fbe7c41f0931"}, -] - -[package.dependencies] -docutils = "<0.19" -sphinx = ">=1.6,<8" -sphinxcontrib-jquery = ">=4,<5" - -[package.extras] -dev = ["bump2version", "sphinxcontrib-httpdomain", "transifex-client", "wheel"] - -[[package]] -name = "sphinx-tabs" -version = "3.4.5" -description = "Tabbed views for Sphinx" -optional = false -python-versions = "~=3.7" -groups = ["dev"] -files = [ - {file = "sphinx-tabs-3.4.5.tar.gz", hash = "sha256:ba9d0c1e3e37aaadd4b5678449eb08176770e0fc227e769b6ce747df3ceea531"}, - {file = "sphinx_tabs-3.4.5-py3-none-any.whl", hash = "sha256:92cc9473e2ecf1828ca3f6617d0efc0aa8acb06b08c56ba29d1413f2f0f6cf09"}, -] - -[package.dependencies] -docutils = "*" -pygments = "*" -sphinx = "*" - -[package.extras] -code-style = ["pre-commit (==2.13.0)"] -testing = ["bs4", "coverage", "pygments", "pytest (>=7.1,<8)", "pytest-cov", "pytest-regressions", "rinohtype"] - -[[package]] -name = "sphinx-toolbox" -version = "3.10.0" -description = "Box of handy tools for Sphinx 🧰 📔" -optional = false -python-versions = ">=3.7" -groups = ["dev"] -files = [ - {file = "sphinx_toolbox-3.10.0-py3-none-any.whl", hash = "sha256:675e5978eaee31adf21701054fa75bacf820459d56e93ac30ad01eaee047a6ef"}, - {file = "sphinx_toolbox-3.10.0.tar.gz", hash = "sha256:6afea9ac9afabe76bd5bd4d2b01edfdad81d653a1a34768e776e6a56d5a6f572"}, -] - -[package.dependencies] -apeye = ">=0.4.0" -autodocsumm = ">=0.2.0" -beautifulsoup4 = ">=4.9.1" -cachecontrol = {version = ">=0.13.0", extras = ["filecache"]} -dict2css = ">=0.2.3" -docutils = ">=0.16" -domdf-python-tools = ">=2.9.0" -filelock = ">=3.8.0" -html5lib = ">=1.1" -"ruamel.yaml" = ">=0.16.12" -sphinx = ">=3.2.0" -sphinx-autodoc-typehints = ">=1.11.1" -sphinx-jinja2-compat = ">=0.1.0" -sphinx-prompt = ">=1.1.0" -sphinx-tabs = ">=1.2.1,<3.4.7" -tabulate = ">=0.8.7" -typing-extensions = ">=3.7.4.3,<3.10.0.1 || >3.10.0.1" - -[package.extras] -all = ["coincidence (>=0.4.3)", "pygments (>=2.7.4,<=2.13.0)"] -testing = ["coincidence (>=0.4.3)", "pygments (>=2.7.4,<=2.13.0)"] - -[[package]] -name = "sphinxcontrib-applehelp" -version = "2.0.0" -description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5"}, - {file = "sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1"}, -] - -[package.extras] -lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] -standalone = ["Sphinx (>=5)"] -test = ["pytest"] - -[[package]] -name = "sphinxcontrib-devhelp" -version = "2.0.0" -description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp documents" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2"}, - {file = "sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad"}, -] - -[package.extras] -lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] -standalone = ["Sphinx (>=5)"] -test = ["pytest"] - -[[package]] -name = "sphinxcontrib-htmlhelp" -version = "2.1.0" -description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8"}, - {file = "sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9"}, -] - -[package.extras] -lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] -standalone = ["Sphinx (>=5)"] -test = ["html5lib", "pytest"] - -[[package]] -name = "sphinxcontrib-jquery" -version = "4.1" -description = "Extension to include jQuery on newer Sphinx releases" -optional = false -python-versions = ">=2.7" -groups = ["dev"] -files = [ - {file = "sphinxcontrib-jquery-4.1.tar.gz", hash = "sha256:1620739f04e36a2c779f1a131a2dfd49b2fd07351bf1968ced074365933abc7a"}, - {file = "sphinxcontrib_jquery-4.1-py2.py3-none-any.whl", hash = "sha256:f936030d7d0147dd026a4f2b5a57343d233f1fc7b363f68b3d4f1cb0993878ae"}, -] - -[package.dependencies] -Sphinx = ">=1.8" - -[[package]] -name = "sphinxcontrib-jsmath" -version = "1.0.1" -description = "A sphinx extension which renders display math in HTML via JavaScript" -optional = false -python-versions = ">=3.5" -groups = ["dev"] -files = [ - {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"}, - {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, -] - -[package.extras] -test = ["flake8", "mypy", "pytest"] - -[[package]] -name = "sphinxcontrib-qthelp" -version = "2.0.0" -description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp documents" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb"}, - {file = "sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab"}, -] - -[package.extras] -lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] -standalone = ["Sphinx (>=5)"] -test = ["defusedxml (>=0.7.1)", "pytest"] - -[[package]] -name = "sphinxcontrib-serializinghtml" -version = "2.0.0" -description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331"}, - {file = "sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d"}, -] - -[package.extras] -lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] -standalone = ["Sphinx (>=5)"] -test = ["pytest"] - -[[package]] -name = "standard-imghdr" -version = "3.10.14" -description = "Standard library imghdr redistribution. \"dead battery\"." -optional = false -python-versions = "*" -groups = ["dev"] -markers = "python_version >= \"3.13\"" -files = [ - {file = "standard_imghdr-3.10.14-py3-none-any.whl", hash = "sha256:cdf6883163349624dee9a81d2853a20260337c4cd41c04e99c082e01833a08e2"}, - {file = "standard_imghdr-3.10.14.tar.gz", hash = "sha256:2598fe2e7c540dbda34b233295e10957ab8dc8ac6f3bd9eaa8d38be167232e52"}, -] - [[package]] name = "stringcase" version = "1.2.0" @@ -2861,21 +2159,6 @@ files = [ {file = "stringcase-1.2.0.tar.gz", hash = "sha256:48a06980661908efe8d9d34eab2b6c13aefa2163b3ced26972902e3bdfd87008"}, ] -[[package]] -name = "tabulate" -version = "0.9.0" -description = "Pretty-print tabular data" -optional = false -python-versions = ">=3.7" -groups = ["dev"] -files = [ - {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, - {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, -] - -[package.extras] -widechars = ["wcwidth"] - [[package]] name = "tomli" version = "2.2.1" @@ -2930,6 +2213,7 @@ files = [ {file = "typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c"}, {file = "typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef"}, ] +markers = {dev = "python_version < \"3.11\""} [[package]] name = "tzdata" @@ -3058,18 +2342,6 @@ files = [ [package.extras] watchmedo = ["PyYAML (>=3.10)"] -[[package]] -name = "webencodings" -version = "0.5.1" -description = "Character encoding aliases for legacy web content" -optional = false -python-versions = "*" -groups = ["dev"] -files = [ - {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, - {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, -] - [[package]] name = "wrapt" version = "1.17.2" @@ -3185,4 +2457,4 @@ hdf5 = ["h5py"] [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "2cc578caf8bdf9a2a86a39d2e62c30d6cd0929e34b29a664dcb89cbd391171ec" +content-hash = "47e85f3ba2f74900790a25e8d4aef25eef27205115bf054e928f12d4f8cb81ce" diff --git a/pyproject.toml b/pyproject.toml index 4f894ee30..b66278854 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,17 +52,13 @@ pytest = "^7" # Code quality pre-commit = "^2.17" coverage = "^5" +ruff = "^0.6.9" # Template app dependencies numpy = "^2.2.1" pandas = "^2.2.3" dateparser = "1.1.1" stringcase = "1.2.0" # Documentation -Sphinx = ">=5,<8" -sphinx-rtd-theme = ">=1,<2" -sphinx-tabs = ">=3.4.0,<4" -sphinx-toolbox = "^3" -ruff = "^0.6.9" mkdocs-material = "^9.6.17" mkdocs-glightbox = "^0.4.0" mike = "^2.1.3" From 9421f0e493c164a0b7ebb87880731ba6a73c2fe5 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 18:42:07 -0400 Subject: [PATCH 50/69] DOC: Add old twined docs --- docs/docs/twines/about/about_digital_twins.md | 53 +++ .../about/about_introducing_json_schema.md | 126 ++++++ .../about/about_other_considerations.md | 112 +++++ docs/docs/twines/about/about_requirements.md | 37 ++ docs/docs/twines/about/index.md | 16 + docs/docs/twines/anatomy.md | 109 +++++ docs/docs/twines/anatomy_children.md | 9 + docs/docs/twines/anatomy_credentials.md | 83 ++++ docs/docs/twines/anatomy_manifest.md | 420 ++++++++++++++++++ docs/docs/twines/anatomy_monitors.md | 57 +++ docs/docs/twines/anatomy_values.md | 157 +++++++ docs/docs/twines/examples.md | 200 +++++++++ docs/docs/twines/index.md | 109 +++++ docs/docs/twines/lifecycle.md | 34 ++ docs/docs/twines/quick_start.md | 5 + .../quick_start_create_your_first_twine.md | 112 +++++ docs/mkdocs.yml | 11 + 17 files changed, 1650 insertions(+) create mode 100644 docs/docs/twines/about/about_digital_twins.md create mode 100644 docs/docs/twines/about/about_introducing_json_schema.md create mode 100644 docs/docs/twines/about/about_other_considerations.md create mode 100644 docs/docs/twines/about/about_requirements.md create mode 100644 docs/docs/twines/about/index.md create mode 100644 docs/docs/twines/anatomy.md create mode 100644 docs/docs/twines/anatomy_children.md create mode 100644 docs/docs/twines/anatomy_credentials.md create mode 100644 docs/docs/twines/anatomy_manifest.md create mode 100644 docs/docs/twines/anatomy_monitors.md create mode 100644 docs/docs/twines/anatomy_values.md create mode 100644 docs/docs/twines/examples.md create mode 100644 docs/docs/twines/index.md create mode 100644 docs/docs/twines/lifecycle.md create mode 100644 docs/docs/twines/quick_start.md create mode 100644 docs/docs/twines/quick_start_create_your_first_twine.md diff --git a/docs/docs/twines/about/about_digital_twins.md b/docs/docs/twines/about/about_digital_twins.md new file mode 100644 index 000000000..ade802551 --- /dev/null +++ b/docs/docs/twines/about/about_digital_twins.md @@ -0,0 +1,53 @@ +# Digital Twins {#digital_twins} + +A digital twin is a virtual representation of a real life being - a +physical asset like a wind turbine or car - or even a human. + +There are three reasons why you might want to create a digital twin: + +: - Monitoring - Prediction - Optimisation + +On its own, a digital twin can be quite useful. For example, a twin +might embody an AI-based analysis to predict power output of a turbine. + +
+ +
A digital twin consists of some kind of analysis or +processing task, which could be run many times per second, or daily, +down to occasionally or sometimes only once (the same as a "normal" +analysis).
+
+ +Coupling digital twins is generally even more useful. You might wish to +couple your turbine twin with a representation of the local power grid, +and a representation of a factory building to determine power demand\... +enabling you to optimise your factory plant for lowest energy cost +whilst intelligently selling surplus power to the grid. + +
+ +
A hierarchy of digital twins. Each blue circle represents a +twin, coupled to its neighbours. Yellow nodes are where schema are used +to connect twins.
+
+ +## Gemini Principles {#gemini_principles} + +The Gemini Principles have been derived by the [Centre for Digital Built +Britain +(CDBB)](https://www.cdbb.cam.ac.uk/system/files/documents/TheGeminiPrinciples.pdf). +We strongly recommend you give them a read if embarking on a digital +twins project. + +The aim of **twined** is to enable the following principles. In +particular: + +1. Openness (open-source project to create schema for twins that can be + run anywhere, anywhen) +2. Federation (encouraging a standardised way of connecting twins + together) +3. Security (making sure schemas and data can be read safely) +4. Public Good (see our nano-rant about climate change in + `reason_for_being`{.interpreted-text role="ref"}) diff --git a/docs/docs/twines/about/about_introducing_json_schema.md b/docs/docs/twines/about/about_introducing_json_schema.md new file mode 100644 index 000000000..b75ab93e8 --- /dev/null +++ b/docs/docs/twines/about/about_introducing_json_schema.md @@ -0,0 +1,126 @@ +# Introducing JSON Schema {#introducing_json_schema} + +`JSON` is a data interchange format that has rapidly taken over as the +defacto web-based data communication standard in recent years. + +`JSONSchema` is a way of specifying what a `JSON` document should +contain. The Schema are, themselves, written in `JSON`! + +Whilst schema can become extremely complicated in some scenarios, they +are best designed to be quite succinct. See below for the schema (and +matching `JSON`) for an integer and a string variable. + +**JSON:** + +```json +{ + "id": 1, + "name": "Tom" +} +``` + +**Schema:** + +```json +{ + "type": "object", + "title": "An id number and a name", + "properties": { + "id": { + "type": "integer", + "title": "An integer number", + "default": 0 + }, + "name": { + "type": "string", + "title": "A string name", + "default": "" + } + } +} +``` + +## Useful resources {#useful_resources} + +Link Resource + +--- + + Useful web tool for inferring schema from existing json + A powerful online editor for json, allowing manipulation of large documents better than most text editors + The JSON standard spec + The (draft standard) JSONSchema spec + A front end library for generating webforms directly from a schema + +## Human readability {#human_readbility} + +Back in our `requirements`{.interpreted-text role="ref"} section, we +noted it was important for humans to read and understand schema. + +The actual documents themselves are pretty easy to read by technical +users. But, for non technical users, readability can be enhanced even +further by the ability to turn `JSONSchema` into web forms +automatically. For our example above, we can autogenerate a web form +straight from the schema: + +
+ +
Web form generated from the example schema +above.
+
+ +Thus, we can take a schema (or a part of a schema) and use it to +generate a control form for a digital twin in a web interface without +writing a separate form component - great for ease and maintainability. + +## Why not XML? {#why_not_xml} + +In a truly excellent [three-part +blog](https://www.toptal.com/web/json-vs-xml-part-3), writer Seva Savris +takes us through the ups and downs of `JSON` versus `XML`; well worth a +read if wishing to understand the respective technologies better. + +In short, both `JSON` and `XML` are generalised data interchange +specifications and can both can do what we want here. We choose `JSON` +because: + +1. Textual representation is much more concise and easy to understand + (very important where non-developers like engineers and scientists + must be expected to interpret schema) +2. [Attack + vectors](https://www.opswat.com/blog/depth-look-xml-document-attack-vectors). + Because entities in `XML` are not necessarily primitives (unlike in + `JSON`), an `XML` document parser in its default state may leave a + system open to XXE injection attacks and DTD validation attacks, and + therefore requires hardening. `JSON` documents are similarly + afflicated (just like any kind of serialized data) but default + parsers, operating on the premise of only deserializing to primitive + types, are safe by default - it is only when nondefault parsering or + deserialization techniques (such as `JSONP`) are used that the + application becomes vulnerable. By utilising a default `JSON` parser + we can therefore significantly shrink the attack surface of the + system. See [this blog + post](https://blog.securityevaluators.com/xml-vs-json-security-risks-22e5320cf529) + for further discussion. +3. `XML` is powerful\... perhaps too powerful. The standard can be + adapted greatly, resulting in high encapsulation and a high + resilience to future unknowns. Both beneficial. However, this + requires developers of twins to maintain interfaces of very high + complexity, adaptable to a much wider variety of input. To enable + developers to progress, we suggest handling changes and future + unknowns through well-considered versioning, whilst keeping their + API simple. +4. `XML` allows baked-in validation of data and attributes. Whilst + advantageous in some situations, this is not a benefit here. We wish + validation to be one-sided: validation of data accepted/generated by + a digital twin should be occur within (at) the boundaries of that + twin. +5. Required validation capabilities, built into `XML` are achievable + with `JSONSchema` (otherwise missing from the pure `JSON` standard) +6. `JSON` is a more compact expression than XML, significantly reducing + memory and bandwidth requirements. Whilst not a major issue for most + modern PCS, sensors on the edge may have limited memory, and both + memory and bandwidth at scale are extremely expensive. Thus for + extremely large networks of interconnected systems there could be + significant speed and cost savings. diff --git a/docs/docs/twines/about/about_other_considerations.md b/docs/docs/twines/about/about_other_considerations.md new file mode 100644 index 000000000..03d4ddecb --- /dev/null +++ b/docs/docs/twines/about/about_other_considerations.md @@ -0,0 +1,112 @@ +# Other Considerations {#other_considerations} + +A variety of thoughts that arose whilst architecting **twined**. + +## Bash-style stdio {#bash_style_stdio} + +Some thought was given to using a very old-school-unix approach to +piping data between twins, via stdout. + +Whilst attractive (as being a wildly fast way of piping data between +twins on the same machine) it was felt this was insufficiently general, +eg: + +> - where twins don\'t exist on the same machine or container, making it +> cumbersome to engineer common iostreams +> - where slight differences between different shells might lead to +> incompatibilities or changes in behaviour + +And also unfriendly, eg: + +> - engineers or scientists unfamiliar with subtleties of bash shell +> scripting encounter difficulty piping data around +> - difficult to build friendly web based tools to introspect the data +> and configuration +> - bound to be headaches on windows platforms, even though windows now +> supports bash +> - easy to corrupt using third party libraries (e.g. which print to +> stdout) + +## Units {#Units} + +Being used (mostly) for engineering and scientific analysis, it was +tempting to add in a specified sub-schema for units. For example, +mandating that where values can be given in units, they be specified in +a certain way, like: + +```javascript +{ + "wind_speed": { + "value": 10.2, + "units": "mph" + } +} +``` + +or (more succinct): + +```javascript +{ + "wind_speed": 10.2, + "wind_speed_units": "mph" +} +``` + +It\'s still extremely tempting to provide this facility; or at least +provide some way of specifying in the schema what units a value should +be provided in. Thinking about it but don\'t have time right now. If +anybody wants to start crafting a PR with an extension or update to +**twined** that facilitates this; please raise an issue to start +progressing it. + +## Variable Style {#variable_style} + +A premptive stamp on the whinging\... + +Note that in the `JSON` descriptions above, all variables are named in +`snake_case` rather than `camelCase`. This decision, more likely than +even Brexit to divide opinions, is based on: + +- The languages we anticipate being most popular for building twins seem to trend toward snake case (eg + + : [python](https://www.python.org/dev/peps/pep-0008/), + [c++](https://google.github.io/styleguide/cppguide.html)) although + to be fair we might\'ve woefully misjudged which languages start + emerging. + +- The reservation of snake case for the schema spec has the subtle advantage that in future, we might be able to use + + : camelCase within the spec to denote class types in some useful + way, just like in python. Not sure yet; just mulling. + +- The `requirements`{.interpreted-text role="ref"} mention human-readability as a must; + + : [this + paper](https://ieeexplore.ieee.org/document/5521745?tp=&arnumber=5521745&url=http:%2F%2Fieeexplore.ieee.org%2Fxpls%2Fabs_all.jsp%3Farnumber%3D5521745) + suggests a 20% slower comprehension of camel case than snake, + although to be fair that\'s probably arguable. + +- We\'re starting in Python so are taking a lead from PEP8, which is bar none the most successful style guide on the + + : planet, because it got everybody on the same page really early on. + +If existing code that you\'re dropping in uses camelCase, please don\'t +file that as an issue\... converting property names automatically after +schema validation generation is trivial, there are tons of libraries +(like [humps](https://humps.readthedocs.io/en/latest/)) to do it. + +We\'d also consider a pull request for a built-in utility converting +[to](https://pypi.org/project/camelcase/) and +[from](https://pypi.org/project/snakecase/) that does this following +validation and prior to returning results. Suggest your proposed +approach on the [issues board](https://github.com/octue/twined). + +## Language Choice {#language_choice} + +**twined** is presently released in python only. It won\'t be too hard +to replicate functionality in other languages, and we\'re considering +other languages at present, so might be easily persuadable ;) + +If you require implementation of **twined** in a different language, and +are willing to consider sponsorship of development and maintenance of +that library, please [file an issue](https://github.com/octue/twined). diff --git a/docs/docs/twines/about/about_requirements.md b/docs/docs/twines/about/about_requirements.md new file mode 100644 index 000000000..9727e323d --- /dev/null +++ b/docs/docs/twines/about/about_requirements.md @@ -0,0 +1,37 @@ +# Requirements of the framework {#requirements} + +A _twine_ must describe a digital twin, and have multiple roles. It +must: + +1. Define what data is required by a digital twin, in order to run +2. Define what data will be returned by the twin following a successful + run +3. Define the formats of these data, in such a way that incoming data + can be validated +4. Define what other (1st or 3rd party) twins / services are required + by this one in order for it to run. + +If this weren\'t enough, the description: + +1. Must be trustable (i.e. a _twine_ from an untrusted, corrupt or + malicious third party should be safe to at least read) +2. Must be machine-readable _and machine-understandable_[^1] +3. Must be human-readable _and human-understandable_[^2] +4. Must be discoverable (that is, searchable/indexable) otherwise + people won\'t know it\'s there in orer to use it. + +Fortunately for digital twin developers, several of these requirements +have already been seen for data interchange formats developed for the +web. **twined** uses `JSON` and `JSONSchema` to help interchange data. + +If you\'re not already familiar with `JSONSchema` (or wish to know why +**twined** uses `JSON` over the seemingly more appropriate `XML` +standard), see `introducing_json_schema`{.interpreted-text role="ref"}. + +[^1]: + _Understandable_ essentially means that, once read, the machine or + human knows what it actually means and what to do with it. + +[^2]: + _Understandable_ essentially means that, once read, the machine or + human knows what it actually means and what to do with it. diff --git a/docs/docs/twines/about/index.md b/docs/docs/twines/about/index.md new file mode 100644 index 000000000..fce02f56c --- /dev/null +++ b/docs/docs/twines/about/index.md @@ -0,0 +1,16 @@ +# About Twines {#about} + +**Twined** is a framework for describing a digital twin or data service. + +We call these descriptions \"twines\". To just get started building a +_twine_, check out the `quick_start`{.interpreted-text role="ref"}. To +get into the detail of what\'s in a _twine_, see +`anatomy`{.interpreted-text role="ref"}. + +Here, we look at requirements for the framework, our motivations and +background, and some of the decisions made while developing **twined**. + +::: {.toctree maxdepth="1"} +about_digital_twins about_requirements about_introducing_json_schema +about_other_considerations +::: diff --git a/docs/docs/twines/anatomy.md b/docs/docs/twines/anatomy.md new file mode 100644 index 000000000..96952b1c1 --- /dev/null +++ b/docs/docs/twines/anatomy.md @@ -0,0 +1,109 @@ +# Anatomy Of The Twine File {#anatomy} + +The main point of **twined** is to enable engineers and scientists to +easily (and rigorously) define a digital twin or data service. + +This is done by adding a `twine.json` file to the repository containing +your code. Adding a _twine_ means you can: + +- communicate (to you or a colleague) what data is required by this + service +- communicate (to another service / machine) what data is required +- deploy services automatically with a provider like + [Octue](https://www.octue.com). + +To just get started building a _twine_, check out the +`quick_start`{.interpreted-text role="ref"}. To learn more about twines +in general, see `about`{.interpreted-text role="ref"}. Here, we describe +the parts of a _twine_ (\"strands\") and what they mean. + +## Strands + +A _twine_ has several sections, called _strands_. Each defines a +different kind of data required (or produced) by the twin. + +--- + +Strand Describes the twin\'s requirements for\... + +--- + +`Configuration Values `{.interpreted-text Data, in JSON form, used for configuration of the +role="ref"} twin/service. + +`Configuration Manifest `{.interpreted-text Files/datasets required by the twin at +role="ref"} configuration/startup + +`Input Values `{.interpreted-text Data, in JSON form, passed to the twin in order +role="ref"} to trigger an analysis + +`Input Manifest `{.interpreted-text role="ref"} Files/datasets passed with Input Values to +trigger an analysis + +`Output Values `{.interpreted-text Data, in JSON form, that will be produced by the +role="ref"} twin (in response to inputs) + +`Output Manifest `{.interpreted-text Files/datasets that will be produced by the twin +role="ref"} (in response to inputs) + +`Credentials `{.interpreted-text role="ref"} Credentials that are required by the twin in +order to access third party services + +`Children `{.interpreted-text role="ref"} Other twins, access to which are required for +this twin to function + +`Monitors `{.interpreted-text role="ref"} Visual and progress outputs from an analysis + +--- + +::: {.toctree maxdepth="1" hidden=""} +anatomy_values anatomy_manifest anatomy_credentials anatomy_monitors +anatomy_children +::: + +## Twine File Schema {#twine_file_schema} + +Because the `twine.json` file itself is in `JSON` format with a strict +structure, **twined** uses a schema to make that twine files are +correctly written (a \"schema-schema\", if you will, since a twine +already contains schema). Try not to think about it. But if you must, +the _twine_ schema is +[here](https://github.com/octue/twined/blob/master/twined/schema/twine_schema.json). + +The first thing **twined** always does is check that the `twine.json` +file itself is valid, and give you a descriptive error if it isn\'t. + +## Other External I/O {#other_external_io} + +A twin might: + +- GET/POST data from/to an external API, +- query/update a database, +- upload files to an object store, +- trigger events in another network, or +- perform pretty much any interaction you can think of with other + applications over the web. + +However, such data exchange may not be controllable by **twined** (which +is intended to operate at the boundaries of the twin) unless the +resulting data is returned from the twin (and must therefore be +compliant with the schema). + +So, there\'s nothing for **twined** to do here, and no need for a strand +in the _twine_ file. However, interacting with third party APIs or +databases might require some credentials. See +`credentials_strand`{.interpreted-text role="ref"} for help with that. + +:::: note +::: title +Note +::: + +This is actually a very common scenario. For example, the purpose of the +twin might be to fetch data (like a weather forecast) from some external +API then return it in the `output_values` for use in a network of +digital twins. But its the twin developer\'s job to do the fetchin\' and +make sure the resulting data is compliant with the +`output_values_schema` (see `values_based_strands`{.interpreted-text +role="ref"}). +:::: diff --git a/docs/docs/twines/anatomy_children.md b/docs/docs/twines/anatomy_children.md new file mode 100644 index 000000000..29e5fbb45 --- /dev/null +++ b/docs/docs/twines/anatomy_children.md @@ -0,0 +1,9 @@ +# Children Strand {#children_strand} + +:::: attention +::: title +Attention +::: + +Coming Soon! +:::: diff --git a/docs/docs/twines/anatomy_credentials.md b/docs/docs/twines/anatomy_credentials.md new file mode 100644 index 000000000..8ed890dc8 --- /dev/null +++ b/docs/docs/twines/anatomy_credentials.md @@ -0,0 +1,83 @@ +# Credentials Strand {#credentials_strand} + +In order to: + +- GET/POST data from/to an API, +- query a database, or +- connect to a socket (for receiving Values or emitting Values, Monitors + or Logs), + +A digital twin must have _access_ to it. API keys, database URIs, etc +must be supplied to the digital twin but treated with best practice with +respect to security considerations. The purpose of the `credentials` +strand is to dictate what credentials the twin requires in order to +function. + +## Defining the Credentials Strand {#defining_the_credentials_strand} + +This is the simplest of the strands, containing a list of credentials +(whose `NAMES_SHOULD_BE_SHOUTY_SNAKE_CASE`) with a reminder of the +purpose. + +```javascript +{ + "credentials": [ + { + "name": "SECRET_THE_FIRST", + "purpose": "Token for accessing a 3rd party API service" + }, + { + "name": "SECRET_THE_SECOND", + "purpose": "Token for accessing a 3rd party API service" + }, + { + "name": "SECRET_THE_THIRD", + "purpose": "Another secret, like a password for a sandbox or local database" + } + ] +} +``` + +## Supplying Credentials {#supplying_credentials} + +:::: attention +::: title +Attention +::: + +_Credentials should never be hard-coded into application code_ + +Do you trust the twin code? If you insert credentials to your own +database into a digital twin provided by a third party, you better be +very sure that twin isn\'t going to scrape all that data out then send +it elsewhere! + +Alternatively, if you\'re building a twin requiring such credentials, +it\'s your responsibility to give the end users confidence that you\'re +not abusing their access. + +There\'ll be a lot more discussion on these issues, but it\'s outside +the scope of **twined** - all we do here is make sure a twin has the +credentials it requires. +:::: + +Credentials should be securely managed by whatever system is managing +the twin, then made accessible to the twin in the form of environment +variables: + +```javascript +SERVICE_API_KEY = + someLongTokenTHatYouProbablyHaveToPayTheThirdPartyProviderLoadsOfMoneyFor; +``` + +Credentials may also reside in a `.env` file in the current directory, +either in the format above (with a new line for each variable) or, for +convenience, as bash exports like: + +```javascript +export SERVICE_API_KEY=someLongTokenTHatYouProbablyHaveToPayTheThirdPartyProviderLoadsOfMoneyFor +``` + +The `validate_credentials()` method of the `Twine` class checks for +their presence and, where contained in a `.env` file, ensures they are +loaded into the environment. diff --git a/docs/docs/twines/anatomy_manifest.md b/docs/docs/twines/anatomy_manifest.md new file mode 100644 index 000000000..dfb3082cd --- /dev/null +++ b/docs/docs/twines/anatomy_manifest.md @@ -0,0 +1,420 @@ +# Manifest-based Strands {#manifest_strands} + +Frequently, twins operate on files containing some kind of data. These +files need to be made accessible to the code running in the twin, in +order that their contents can be read and processed. Conversely, a twin +might produce an output dataset which must be understood by users. + +The `configuration_manifest`, `input_manifest` and `output_manifest` +strands describe what kind of datasets (and associated files) are +required / produced. + +:::: note +::: title +Note +::: + +Files are always contained in datasets, even if there\'s only one file. +It\'s so that we can keep nitty-gritty file metadata separate from the +more meaningful, higher level metadata like what a dataset is for. +:::: + +:::::: tabs +::: group-tab +Configuration Manifest Strand + +This describes datasets/files that are required at startup of the twin / +service. They typically contain a resource that the twin might use +across many analyses. + +For example, a twin might predict failure for a particular component, +given an image. It will require a trained ML model (saved in a +`*.pickle` or `*.json`). While many thousands of predictions might be +done over the period that the twin is deployed, all predictions are done +using this version of the model - so the model file is supplied at +startup. +::: + +::: group-tab +Input Manifest Strand + +These files are made available for the twin to run a particular analysis +with. Each analysis will likely have different input datasets. + +For example, a twin might be passed a dataset of LiDAR `*.scn` files and +be expected to compute atmospheric flow properties as a timeseries +(which might be returned in the +`output values `{.interpreted-text role="ref"} for +onward processing and storage). +::: + +::: group-tab +Output Manifest Strand + +Files are created by the twin during an analysis, tagged and stored as +datasets for some onward purpose. This strand is not used for sourcing +data; it enables users or other services to understand appropriate +search terms to retrieve datasets produced. +::: +:::::: + +## Describing Manifests {#describing_manifests} + +Manifest-based strands are a **description of what files are needed**. +The purpose of the manifest strands is to provide a helper to a wider +system providing datafiles to digital twins. + +::::::::::::::::::::: tabs +:::::::: group-tab +Configuration Manifest Strand + +::::::: accordion +:::: accordion-row +Show twine containing this strand + +::: {.literalinclude language="javascript"} +../../octue/twined/examples/damage_classifier_service/twine.json +::: +:::: + +:::: accordion-row +Show a matching file manifest + +::: {.literalinclude language="javascript"} +../../octue/twined/examples/damage_classifier_service/data/configuration_manifest.json +::: +:::: +::::::: +:::::::: + +:::::::: group-tab +Input Manifest Strand + +Here we specify that two datasets (and all or some of the files +associated with them) are required, for a service that cross-checks +meteorological mast data and power output data for a wind farm. + +::::::: accordion +:::: accordion-row +Show twine containing this strand + +::: {.literalinclude language="javascript"} +../../octue/twined/examples/met_mast_scada_service/strands/input_manifest.json +::: +:::: + +:::: accordion-row +Show a matching file manifest + +::: {.literalinclude language="javascript"} +../../octue/twined/examples/met_mast_scada_service/data/input_manifest.json +::: +:::: +::::::: +:::::::: + +:::::::: group-tab +Output Manifest Strand + +::::::: accordion +:::: accordion-row +Show twine containing this strand + +::: {.literalinclude language="javascript"} +../../octue/twined/examples/met_mast_scada_service/strands/output_manifest.json +::: +:::: + +:::: accordion-row +Show a matching file manifest + +::: {.literalinclude language="javascript"} +../../octue/twined/examples/met_mast_scada_service/data/output_manifest.json +::: +:::: +::::::: +:::::::: +::::::::::::::::::::: + +## File tag templates {#file_tag_templates} + +Datafiles can be tagged with key-value pairs of relevant metadata that +can be used in analyses. Certain datasets might need one set of metadata +on each file, while others might need a different set. The required (or +optional) file tags can be specified in the twine in the +`file_tags_template` property of each dataset of any `manifest` strand. +Each file in the corresponding manifest strand is then validated against +its dataset\'s file tag template to ensure the required tags are +present. + +::::::::::: tabs +:::::: group-tab +Manifest strand with file tag template + +The example below is for an input manifest, but the format is the same +for configuration and output manifests. + +::::: accordion +::: accordion-row +Show twine containing a manifest strand with a file tag template + +```javascript +{ + "input_manifest": { + "datasets": [ + { + "key": "met_mast_data", + "purpose": "A dataset containing meteorological mast data", + "file_tags_template": { + "type": "object", + "properties": { + "manufacturer": {"type": "string"}, + "height": {"type": "number"}, + "is_recycled": {"type": "boolean"} + }, + "required": ["manufacturer", "height", "is_recycled"] + } + } + ] + } +} +``` + +::: + +::: accordion-row +Show a matching file manifest + +```javascript +{ + "id": "8ead7669-8162-4f64-8cd5-4abe92509e17", + "datasets": [ + { + "id": "7ead7669-8162-4f64-8cd5-4abe92509e17", + "name": "met_mast_data", + "tags": {}, + "labels": ["met", "mast", "wind"], + "files": [ + { + "path": "input/datasets/7ead7669/file_1.csv", + "cluster": 0, + "sequence": 0, + "extension": "csv", + "labels": ["mykeyword1", "mykeyword2"], + "tags": { + "manufacturer": "vestas", + "height": 500, + "is_recycled": true + }, + "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", + "name": "file_1.csv" + }, + { + "path": "input/datasets/7ead7669/file_1.csv", + "cluster": 0, + "sequence": 1, + "extension": "csv", + "labels": [], + "tags": { + "manufacturer": "vestas", + "height": 500, + "is_recycled": true + }, + "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", + "name": "file_1.csv" + } + ] + } + ] +} +``` + +::: +::::: +:::::: + +:::::: group-tab +Manifest strand with a remote file tag template + +A remote reference can also be given for a file tag template. If the tag +template somewhere public, this is useful for sharing the template +between one or more teams working on the same type of data. + +The example below is for an input manifest, but the format is the same +for configuration and output manifests. It also shows two different tag +templates being specified for two different types of dataset required by +the manifest. + +::::: accordion +::: accordion-row +Show twine using a remote tag template + +```javascript +{ + "input_manifest": { + "datasets": [ + { + "key": "met_mast_data", + "purpose": "A dataset containing meteorological mast data", + "file_tags_template": { + "$ref": "https://refs.schema.octue.com/octue/my-file-type-tag-template/0.0.0.json" + } + }, + { + "key": "some_other_kind_of_dataset", + "purpose": "A dataset containing something else", + "file_tags_template": { + "$ref": "https://refs.schema.octue.com/octue/another-file-type-tag-template/0.0.0.json" + } + } + ] + } +} +``` + +::: + +::: accordion-row +Show a matching file manifest + +```javascript +{ + "id": "8ead7669-8162-4f64-8cd5-4abe92509e17", + "datasets": [ + { + "id": "7ead7669-8162-4f64-8cd5-4abe92509e17", + "name": "met_mast_data", + "tags": {}, + "labels": ["met", "mast", "wind"], + "files": [ + { + "path": "input/datasets/7ead7669/file_1.csv", + "cluster": 0, + "sequence": 0, + "extension": "csv", + "labels": ["mykeyword1", "mykeyword2"], + "tags": { + "manufacturer": "vestas", + "height": 500, + "is_recycled": true + }, + "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", + "name": "file_1.csv" + }, + { + "path": "input/datasets/7ead7669/file_1.csv", + "cluster": 0, + "sequence": 1, + "extension": "csv", + "labels": [], + "tags": { + "manufacturer": "vestas", + "height": 500, + "is_recycled": true + }, + "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", + "name": "file_1.csv" + } + ] + }, + { + "id": "7ead7669-8162-4f64-8cd5-4abe92509e29", + "name": "some_other_kind_of_dataset", + "tags": {}, + "labels": ["my-label"], + "files": [ + { + "path": "input/datasets/7eadpp9/interesting_file.dat", + "cluster": 0, + "sequence": 0, + "extension": "dat", + "labels": [], + "tags": { + "length": 864, + "orientation_angle": 85 + }, + "id": "abff07bc-7c19-4ed5-be6d-a6546eae9071", + "name": "interesting_file.csv" + }, + } + ] +} +``` + +::: +::::: +:::::: +::::::::::: + +TODO - clean up or remove this section + +## How Filtering Works {#how_filtering_works} + +It\'s the job of **twined** to make sure of two things: + +1. make sure the _twine_ file itself is valid, + + > **File data (input, output)** + > + > Files are not streamed directly to the digital twin (this would + > require extreme bandwidth in whatever system is orchestrating all + > the twins). Instead, files should be made available on the local + > storage system; i.e. a volume mounted to whatever container or VM + > the digital twin runs in. + > + > Groups of files are described by a `manifest`, where a manifest is + > (in essence) a catalogue of files in a dataset. + > + > A digital twin might receive multiple manifests, if it uses + > multiple datasets. For example, it could use a 3D point cloud + > LiDAR dataset, and a meteorological dataset. + > + > ```javascript + > { + > "manifests": [ + > { + > "type": "dataset", + > "id": "3c15c2ba-6a32-87e0-11e9-3baa66a632fe", // UUID of the manifest + > "files": [ + > { + > "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", // UUID of that file + > "sha1": "askjnkdfoisdnfkjnkjsnd" // for quality control to check correctness of file contents + > "name": "Lidar - 4 to 10 Dec.csv", + > "path": "local/file/path/to/folder/containing/it/", + > "type": "csv", + > "metadata": { + > }, + > "size_bytes": 59684813, + > "tags": {"special_number": 1}, + > "labels": ["lidar", "helpful", "information", "like"], // Searchable, parsable and filterable + > }, + > { + > "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", + > "name": "Lidar - 11 to 18 Dec.csv", + > "path": "local/file/path/to/folder/containing/it/", + > "type": "csv", + > "metadata": { + > }, + > "size_bytes": 59684813, + > "tags": {"special_number": 2}, + > "labels": ["lidar", "helpful", "information", "like"] // Searchable, parsable and filterable + > }, + > { + > "id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86", + > "name": "Lidar report.pdf", + > "path": "local/file/path/to/folder/containing/it/", + > "type": "pdf", + > "metadata": { + > }, + > "size_bytes": 484813, + > "tags": {}, + > "labels": ["report"] // Searchable, parsable and filterable + > } + > ] + > }, + > { + > // ... another dataset manifest ... + > } + > ] + > } + > ``` diff --git a/docs/docs/twines/anatomy_monitors.md b/docs/docs/twines/anatomy_monitors.md new file mode 100644 index 000000000..ee8f0a7e1 --- /dev/null +++ b/docs/docs/twines/anatomy_monitors.md @@ -0,0 +1,57 @@ +# Monitor Message Strand {#monitors_strand} + +The `monitor_message_schema` strand is _values-based_ meaning the data +that matches the strand is in JSON form. It is a _json schema_ which +describes a monitor message. + +:::: tabs +::: group-tab +Monitors Strand + +There are two kinds of monitoring data required from a digital twin. + +**Monitor data (output)** + +Values for health and progress monitoring of the twin, for example +percentage progress, iteration number and status - perhaps even +residuals graphs for a converging calculation. Broadly speaking, this +should be user-facing information. + +_This kind of monitoring data can be in a suitable form for display on a +dashboard_ + +**Log data (output)** + +Logged statements, typically in iostream form, produced by the twin +(e.g. via python\'s `logging` module) must be capturable as an output +for debugging and monitoring purposes. Broadly speaking, this should be +developer-facing information. +::: +:::: + +Let\'s look at basic examples for twines containing each of these +strands: + +:::: tabs +::: group-tab +Monitors Strand + +**Monitor data (output)** + +```javascript +{ + "monitor_message_schema": { + "type": "object", + "properties": { + "my_property": { + "type": "number" + } + }, + "required": ["my_property"] + } +} +``` + +**Log data (output)** +::: +:::: diff --git a/docs/docs/twines/anatomy_values.md b/docs/docs/twines/anatomy_values.md new file mode 100644 index 000000000..2a2c14b9b --- /dev/null +++ b/docs/docs/twines/anatomy_values.md @@ -0,0 +1,157 @@ +# Values-based Strands {#values_based_strands} + +The `configuration_values_schema`, `input_values_schema` and +`output_values_schema` strands are _values-based_, meaning the data that +matches these strands is in JSON form. + +Each of these strands is a _json schema_ which describes that data. + +:::::::: tabs +::: group-tab +Configuration Values Strand + +This strand is a `configuration_values_schema`, that is used to check +validity of any `configuration_values` data supplied to the twin at +startup. + +The Configuration Values Strand is generally used to define control +parameters relating to what the twin should do, or how it should +operate. + +For example, should it produce output images as low resolution PNGs or +as SVGs? How many iterations of a fluid flow solver should be used? What +is the acceptable error level on an classifier algorithm? +::: + +::::: group-tab +Input Values Strand + +This strand is an `input_values_schema`, that is used to check validity +of `input_values` data supplied to the twin at the beginning of an +analysis task. + +The Input Values Strand is generally used to define actual data which +will be processed by the twin. Sometimes, it may be used to define +control parameters specific to an analysis. + +For example, if a twin cleans and detects anomalies in a 10-minute +timeseries of 1Hz data, the `input_values` might contain an array of +data and a list of corresponding timestamps. It may also contain a +control parameter specifying which algorithm is used to do the +detection. + +:::: note +::: title +Note +::: + +Depending on the way the twin is deployed (see +`deployment`{.interpreted-text role="ref"}), the `input_values` might +come in from a web request, over a websocket or called directly from the +command line or another library. + +However they come, if the new `input_values` validate against the +`input_values_schema` strand, then analysis can proceed. +:::: +::::: + +::: group-tab +Output Values Strand + +This strand is an `output_values_schema`, that is used to check results +(`output_values`) computed during an analysis. This ensures that the +application wrapped up within the _twine_ is operating correctly, and +enables other twins/services or the end users to see what outputs they +will get. + +For example,if a twin cleans and detects anomalies in a 10-minute +timeseries of 1Hz data, the `output_values` might contain an array of +data interpolated onto regular timestamps, with missing values filled in +and a list of warnings where anomalies were found. +::: +:::::::: + +Let\'s look at basic examples for twines containing each of these +strands: + +:::::: tabs +::: group-tab +Configuration Values Strand + +This _twine_ contains an example `configuration_values_schema` with one +control parameter. + +[Many more detailed and specialised examples are available in the GitHub +repository](https://github.com/octue/twined/tree/main/examples) + +```javascript +{ + "configuration_values_schema": { + "title": "The example configuration form", + "description": "The Configuration Values Strand of an example twine", + "type": "object", + "properties": { + "n_iterations": { + "description": "An example of an integer configuration variable, called 'n_iterations'.", + "type": "integer", + "minimum": 1, + "maximum": 10, + "default": 5 + } + } + } +} +``` + +Matching `configuration_values` data could look like this: + +```javascript +{ + "n_iterations": 8, +} +``` + +::: + +::: group-tab +Input Values Strand + +This _twine_ contains an example `input_values_schema` with one input +value, which marked as required. + +Many more detailed and specialised examples are available in +`examples`{.interpreted-text role="ref"}. + +```javascript +{ + "input_values_schema": { + "title": "Input Values", + "description": "The input values strand of an example twine, with a required height value", + "type": "object", + "properties": { + "height": { + "description": "An example of an integer value called 'height'", + "type": "integer", + "minimum": 2 + } + }, + "required": ["height"] + }, +``` + +Matching `input_values` data could look like this: + +```javascript +{ + "height": 13, +} +``` + +::: + +::: group-tab +Output Values Strand + +Stuff +::: +:::::: diff --git a/docs/docs/twines/examples.md b/docs/docs/twines/examples.md new file mode 100644 index 000000000..974c05706 --- /dev/null +++ b/docs/docs/twines/examples.md @@ -0,0 +1,200 @@ +# Examples + +Here, we look at example use cases for the library, and show how to use +it in python. + +It\'s also well worth looking at the unit test cases copied straight +from the unit test cases, so you can always check there to see how +everything hooks up. + +## \[Simple\] Equipment installation cost {#example_equipment_installation_cost} + +::::: tabs +::: group-tab +Scenario + +You need to provide your team with an estimate for installation cost of +an equipment foundation. + +It\'s a straightforward calculation for you, but the Logistics Team +keeps changing the installation position, to try and optimise the +overall project logistics. + +Each time the locations change, the GIS team gives you an updated +embedment depth, which is what you use (along with steel cost and +foundation type), to calculate cost and report it back. + +This twine allows you to define to create a wrapper around your scripts +that communicates to the GIS team what you need as an input, communicate +to the logistics team what they can expect as an output. + +When deployed as a digital twin, the calculation gets automatically +updated, leaving you free to get on with all the other work! +::: + +::: group-tab +Twine + +We specify the `steel_cost` and `foundation_type` as `configuration` +values, which you can set on startup of the twin. + +Once the twin is running, it requires the `embedment_depth` as an +`input_value` from the GIS team. A member of the GIS team can use your +twin to get `foundation_cost` directly. + +```javascript +{ + "title": "Foundation Cost Model", + "description": "This twine helps compute the cost of an installed foundation.", + "children": [ + ], + "configuration_values_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Foundation cost twin configuration", + "description": "Set config parameters and constants at startup of the twin.", + "type": "object", + "properties": { + "steel_cost": { + "description": "The cost of steel in GBP/m^3. To get a better predictive model, you could add an economic twin that forecasts the cost of steel using the project timetable.", + "type": "number", + "minimum": 0, + "default": 3000 + }, + "foundation_type": { + "description": "The type of foundation being used.", + "type": "string", + "pattern": "^(monopile|twisted-jacket)$", + "default": "monopile" + } + } + }, + "input_values_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Input Values schema for the foundation cost twin", + "description": "These values are supplied to the twin asynchronously over a web socket. So as these values change, the twin can reply with an update.", + "type": "object", + "properties": { + "embedment_depth": { + "description": "Embedment depth in metres", + "type": "number", + "minimum": 10, + "maximum": 500 + } + } + }, + "output_manifest": { + "datasets": [] + }, + "output_values_schema": { + "title": "Output Values schema for the foundation cost twin", + "description": "The response supplied to a change in input values will always conform to this schema.", + "type": "object", + "properties": { + "foundation_cost": { + "description": "The foundation cost.", + "type": "integer", + "minimum": 2 + } + } + } +} +``` + +::: +::::: + +## \[Simple\] Site weather conditions {#example_site_weather_conditions} + +::::: tabs +::: group-tab +Scenario + +You need to be able to get characteristic weather conditions at a +specific location, for a range of reasons including assessing extreme +design loads. The values you need are computed in a script, which calls +a Weather API (provided by a third party), but also needs a dataset of +\"Wind Resource\" files. +::: + +::: group-tab +Twine + +```javascript +{ + "title": "Weather Service Digital Twin", + "description": "Provides a model for design extreme weather conditions given a location", + "notes": "Easily extendable with children to add forecast and historical data of different types.", + "credentials": [ + { + "name": "WEATHER_API_SECRET_KEY", + "purpose": "Token for accessing a 3rd party weather API service" + } + ], + "input_manifest": { + "datasets": [ + { + "key": "wind_resource_data", + "purpose": "A dataset containing Wind Resource Grid files" + } + ] + }, + "input_values_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Input Values for the weather service twin", + "description": "This is a simple example for getting metocean conditions at a single location", + "type": "object", + "properties": { + "location": { + "description": "Location", + "type": "object", + "properties": { + "latitude": { + "type": "number", + "minimum": -90, + "maximum": 90 + }, + "longitude": { + "type": "number", + "minimum": -180, + "maximum": 180 + }, + "srid": { + "description": "The Spatial Reference System ID for the coordinate. Default is 4326 (WGS84)", + "type": "integer", + "default": 4326 + } + } + } + } + }, + "output_manifest": { + "datasets": [ + { + "key": "production_data", + "purpose": "A dataset containing production data", + "tags": {"cleaned": true}, + "labels": ["production", "wind"] + } + ] + }, + "output_values_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Output Values for the metocean service twin", + "description": "The output values strand of an example twine", + "type": "object", + "properties": { + "water_depth": { + "description": "Design water depth for use in concept calculations", + "type": "number" + }, + "extreme_wind_speed": { + "description": "Extreme wind speed value for use in concept calculations", + "type": "number" + } + } + } +} +``` + +::: +::::: diff --git a/docs/docs/twines/index.md b/docs/docs/twines/index.md new file mode 100644 index 000000000..6722bf27e --- /dev/null +++ b/docs/docs/twines/index.md @@ -0,0 +1,109 @@ +:::: attention +::: title +Attention +::: + +This library is in very early stages. Like the idea of it? Please [star +us on GitHub](https://github.com/octue/twined) and contribute via the +[issues board](https://github.com/octue/twined/issues) and +[roadmap](https://github.com/octue/twined/projects/1). +:::: + +# Twined + +**twined** is a library to help create and connect +`digital_twins`{.interpreted-text role="ref"} and data services. + +> + +A digital twin is a virtual representation of a real life being - a +physical asset like a wind turbine or car - or even a human. Like real +things, digital twins need to interact, so can be connected together, +but need a common communication framework to do so. + +**twined** helps you to define a single file, a \"twine\", that defines +a digital twin / data service. It specifies specifying its data +interfaces, connections to other twins, and other requirements. + +Any person, or any computer, can read a twine and understand +_what-goes-in_ and _what-comes-out_. That makes it easy to collaborate +with other teams, since everybody is crystal clear about what\'s needed. + +
+ +
Digital twins / data services connected in a hierarchy. Each +blue circle represents a twin, coupled to its neighbours. Yellow nodes +are where schema are used to connect twins.
+
+ +## Aims + +**twined** provides a toolkit to help create and validate \"twines\" - +descriptions of a digital twin, what data it requires, what it does and +how it works. + +The goals of this **twined** library are as follows: + +: - Provide a clear framework for what a _twine_ can and/or must +contain - Provide functions to validate incoming data against a known +_twine_ - Provide functions to check that a _twine_ itself is valid - Provide (or direct you to) tools to create _twines_ describing +what you require + +In `anatomy`{.interpreted-text role="ref"}, we describe the different +parts of a twine (examining how digital twins connect and interact\... +building them together in hierarchies and networks). But you may prefer +to dive straight in with the `quick_start`{.interpreted-text role="ref"} +guide. + +The scope of **twined** is not large. Many other libraries will deal +with hosting and deploying digital twins, still more will deal with the +actual analyses done within them. **twined** purely deals with parsing +and checking the information exchanged. + +## Raison d\'etre {#reason_for_being} + +Octue believes that a lynchpin of solving climate change is the ability +for all engineering, manufacturing, supply chain and infrastructure +plant to be connected together, enabling strong optimisation and +efficient use of these systems. + +To enable engineers and scientists to build, connect and run digital +twins in large networks (or even in small teams!) it is necessary for +everyone to be on the same page - the +`gemini_principles`{.interpreted-text role="ref"} are a great way to +start with that, which is why we\'ve released this part of our +technology stack as open source, to support those principles and help +develop a wider ecosystem. + +The main goal is to **help engineers and scientists focus on doing +engineering and science** - instead of apis, data cleaning/management, +and all this cloud-pipeline-devops-test-ci-ml BS that takes up 90% of a +scientist\'s time, when they should be spending their valuable time +researching migratory patterns of birds, or cell structures, or wind +turbine performance, or whatever excites them. + +## Uses + +At [Octue](https://www.octue.com), **twined** is used as a core part of +our application creation process: + +> - As a format to communicate requirements to our partners in research +> projects +> - As a tool to validate incoming data to digital twins +> - As a framework to help establish schema when designing digital twins +> - As a source of information on digital twins in our network, to help +> map and connect twins together + +We\'d like to hear about your use case. Please get in touch! + +We use the [GitHub Issue Tracker](https://github.com/octue/twined) to +manage bug reports and feature requests. Please note, this is not a +\"general help\" forum; we recommend Stack Overflow for such questions. +For really gnarly issues or for help designing digital twin schema, +Octue is able to provide application support services for those building +digital twins using **twined**. + +::: {.toctree maxdepth="2"} +self quick_start anatomy about deployment license version_history +::: diff --git a/docs/docs/twines/lifecycle.md b/docs/docs/twines/lifecycle.md new file mode 100644 index 000000000..133e7f096 --- /dev/null +++ b/docs/docs/twines/lifecycle.md @@ -0,0 +1,34 @@ +> Data matching the `configuration_values_schema` is supplied to the +> digital twin / data service at startup. +> +> It\'s generally used to define control parameters relating to what the +> service should do, or how it should operate. For example, should it +> produce output images as low resolution PNGs or as SVGs? How many +> iterations of a fluid flow solver should be used? What is the +> acceptable error level on an classifier algorithm? +> +> Input Values +> +> Once configuration data supplied to a service has been validated, it +> can accept inputs and run analyses using them. +> +> Depending on the way it\'s deployed (see +> `deployment`{.interpreted-text role="ref"}), the `input_values` might +> come in from a web request, over a websocket or called directly from +> the command line or another library. +> +> However it comes, new `input_values`, which are in `JSON` format, are +> checked against the `input_values_schema` strand of the twine. If they +> match, then analysis can proceed. +> +> Output Values +> +> Once a service has Data matching the `output_values_schema` is +> supplied to the service while it\'s running. Depending on the way +> it\'s deployed, the values might come in from a web request, over a +> websocket or called directly from another library +> +> Input For example current rotor speed, or forecast wind direction. +> +> Values might be passed at instantiation of a twin (typical +> application-like process) or via a socket. diff --git a/docs/docs/twines/quick_start.md b/docs/docs/twines/quick_start.md new file mode 100644 index 000000000..c8bee73f0 --- /dev/null +++ b/docs/docs/twines/quick_start.md @@ -0,0 +1,5 @@ +# Quick Start {#quick_start} + +::: {.toctree maxdepth="2"} +quick_start_installation quick_start_create_your_first_twine +::: diff --git a/docs/docs/twines/quick_start_create_your_first_twine.md b/docs/docs/twines/quick_start_create_your_first_twine.md new file mode 100644 index 000000000..0d0f997a0 --- /dev/null +++ b/docs/docs/twines/quick_start_create_your_first_twine.md @@ -0,0 +1,112 @@ +# Create your first twine {#create_your_first_twine} + +Let\'s say we want a digital twin that accepts two values, uses them to +make a calculation, then gives the result. Anyone connecting to the twin +will need to know what values it requires, and what it responds with. + +First, create a blank text file, call it [twine.json]{.title-ref}. +We\'ll give the twin a title and description. Paste in the following: + +```javascript +{ + "title": "My first digital twin... of an atomising discombobulator", + "description": "A simple example... estimates the `foz` value of an atomising discombobulator." +} +``` + +Now, let\'s define an input values strand, to specify what values are +required by the twin. For this we use a json schema (you can read more +about them in `introducing_json_schema`{.interpreted-text role="ref"}). +Add the `input_values` field, so your twine looks like this: + +```javascript +{ + "title": "My first digital twin", + "description": "A simple example to build on..." + "input_values_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Input Values schema for my first digital twin", + "description": "These values are supplied to the twin by another program (often over a websocket, depending on your integration provider). So as these values change, the twin can reply with an update.", + "type": "object", + "properties": { + "foo": { + "description": "The foo value... speed of the discombobulator's input bobulation module, in m/s", + "type": "number", + "minimum": 10, + "maximum": 500 + }, + "baz": { + "description": "The baz value... period of the discombobulator's recombulation unit, in s", + "type": "number", + "minimum": 0, + "maximum": 1000 + } + } + } +} +``` + +Finally, let\'s define an output values strand, to define what kind of +data is returned by the twin: + +```javascript +"output_values_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Output Values schema for my first digital twin", + "description": "The twin will output data that matches this schema", + "type": "object", + "properties": { + "foz": { + "description": "Estimate of the foz value... efficiency of the discombobulator in %", + "type": "number", + "minimum": 10, + "maximum": 500 + } + } +} +``` + +# Load the twine {#load_the_twine} + +**twined** provides a [Twine()]{.title-ref} class to load a twine (from +a file or a json string). The loading process checks the twine itself is +valid. It\'s as simple as: + +```py +from octue.twined import Twine + +my_twine = Twine(source='twine.json') +``` + +# Validate some inputs {#validate_some_inputs} + +Say we have some json that we want to parse and validate, to make sure +it matches what\'s required for input values. + +```py +my_input_values = my_twine.validate_input_values(json='{"foo": 30, "baz": 500}') +``` + +You can read the values from a file too. Paste the following into a file +named `input_values.json`: + +```javascript +{ + "foo": 30, + "baz": 500 +} +``` + +Then parse and validate directly from the file: + +```py +my_input_values = my_twine.validate_input_values(source="input_values.json") +``` + +:::: attention +::: title +Attention +::: + +LIBRARY IS UNDER CONSTRUCTION! WATCH THIS SPACE FOR MORE! +:::: diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index f3889e8ef..f29aaba87 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -14,6 +14,7 @@ theme: - navigation.tracking - navigation.path - navigation.footer + - navigation.indexes - search.suggest - search.highlight - search.share @@ -56,6 +57,16 @@ nav: - services.md - asking_questions.md - creating_services.md + - Twines: + - twines/index.md + - Quickstart: twines/quick_start_create_your_first_twine.md + - twines/anatomy.md + - About Twines: + - twines/about/index.md + - twines/about/about_digital_twins.md + - twines/about/about_requirements.md + - twines/about/about_introducing_json_schema.md + - twines/about/about_other_considerations.md - updating_services.md - running_services_locally.md - deploying_services.md From 6f0ed220f726297177a7958dea9b60a3784ba863 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 18:48:41 -0400 Subject: [PATCH 51/69] DOC: Move `CONTRIBUTING.md` to repo root --- docs/contributing.md => CONTRIBUTING.md | 0 README.md | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename docs/contributing.md => CONTRIBUTING.md (100%) diff --git a/docs/contributing.md b/CONTRIBUTING.md similarity index 100% rename from docs/contributing.md rename to CONTRIBUTING.md diff --git a/README.md b/README.md index 8ef8a1358..2b68bd6fd 100644 --- a/README.md +++ b/README.md @@ -85,4 +85,4 @@ python3 -m unittest ## Contributing -Take a look at our [contributing](/docs/contributing.md) page. +Take a look at our [contributing](/CONTRIBUTING.md) page. From e4740c429578e04b12683521fe04c8d3eda13e8f Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 18:58:02 -0400 Subject: [PATCH 52/69] DOC: Move docs up a directory skipci --- docs/.gitignore | 2 -- docs/{docs => }/asking_questions.md | 4 ++-- docs/{docs => }/authentication.md | 0 docs/{docs => }/available_filters.md | 0 docs/{docs => }/creating_apps.md | 0 docs/{docs => }/creating_services.md | 2 +- docs/{docs => }/data_containers.md | 0 docs/{docs => }/datafile.md | 0 docs/{docs => }/dataset.md | 0 docs/{docs => }/deploying_services.md | 2 +- docs/{docs => }/downloading_datafiles.md | 0 .../images/213_purple-fruit-snake-transparent.gif | Bin docs/{docs => }/images/coloured_logs.png | Bin docs/{docs => }/images/datafile_use_cases.png | Bin .../add_roles_to_service_account.png | Bin .../choose_dockerfile.png | Bin .../choose_repository.png | Bin .../deploying_services_advanced/create_service.png | Bin .../create_service_account.png | Bin .../deploying_services_advanced/create_trigger.png | Bin .../low_carbon_regions.png | Bin .../deploying_services_advanced/save_and_create.png | Bin .../service_name_and_region.png | Bin .../deploying_services_advanced/set_traffic.png | Bin .../set_up_with_cloud_build.png | Bin .../images/digital_twin_component_basic.svg | 0 .../digital_twin_component_for_simulation.svg | 0 docs/{docs => }/images/digital_twin_hierarchy.svg | 0 .../images/digital_twin_hierarchy_extended.svg | 0 docs/{docs => }/images/logos_square-dark.png | Bin docs/{docs => }/images/schema_form_example.png | Bin docs/{docs => }/images/updating_services/checks.png | Bin .../images/updating_services/deployment.png | Bin docs/{docs => }/images/updating_services/diff.png | Bin docs/{docs => }/images/updating_services/pytest.png | Bin docs/{docs => }/index.md | 0 docs/{docs => }/installation.md | 0 docs/{docs => }/inter_service_compatibility.md | 0 docs/{docs => }/license.md | 0 docs/{docs => }/logging.md | 0 docs/{docs => }/manifest.md | 2 +- docs/{docs => }/running_services_locally.md | 0 docs/{docs => }/services.md | 0 docs/{docs => }/stylesheets/extra.css | 0 docs/{docs => }/testing_services.md | 0 docs/{docs => }/troubleshooting_services.md | 0 docs/{docs => }/twines/about/about_digital_twins.md | 0 .../twines/about/about_introducing_json_schema.md | 0 .../twines/about/about_other_considerations.md | 0 docs/{docs => }/twines/about/about_requirements.md | 0 docs/{docs => }/twines/about/index.md | 0 docs/{docs => }/twines/anatomy.md | 0 docs/{docs => }/twines/anatomy_children.md | 0 docs/{docs => }/twines/anatomy_credentials.md | 0 docs/{docs => }/twines/anatomy_manifest.md | 0 docs/{docs => }/twines/anatomy_monitors.md | 0 docs/{docs => }/twines/anatomy_values.md | 0 docs/{docs => }/twines/examples.md | 0 docs/{docs => }/twines/index.md | 0 docs/{docs => }/twines/lifecycle.md | 0 docs/{docs => }/twines/quick_start.md | 0 .../twines/quick_start_create_your_first_twine.md | 0 docs/{docs => }/updating_services.md | 0 docs/{docs => }/version_history.md | 0 docs/mkdocs.yml => mkdocs.yml | 0 65 files changed, 5 insertions(+), 7 deletions(-) delete mode 100644 docs/.gitignore rename docs/{docs => }/asking_questions.md (98%) rename docs/{docs => }/authentication.md (100%) rename docs/{docs => }/available_filters.md (100%) rename docs/{docs => }/creating_apps.md (100%) rename docs/{docs => }/creating_services.md (98%) rename docs/{docs => }/data_containers.md (100%) rename docs/{docs => }/datafile.md (100%) rename docs/{docs => }/dataset.md (100%) rename docs/{docs => }/deploying_services.md (96%) rename docs/{docs => }/downloading_datafiles.md (100%) rename docs/{docs => }/images/213_purple-fruit-snake-transparent.gif (100%) rename docs/{docs => }/images/coloured_logs.png (100%) rename docs/{docs => }/images/datafile_use_cases.png (100%) rename docs/{docs => }/images/deploying_services_advanced/add_roles_to_service_account.png (100%) rename docs/{docs => }/images/deploying_services_advanced/choose_dockerfile.png (100%) rename docs/{docs => }/images/deploying_services_advanced/choose_repository.png (100%) rename docs/{docs => }/images/deploying_services_advanced/create_service.png (100%) rename docs/{docs => }/images/deploying_services_advanced/create_service_account.png (100%) rename docs/{docs => }/images/deploying_services_advanced/create_trigger.png (100%) rename docs/{docs => }/images/deploying_services_advanced/low_carbon_regions.png (100%) rename docs/{docs => }/images/deploying_services_advanced/save_and_create.png (100%) rename docs/{docs => }/images/deploying_services_advanced/service_name_and_region.png (100%) rename docs/{docs => }/images/deploying_services_advanced/set_traffic.png (100%) rename docs/{docs => }/images/deploying_services_advanced/set_up_with_cloud_build.png (100%) rename docs/{docs => }/images/digital_twin_component_basic.svg (100%) rename docs/{docs => }/images/digital_twin_component_for_simulation.svg (100%) rename docs/{docs => }/images/digital_twin_hierarchy.svg (100%) rename docs/{docs => }/images/digital_twin_hierarchy_extended.svg (100%) rename docs/{docs => }/images/logos_square-dark.png (100%) rename docs/{docs => }/images/schema_form_example.png (100%) rename docs/{docs => }/images/updating_services/checks.png (100%) rename docs/{docs => }/images/updating_services/deployment.png (100%) rename docs/{docs => }/images/updating_services/diff.png (100%) rename docs/{docs => }/images/updating_services/pytest.png (100%) rename docs/{docs => }/index.md (100%) rename docs/{docs => }/installation.md (100%) rename docs/{docs => }/inter_service_compatibility.md (100%) rename docs/{docs => }/license.md (100%) rename docs/{docs => }/logging.md (100%) rename docs/{docs => }/manifest.md (98%) rename docs/{docs => }/running_services_locally.md (100%) rename docs/{docs => }/services.md (100%) rename docs/{docs => }/stylesheets/extra.css (100%) rename docs/{docs => }/testing_services.md (100%) rename docs/{docs => }/troubleshooting_services.md (100%) rename docs/{docs => }/twines/about/about_digital_twins.md (100%) rename docs/{docs => }/twines/about/about_introducing_json_schema.md (100%) rename docs/{docs => }/twines/about/about_other_considerations.md (100%) rename docs/{docs => }/twines/about/about_requirements.md (100%) rename docs/{docs => }/twines/about/index.md (100%) rename docs/{docs => }/twines/anatomy.md (100%) rename docs/{docs => }/twines/anatomy_children.md (100%) rename docs/{docs => }/twines/anatomy_credentials.md (100%) rename docs/{docs => }/twines/anatomy_manifest.md (100%) rename docs/{docs => }/twines/anatomy_monitors.md (100%) rename docs/{docs => }/twines/anatomy_values.md (100%) rename docs/{docs => }/twines/examples.md (100%) rename docs/{docs => }/twines/index.md (100%) rename docs/{docs => }/twines/lifecycle.md (100%) rename docs/{docs => }/twines/quick_start.md (100%) rename docs/{docs => }/twines/quick_start_create_your_first_twine.md (100%) rename docs/{docs => }/updating_services.md (100%) rename docs/{docs => }/version_history.md (100%) rename docs/mkdocs.yml => mkdocs.yml (100%) diff --git a/docs/.gitignore b/docs/.gitignore deleted file mode 100644 index 10971d675..000000000 --- a/docs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -doctrees -html diff --git a/docs/docs/asking_questions.md b/docs/asking_questions.md similarity index 98% rename from docs/docs/asking_questions.md rename to docs/asking_questions.md index 33589c198..e1d78224e 100644 --- a/docs/docs/asking_questions.md +++ b/docs/asking_questions.md @@ -21,7 +21,7 @@ to a child for processing/analysis. Questions can be: or custom logic in your own webserver. Questions are always asked to a _revision_ of a service. You can ask a -service a question if you have its [SRUID](services.md/#service-names), project ID, and the necessary permissions. +service a question if you have its [SRUID](/services/#service-names), project ID, and the necessary permissions. ## Asking a question @@ -49,7 +49,7 @@ answer["output_manifest"]["my_dataset"].files If you're using an environment other than the `main` environment, then before asking any questions to your Twined services, set the `TWINED_SERVICES_TOPIC_NAME` environment variable to the name of the Twined services Pub/Sub topic - (this is set when [deploying a service network](deploying_services.md/#deploying-services-advanced-developers-guide). + (this is set when [deploying a service network](/deploying_services/#deploying-services-advanced-developers-guide). It will be in the form `.octue.twined.services` !!! note diff --git a/docs/docs/authentication.md b/docs/authentication.md similarity index 100% rename from docs/docs/authentication.md rename to docs/authentication.md diff --git a/docs/docs/available_filters.md b/docs/available_filters.md similarity index 100% rename from docs/docs/available_filters.md rename to docs/available_filters.md diff --git a/docs/docs/creating_apps.md b/docs/creating_apps.md similarity index 100% rename from docs/docs/creating_apps.md rename to docs/creating_apps.md diff --git a/docs/docs/creating_services.md b/docs/creating_services.md similarity index 98% rename from docs/docs/creating_services.md rename to docs/creating_services.md index 4d74e3751..d9eef87ba 100644 --- a/docs/docs/creating_services.md +++ b/docs/creating_services.md @@ -102,7 +102,7 @@ are supported. ### Where to specify the namespace, name, and revision tag -See [here](services.md/#service-names) for service naming requirements. +See [here](/services/#service-names) for service naming requirements. **Namespace** diff --git a/docs/docs/data_containers.md b/docs/data_containers.md similarity index 100% rename from docs/docs/data_containers.md rename to docs/data_containers.md diff --git a/docs/docs/datafile.md b/docs/datafile.md similarity index 100% rename from docs/docs/datafile.md rename to docs/datafile.md diff --git a/docs/docs/dataset.md b/docs/dataset.md similarity index 100% rename from docs/docs/dataset.md rename to docs/dataset.md diff --git a/docs/docs/deploying_services.md b/docs/deploying_services.md similarity index 96% rename from docs/docs/deploying_services.md rename to docs/deploying_services.md index 826d29f69..91a658412 100644 --- a/docs/docs/deploying_services.md +++ b/docs/deploying_services.md @@ -42,7 +42,7 @@ done automatically: including automated pre-deployment testing and creation of a GitHub release -You can now [ask your service some questions](asking_questions.md)! It will be available in the service network as +You can now [ask your service some questions](/asking_questions)! It will be available in the service network as `/:` (e.g. `octue/example-service-kueue:0.1.1`). ## Deploying the infrastructure diff --git a/docs/docs/downloading_datafiles.md b/docs/downloading_datafiles.md similarity index 100% rename from docs/docs/downloading_datafiles.md rename to docs/downloading_datafiles.md diff --git a/docs/docs/images/213_purple-fruit-snake-transparent.gif b/docs/images/213_purple-fruit-snake-transparent.gif similarity index 100% rename from docs/docs/images/213_purple-fruit-snake-transparent.gif rename to docs/images/213_purple-fruit-snake-transparent.gif diff --git a/docs/docs/images/coloured_logs.png b/docs/images/coloured_logs.png similarity index 100% rename from docs/docs/images/coloured_logs.png rename to docs/images/coloured_logs.png diff --git a/docs/docs/images/datafile_use_cases.png b/docs/images/datafile_use_cases.png similarity index 100% rename from docs/docs/images/datafile_use_cases.png rename to docs/images/datafile_use_cases.png diff --git a/docs/docs/images/deploying_services_advanced/add_roles_to_service_account.png b/docs/images/deploying_services_advanced/add_roles_to_service_account.png similarity index 100% rename from docs/docs/images/deploying_services_advanced/add_roles_to_service_account.png rename to docs/images/deploying_services_advanced/add_roles_to_service_account.png diff --git a/docs/docs/images/deploying_services_advanced/choose_dockerfile.png b/docs/images/deploying_services_advanced/choose_dockerfile.png similarity index 100% rename from docs/docs/images/deploying_services_advanced/choose_dockerfile.png rename to docs/images/deploying_services_advanced/choose_dockerfile.png diff --git a/docs/docs/images/deploying_services_advanced/choose_repository.png b/docs/images/deploying_services_advanced/choose_repository.png similarity index 100% rename from docs/docs/images/deploying_services_advanced/choose_repository.png rename to docs/images/deploying_services_advanced/choose_repository.png diff --git a/docs/docs/images/deploying_services_advanced/create_service.png b/docs/images/deploying_services_advanced/create_service.png similarity index 100% rename from docs/docs/images/deploying_services_advanced/create_service.png rename to docs/images/deploying_services_advanced/create_service.png diff --git a/docs/docs/images/deploying_services_advanced/create_service_account.png b/docs/images/deploying_services_advanced/create_service_account.png similarity index 100% rename from docs/docs/images/deploying_services_advanced/create_service_account.png rename to docs/images/deploying_services_advanced/create_service_account.png diff --git a/docs/docs/images/deploying_services_advanced/create_trigger.png b/docs/images/deploying_services_advanced/create_trigger.png similarity index 100% rename from docs/docs/images/deploying_services_advanced/create_trigger.png rename to docs/images/deploying_services_advanced/create_trigger.png diff --git a/docs/docs/images/deploying_services_advanced/low_carbon_regions.png b/docs/images/deploying_services_advanced/low_carbon_regions.png similarity index 100% rename from docs/docs/images/deploying_services_advanced/low_carbon_regions.png rename to docs/images/deploying_services_advanced/low_carbon_regions.png diff --git a/docs/docs/images/deploying_services_advanced/save_and_create.png b/docs/images/deploying_services_advanced/save_and_create.png similarity index 100% rename from docs/docs/images/deploying_services_advanced/save_and_create.png rename to docs/images/deploying_services_advanced/save_and_create.png diff --git a/docs/docs/images/deploying_services_advanced/service_name_and_region.png b/docs/images/deploying_services_advanced/service_name_and_region.png similarity index 100% rename from docs/docs/images/deploying_services_advanced/service_name_and_region.png rename to docs/images/deploying_services_advanced/service_name_and_region.png diff --git a/docs/docs/images/deploying_services_advanced/set_traffic.png b/docs/images/deploying_services_advanced/set_traffic.png similarity index 100% rename from docs/docs/images/deploying_services_advanced/set_traffic.png rename to docs/images/deploying_services_advanced/set_traffic.png diff --git a/docs/docs/images/deploying_services_advanced/set_up_with_cloud_build.png b/docs/images/deploying_services_advanced/set_up_with_cloud_build.png similarity index 100% rename from docs/docs/images/deploying_services_advanced/set_up_with_cloud_build.png rename to docs/images/deploying_services_advanced/set_up_with_cloud_build.png diff --git a/docs/docs/images/digital_twin_component_basic.svg b/docs/images/digital_twin_component_basic.svg similarity index 100% rename from docs/docs/images/digital_twin_component_basic.svg rename to docs/images/digital_twin_component_basic.svg diff --git a/docs/docs/images/digital_twin_component_for_simulation.svg b/docs/images/digital_twin_component_for_simulation.svg similarity index 100% rename from docs/docs/images/digital_twin_component_for_simulation.svg rename to docs/images/digital_twin_component_for_simulation.svg diff --git a/docs/docs/images/digital_twin_hierarchy.svg b/docs/images/digital_twin_hierarchy.svg similarity index 100% rename from docs/docs/images/digital_twin_hierarchy.svg rename to docs/images/digital_twin_hierarchy.svg diff --git a/docs/docs/images/digital_twin_hierarchy_extended.svg b/docs/images/digital_twin_hierarchy_extended.svg similarity index 100% rename from docs/docs/images/digital_twin_hierarchy_extended.svg rename to docs/images/digital_twin_hierarchy_extended.svg diff --git a/docs/docs/images/logos_square-dark.png b/docs/images/logos_square-dark.png similarity index 100% rename from docs/docs/images/logos_square-dark.png rename to docs/images/logos_square-dark.png diff --git a/docs/docs/images/schema_form_example.png b/docs/images/schema_form_example.png similarity index 100% rename from docs/docs/images/schema_form_example.png rename to docs/images/schema_form_example.png diff --git a/docs/docs/images/updating_services/checks.png b/docs/images/updating_services/checks.png similarity index 100% rename from docs/docs/images/updating_services/checks.png rename to docs/images/updating_services/checks.png diff --git a/docs/docs/images/updating_services/deployment.png b/docs/images/updating_services/deployment.png similarity index 100% rename from docs/docs/images/updating_services/deployment.png rename to docs/images/updating_services/deployment.png diff --git a/docs/docs/images/updating_services/diff.png b/docs/images/updating_services/diff.png similarity index 100% rename from docs/docs/images/updating_services/diff.png rename to docs/images/updating_services/diff.png diff --git a/docs/docs/images/updating_services/pytest.png b/docs/images/updating_services/pytest.png similarity index 100% rename from docs/docs/images/updating_services/pytest.png rename to docs/images/updating_services/pytest.png diff --git a/docs/docs/index.md b/docs/index.md similarity index 100% rename from docs/docs/index.md rename to docs/index.md diff --git a/docs/docs/installation.md b/docs/installation.md similarity index 100% rename from docs/docs/installation.md rename to docs/installation.md diff --git a/docs/docs/inter_service_compatibility.md b/docs/inter_service_compatibility.md similarity index 100% rename from docs/docs/inter_service_compatibility.md rename to docs/inter_service_compatibility.md diff --git a/docs/docs/license.md b/docs/license.md similarity index 100% rename from docs/docs/license.md rename to docs/license.md diff --git a/docs/docs/logging.md b/docs/logging.md similarity index 100% rename from docs/docs/logging.md rename to docs/logging.md diff --git a/docs/docs/manifest.md b/docs/manifest.md similarity index 98% rename from docs/docs/manifest.md rename to docs/manifest.md index 3951bea1b..e04310072 100644 --- a/docs/docs/manifest.md +++ b/docs/manifest.md @@ -46,7 +46,7 @@ child = Child( answer, question_uuid = child.ask(input_manifest=manifest) ``` -See [here](asking_questions.md) for more information. +See [here](/asking_questions) for more information. ### Receive datasets from a service diff --git a/docs/docs/running_services_locally.md b/docs/running_services_locally.md similarity index 100% rename from docs/docs/running_services_locally.md rename to docs/running_services_locally.md diff --git a/docs/docs/services.md b/docs/services.md similarity index 100% rename from docs/docs/services.md rename to docs/services.md diff --git a/docs/docs/stylesheets/extra.css b/docs/stylesheets/extra.css similarity index 100% rename from docs/docs/stylesheets/extra.css rename to docs/stylesheets/extra.css diff --git a/docs/docs/testing_services.md b/docs/testing_services.md similarity index 100% rename from docs/docs/testing_services.md rename to docs/testing_services.md diff --git a/docs/docs/troubleshooting_services.md b/docs/troubleshooting_services.md similarity index 100% rename from docs/docs/troubleshooting_services.md rename to docs/troubleshooting_services.md diff --git a/docs/docs/twines/about/about_digital_twins.md b/docs/twines/about/about_digital_twins.md similarity index 100% rename from docs/docs/twines/about/about_digital_twins.md rename to docs/twines/about/about_digital_twins.md diff --git a/docs/docs/twines/about/about_introducing_json_schema.md b/docs/twines/about/about_introducing_json_schema.md similarity index 100% rename from docs/docs/twines/about/about_introducing_json_schema.md rename to docs/twines/about/about_introducing_json_schema.md diff --git a/docs/docs/twines/about/about_other_considerations.md b/docs/twines/about/about_other_considerations.md similarity index 100% rename from docs/docs/twines/about/about_other_considerations.md rename to docs/twines/about/about_other_considerations.md diff --git a/docs/docs/twines/about/about_requirements.md b/docs/twines/about/about_requirements.md similarity index 100% rename from docs/docs/twines/about/about_requirements.md rename to docs/twines/about/about_requirements.md diff --git a/docs/docs/twines/about/index.md b/docs/twines/about/index.md similarity index 100% rename from docs/docs/twines/about/index.md rename to docs/twines/about/index.md diff --git a/docs/docs/twines/anatomy.md b/docs/twines/anatomy.md similarity index 100% rename from docs/docs/twines/anatomy.md rename to docs/twines/anatomy.md diff --git a/docs/docs/twines/anatomy_children.md b/docs/twines/anatomy_children.md similarity index 100% rename from docs/docs/twines/anatomy_children.md rename to docs/twines/anatomy_children.md diff --git a/docs/docs/twines/anatomy_credentials.md b/docs/twines/anatomy_credentials.md similarity index 100% rename from docs/docs/twines/anatomy_credentials.md rename to docs/twines/anatomy_credentials.md diff --git a/docs/docs/twines/anatomy_manifest.md b/docs/twines/anatomy_manifest.md similarity index 100% rename from docs/docs/twines/anatomy_manifest.md rename to docs/twines/anatomy_manifest.md diff --git a/docs/docs/twines/anatomy_monitors.md b/docs/twines/anatomy_monitors.md similarity index 100% rename from docs/docs/twines/anatomy_monitors.md rename to docs/twines/anatomy_monitors.md diff --git a/docs/docs/twines/anatomy_values.md b/docs/twines/anatomy_values.md similarity index 100% rename from docs/docs/twines/anatomy_values.md rename to docs/twines/anatomy_values.md diff --git a/docs/docs/twines/examples.md b/docs/twines/examples.md similarity index 100% rename from docs/docs/twines/examples.md rename to docs/twines/examples.md diff --git a/docs/docs/twines/index.md b/docs/twines/index.md similarity index 100% rename from docs/docs/twines/index.md rename to docs/twines/index.md diff --git a/docs/docs/twines/lifecycle.md b/docs/twines/lifecycle.md similarity index 100% rename from docs/docs/twines/lifecycle.md rename to docs/twines/lifecycle.md diff --git a/docs/docs/twines/quick_start.md b/docs/twines/quick_start.md similarity index 100% rename from docs/docs/twines/quick_start.md rename to docs/twines/quick_start.md diff --git a/docs/docs/twines/quick_start_create_your_first_twine.md b/docs/twines/quick_start_create_your_first_twine.md similarity index 100% rename from docs/docs/twines/quick_start_create_your_first_twine.md rename to docs/twines/quick_start_create_your_first_twine.md diff --git a/docs/docs/updating_services.md b/docs/updating_services.md similarity index 100% rename from docs/docs/updating_services.md rename to docs/updating_services.md diff --git a/docs/docs/version_history.md b/docs/version_history.md similarity index 100% rename from docs/docs/version_history.md rename to docs/version_history.md diff --git a/docs/mkdocs.yml b/mkdocs.yml similarity index 100% rename from docs/mkdocs.yml rename to mkdocs.yml From 2c0c9eb27cfec6c574b761b12d1e0f276f6ec435 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 19:06:31 -0400 Subject: [PATCH 53/69] DOC: Update `CONTRIBUTING.md` --- CONTRIBUTING.md | 30 +++--------------------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 844c737b4..c2e779ff5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -84,36 +84,12 @@ You can run pre-commit hooks without making a commit, too, like: pre-commit run black --all-files ``` -or - -``` -# -v gives verbose output, useful for figuring out why docs won't build -pre-commit run build-docs -v -``` - ## Documentation -### Building documents automatically - -The documentation will build automatically in a pre-configured environment when you make a commit. - -In fact, the way `pre-commit` works, you won't be allowed to make the commit unless the documentation builds. This way -we avoid getting broken documentation pushed to the main repository on any commit so we can rely on builds working. - -### Building documents manually - -**If you did need to build the documentation** - -Install `doxygen`. On a mac, that's `brew install doxygen`; other systems may differ. - -Install sphinx and other requirements for building the docs: +To serve the docs locally, run: ``` -pip install -r docs/requirements.txt +mkdocs serve ``` -Run the build process: - -``` -sphinx-build -b html docs/source docs/html -``` +This makes the docs available on `localhost` via a hot-reloading server From 8f8120a7cc9d9099d7d5e343c698938a8c8c7d68 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 19:14:13 -0400 Subject: [PATCH 54/69] OPS: Add CI for deploying docs --- .github/workflows/release.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ffafda68f..2c20c6135 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -129,3 +129,37 @@ jobs: - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@v1.12.4 + + docs: + needs: publish + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - name: Configure Git Credentials + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + + - uses: actions/setup-python@v5 + with: + python-version: 3.x + + - name: Create cache key + run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + + - name: Check cache + uses: actions/cache@v4 + with: + key: mkdocs-material-${{ env.cache_id }} + path: ~/.cache + restore-keys: | + mkdocs-material- + + - name: Install material for mkdocs + run: pip install mkdocs-material + + - name: Build and deploy docs + run: mkdocs gh-deploy --force From 080be20662ba72f45c63adf76fb7d4c5d1a5fab7 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 19:15:34 -0400 Subject: [PATCH 55/69] WIP: Test building docs skipci --- .github/workflows/test-build-docs.yml | 38 +++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/test-build-docs.yml diff --git a/.github/workflows/test-build-docs.yml b/.github/workflows/test-build-docs.yml new file mode 100644 index 000000000..f022e0e1e --- /dev/null +++ b/.github/workflows/test-build-docs.yml @@ -0,0 +1,38 @@ +name: Test build docs + +on: + push + +jobs: + docs: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - name: Configure Git Credentials + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + + - uses: actions/setup-python@v5 + with: + python-version: 3.x + + - name: Create cache key + run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + + - name: Check cache + uses: actions/cache@v4 + with: + key: mkdocs-material-${{ env.cache_id }} + path: ~/.cache + restore-keys: | + mkdocs-material- + + - name: Install material for mkdocs + run: pip install mkdocs-material mkdocs-glightbox mike + + - name: Build and deploy docs + run: mkdocs gh-deploy --force From 8057d9ee9a1dabd0909392417a16997452104da1 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 19:23:24 -0400 Subject: [PATCH 56/69] OPS: Fix docs deployment --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2c20c6135..3a92f63eb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -159,7 +159,7 @@ jobs: mkdocs-material- - name: Install material for mkdocs - run: pip install mkdocs-material + run: pip install mkdocs-material mkdocs-glightbox mike - name: Build and deploy docs run: mkdocs gh-deploy --force From 2e18088a24a73394076db97ec0f8ce22d30e8ddf Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 19:23:37 -0400 Subject: [PATCH 57/69] REV: Revert "WIP: Test building docs" This reverts commit 080be20662ba72f45c63adf76fb7d4c5d1a5fab7. --- .github/workflows/test-build-docs.yml | 38 --------------------------- 1 file changed, 38 deletions(-) delete mode 100644 .github/workflows/test-build-docs.yml diff --git a/.github/workflows/test-build-docs.yml b/.github/workflows/test-build-docs.yml deleted file mode 100644 index f022e0e1e..000000000 --- a/.github/workflows/test-build-docs.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Test build docs - -on: - push - -jobs: - docs: - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - - - name: Configure Git Credentials - run: | - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - - - uses: actions/setup-python@v5 - with: - python-version: 3.x - - - name: Create cache key - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV - - - name: Check cache - uses: actions/cache@v4 - with: - key: mkdocs-material-${{ env.cache_id }} - path: ~/.cache - restore-keys: | - mkdocs-material- - - - name: Install material for mkdocs - run: pip install mkdocs-material mkdocs-glightbox mike - - - name: Build and deploy docs - run: mkdocs gh-deploy --force From 7404d25bb6225156fbb309712b487c8b35f4816b Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 19:24:19 -0400 Subject: [PATCH 58/69] DOC: Move `nav` section in `mkdocs.yml` to top --- mkdocs.yml | 64 +++++++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index f29aaba87..228151bf7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,6 +1,38 @@ site_name: Octue Twined site_url: https://docs.twined.octue.com +nav: + - index.md + - installation.md + - data_containers.md + - datafile.md + - dataset.md + - manifest.md + - services.md + - asking_questions.md + - creating_services.md + - Twines: + - twines/index.md + - Quickstart: twines/quick_start_create_your_first_twine.md + - twines/anatomy.md + - About Twines: + - twines/about/index.md + - twines/about/about_digital_twins.md + - twines/about/about_requirements.md + - twines/about/about_introducing_json_schema.md + - twines/about/about_other_considerations.md + - updating_services.md + - running_services_locally.md + - deploying_services.md + - testing_services.md + - troubleshooting_services.md + - logging.md + - authentication.md + - inter_service_compatibility.md + # - api.md + - license.md + - version_history.md + theme: name: material logo: images/logos_square-dark.png @@ -47,38 +79,6 @@ plugins: - glightbox - search -nav: - - index.md - - installation.md - - data_containers.md - - datafile.md - - dataset.md - - manifest.md - - services.md - - asking_questions.md - - creating_services.md - - Twines: - - twines/index.md - - Quickstart: twines/quick_start_create_your_first_twine.md - - twines/anatomy.md - - About Twines: - - twines/about/index.md - - twines/about/about_digital_twins.md - - twines/about/about_requirements.md - - twines/about/about_introducing_json_schema.md - - twines/about/about_other_considerations.md - - updating_services.md - - running_services_locally.md - - deploying_services.md - - testing_services.md - - troubleshooting_services.md - - logging.md - - authentication.md - - inter_service_compatibility.md - # - api.md - - license.md - - version_history.md - copyright: Copyright © 2025 Octue extra: From 097923259ad092ed24d09d4fd33f291e9f77401e Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 19:28:08 -0400 Subject: [PATCH 59/69] DOC: Move data containers docs into section --- docs/data_containers.md | 10 ---------- docs/{ => data_containers}/datafile.md | 2 +- docs/{ => data_containers}/dataset.md | 0 docs/data_containers/index.md | 10 ++++++++++ docs/{ => data_containers}/manifest.md | 0 docs/downloading_datafiles.md | 2 +- mkdocs.yml | 9 +++++---- 7 files changed, 17 insertions(+), 16 deletions(-) delete mode 100644 docs/data_containers.md rename docs/{ => data_containers}/datafile.md (99%) rename docs/{ => data_containers}/dataset.md (100%) create mode 100644 docs/data_containers/index.md rename docs/{ => data_containers}/manifest.md (100%) diff --git a/docs/data_containers.md b/docs/data_containers.md deleted file mode 100644 index 7e816b73b..000000000 --- a/docs/data_containers.md +++ /dev/null @@ -1,10 +0,0 @@ -# Datafiles, datasets, and manifests - -One of the main features of Twined is making using, creating, and -sharing scientific datasets easy. There are three main data classes in -the that do this. - -- **Datafile** - [a single local or cloud file](/datafile) and its metadata. -- **Dataset** - [a set of related datafiles](/dataset) that exist in the same location, plus metadata. -- **Manifest** - [a set of related datasets](/manifest) that exist anywhere, plus metadata. Typically produced by or for - one analysis. diff --git a/docs/datafile.md b/docs/data_containers/datafile.md similarity index 99% rename from docs/datafile.md rename to docs/data_containers/datafile.md index c605656cb..31093f691 100644 --- a/docs/datafile.md +++ b/docs/data_containers/datafile.md @@ -199,7 +199,7 @@ exiting the context (the `with` block), it closes the datafile locally and, if the datafile also exists in the cloud, updates the cloud object with any data or metadata changes. -![image](images/datafile_use_cases.png) +![image](../images/datafile_use_cases.png) ### Example A diff --git a/docs/dataset.md b/docs/data_containers/dataset.md similarity index 100% rename from docs/dataset.md rename to docs/data_containers/dataset.md diff --git a/docs/data_containers/index.md b/docs/data_containers/index.md new file mode 100644 index 000000000..f257a2ae6 --- /dev/null +++ b/docs/data_containers/index.md @@ -0,0 +1,10 @@ +# Datafiles, datasets, and manifests + +One of the main features of Twined is making using, creating, and +sharing scientific datasets easy. There are three main data classes in +the that do this. + +- **Datafile** - [a single local or cloud file](/data_containers/datafile) and its metadata. +- **Dataset** - [a set of related datafiles](/data_containers/dataset) that exist in the same location, plus metadata. +- **Manifest** - [a set of related datasets](/data_containers/manifest) that exist anywhere, plus metadata. Typically produced by or for + one analysis. diff --git a/docs/manifest.md b/docs/data_containers/manifest.md similarity index 100% rename from docs/manifest.md rename to docs/data_containers/manifest.md diff --git a/docs/downloading_datafiles.md b/docs/downloading_datafiles.md index 38a5aa883..7938653f6 100644 --- a/docs/downloading_datafiles.md +++ b/docs/downloading_datafiles.md @@ -1,7 +1,7 @@ # More information on downloading datafiles - To avoid unnecessary data transfer and costs, cloud datafiles are not - downloaded locally [until necessary](/datafile/#automatic-lazy-downloading) + downloaded locally [until necessary](/data_containers/datafile/#automatic-lazy-downloading) - When downloaded, they are downloaded by default to a temporary local file that will exist at least as long as the python session is running - Calling `Datafile.download` or using `Datafile.local_path` again will diff --git a/mkdocs.yml b/mkdocs.yml index 228151bf7..17b011c21 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,10 +4,11 @@ site_url: https://docs.twined.octue.com nav: - index.md - installation.md - - data_containers.md - - datafile.md - - dataset.md - - manifest.md + - Datafiles, datasets, and manifests: + - data_containers/index.md + - data_containers/datafile.md + - data_containers/dataset.md + - data_containers/manifest.md - services.md - asking_questions.md - creating_services.md From b682f24c39d7e42e56c0dde1a9f156f930ff293f Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 19:38:58 -0400 Subject: [PATCH 60/69] CHO: Update version compatibility metadata --- docs/inter_service_compatibility.md | 121 ++++----- .../metadata/version_compatibilities.json | 235 +++++++++++++----- 2 files changed, 238 insertions(+), 118 deletions(-) diff --git a/docs/inter_service_compatibility.md b/docs/inter_service_compatibility.md index df6215a54..4079e0ad5 100644 --- a/docs/inter_service_compatibility.md +++ b/docs/inter_service_compatibility.md @@ -14,63 +14,64 @@ question. - `0` = incompatible - `1` = compatible -| | 0.67.0 | 0.66.1 | 0.66.0 | 0.65.0 | 0.64.0 | 0.63.0 | 0.62.1 | 0.62.0 | 0.61.2 | 0.61.1 | 0.61.0 | 0.60.2 | 0.60.1 | 0.60.0 | 0.59.1 | 0.59.0 | 0.58.0 | 0.57.2 | 0.57.1 | 0.57.0 | 0.56.0 | 0.55.0 | 0.54.0 | 0.53.0 | 0.52.2 | 0.52.1 | 0.52.0 | 0.51.0 | 0.50.1 | 0.50.0 | 0.49.2 | 0.49.1 | 0.49.0 | 0.48.0 | 0.47.2 | 0.47.1 | 0.47.0 | 0.46.3 | 0.46.2 | 0.46.1 | 0.46.0 | 0.45.0 | 0.44.0 | 0.43.7 | 0.43.6 | 0.43.5 | 0.43.4 | 0.43.3 | 0.43.2 | 0.43.1 | 0.43.0 | 0.42.1 | 0.42.0 | 0.41.1 | 0.41.0 | 0.40.2 | 0.40.1 | 0.40.0 | -| :----- | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -| 0.67.0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.66.1 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.66.0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.65.0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.64.0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.63.0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.62.1 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.62.0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.61.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.61.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.61.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.60.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.60.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.60.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.59.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.59.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.58.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.57.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.57.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.57.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.56.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.55.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.54.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.53.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.52.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.52.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.52.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.51.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | -| 0.50.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.50.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.49.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.49.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.49.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.48.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.47.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.47.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.47.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.46.3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.46.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.46.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.46.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.45.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.44.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.43.7 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.43.6 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.43.5 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.43.4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.43.3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.43.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.43.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.43.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.42.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.42.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.41.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.41.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.40.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.40.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | -| 0.40.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| | 0.68.0 | 0.67.0 | 0.66.1 | 0.66.0 | 0.65.0 | 0.64.0 | 0.63.0 | 0.62.1 | 0.62.0 | 0.61.2 | 0.61.1 | 0.61.0 | 0.60.2 | 0.60.1 | 0.60.0 | 0.59.1 | 0.59.0 | 0.58.0 | 0.57.2 | 0.57.1 | 0.57.0 | 0.56.0 | 0.55.0 | 0.54.0 | 0.53.0 | 0.52.2 | 0.52.1 | 0.52.0 | 0.51.0 | 0.50.1 | 0.50.0 | 0.49.2 | 0.49.1 | 0.49.0 | 0.48.0 | 0.47.2 | 0.47.1 | 0.47.0 | 0.46.3 | 0.46.2 | 0.46.1 | 0.46.0 | 0.45.0 | 0.44.0 | 0.43.7 | 0.43.6 | 0.43.5 | 0.43.4 | 0.43.3 | 0.43.2 | 0.43.1 | 0.43.0 | 0.42.1 | 0.42.0 | 0.41.1 | 0.41.0 | 0.40.2 | 0.40.1 | 0.40.0 | +| :----- | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | -----: | +| 0.68.0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.67.0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.66.1 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.66.0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.65.0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.64.0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.63.0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.62.1 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.62.0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.61.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.61.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.61.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.60.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.60.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.60.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.59.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.59.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.58.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.57.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.57.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.57.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.56.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.55.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.54.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.53.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.52.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.52.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.52.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.51.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| 0.50.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.50.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.49.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.49.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.49.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.48.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.47.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.47.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.47.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.46.3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.46.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.46.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.46.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.45.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.44.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.43.7 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.43.6 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.43.5 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.43.4 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.43.3 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.43.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.43.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.43.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.42.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.42.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.41.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.41.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.40.2 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.40.1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | +| 0.40.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | diff --git a/octue/twined/metadata/version_compatibilities.json b/octue/twined/metadata/version_compatibilities.json index f288b7f55..34ed07ec5 100644 --- a/octue/twined/metadata/version_compatibilities.json +++ b/octue/twined/metadata/version_compatibilities.json @@ -57,7 +57,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.40.1": { "0.40.1": true, @@ -117,7 +118,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.40.2": { "0.41.0": true, @@ -177,7 +179,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.41.0": { "0.41.0": true, @@ -237,7 +240,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.41.1": { "0.41.1": true, @@ -297,7 +301,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.42.0": { "0.42.0": true, @@ -357,7 +362,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.42.1": { "0.43.2": true, @@ -417,7 +423,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.43.0": { "0.43.2": true, @@ -477,7 +484,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.43.1": { "0.43.2": true, @@ -537,7 +545,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.43.2": { "0.43.2": true, @@ -597,7 +606,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.43.3": { "0.43.3": true, @@ -657,7 +667,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.43.4": { "0.43.4": true, @@ -717,7 +728,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.43.5": { "0.43.5": true, @@ -777,7 +789,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.43.6": { "0.43.6": true, @@ -837,7 +850,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.43.7": { "0.43.7": true, @@ -897,7 +911,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.44.0": { "0.44.0": true, @@ -957,7 +972,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.45.0": { "0.45.0": true, @@ -1017,7 +1033,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.46.0": { "0.46.0": true, @@ -1077,7 +1094,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.46.1": { "0.46.1": true, @@ -1137,7 +1155,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.46.2": { "0.46.2": true, @@ -1197,7 +1216,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.46.3": { "0.46.3": true, @@ -1257,7 +1277,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.47.0": { "0.47.0": true, @@ -1317,7 +1338,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.47.1": { "0.47.1": true, @@ -1377,7 +1399,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.47.2": { "0.47.2": true, @@ -1437,7 +1460,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.48.0": { "0.48.0": true, @@ -1497,7 +1521,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.49.0": { "0.49.1": true, @@ -1557,7 +1582,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.49.1": { "0.49.1": true, @@ -1617,7 +1643,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.49.2": { "0.49.2": true, @@ -1677,7 +1704,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.50.0": { "0.50.0": true, @@ -1737,7 +1765,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.50.1": { "0.51.0": false, @@ -1797,7 +1826,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.51.0": { "0.51.0": true, @@ -1857,7 +1887,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.52.0": { "0.51.0": true, @@ -1917,7 +1948,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.52.1": { "0.51.0": true, @@ -1977,7 +2009,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.52.2": { "0.51.0": true, @@ -2037,7 +2070,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.53.0": { "0.51.0": false, @@ -2097,7 +2131,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.54.0": { "0.51.0": false, @@ -2157,7 +2192,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.55.0": { "0.51.0": false, @@ -2217,7 +2253,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.56.0": { "0.51.0": false, @@ -2277,7 +2314,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.57.0": { "0.51.0": false, @@ -2337,7 +2375,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.57.1": { "0.51.0": false, @@ -2397,7 +2436,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.57.2": { "0.51.0": false, @@ -2457,7 +2497,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.58.0": { "0.51.0": false, @@ -2517,7 +2558,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.59.0": { "0.51.0": false, @@ -2577,7 +2619,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.59.1": { "0.51.0": false, @@ -2637,7 +2680,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.60.0": { "0.51.0": false, @@ -2697,7 +2741,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.60.1": { "0.51.0": false, @@ -2757,7 +2802,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.60.2": { "0.51.0": false, @@ -2817,7 +2863,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.61.0": { "0.51.0": false, @@ -2877,7 +2924,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.61.1": { "0.51.0": false, @@ -2937,7 +2985,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.61.2": { "0.51.0": false, @@ -2997,7 +3046,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.62.0": { "0.51.0": false, @@ -3057,7 +3107,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.62.1": { "0.51.0": false, @@ -3117,7 +3168,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.63.0": { "0.51.0": false, @@ -3177,7 +3229,8 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.64.0": { "0.51.0": false, @@ -3237,7 +3290,8 @@ "0.65.0": true, "0.66.0": true, "0.66.1": true, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.65.0": { "0.51.0": false, @@ -3297,7 +3351,8 @@ "0.65.0": true, "0.66.0": true, "0.66.1": true, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.66.0": { "0.51.0": false, @@ -3357,7 +3412,8 @@ "0.65.0": true, "0.66.0": true, "0.66.1": true, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.66.1": { "0.51.0": false, @@ -3417,7 +3473,8 @@ "0.65.0": true, "0.66.0": true, "0.66.1": true, - "0.67.0": false + "0.67.0": false, + "0.68.0": false }, "0.67.0": { "0.51.0": false, @@ -3477,6 +3534,68 @@ "0.65.0": false, "0.66.0": false, "0.66.1": false, - "0.67.0": true + "0.67.0": true, + "0.68.0": true + }, + "0.68.0": { + "0.51.0": false, + "0.50.1": false, + "0.50.0": false, + "0.49.2": false, + "0.49.1": false, + "0.49.0": false, + "0.48.0": false, + "0.47.2": false, + "0.47.1": false, + "0.47.0": false, + "0.46.3": false, + "0.46.2": false, + "0.46.1": false, + "0.46.0": false, + "0.45.0": false, + "0.44.0": false, + "0.43.7": false, + "0.43.6": false, + "0.43.5": false, + "0.43.4": false, + "0.43.3": false, + "0.43.2": false, + "0.43.1": false, + "0.43.0": false, + "0.42.1": false, + "0.42.0": false, + "0.41.1": false, + "0.41.0": false, + "0.40.2": false, + "0.40.1": false, + "0.40.0": false, + "0.52.0": false, + "0.52.1": false, + "0.52.2": false, + "0.53.0": false, + "0.54.0": false, + "0.55.0": false, + "0.56.0": false, + "0.57.0": false, + "0.57.1": false, + "0.57.2": false, + "0.58.0": false, + "0.59.0": false, + "0.59.1": false, + "0.60.0": false, + "0.60.1": false, + "0.60.2": false, + "0.61.0": false, + "0.61.1": false, + "0.61.2": false, + "0.62.0": false, + "0.62.1": false, + "0.63.0": false, + "0.64.0": false, + "0.65.0": false, + "0.66.0": false, + "0.66.1": false, + "0.67.0": true, + "0.68.0": true } } From b88ce12d14ca85ec06d5e8ebbbc6f41ce9359522 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Mon, 18 Aug 2025 19:39:39 -0400 Subject: [PATCH 61/69] DEP: Add `tabulate` for version table script skipci --- poetry.lock | 17 ++++++++++++++++- pyproject.toml | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 148c2feb2..d5e79448d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2159,6 +2159,21 @@ files = [ {file = "stringcase-1.2.0.tar.gz", hash = "sha256:48a06980661908efe8d9d34eab2b6c13aefa2163b3ced26972902e3bdfd87008"}, ] +[[package]] +name = "tabulate" +version = "0.9.0" +description = "Pretty-print tabular data" +optional = false +python-versions = ">=3.7" +groups = ["dev"] +files = [ + {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, + {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, +] + +[package.extras] +widechars = ["wcwidth"] + [[package]] name = "tomli" version = "2.2.1" @@ -2457,4 +2472,4 @@ hdf5 = ["h5py"] [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "47e85f3ba2f74900790a25e8d4aef25eef27205115bf054e928f12d4f8cb81ce" +content-hash = "4a0c2044c5b1e7745d6ce9fb557987097707abdc9c11014d6a18b1c9bc1e23c0" diff --git a/pyproject.toml b/pyproject.toml index b66278854..76fb389ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ stringcase = "1.2.0" mkdocs-material = "^9.6.17" mkdocs-glightbox = "^0.4.0" mike = "^2.1.3" +tabulate = "^0.9.0" [tool.ruff] line-length = 120 From a77165b0a7c9f047ce6781e9cc52731157b66eed Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 19 Aug 2025 12:06:53 -0400 Subject: [PATCH 62/69] DOC: Add readthedocs config skipci --- .readthedocs.yaml | 12 ++++++++++++ docs/requirements.txt | 3 +++ mkdocs.yml | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 .readthedocs.yaml create mode 100644 docs/requirements.txt diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..73b1f125f --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,12 @@ +version: 2 + +build: + os: "ubuntu-24.04" + tools: + python: "3" + jobs: + pre_install: + - pip install -r docs/requirements.txt + +mkdocs: + configuration: mkdocs.yml diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..c2a64cb64 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +mkdocs-material>=9.6.17,<10 +mkdocs-glightbox>=0.4.0,<0.5.0 +mike>=2.1.3,<3 diff --git a/mkdocs.yml b/mkdocs.yml index 17b011c21..2cd1d178d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,5 @@ site_name: Octue Twined -site_url: https://docs.twined.octue.com +site_url: https://octue-python-sdk.readthedocs.io nav: - index.md From f793455aa3d51931ebfe34f52eff007571e5ba0e Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 19 Aug 2025 12:13:02 -0400 Subject: [PATCH 63/69] DOC: Pin python version used to build docs --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 73b1f125f..f4fa6ea5a 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,7 +3,7 @@ version: 2 build: os: "ubuntu-24.04" tools: - python: "3" + python: "3.13.3" jobs: pre_install: - pip install -r docs/requirements.txt From 8e9c8e9c024e31eddabfb462d54a7cb884999db4 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 19 Aug 2025 12:13:16 -0400 Subject: [PATCH 64/69] DOC: Remove `mike` version handler from docs skipci --- docs/requirements.txt | 1 - mkdocs.yml | 2 -- 2 files changed, 3 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index c2a64cb64..391ea0bd7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,2 @@ mkdocs-material>=9.6.17,<10 mkdocs-glightbox>=0.4.0,<0.5.0 -mike>=2.1.3,<3 diff --git a/mkdocs.yml b/mkdocs.yml index 2cd1d178d..feeccf73b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -90,8 +90,6 @@ extra: link: https://www.linkedin.com/company/octue - icon: material/web link: https://octue.com - version: - provider: mike extra_css: - stylesheets/extra.css From 93b8811b3cbafabd69b417299bed8c84d9979f5c Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 19 Aug 2025 12:14:31 -0400 Subject: [PATCH 65/69] DOC: Loosen docs python pin slightly skipci --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f4fa6ea5a..2f71aea47 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,7 +3,7 @@ version: 2 build: os: "ubuntu-24.04" tools: - python: "3.13.3" + python: "3.13" jobs: pre_install: - pip install -r docs/requirements.txt From da7e4a573cdd904d19687ddef126143e19ca35f4 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 19 Aug 2025 12:20:20 -0400 Subject: [PATCH 66/69] DEP: Remove `mike` --- poetry.lock | 82 ++------------------------------------------------ pyproject.toml | 1 - 2 files changed, 3 insertions(+), 80 deletions(-) diff --git a/poetry.lock b/poetry.lock index d5e79448d..c4c575650 100644 --- a/poetry.lock +++ b/poetry.lock @@ -911,7 +911,7 @@ version = "8.6.1" description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main"] files = [ {file = "importlib_metadata-8.6.1-py3-none-any.whl", hash = "sha256:02a89390c1e15fdfdc0d7c6b25cb3e62650d0494005c97d6f148bf5b9787525e"}, {file = "importlib_metadata-8.6.1.tar.gz", hash = "sha256:310b41d755445d74569f993ccfc22838295d9fe005425094fad953d7f15c8580"}, @@ -929,26 +929,6 @@ perf = ["ipython"] test = ["flufl.flake8", "importlib_resources (>=1.3) ; python_version < \"3.9\"", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] type = ["pytest-mypy"] -[[package]] -name = "importlib-resources" -version = "6.5.2" -description = "Read resources from Python packages" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "importlib_resources-6.5.2-py3-none-any.whl", hash = "sha256:789cfdc3ed28c78b67a06acb8126751ced69a3d5f79c095a98298cd8a760ccec"}, - {file = "importlib_resources-6.5.2.tar.gz", hash = "sha256:185f87adef5bcc288449d98fb4fba07cea78bc036455dd44c5fc4a2fe78fed2c"}, -] - -[package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] -cover = ["pytest-cov"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -enabler = ["pytest-enabler (>=2.2)"] -test = ["jaraco.test (>=5.4)", "pytest (>=6,!=8.1.*)", "zipp (>=3.17)"] -type = ["pytest-mypy"] - [[package]] name = "iniconfig" version = "2.1.0" @@ -1115,32 +1095,6 @@ files = [ {file = "mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8"}, ] -[[package]] -name = "mike" -version = "2.1.3" -description = "Manage multiple versions of your MkDocs-powered documentation" -optional = false -python-versions = "*" -groups = ["dev"] -files = [ - {file = "mike-2.1.3-py3-none-any.whl", hash = "sha256:d90c64077e84f06272437b464735130d380703a76a5738b152932884c60c062a"}, - {file = "mike-2.1.3.tar.gz", hash = "sha256:abd79b8ea483fb0275b7972825d3082e5ae67a41820f8d8a0dc7a3f49944e810"}, -] - -[package.dependencies] -importlib-metadata = "*" -importlib-resources = "*" -jinja2 = ">=2.7" -mkdocs = ">=1.0" -pyparsing = ">=3.0" -pyyaml = ">=5.1" -pyyaml-env-tag = "*" -verspec = "*" - -[package.extras] -dev = ["coverage", "flake8 (>=3.0)", "flake8-quotes", "shtab"] -test = ["coverage", "flake8 (>=3.0)", "flake8-quotes", "shtab"] - [[package]] name = "mkdocs" version = "1.6.1" @@ -1667,21 +1621,6 @@ pyyaml = "*" [package.extras] extra = ["pygments (>=2.19.1)"] -[[package]] -name = "pyparsing" -version = "3.2.3" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -optional = false -python-versions = ">=3.9" -groups = ["dev"] -files = [ - {file = "pyparsing-3.2.3-py3-none-any.whl", hash = "sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf"}, - {file = "pyparsing-3.2.3.tar.gz", hash = "sha256:b9c13f1ab8b3b542f72e28f634bad4de758ab3ce4546e4301970ad6fa77c38be"}, -] - -[package.extras] -diagrams = ["jinja2", "railroad-diagrams"] - [[package]] name = "pytest" version = "7.4.4" @@ -2278,21 +2217,6 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] -[[package]] -name = "verspec" -version = "0.1.0" -description = "Flexible version handling" -optional = false -python-versions = "*" -groups = ["dev"] -files = [ - {file = "verspec-0.1.0-py3-none-any.whl", hash = "sha256:741877d5633cc9464c45a469ae2a31e801e6dbbaa85b9675d481cda100f11c31"}, - {file = "verspec-0.1.0.tar.gz", hash = "sha256:c4504ca697b2056cdb4bfa7121461f5a0e81809255b41c03dda4ba823637c01e"}, -] - -[package.extras] -test = ["coverage", "flake8 (>=3.7)", "mypy", "pretend", "pytest"] - [[package]] name = "virtualenv" version = "20.31.2" @@ -2452,7 +2376,7 @@ version = "3.21.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" -groups = ["main", "dev"] +groups = ["main"] files = [ {file = "zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931"}, {file = "zipp-3.21.0.tar.gz", hash = "sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4"}, @@ -2472,4 +2396,4 @@ hdf5 = ["h5py"] [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "4a0c2044c5b1e7745d6ce9fb557987097707abdc9c11014d6a18b1c9bc1e23c0" +content-hash = "42da87c7b35bc255a87855d6f2d54b3220cad8e126cc09634b607ae3a1ef344c" diff --git a/pyproject.toml b/pyproject.toml index 76fb389ea..63fb88f31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,6 @@ stringcase = "1.2.0" # Documentation mkdocs-material = "^9.6.17" mkdocs-glightbox = "^0.4.0" -mike = "^2.1.3" tabulate = "^0.9.0" [tool.ruff] From 9e040ad361217cf5a6c4bf097416aaa1f8d42800 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 19 Aug 2025 12:20:28 -0400 Subject: [PATCH 67/69] OPS: Remove GitHub pages CI job --- .github/workflows/release.yml | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3a92f63eb..ffafda68f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -129,37 +129,3 @@ jobs: - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@v1.12.4 - - docs: - needs: publish - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - - - name: Configure Git Credentials - run: | - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - - - uses: actions/setup-python@v5 - with: - python-version: 3.x - - - name: Create cache key - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV - - - name: Check cache - uses: actions/cache@v4 - with: - key: mkdocs-material-${{ env.cache_id }} - path: ~/.cache - restore-keys: | - mkdocs-material- - - - name: Install material for mkdocs - run: pip install mkdocs-material mkdocs-glightbox mike - - - name: Build and deploy docs - run: mkdocs gh-deploy --force From a5e96bc1329c0038d297d64b9eb09ea69dad6fab Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 19 Aug 2025 16:55:59 -0400 Subject: [PATCH 68/69] DOC: Fix path to image in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2b68bd6fd..0a7835c48 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10961975.svg)](https://doi.org/10.5281/zenodo.10961975) -# Octue Python SDK Purple Fruit Snake +# Octue Python SDK Purple Fruit Snake The python SDK for running [Octue](https://octue.com) data services, digital twins, and applications - get faster data groundwork so you have more time for the science! From daddfdd4ddb5843117cb6d25d85c051b77528a14 Mon Sep 17 00:00:00 2001 From: cortadocodes Date: Tue, 19 Aug 2025 16:58:35 -0400 Subject: [PATCH 69/69] DOC: Update documentation URL skipci --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index feeccf73b..8d9eff6b6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,5 @@ site_name: Octue Twined -site_url: https://octue-python-sdk.readthedocs.io +site_url: https://twined.octue.com nav: - index.md