diff --git a/.github/workflows/mixed_mode_test.yml b/.github/workflows/mixed_mode_test.yml index 469b2ecf13..6320fc7fc1 100644 --- a/.github/workflows/mixed_mode_test.yml +++ b/.github/workflows/mixed_mode_test.yml @@ -40,7 +40,7 @@ jobs: - name: Checkout Main run: git checkout main - name: Update release notes - run: python build/publish-mixed-mode-results.py ${{ inputs.tag }} --release-notes docs/ReleaseNotes.md --commit --run-link ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: python build/publish-mixed-mode-results.py ${{ inputs.tag }} --release-notes docs/sphinx/source/ReleaseNotes.md --commit --run-link ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - name: Push release notes update run: git push diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e5d5f854fd..096e9014e6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,6 @@ jobs: checks: write contents: write packages: write - pages: write pull-requests: write steps: - name: Checkout sources @@ -18,6 +17,7 @@ jobs: with: ssh-key: ${{ secrets.DEPLOY_KEY }} - name: Setup Base Environment + id: setup-base uses: ./actions/setup-base-env - name: Setup FDB uses: ./actions/setup-fdb @@ -44,4 +44,31 @@ jobs: MAVEN_USER: ${{ secrets.MAVEN_USER }} MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }} - # TODO: Publish documentation updates + # Build documentation. + - name: Cache Python Environment + uses: actions/cache@v4 + with: + path: docs/sphinx/.venv + key: ${{ runner.os }}-sphinx-python-${{ steps.setup-base.outputs.python-version }}-${{ hashFiles('docs/sphinx/requirements.txt') }} + - name: Build Documentation Site + uses: ./actions/run-gradle + with: + gradle_command: documentationSite -PreleaseBuild=true + - name: Upload Documentation + id: doc_upload + uses: actions/upload-pages-artifact@v3 + with: + path: docs/sphinx/.out/html/ + + deploy_docs: + runs-on: ubuntu-latest + needs: gradle + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.doc_upload.outputs.page_url }} + steps: + - name: Deploy Documentation + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 9c87e71f27..ab509c8978 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,11 @@ atlassian-ide-plugin.xml .idea/scala_settings.xml .idea/shelf +# Built during documentation +/docs/sphinx/.venv +/docs/sphinx/source/api/**/index.md +*.diagram.svg + # VSCode specific files/directories .vscode **/bin diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 55d47dbd39..c3aa9be660 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,7 +83,7 @@ it is addressing and, upon merging of the PR into the main code line, will automatically mark the issue as resolved. If your pull request results in a user-visible change to the Record Layer, you should -also update the [release notes](docs/ReleaseNotes.md). For most changes, it +also update the [release notes](docs/sphinx/source/ReleaseNotes.md). For most changes, it is sufficient to fill in one of the bullets in the "next release" section of that document. You should include a short description of the change as well as filling in the issue number. The "next release" section is commented out, so the change won't diff --git a/actions/release-build-publish/action.yml b/actions/release-build-publish/action.yml index c7fc308c1b..7756d78332 100644 --- a/actions/release-build-publish/action.yml +++ b/actions/release-build-publish/action.yml @@ -51,8 +51,12 @@ runs: id: push_updates shell: bash run: git push origin + # Continue the build (including downstream steps). If the push fails, we'll create a PR + continue-on-error: true - name: Create Merge PR if conflict - if: failure() && steps.push_updates.conclusion == 'failure' + # Only create the PR if we've otherwise been successful, but the push failed. Note that + # we're checking the .outcome of the push step, which is applied before continue-on-error. + if: success() && steps.push_updates.outcome == 'failure' uses: peter-evans/create-pull-request@bb88e27d3f9cc69c8bc689eba126096c6fe3dded id: pr_on_conflict with: @@ -62,3 +66,8 @@ runs: sign-commits: true body: | Updates from release for version ${{ steps.get_version.outputs.version }}. Conflicts during the build prevented automatic updating. Please resolve conflicts by checking out the current branch, merging, and then deleting this branch. + + # Creating the PR can change the current branch. Explicitly check out the tag here for downstream builds + - name: Revert to tag + shell: bash + run: git checkout "${{ steps.get_version.outputs.version }}" diff --git a/actions/setup-base-env/action.yml b/actions/setup-base-env/action.yml index 8598615a1f..b9f9e41c25 100644 --- a/actions/setup-base-env/action.yml +++ b/actions/setup-base-env/action.yml @@ -1,5 +1,9 @@ name: Setup Base Environment +outputs: + python-version: + value: ${{ steps.setup-python.outputs.python-version }} + runs: using: "composite" steps: @@ -11,6 +15,7 @@ runs: - name: Setup Gradle uses: gradle/actions/setup-gradle@0bdd871935719febd78681f197cd39af5b6e16a6 - name: Setup Python + id: setup-python uses: actions/setup-python@v5 with: python-version: '3.13' diff --git a/build.gradle b/build.gradle index a5cb6308c8..31ff712aee 100644 --- a/build.gradle +++ b/build.gradle @@ -240,8 +240,8 @@ subprojects { linksOffline "https://developers.google.com/protocol-buffers/docs/reference/java/", "${packageListDir}/protobuf/" // This is for dependent sub-projects so that their links to things in the Record Layer work - linksOffline "https://static.javadoc.io/org.foundationdb/fdb-extensions/${project.version}/", "${packageListDir}/fdb-extensions/" - linksOffline "https://static.javadoc.io/org.foundationdb/fdb-record-layer-core/${project.version}/", "${packageListDir}/fdb-record-layer-core/" + linksOffline "https://foundationdb.github.io/fdb-record-layer/api/fdb-extensions/", "${packageListDir}/fdb-extensions/" + linksOffline "https://foundationdb.github.io/fdb-record-layer/api/fdb-record-layer-core/", "${packageListDir}/fdb-record-layer-core/" } } compileJava { @@ -333,6 +333,8 @@ clean.doLast { } } +apply from: 'gradle/sphinx.gradle' + if (!JavaVersion.current().isJava8Compatible()) { throw new Exception("Java 8 is required to build fdb-record-layer") } diff --git a/build/update_release_notes.bash b/build/update_release_notes.bash index 3080b1c6e2..adf012c8c6 100755 --- a/build/update_release_notes.bash +++ b/build/update_release_notes.bash @@ -31,7 +31,7 @@ script_dir="$( dirname $0 )" success=0 -release_notes_file="${script_dir}/../docs/ReleaseNotes.md" +release_notes_file="${script_dir}/../docs/sphinx/source/ReleaseNotes.md" if [[ -n "${GIT_BRANCH:-}" ]] ; then branch="${GIT_BRANCH#*/}" diff --git a/docs/_config.yml b/docs/_config.yml deleted file mode 100644 index 2df73f5591..0000000000 --- a/docs/_config.yml +++ /dev/null @@ -1,7 +0,0 @@ -title: FoundationDB Record Layer -footer: - copy: Copyright 2015-2019 Apple, Inc and the FoundationDB project authors. -markdown: kramdown - -kramdown: - parse_block_html: true \ No newline at end of file diff --git a/docs/_includes/footer.html b/docs/_includes/footer.html deleted file mode 100644 index 83cc28ae38..0000000000 --- a/docs/_includes/footer.html +++ /dev/null @@ -1,5 +0,0 @@ -<div class="footer border-top border-gray-light mt-5 pt-3 text-right text-gray"> - {%- if site.footer.copy -%} - <p>© {{ site.footer.copy | escape }}</p> - {%- endif -%} -</div> \ No newline at end of file diff --git a/docs/_layouts/default.html b/docs/_layouts/default.html deleted file mode 100644 index 8200d9e3bd..0000000000 --- a/docs/_layouts/default.html +++ /dev/null @@ -1,14 +0,0 @@ -<!doctype html> -<html lang="en"> - <head> - <meta charset="utf-8"> - <meta name="viewport" content="width=device-width"> - <title>{{ page.title | default: site.title }}</title> - <link rel="stylesheet" href="/fdb-record-layer/assets/css/style.css"> - </head> - <body> - <div class="container-lg px-3 my-5 markdown-body"> - {{ content }} - </div> - </body> -</html> \ No newline at end of file diff --git a/docs/_layouts/page.html b/docs/_layouts/page.html deleted file mode 100644 index 50c981dad7..0000000000 --- a/docs/_layouts/page.html +++ /dev/null @@ -1,11 +0,0 @@ ---- -layout: default ---- -<article class="post"> - <div class="post-content"> - {{ content }} - </div> - -</article> - -{%- include footer.html -%} \ No newline at end of file diff --git a/docs/sphinx/generate_railroad_svg.py b/docs/sphinx/generate_railroad_svg.py new file mode 100644 index 0000000000..361394e7e7 --- /dev/null +++ b/docs/sphinx/generate_railroad_svg.py @@ -0,0 +1,41 @@ +#!/bin/python +# +# generate_railroad_svg.py +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2015-2025 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from os import walk, path +from railroad import Diagram, Choice, Terminal, NonTerminal, OneOrMore, Sequence, Optional, Stack +import railroad + +# Vertical space +railroad.VS = 15 + +# Arc radius +railroad.AR = 12 + +# Find all diagram files and generate svgs +base_dir = path.dirname(path.abspath(__file__)) +for root, dirs, files in walk('.'): + for file in files: + if file.endswith('.diagram'): + with open(path.join(root, file), 'r') as diagram_file: + diagram = eval(diagram_file.read()) + with open(path.join(root, file + ".svg"), 'w') as svg_file: + diagram.writeStandalone(svg_file.write) + diff --git a/docs/sphinx/requirements.txt b/docs/sphinx/requirements.txt new file mode 100644 index 0000000000..d24bf5d264 --- /dev/null +++ b/docs/sphinx/requirements.txt @@ -0,0 +1,27 @@ +alabaster==1.0.0 +babel==2.16.0 +beautifulsoup4==4.12.3 +certifi==2024.7.4 +charset-normalizer==3.3.2 +docutils==0.21.2 +furo==2024.8.6 +idna==3.7 +imagesize==1.4.1 +Jinja2==3.1.4 +MarkupSafe==2.1.5 +myst-parser==4.0.0 +packaging==24.1 +Pygments==2.18.0 +railroad-diagrams==3.0.1 +requests==2.32.3 +snowballstemmer==2.2.0 +soupsieve==2.5 +sphinx==8.1.3 +sphinx-basic-ng==1.0.0b2 +sphinxcontrib-devhelp==2.0.0 +sphinxcontrib-htmlhelp==2.1.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==2.0.0 +sphinxcontrib-serializinghtml==2.0.0 +tomli==2.0.1 +urllib3==2.2.2 diff --git a/docs/Building.md b/docs/sphinx/source/Building.md similarity index 100% rename from docs/Building.md rename to docs/sphinx/source/Building.md diff --git a/docs/Coding_Best_Practices.md b/docs/sphinx/source/Coding_Best_Practices.md similarity index 100% rename from docs/Coding_Best_Practices.md rename to docs/sphinx/source/Coding_Best_Practices.md diff --git a/docs/Extending.md b/docs/sphinx/source/Extending.md similarity index 100% rename from docs/Extending.md rename to docs/sphinx/source/Extending.md diff --git a/docs/FAQ.md b/docs/sphinx/source/FAQ.md similarity index 100% rename from docs/FAQ.md rename to docs/sphinx/source/FAQ.md diff --git a/docs/FDB_logo.png b/docs/sphinx/source/FDB_logo.png similarity index 100% rename from docs/FDB_logo.png rename to docs/sphinx/source/FDB_logo.png diff --git a/docs/GettingStarted.md b/docs/sphinx/source/GettingStarted.md similarity index 99% rename from docs/GettingStarted.md rename to docs/sphinx/source/GettingStarted.md index 6360945d5f..cdfce853cf 100644 --- a/docs/GettingStarted.md +++ b/docs/sphinx/source/GettingStarted.md @@ -14,7 +14,7 @@ To keep the guide simple, we will not cover: * Modifying a schema * Using advanced [index](Overview.md#secondary-indexes) and [query](Overview.md#querying-for-records) functionality -### Setup +## Setup Install Java JDK for version 8 or above; for this demo we are using JDK `1.8.0_181`. @@ -52,7 +52,7 @@ Now we should add the Record Layer as a dependency of our project. The Record La are published to Maven Central, so we can declare the dependency by adding the following to our project's `build.gradle` file: -```gradle +```groovy repositories { mavenCentral() maven { @@ -64,11 +64,12 @@ dependencies { implementation 'org.foundationdb:fdb-record-layer-core-pb3:VERSION_NUMBER' } ``` + Replace `VERSION_NUMBER` with a recent version of the artifact from Maven Central (See [mvnrepository](https://mvnrepository.com/artifact/org.foundationdb/fdb-record-layer-core-pb3) for a listing of published versions). -### ProtoBuf Configuration +## Protobuf Configuration Our sample project will use [Protocol Buffers](https://developers.google.com/protocol-buffers/) (protobuf) to define our record meta-data. First, since we are using Gradle, let's include the @@ -77,7 +78,7 @@ which will allow us to add protobuf compilation as a step in our build process. Add this to the top of your `build.gradle`, ahead of the above `repositories` and `dependencies` added above: -```gradle +```groovy plugins { id 'java' id 'com.google.protobuf' version "0.8.19" @@ -88,7 +89,7 @@ include `id'java'`. Additionally, add the following: -```gradle +```groovy protobuf { generatedFilesBaseDir = "${projectDir}/src/main/generated" protoc { @@ -119,7 +120,7 @@ One last step might be necessary to configure your IDE of choice to discover the offer auto-complete suggestions. The additional generated source directory can be added to the list of Java sources by adding the following to the project's `build.gradle` file: -```gradle +```groovy sourceSets { main { java { @@ -180,7 +181,7 @@ Finally, the Record Layer requires we have a `UnionDescriptor` message which con record types to be stored in our record store (here only `Order`). We must either set the `usage = UNION` option for this message or we can omit the option and instead name the message `RecordTypeUnion`. -### Creating an Application +## Creating an Application Run `./gradlew generateProto` to see that the above configuration is correct. You should see the generated code put into `src/main/generated/main/java/RecordLayerDemoProto.java`. @@ -411,7 +412,7 @@ flower { price: 34 ``` -### Summary +## Summary We've covered creating a very simple demo app from scratch using the FDB Record Layer. We downloaded and started up an FDB server, created a new Java project, brought in the Record Layer dependency, diff --git a/docs/Overview.md b/docs/sphinx/source/Overview.md similarity index 100% rename from docs/Overview.md rename to docs/sphinx/source/Overview.md diff --git a/docs/ReleaseNotes.md b/docs/sphinx/source/ReleaseNotes.md similarity index 99% rename from docs/ReleaseNotes.md rename to docs/sphinx/source/ReleaseNotes.md index cd49e73f04..dc95f9cf7e 100644 --- a/docs/ReleaseNotes.md +++ b/docs/sphinx/source/ReleaseNotes.md @@ -132,7 +132,7 @@ The Apache Commons library has been removed as a dependency. There were a few lo ### Breaking Changes -Support for the Protobuf 2 runtime has been removed as of this version. All artifacts now use Protobuf version 3. Note that the choice of Protobuf runtime version is distinct from the choice of Protobuf message syntax, and that users wishing to retain Protobuf 2 behavior can still achieve the same semantics (including [optional field behavior]()) as long as they specify the syntax on their Protobuf file as `proto2`. Note that the Maven artifacts using Protobuf version 3 used to be suffixed with `-pb3`. Existing Protobuf 3 users must remove that suffix from their dependency declarations (e.g., `fdb-record-layer-core-pb3` should now be `fdb-record-layer-core`). +Support for the Protobuf 2 runtime has been removed as of this version. All artifacts now use Protobuf version 3. Note that the choice of Protobuf runtime version is distinct from the choice of Protobuf message syntax, and that users wishing to retain Protobuf 2 behavior can still achieve the same semantics (including [optional field behavior](Overview.md#indexing-and-querying-of-missing--null-values)) as long as they specify the syntax on their Protobuf file as `proto2`. Note that the Maven artifacts using Protobuf version 3 used to be suffixed with `-pb3`. Existing Protobuf 3 users must remove that suffix from their dependency declarations (e.g., `fdb-record-layer-core-pb3` should now be `fdb-record-layer-core`). Starting with version [3.4.455.0](#344550), the semantics of `UnnestedRecordType` were changed in response to [Issue #2512](https://github.com/FoundationDB/fdb-record-layer/issues/2512). It was identified that extraneous synthetic records were being produced when one of the children was empty. This did not match the semantics of `FanOut` expressions, and so the unnesting calculation was changed. This means that any index on an existing `UnnestedRecordType` requires rebuilding to clear out any such entries from older indexes. @@ -2168,7 +2168,7 @@ The `FDBDatabase::getReadVersion()` method has been replaced with the `FDBRecord ### Breaking Changes -The Guava version has been updated to version 27. Users of the [shaded variants](Shaded.html#Variants) of the Record Layer dependencies should not be affected by this change. Users of the unshaded variant using an older version of Guava as part of their own project may find that their own Guava dependency is updated automatically by their dependency manager. +The Guava version has been updated to version 27. Users of the [shaded variants](Versioning.md#variants) of the Record Layer dependencies should not be affected by this change. Users of the unshaded variant using an older version of Guava as part of their own project may find that their own Guava dependency is updated automatically by their dependency manager. ### 2.7.79.0 diff --git a/docs/sphinx/source/SQL_Reference.md b/docs/sphinx/source/SQL_Reference.md new file mode 100644 index 0000000000..f22ead7db0 --- /dev/null +++ b/docs/sphinx/source/SQL_Reference.md @@ -0,0 +1,31 @@ +# SQL Reference + +The FDB relational subproject provides a SQL API for interacting with Record Layer databases. This SQL API is currently under active development and is expected +to change frequently. This reference is currently also a work in progress to be updated as the SQL engine develops. + + +```{toctree} +:caption: SQL +:maxdepth: 2 + +reference/sql_types +reference/sql_commands +reference/Functions +``` + +```{toctree} +:caption: Data Model +:maxdepth: 2 + +reference/Databases_Schemas_SchemaTemplates +reference/Tables +reference/Indexes +``` + +```{toctree} +:caption: Miscellaneous +:maxdepth: 2 + +reference/Direct_Access_Api +reference/understanding_bitmap +``` diff --git a/docs/SchemaEvolution.md b/docs/sphinx/source/SchemaEvolution.md similarity index 100% rename from docs/SchemaEvolution.md rename to docs/sphinx/source/SchemaEvolution.md diff --git a/docs/Versioning.md b/docs/sphinx/source/Versioning.md similarity index 90% rename from docs/Versioning.md rename to docs/sphinx/source/Versioning.md index f3c83a1022..49aaf9b0ba 100644 --- a/docs/Versioning.md +++ b/docs/sphinx/source/Versioning.md @@ -110,10 +110,10 @@ Classes and methods annotations using `@API` determine when they can be changed using the starting version. -* In `docs/ReleaseNotes.md`, move the template for release notes for the next release to be just above the release note for the starting version `a.b.c`. +* In `docs/sphinx/source/ReleaseNotes.md`, move the template for release notes for the next release to be just above the release note for the starting version `a.b.c`. * Commit and push the new branch upstream. * Create patch fix pull requests against this new branch. -* There may be conflicts in the `gradle.properties` and `docs/ReleaseNotes.md` files when the patch branch is merged into main. Be sure whenever merging that branch in that (1) the version on main is not accidentally changed and (2) the release notes file contains all release notes from both branches after the merge. +* There may be conflicts in the `gradle.properties` and `docs/sphinx/source/ReleaseNotes.md` files when the patch branch is merged into main. Be sure whenever merging that branch in that (1) the version on main is not accidentally changed and (2) the release notes file contains all release notes from both branches after the merge. diff --git a/docs/sphinx/source/api/api.md.template b/docs/sphinx/source/api/api.md.template new file mode 100644 index 0000000000..91c7614bb5 --- /dev/null +++ b/docs/sphinx/source/api/api.md.template @@ -0,0 +1 @@ +Placeholder to allow for doc build. This will be replaced by Javadoc by the build. \ No newline at end of file diff --git a/docs/sphinx/source/api/index.md.template b/docs/sphinx/source/api/index.md.template new file mode 100644 index 0000000000..cc0ba4afd5 --- /dev/null +++ b/docs/sphinx/source/api/index.md.template @@ -0,0 +1,4 @@ +# API Documentation + +Below find the current Javadoc documentation for the various Record Layer +subprojects. diff --git a/docs/sphinx/source/conf.py b/docs/sphinx/source/conf.py new file mode 100644 index 0000000000..c4424af20b --- /dev/null +++ b/docs/sphinx/source/conf.py @@ -0,0 +1,69 @@ +# +# conf.py +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2015-2025 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'FoundationDB Record Layer' +copyright = '2025, Apple Inc.' +author = 'Apple Inc.' + +extensions = ['myst_parser'] + +templates_path = ['_templates'] +exclude_patterns = [] + + +html_theme = 'furo' +html_theme_options = { +} + +html_static_path = ['_static'] +html_show_sphinx = False + +source_suffix = { + '.rst': 'restructuredtext', + '.md': 'markdown', +} + +root_doc = 'index' + +suppress_warnings = ['misc.highlighting_failure'] + +rst_prolog = """ +.. role:: sql(code) + :language: sql + + +.. role:: json(code) + :language: json + + +.. role:: java(code) + :language: java +""" + +myst_heading_anchors = 4 + diff --git a/docs/index.md b/docs/sphinx/source/index.md similarity index 53% rename from docs/index.md rename to docs/sphinx/source/index.md index 01cb2404de..0f5d994605 100644 --- a/docs/index.md +++ b/docs/sphinx/source/index.md @@ -22,23 +22,20 @@ reliability, and performance in a distributed setting. ## Documentation -* [Overview and Examples](Overview.md) -* [Getting Started](GettingStarted.md) -* [Building](Building.md) -* [Schema Evolution and Meta-data Maintenance](SchemaEvolution.md) -* [Extending the Record Layer](Extending.md) -* [Versioning Guide](Versioning.md) -* [Coding Best Practices](Coding_Best_Practices.md) -* [Release Notes](ReleaseNotes.md) -* [Frequently Asked Questions](FAQ.md) -* [Contributing](https://github.com/FoundationDB/fdb-record-layer/blob/main/CONTRIBUTING.md) -* [Code of Conduct](https://github.com/FoundationDB/fdb-record-layer/blob/main/CODE_OF_CONDUCT.md) -* [License](https://github.com/FoundationDB/fdb-record-layer/blob/main/LICENSE) - -The API documentation can be found here for the most recent releases of each of this -project's published libraries: - -* [FDB extensions utility library](https://javadoc.io/doc/org.foundationdb/fdb-extensions/) -* [FDB Record Layer core library (proto2)](https://javadoc.io/doc/org.foundationdb/fdb-record-layer-core/) -* [FDB Record Layer core library (proto3)](https://javadoc.io/doc/org.foundationdb/fdb-record-layer-core-pb3/) - +```{toctree} +:maxdepth: 1 +Overview and Examples <Overview> +GettingStarted +Building <Building> +SchemaEvolution +Extending +Versioning Guide <Versioning> +Coding Best Practices <Coding_Best_Practices> +ReleaseNotes +Frequently Asked Questions <FAQ> +SQL Reference <SQL_Reference> +API <api/index> +Contributing <https://github.com/FoundationDB/fdb-record-layer/blob/main/CONTRIBUTING.md> +Code of Conduct <https://github.com/FoundationDB/fdb-record-layer/blob/main/CODE_OF_CONDUCT.md> +License <https://github.com/FoundationDB/fdb-record-layer/blob/main/LICENSE> +``` diff --git a/docs/sphinx/source/reference/Databases_Schemas_SchemaTemplates.rst b/docs/sphinx/source/reference/Databases_Schemas_SchemaTemplates.rst new file mode 100644 index 0000000000..5fb9cb5baa --- /dev/null +++ b/docs/sphinx/source/reference/Databases_Schemas_SchemaTemplates.rst @@ -0,0 +1,100 @@ +==================================== +Databases, Schemas, Schema Templates +==================================== + +.. _schema: + +Schemas +####### + +A *schema* in the Relational Layer is a collection of tables which share a semantic meaning *to the application*. These are +tables that the administrator wants grouped close together on disk, and which share logical relationships with one another. +This is in line with the definition of "schema" that is used by JDBC. Schemas play a large role in how the Relational Layer +is implemented, but from the API perspective they are really just a grouping of related data. + +Schemas are represented by a single string label, and schemas are unique *within a database*. That is, for the same database, +no two schemas can share a name. However, different databases may each have a schema with identical names. + +.. _database: + +Databases +######### + +A *database* is a grouping of data which shares access privileges and/or application constraints. Databases are represented +using a URI, which exposes a tree-like structure, so that applications can lay out their databases in a logical way. +Databases are uniquely identified by their path. Only one database *globally* can be represented using that specific path segment. + +.. note:: + + The mapping of database URI paths to their physical locations in a FoundationDB cluster is currently in flux. + Adopters should expect changes to allow them more control over how databases are stored. + +From a security standpoint, databases are the focal point of security--a user must be authorized to see the contents of a database. + +.. warning:: + + Authentication and authorization are currently not supported natively by the Relational Layer. For the moment, + adopters must provide their own solution to validate users prior to connecting to the database. However, aligning + databases with authorization goals may ready an existing deployment with future capabilities and allow an + easier migration in the future. + +It is assumed that each underlying FoundationDB cluster will maintain millions of databases, and that globally there may be billions +of unique databases within the entirety of a single deployment of the Relational Layer. + +.. _schema_template: + +Schema Templates +################ + +One important way that the FDB Relational Layer differs from a "normal" relational database in the DDL language is that +it does *not* support direct schema manipulation. In a typical relational database, one would create a single database, +then within that database define a single schema, and then within that schema define specific tables as a sequence of manual steps. + +Instead, in the FDB Relational Layer, we define *schema templates*. A schema template is simply a blueprint for how schemas +are intended to be constructed. To create a schema, you first create a schema template which defines all the tables, indexes, +etc., which are to be defined within any matching schema. Then, you create a schema and explicitly link it by name to schema +template. From then on, the Relational Layer will ensure that the data contained in the schema will be laid out in the +way specifed by the schema template definition. + +Functionally, this means that all the table, index, struct, etc., creation statements are within a schema template, rather +than explicit top-level operations. For example, to create a simple `Restaurant` schema, you would do the following: + +.. code-block:: sql + + CREATE SCHEMA TEMPLATE restaurant_template + CREATE TYPE AS STRUCT Location (latitude string, longitude string) + CREATE TABLE restaurant (rest_no integer, name string, location Location, primary key(rest_no)); + + + CREATE DATABASE /<uniqueDbPath> + CREATE SCHEMA /<uniqueDbPath>/<schemaName> WITH TEMPLATE restaurant_template + + +When you wish to modify a schema's layout (say, to add an index to a specific table), you make the modification instead to +the schema template. The Relational Layer will then automatically migrate *all* schemas which are linked to the specified +template to reflect the modification. + +.. warning:: + + The storage solution for the schema templates is currently in active development and may change in backwards + incompatible ways. Adopters should be aware that they may need to migrate any schema templates they write at + present to a new solution when it becomes available. + +It is important to note that it is not possible to modify a schema template without affecting *all* connected schemas. By +similar reasoning, it is not possible to modify an existing schema without modifying its connected schema template. + +Though this approach can be cumbersome for some simple use cases that only want to manage a single database, the benefit +of this approach is that it can make it much easier to manage a collection of similar databases. For example, by +sharding data into different databases according to access requirements, users can be granted access only to their +relevant databases, while the database administrator can still easily make changes that are applied to all users. + +Bridging a pre-existing Record Layer setup +########################################## + +Because of the current limitations around organizing databases and managing schema templates, adopters are advised +that they may be required to use the FDB Record Layer for schema management as this part of the system becomes +more mature. For that reason, they may need to construct a `RecordMetaData` and use a `KeySpacePath` to construct +an `FDBRecordStore` (these Record Layer classes corresponding roughly to a schema template, a database, and a +schema). They can then use the `TransactionBoundDatabase` to use the SQL API to interact with their pre-existing +Record Layer databases. This can allow the adopter to transition to the SQL-based API or make use of new capabilities +only available in the SQL query engine without needing to transition fully to the Relational Layer. \ No newline at end of file diff --git a/docs/sphinx/source/reference/Direct_Access_Api.rst b/docs/sphinx/source/reference/Direct_Access_Api.rst new file mode 100644 index 0000000000..572a9bcaa4 --- /dev/null +++ b/docs/sphinx/source/reference/Direct_Access_Api.rst @@ -0,0 +1,108 @@ +================= +Direct Access API +================= + +The Relational Layer's Direct Access API is a simplified API for accessing data within specific ranges. +It avoids using SQL directly, but it uses the same interface as the SQL connection to allow intermingling of +SQL queries with these simpler operations. The intent is to provide highly simplified access for common +current use-cases (e.g., scanning records within a specific range), but it is *highly* limited. For more +powerful query execution, use the :doc:`SQL query language <sql_commands>` instead. + +The Direct Access API consists of the following operations: + +* Scan +* Get +* Insert +* Delete + +Scan +#### + +Scans can access all records which fit within a specified primary key or index key range. These scans can be arbitrarily large, based on the data and the range of interest, and thus continuations are provided to enable paginated queries. For example + +.. code-block:: java + + try (RelationalStatement statement = createStatement()) { + TableScan scan = TableScan.newBuilder() + .setStartKey(<startKeyName>, <startKeyValue>) + // fill in the rest of the starting keys... + .setEndKey(<endKeyName>, <endKeyValue>) + // fill in the rest of the ending keys... + .build(); + Continuation continuation = Continuation.BEGIN; + int continuationBatchSize = 10; + while (!continuation.isEnd()) { + int rowCount = 0; + try (RelationalResultSet scanResultSet = statement.executeScan(<tableName>, scan, <options>) { + while(scanResultSet.next()) { + //process a returned record + rowCount++; + if (rowCount == continuationBatchSize) { + continuation = scanResultSet.getContinuation(); + break; + } + } + } + } + } + + +Scan keys can be partial, as long as they are contiguous. You can specify :java:`pk1, pk2`, but not :java:`pk1, pk3`; +otherwise, the API will throw an error. + +The returned :java:`ResultSet` can access a continuation, which is a pointer to the *first row which has not yet been read*. +Put another way, if the *current* location in the :java:`ResultSet` is position :java:`X`, then the continuation will point to position :java:`X+1`. +This can be used as a marker for restarting a given query. By passing the continuation to the API when executing the +same statement again, the continuation will be used to resume the execution at the specified location. + +Get +### + +The Get operation provides the ability to specify the primary key of a certain record, and quickly return the record +associated with that Primary Key from a specific table. If the record does not exist, then the result set will be empty. + +.. code-block:: java + + try (RelationalStatement statement = createStatement()) { + KeySet primaryKey = new KeySet() + .setKeyColumn("<pkColumn1Name>", <pkColumn1Value>) + // set the other values for the primary key of the table... + try (RelationalResultSet recordResultSet = statement.executeGet(<tableName>, primaryKey, <options>)) { + if (recordResultSet.next()){ + //process the returned record -- there will always be no more than 1 record in the returned result set. + } + } + } + +If you specify an incomplete :java:`KeySet` (i.e., an incomplete primary key), then the API will throw an error. + +Insert +###### + +The Insert API provides a way to insert a data element into a specific table using programmatic API. The API requires +building a :java:`DynamicMessage`, as follows: + +.. code-block:: java + + try (RelationalStatement statement = createStatement()) { + DynamicMessageBuilder messageBuilder = statement.getMessageBuilder(<tableName>) + .setField(<fieldName>, <fieldValue>) + // set the other fields in the record, including nested or repeated structures... + statement.executeInsert(<tableName>, messageBuilder.build(), <options>); + } + +You can also insert multiple records together in batch, using an iterable interface of build records. + +Delete +###### + +Deletes are very similar to inserts, except that you specify the primary keys of the rows that you want to delete: + +.. code-block:: java + + try (RelationalStatement statement = createStatement()) { + KeySet primaryKey = new KeySet() + .setKeyColumn("<pkColumn1Name>", <pkColumn1Value>) + // set the other values for the primary key of the table... + statement.executeDelete(<tableName>, primaryKey, <options>); + } diff --git a/docs/sphinx/source/reference/Functions.rst b/docs/sphinx/source/reference/Functions.rst new file mode 100644 index 0000000000..2767e81ec2 --- /dev/null +++ b/docs/sphinx/source/reference/Functions.rst @@ -0,0 +1,11 @@ +========= +Functions +========= + +This topic provides reference information for the system-defined functions. + +.. toctree:: + :maxdepth: 3 + + Functions/aggregate_functions + Functions/scalar_functions diff --git a/docs/sphinx/source/reference/Functions/aggregate_functions.rst b/docs/sphinx/source/reference/Functions/aggregate_functions.rst new file mode 100644 index 0000000000..5a5e4612f6 --- /dev/null +++ b/docs/sphinx/source/reference/Functions/aggregate_functions.rst @@ -0,0 +1,13 @@ +=================== +Aggregate Functions +=================== + +Aggregate functions perform calculations on zero, one, or multiple rows of values, and they return a single value. + +List of functions (by sub-category) +################################### + +.. toctree:: + :maxdepth: 1 + + aggregate_functions/bitmap_construct_agg \ No newline at end of file diff --git a/docs/sphinx/source/reference/Functions/aggregate_functions/bitmap_construct_agg.rst b/docs/sphinx/source/reference/Functions/aggregate_functions/bitmap_construct_agg.rst new file mode 100644 index 0000000000..45ea43a1b6 --- /dev/null +++ b/docs/sphinx/source/reference/Functions/aggregate_functions/bitmap_construct_agg.rst @@ -0,0 +1,45 @@ +==================== +BITMAP_CONSTRUCT_AGG +==================== + +.. _bitmap-construct-agg: + +See: :ref:`Understanding How Bitmaps Identify Distinct Values <understanding-bitmaps>`. + +Syntax +====== + +.. code-block:: sql + + BITMAP_CONSTRUCT_AGG( <bit_position> ) + +Parameters +========== + +``bit_position`` + A numeric value that represents the BITMAP_BIT_POSITION of an input. + +Returns +======= +The function takes the relative position of multiple inputs in the bitmap, and returns a byte array that is a bitmap with bits set for each distinct input. +The result should be grouped by :ref:`BITMAP_BUCKET_OFFSET <bitmap-bucket-offset>` or :ref:`BITMAP_BUCKET_NUMBER <bitmap-bucket-number>`, so that values in different "buckets" are not merged together. +For example: + +.. code-block:: sql + + SELECT BITMAP_BUCKET_OFFSET(UID) as offset, BITMAP_CONSTRUCT_AGG(BITMAP_BIT_POSITION(UID)) as bitmap FROM T + GROUP BY BITMAP_BUCKET_OFFSET(UID) + + ``UID = 0, 1, 9999, 10000, -1`` + +.. list-table:: + :header-rows: 1 + + * - :sql:`OFFSET` + - :sql:`BITMAP` + * - :json:`0` + - :json:`[b'00000011, 0, ..., b'10000000]` + * - :json:`1` + - :json:`[b'00000001, 0, ..., 0]` + * - :json:`-1` + - :json:`[0, 0, ..., b'10000000]` diff --git a/docs/sphinx/source/reference/Functions/scalar_functions.rst b/docs/sphinx/source/reference/Functions/scalar_functions.rst new file mode 100644 index 0000000000..0e6da8fe17 --- /dev/null +++ b/docs/sphinx/source/reference/Functions/scalar_functions.rst @@ -0,0 +1,15 @@ +================ +Scalar Functions +================ + +Scalar functions are functions that take an input of one row and produce a single value. + +List of functions (by sub-category) +################################### + +.. toctree:: + :maxdepth: 1 + + scalar_functions/bitmap_bit_position + scalar_functions/bitmap_bucket_number + scalar_functions/bitmap_bucket_offset diff --git a/docs/sphinx/source/reference/Functions/scalar_functions/bitmap_bit_position.rst b/docs/sphinx/source/reference/Functions/scalar_functions/bitmap_bit_position.rst new file mode 100644 index 0000000000..ef1e4f7dda --- /dev/null +++ b/docs/sphinx/source/reference/Functions/scalar_functions/bitmap_bit_position.rst @@ -0,0 +1,48 @@ +=================== +BITMAP_BIT_POSITION +=================== + +.. _bitmap-bit-position: + +See: :ref:`Understanding How Bitmaps Identify Distinct Values <understanding-bitmaps>`. + +Syntax +====== + +.. code-block:: sql + + BITMAP_BIT_POSITION( <expr> ) + +Parameters +========== + +``expr`` + A numeric expression + +Returns +======= + +Returns the zero-based position of the numeric input in a bitmap (fixed length = 10000). + +Example +======= + +.. code-block:: sql + + SELECT input, BITMAP_BIT_POSITION(input) FROM T + +.. list-table:: + :header-rows: 1 + + * - :sql:`input` + - :sql:`BITMAP_BIT_POSITION` + * - :json:`0` + - :json:`0` + * - :json:`1` + - :json:`1` + * - :json:`9999` + - :json:`9999` + * - :json:`10000` + - :json:`0` + * - :json:`-1` + - :json:`9999` diff --git a/docs/sphinx/source/reference/Functions/scalar_functions/bitmap_bucket_number.rst b/docs/sphinx/source/reference/Functions/scalar_functions/bitmap_bucket_number.rst new file mode 100644 index 0000000000..aa35becfcb --- /dev/null +++ b/docs/sphinx/source/reference/Functions/scalar_functions/bitmap_bucket_number.rst @@ -0,0 +1,51 @@ +==================== +BITMAP_BUCKET_NUMBER +==================== + +.. _bitmap-bucket-number: + +See: :ref:`Understanding How Bitmaps Identify Distinct Values <understanding-bitmaps>`. + +Syntax +====== + +.. code-block:: sql + + BITMAP_BUCKET_NUMBER( <expr> ) + +Parameters +========== + +``expr`` + A numeric expression + +Returns +======= + +Given a numeric input, returns a “bucket number” that finds the bitmap (fixed length = 10000) that has the input value. +If the input in range [0, 9999], output = 0; +if the input in range [10000, 19999], output = 1; +if the input is negative, the output is negative as well -- if input in range [-10000, 0), output = -1, etc. + +Example +======= + +.. code-block:: sql + + SELECT input, BITMAP_BUCKET_NUMBER(input) FROM T + +.. list-table:: + :header-rows: 1 + + * - :sql:`input` + - :sql:`BITMAP_BUCKET_NUMBER` + * - :json:`0` + - :json:`0` + * - :json:`1` + - :json:`0` + * - :json:`9999` + - :json:`0` + * - :json:`10000` + - :json:`1` + * - :json:`-1` + - :json:`-1` diff --git a/docs/sphinx/source/reference/Functions/scalar_functions/bitmap_bucket_offset.rst b/docs/sphinx/source/reference/Functions/scalar_functions/bitmap_bucket_offset.rst new file mode 100644 index 0000000000..efcd5bd68a --- /dev/null +++ b/docs/sphinx/source/reference/Functions/scalar_functions/bitmap_bucket_offset.rst @@ -0,0 +1,48 @@ +==================== +BITMAP_BUCKET_OFFSET +==================== + +.. _bitmap-bucket-offset: + +See: :ref:`Understanding How Bitmaps Identify Distinct Values <understanding-bitmaps>`. + +Syntax +====== + +.. code-block:: sql + + BITMAP_BUCKET_OFFSET( <expr> ) + +Parameters +========== + +``expr`` + A numeric expression + +Returns +======= + +Given a numeric input, returns the "offset" of the input in the bitmap. Essentially, BITMAP_BUCKET_OFFSET + :ref:`BITMAP_BIT_POSITION <bitmap-bit-position>` = INPUT_VALUE. + +Example +======= + +.. code-block:: sql + + SELECT input, BITMAP_BUCKET_OFFSET(input) FROM T + +.. list-table:: + :header-rows: 1 + + * - :sql:`input` + - :sql:`BITMAP_BUCKET_OFFSET` + * - :json:`0` + - :json:`0` + * - :json:`1` + - :json:`0` + * - :json:`9999` + - :json:`0` + * - :json:`10000` + - :json:`10000` + * - :json:`-1` + - :json:`-10000` \ No newline at end of file diff --git a/docs/sphinx/source/reference/Indexes.rst b/docs/sphinx/source/reference/Indexes.rst new file mode 100644 index 0000000000..cd23d57ed0 --- /dev/null +++ b/docs/sphinx/source/reference/Indexes.rst @@ -0,0 +1,130 @@ +======= +Indexes +======= + +.. _index_definition: + +Defining indexes +################ + +It is possible to define indexes in the Relational Layer as materialized views of :doc:`SELECT <sql_commands/DQL/SELECT>` +statements.To be used in an index, the query must able to be incrementally updated. That is, when a row on an indexed +table is inserted, updated, or deleted, it must be possible to construct a finite set up modifications to the index +structure to reflect the new query results. + +In practice, that means that indexes are translated into a Record Layer key expression and index type. Indexes currently +only support indexes on a single table. When row is modified, the key expression generates zero or more tuples based +on the index data, and then it stores them in a format optimized for fast lookups. + +The simplest indexes are simple projections with an `ORDER BY` clause. They follow this form: + +.. code-block:: sql + + CREATE INDEX <indexName> AS + SELECT (field1, field2, ... fieldN, fieldN+1, ... fieldM) + FROM <tableName> ORDER BY ( field1, field2, ... fieldN ) + +Where: + +* :sql:`indexName` is the name of the index. +* :sql:`tableName` is the name of the table the index is defined on. +* :sql:`field1, field2, ... fieldN` are the indexed fields. +* :sql:`fieldN+1, fieldN+1, ... fieldM` optional list of non-indexed fields to be attached to each tuple to reduce the chance of fetch (index lookup). + +For example, let us say we have the following table: + +.. code-block:: sql + + CREATE TABLE employee(id INT64, fname STRING, lname STRING, PRIMARY KEY (id)) + +We define the following index: + +.. code-block:: sql + + CREATE INDEX fnameIdx AS SELECT fname, lname FROM employee ORDER BY fname + +This index will contain two-tuples of `(fname, lname)`, but ordered only by `fname`. In the Record Layer, this +corresponds to a `VALUE` index with the `fname` field indexed in the key and the `lname` field indexed in the value. +This index can be used to optimize queries like: + +.. code-block:: sql + + -- Simple index scan + SELECT fname, lname FROM employee ORDER BY fname ASC; + -- Simple index scan, but in reverse + SELECT fname, lname FROM employee ORDER BY fname DESC; + -- Limited scan of the index to single fname value + SELECT lname FROM employee WHERE fname = ?; + -- Limited scan of the index to range of fname values + SELECT fname, lname FROM employee WHERE fname > ? AND fname < ?; + -- Limited scan of the index to single fname value with additional lname predicate + SELECT lname FROM employee WHERE fname = ? AND lname LIKE 'a%'; + +In each case, the fact that index entries are ordered by `fname` is leveraged, either to satisfy the sort or to limit +the scan to a smaller range of index entries. The `lname` field can then be returned to the user without having to +perform an additional lookup of the underlying row. + +Indexes on nested fields +######################## + +The Relational Layer also offers a special version of indexes that can be used to describe how to index nested fields and +:sql:`ARRAY` fields. They are defined using SQL, and they follow a strict set of rules, but before we get into the details, let +us introduce them first by the following simple example. Suppose we have a table :sql:`restaurant` that is defined like this: + +.. code-block:: sql + + CREATE TYPE AS STRUCT restaurant_review (reviewer STRING, rating INT64); + CREATE TABLE restaurant ( + rest_no INT64, name STRING, reviews restaurant_review ARRAY, PRIMARY KEY(rest_no) + ); + +Let us say we have too many queries involving restaurant ratings, so it makes sense to have an index defined on :sql:`rating`. +We can define an index to do exactly that: + +.. code-block:: sql + + CREATE INDEX mv AS + SELECT SQ.rating from restaurant AS RR, (select rating from RR.reviews) SQ; + +At first glance, it looks like the query is performing a join between :sql:`restaurant` and a subquery. However, if we +take a closer look, we see that the table we select from in the subquery is nothing but the nested repeated field in +:sql:`restaurant`. We use the alias :sql:`RR` to link the nested repeated field and its parent together. + + +Index rules +########### + +Defining a index must adhere to these rules: + +* It must involve a single table only, correlated joins that navigate from one nested repeated field to another is possible. +* Predicates are not allowed in any (sub)query. +* Each subquery can have a single _source_ and an arbitrary number of other nested subqueries. A source is effectively a + nested-repeated field. In the example above, :sql:`RR.reviews` is the source of the containing subquery. +* The parent query must have the table itself as a source. +* Propagation of selected fields from subqueries must follow the same order and clustering. Interleaving is not supported. For example, this is illegal: + + .. code-block:: sql + + SELECT T1.a, T2.c, T1.b FROM T t, (SELECT a, b FROM t.X) T1, (SELECT c FROM t.Y) T2 + + However, this is legal: + + .. code-block:: sql + + SELECT T1.a, T1.a, T2.c FROM T t, (SELECT a, b FROM t.X) T1, (SELECT c FROM t.Y) T2 + +Implementation details +###################### + +Indexes have a close relationship with Record Layer key expressions and index types. When the user defines a index +we create logical plan, analyze it, and then pass it to a special visitor that iterates over its nodes generating a +semantically equivalent key expression. Since SQL it a rich language, it is possible to define a index that +can not be translated to a key expression. Therefore, we define the list of rules above to limit, as much as possible, +the possibilities of defining an invalid index. + +If the index can be translated into a key expression, the generated key expression will have a structure +that resembles the structure of the SQL statement: + +* Projected fields :sql:`f1`, :sql:`f2`, ... :sql:`fn` in (sub)queries maps to a :sql:`concat(field(f1), field(f2), ... field(fn))`. +* Projected nested fields (:sql:`f1`, :sql:`f2`, ... :sql:`fn`) from a repeated field :sql:`rf`, i.e. :sql:`select f1, f2, ... fn, ... from FOO.rf` + maps to :sql:`field(rf, FAN_OUT).nest(field(f1), (field(f2), ..., field(fn)))`. diff --git a/docs/sphinx/source/reference/Tables.rst b/docs/sphinx/source/reference/Tables.rst new file mode 100644 index 0000000000..6749a58c1d --- /dev/null +++ b/docs/sphinx/source/reference/Tables.rst @@ -0,0 +1,28 @@ +====== +Tables +====== + +A table in the FDB Relational Layer is treated the same way that a table in an alternative. relational database would be. +A table has a name, columns, and metadata which can be investigated. + +Tables are *required* to have a primary key defined (except for `Single row tables`_), and they may also have one +or more secondary indexes to improve performance on specific queries. Within a :ref:`schema <schema>`, no two rows in the same +table may share the same primary key value, though tables are allowed to duplicate primary key values if they are defined +in different schemas. Additionally, the :ref:`schema template <schema_template>` option `INTERMINGLE_TABLES` further +specifies that no two rows in the same schema can share the same primary key value, even if the rows are from different +tables. The `INTERMINGLE_TABLES` option allows for polymorphic foreign keys (that is, a row can be referenced solely by +its primary key without regard to type), but it is generally counter-intuitive for those coming from traditional +relational systems, and it is therefore discouraged. + +Single row tables +################# + +Tables can also be defined to be constrained to one row only: + +.. code-block:: sql + + CREATE TABLE single_row_table (a INT64, b INT64, SINGLE ROW ONLY) + +Note that there is no explicit primary key, though the "empty value" stands in for the primary key in practice. +Trying to insert more than one row will result in a ``UNIQUE_CONSTRAINT_VIOLATION`` error. + diff --git a/docs/sphinx/source/reference/sql_commands.rst b/docs/sphinx/source/reference/sql_commands.rst new file mode 100644 index 0000000000..02f10d2aed --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands.rst @@ -0,0 +1,10 @@ +============ +SQL Commands +============ + +.. toctree:: + :maxdepth: 3 + + sql_commands/DQL + sql_commands/DDL + sql_commands/DML diff --git a/docs/sphinx/source/reference/sql_commands/DDL.rst b/docs/sphinx/source/reference/sql_commands/DDL.rst new file mode 100644 index 0000000000..5be9b5397e --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL.rst @@ -0,0 +1,12 @@ +DDL +=== + +The DDL defines the syntax for defining the database schema. This includes tables, :ref:`structs <struct_types>`, and enums +as well as indexes. See :doc:`../Databases_Schemas_SchemaTemplates` for more information about DDL concepts. + + +.. toctree:: + :maxdepth: 2 + + DDL/CREATE + DDL/DROP diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE.rst b/docs/sphinx/source/reference/sql_commands/DDL/CREATE.rst new file mode 100644 index 0000000000..3aa3bb2eb1 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE.rst @@ -0,0 +1,13 @@ +====== +CREATE +====== + +.. toctree:: + :maxdepth: 1 + + CREATE/SCHEMA_TEMPLATE + CREATE/DATABASE + CREATE/SCHEMA + CREATE/TABLE + CREATE/TYPE + CREATE/INDEX diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/DATABASE.diagram b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/DATABASE.diagram new file mode 100644 index 0000000000..8167b1cd23 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/DATABASE.diagram @@ -0,0 +1,5 @@ + Diagram( + Terminal('CREATE'), + Terminal('DATABASE'), + NonTerminal('pathToDatabase') + ) diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/DATABASE.rst b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/DATABASE.rst new file mode 100644 index 0000000000..1be7208503 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/DATABASE.rst @@ -0,0 +1,25 @@ +=============== +CREATE DATABASE +=============== + +Create a database. A database can contain multiple schemas. + +Syntax +====== + +.. raw:: html + :file: DATABASE.diagram.svg + +Parameters +========== + +``pathToDatabase`` + The fully-qualified path of the ``DATABASE`` + +Example +======= + +.. code-block:: sql + + -- Create database in domain '/FRL/' + CREATE DATABASE '/FRL/DEMO_DB'; diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/INDEX.diagram b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/INDEX.diagram new file mode 100644 index 0000000000..dfc11e4aa0 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/INDEX.diagram @@ -0,0 +1,8 @@ + Diagram( + Terminal('CREATE'), + Optional('UNIQUE', 'skip'), + Terminal('INDEX'), + NonTerminal('indexName'), + Terminal('AS'), + NonTerminal('query') + ) diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/INDEX.rst b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/INDEX.rst new file mode 100644 index 0000000000..2e81e2b54a --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/INDEX.rst @@ -0,0 +1,22 @@ +============ +CREATE INDEX +============ + +Clause in a :ref:`schema template definition <create-schema-template>` to create an index. The syntax here takes +a query and is inspired by materialized view syntax. However, unlike a general `VIEW`, the index must be able to +maintained incrementally. That means that for any given record insert, update, or delete, it must be possible +to construct the difference that needs to be applied to each index in order to update it without needing to +completely rebuild it. That means that there are a number of limitations to the `query` argument. See +:ref:`index_definition` for more details. + +Syntax +====== + +.. raw:: html + :file: INDEX.diagram.svg + +Parameters +========== + +``indexName`` + The name of the index. Note that the name of the index should be unique in the schema template. diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/SCHEMA.diagram b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/SCHEMA.diagram new file mode 100644 index 0000000000..e6797917f3 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/SCHEMA.diagram @@ -0,0 +1,7 @@ + Diagram( + Terminal('CREATE'), + Terminal('SCHEMA'), + NonTerminal('pathToSchema'), + Terminal('WITH'), + NonTerminal('schemaTemplateName') + ) diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/SCHEMA.rst b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/SCHEMA.rst new file mode 100644 index 0000000000..219ccd6a53 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/SCHEMA.rst @@ -0,0 +1,39 @@ +============= +CREATE SCHEMA +============= + +Create a schema from a schema template. + +Syntax +====== + +.. raw:: html + :file: SCHEMA.diagram.svg + +Parameters +========== + +``pathToSchema`` + The fully-qualified path to the ``SCHEMA`` in a ``DATABASE`` + +``schemaTemplateName`` + The name of the schema template to be used to create this schema + + +Example +======= + +.. code-block:: sql + + -- Create a schema template + CREATE SCHEMA TEMPLATE TEMP + CREATE TABLE T (A BIGINT NULL, B DOUBLE NOT NULL, C STRING, PRIMARY KEY(A, B)) + + -- Create database in domain '/FRL/' + CREATE DATABASE '/FRL/DEMO_DB' + + -- Create a schema in the database from TEMP + CREATE SCHEMA 'FRL/DEMO_DB/DEMO_SCHEMA' WITH TEMP + + + diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/SCHEMA_TEMPLATE.diagram b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/SCHEMA_TEMPLATE.diagram new file mode 100644 index 0000000000..791fd75e87 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/SCHEMA_TEMPLATE.diagram @@ -0,0 +1,23 @@ +Diagram( + Stack( + Sequence( + Terminal('CREATE'), + Terminal('SCHEMA'), + Terminal('TEMPLATE'), + NonTerminal('schemaTemplateName') + ), + Optional( + OneOrMore( + NonTerminal('createTypeClause') + ) + ), + OneOrMore( + NonTerminal('createTableCluase') + ), + Optional( + OneOrMore( + NonTerminal('createIndexCluase') + ) + ) + ) +) \ No newline at end of file diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/SCHEMA_TEMPLATE.rst b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/SCHEMA_TEMPLATE.rst new file mode 100644 index 0000000000..c386cc4e42 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/SCHEMA_TEMPLATE.rst @@ -0,0 +1,21 @@ +====================== +CREATE SCHEMA TEMPLATE +====================== + +.. _create-schema-template: + +A schema template is a predefined structure that defines the set of ``TABLE`` and ``INDEX`` definitions. This can then +used as a blueprint to create ``SCHEMA`` in a database. See: :doc:`../../../Databases_Schemas_SchemaTemplates` for more +information. + +Syntax +====== + +.. raw:: html + :file: SCHEMA_TEMPLATE.diagram.svg + +Parameters +========== + +``schemaTemplateName`` + The name of the schema template. diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TABLE.diagram b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TABLE.diagram new file mode 100644 index 0000000000..8a658f0e1e --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TABLE.diagram @@ -0,0 +1,40 @@ +Diagram( + Stack( + Sequence( + Terminal('CREATE'), + Terminal('TABLE'), + NonTerminal('tableName'), + Terminal('('), + ), + Sequence( + OneOrMore( + Sequence( + NonTerminal('columnName'), + NonTerminal('columnType'), + ), + Terminal(',') + ), + ), + Sequence( + Terminal(','), + Choice(0, + Sequence( + Terminal('PRIMARY'), + Terminal('KEY'), + Terminal('('), + OneOrMore( + NonTerminal('primaryKeyColumnName'), + Terminal(',') + ), + Terminal(')') + ), + Sequence( + Terminal('SINGLE'), + Terminal('ROW'), + Terminal('ONLY') + ) + ), + Terminal(')') + ) + ) +) \ No newline at end of file diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TABLE.rst b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TABLE.rst new file mode 100644 index 0000000000..48bba5bd9f --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TABLE.rst @@ -0,0 +1,111 @@ +============ +CREATE TABLE +============ + +Clause in a :ref:`schema template definition <create-schema-template>` to create a table. +Note that a table can either have a primary key defined from one or multiple columns, or it +must be declared as a ``SINGLE ROW ONLY`` table, in which case only one row can be inserted into it. +This latter type of table can be useful for things like database configuration parameters, which the +application wants to inform its interpretation of all data in the database. + +Syntax +====== + +.. raw:: html + :file: TABLE.diagram.svg + +Parameters +========== + +``tableName`` + The name of the ``TABLE`` + +``columnName`` + The name of a column of the defined ``TABLE`` + +``columnType`` + The associated :ref:`type <sql-types>` of the column + +``primaryKeyColumnName`` + The name of the column to be part of the primary key of the ``TABLE`` + +Examples +======== + +Table with a primary key +------------------------ + +.. code-block:: sql + + CREATE SCHEMA TEMPLATE TEMP + CREATE TABLE T (A BIGINT NULL, B DOUBLE NOT NULL, C STRING, PRIMARY KEY(A, B)) + + -- On a schema that uses the above schema template + INSERT INTO T VALUES + (NULL, 0.0, 'X'), + (1, 1.0, 'A'), + (NULL, 2.0, 'B'); + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :sql:`NULL` + - :json:`0.0` + - :json:`"X"` + * - :sql:`NULL` + - :json:`2.0` + - :json:`"B"` + * - :json:`1` + - :json:`1.0` + - :json:`"A"` + + +Table limited to a single row +----------------------------- + +.. code-block:: sql + + CREATE SCHEMA TEMPLATE TEMP + CREATE TABLE T (A BIGINT NULL, B DOUBLE NOT NULL, C STRING, SINGLE ROW ONLY) + + -- On a schema that uses the above schema template + INSERT INTO T VALUES + (NULL, 0.0, 'X') + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :sql:`NULL` + - :json:`0.0` + - :json:`"X"` + +Attempting to insert a second row in a :sql:`SINGLE ROW ONLY` table will result in a ``UNIQUE_CONSTRAINT_VIOLATION`` error: + +.. code-block:: sql + + INSERT INTO T VALUES + (1, 2.0, 'X') + + SqlException(23505 - UNIQUE_CONSTRAINT_VIOLATION) + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :sql:`NULL` + - :json:`0.0` + - :json:`"X"` diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TYPE.rst b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TYPE.rst new file mode 100644 index 0000000000..5e6da5ffc2 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TYPE.rst @@ -0,0 +1,9 @@ +=========== +CREATE TYPE +=========== + +.. toctree:: + :maxdepth: 1 + + TYPE/STRUCT + TYPE/ENUM diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TYPE/ENUM.diagram b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TYPE/ENUM.diagram new file mode 100644 index 0000000000..cb8cabc87d --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TYPE/ENUM.diagram @@ -0,0 +1,13 @@ +Diagram( + Terminal('CREATE'), + Terminal('TYPE'), + Terminal('AS'), + Terminal('ENUM'), + NonTerminal('enumName'), + Terminal('('), + OneOrMore( + NonTerminal('stringLiteral'), + Terminal(',') + ), + Terminal(')') +) \ No newline at end of file diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TYPE/ENUM.rst b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TYPE/ENUM.rst new file mode 100644 index 0000000000..60d073fb0b --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TYPE/ENUM.rst @@ -0,0 +1,43 @@ +=================== +CREATE TYPE AS ENUM +=================== + +Clause in a :ref:`schema template definition <create-schema-template>` to create an enum type. + +Syntax +====== + +.. raw:: html + :file: ENUM.diagram.svg + + +Parameters +========== + +``enumName`` + The name of the ``ENUM`` type + +``stringLiteral`` + A ``STRING`` representation of an ``ENUM`` value + +Example +======= + +.. code-block:: sql + + CREATE SCHEMA TEMPLATE TEMP + CREATE TYPE AS ENUM color ('blue', 'red', 'yellow') + CREATE TABLE dot(X INTEGER, Y INTEGER, C color, PRIMARY KEY(X, Y)); + + -- On a schema that uses the above schema template + INSERT INTO dot VALUES + (1, 1, 'blue'), + (1, 2, 'yellow'), + (2, 2, 'red'), + (3, 2, 'red'); + + SELECT * FROM dot; + + SELECT * FROM dot WHERE C = 'red'; + +.. TODO: file tickets because the above queries do not work \ No newline at end of file diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TYPE/STRUCT.diagram b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TYPE/STRUCT.diagram new file mode 100644 index 0000000000..1c0f5dd397 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TYPE/STRUCT.diagram @@ -0,0 +1,22 @@ +Diagram( + Stack( + Sequence( + Terminal('CREATE'), + Terminal('TYPE'), + Terminal('AS'), + Terminal('STRUCT'), + NonTerminal('structName'), + ), + Sequence( + Terminal('('), + OneOrMore( + Sequence( + NonTerminal('columnName'), + NonTerminal('columnType'), + ), + Terminal(',') + ), + Terminal(')') + ) + ) +) \ No newline at end of file diff --git a/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TYPE/STRUCT.rst b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TYPE/STRUCT.rst new file mode 100644 index 0000000000..386c06b802 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/CREATE/TYPE/STRUCT.rst @@ -0,0 +1,124 @@ +===================== +CREATE TYPE AS STRUCT +===================== + +.. role:: sql(code) + :language: sql + +Clause in a :ref:`schema template definition <create-schema-template>` to create a struct type. + +Syntax +====== + +.. raw:: html + :file: STRUCT.diagram.svg + + +Parameters +========== + +``structName`` + The name of the :sql:`STRUCT` type + +``columnName`` + The name of a column of the defined :sql:`STRUCT` + +``columnType`` + The associated :ref:`type <sql-types>` of the column + +Example +======= + +.. code-block:: sql + + CREATE SCHEMA TEMPLATE TEMP + CREATE TYPE AS STRUCT coordinates (X DOUBLE, Y DOUBLE) + CREATE TYPE AS STRUCT route(author STRING, steps coordinates ARRAY) + CREATE TABLE T(id BIGINT, R route, PRIMARY KEY(id)) + + -- On a schema that uses the above schema template + INSERT INTO T VALUES + (1, ('Lucy', [(10, 15), (12, 13), (17, 19)])), + (2, ('Lucy', [(10, 15), (10, 19)])), + (3, ('Tim', [(10, 19), (10, 15), (7, 9)])), + (4, ('Tim', [(1, 1), (2, 3), (7, 9)])) + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`ID` + - :sql:`R` + * - :json:`1` + - .. code-block:: json + + { + "AUTHOR" : "Lucy" , + "STEPS" : [ + { "X" : 10.0 , "Y" : 15.0 }, + { "X" : 12.0 , "Y" : 13.0 }, + { "X" : 17.0 , "Y" : 19.0 } + ] + } + * - :json:`2` + - .. code-block:: json + + { + "AUTHOR" : "Lucy" , + "STEPS" : [ + { "X" : 10.0 , "Y" : 15.0 }, + { "X" : 10.0 , "Y" : 19.0 } + ] + } + * - :json:`3` + - .. code-block:: json + + { + "AUTHOR" : "Tim" , + "STEPS" : [ + { "X" : 10.0 , "Y" : 19.0 }, + { "X" : 10.0 , "Y" : 15.0 }, + { "X" : 7.0 , "Y" : 9.0 } + ] + } + * - :json:`4` + - .. code-block:: json + + { + "AUTHOR" : "Tim" , + "STEPS" : [ + { "X" : 1.0 , "Y" : 1.0 }, + { "X" : 2.0 , "Y" : 3.0 }, + { "X" : 7.0 , "Y" : 9.0 } + ] + } + +.. code-block:: sql + + SELECT + ID, R.author, (S.X, S.Y) AS coord + FROM + T, + (SELECT * FROM R.steps) AS S + WHERE S.X < 10; + +.. list-table:: + :header-rows: 1 + + * - :sql:`ID` + - :sql:`AUTHOR` + - :sql:`COORD` + * - :json:`3` + - :json:`"Tim"` + - :json:`{ "X" : 7.0, "Y": 9.0 }` + * - :json:`4` + - :json:`"Tim"` + - :json:`{ "X" : 1.0, "Y": 1.0 }` + * - :json:`4` + - :json:`"Tim"` + - :json:`{ "X" : 2.0, "Y": 3.0 }` + * - :json:`4` + - :json:`"Tim"` + - :json:`{ "X" : 7.0, "Y": 9.0 }` + diff --git a/docs/sphinx/source/reference/sql_commands/DDL/DROP.rst b/docs/sphinx/source/reference/sql_commands/DDL/DROP.rst new file mode 100644 index 0000000000..7c993711d0 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/DROP.rst @@ -0,0 +1,12 @@ +==== +DROP +==== + + + +.. toctree:: + :maxdepth: 1 + + DROP/SCHEMA_TEMPLATE + DROP/DATABASE + DROP/SCHEMA diff --git a/docs/sphinx/source/reference/sql_commands/DDL/DROP/DATABASE.diagram b/docs/sphinx/source/reference/sql_commands/DDL/DROP/DATABASE.diagram new file mode 100644 index 0000000000..b93f1818fb --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/DROP/DATABASE.diagram @@ -0,0 +1,5 @@ + Diagram( + Terminal('DROP'), + Terminal('DATABASE'), + NonTerminal('pathToDatabase') + ) diff --git a/docs/sphinx/source/reference/sql_commands/DDL/DROP/DATABASE.rst b/docs/sphinx/source/reference/sql_commands/DDL/DROP/DATABASE.rst new file mode 100644 index 0000000000..746e4a98da --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/DROP/DATABASE.rst @@ -0,0 +1,25 @@ +============= +DROP DATABASE +============= + +Drop a database. This drops all the schemas that are part of this database. + +Syntax +====== + +.. raw:: html + :file: DATABASE.diagram.svg + +Parameters +========== + +``pathToDatabase`` + The fully-qualified path of the ``DATABASE`` + +Example +======= + +.. code-block:: sql + + -- Create database '/FRL/DEMO_DB' + DROP DATABASE '/FRL/DEMO_DB' diff --git a/docs/sphinx/source/reference/sql_commands/DDL/DROP/SCHEMA.diagram b/docs/sphinx/source/reference/sql_commands/DDL/DROP/SCHEMA.diagram new file mode 100644 index 0000000000..515507f149 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/DROP/SCHEMA.diagram @@ -0,0 +1,5 @@ + Diagram( + Terminal('DROP'), + Terminal('SCHEMA'), + NonTerminal('pathToSchema') + ) diff --git a/docs/sphinx/source/reference/sql_commands/DDL/DROP/SCHEMA.rst b/docs/sphinx/source/reference/sql_commands/DDL/DROP/SCHEMA.rst new file mode 100644 index 0000000000..94613d991e --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/DROP/SCHEMA.rst @@ -0,0 +1,25 @@ +=========== +DROP SCHEMA +=========== + +Drop a schema. This drops all the tables and indexes that are part of this schema. + +Syntax +====== + +.. raw:: html + :file: SCHEMA.diagram.svg + +Parameters +========== + +``pathToSchema`` + The fully-qualified path of the schema in the database + +Example +======= + +.. code-block:: sql + + -- drop schema 'DEMO_SCHEMA' in '/FRL/DEMO_DB' + DROP SCHEMA '/FRL/DEMO_DB/DEMO_SCHEMA' diff --git a/docs/sphinx/source/reference/sql_commands/DDL/DROP/SCHEMA_TEMPLATE.diagram b/docs/sphinx/source/reference/sql_commands/DDL/DROP/SCHEMA_TEMPLATE.diagram new file mode 100644 index 0000000000..6f52c382d9 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/DROP/SCHEMA_TEMPLATE.diagram @@ -0,0 +1,6 @@ + Diagram( + Terminal('DROP'), + Terminal('SCHEMA'), + Terminal('TEMPLATE'), + NonTerminal('schemaTemplateName') + ) diff --git a/docs/sphinx/source/reference/sql_commands/DDL/DROP/SCHEMA_TEMPLATE.rst b/docs/sphinx/source/reference/sql_commands/DDL/DROP/SCHEMA_TEMPLATE.rst new file mode 100644 index 0000000000..588c5980c0 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DDL/DROP/SCHEMA_TEMPLATE.rst @@ -0,0 +1,27 @@ +==================== +DROP SCHEMA TEMPLATE +==================== + +Drops a schema template. + +``NOTE``: This is still under active development and is experimental. The behavior of ``DROP`` is not clearly defined or tested in case there are still schemas that are defined with the schema template that we want to delete. + +Syntax +====== + +.. raw:: html + :file: SCHEMA_TEMPLATE.diagram.svg + +Parameters +========== + +``schemaTemplateName`` + The name of the schema template to be droped. + +Example +======= + +.. code-block:: sql + + -- drop schema template DEMO_SCHEMA_TEMPLATE + DROP SCHEME TEMPLATE DEMO_SCHEMA_TEMPLATE diff --git a/docs/sphinx/source/reference/sql_commands/DML.rst b/docs/sphinx/source/reference/sql_commands/DML.rst new file mode 100644 index 0000000000..c264736791 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DML.rst @@ -0,0 +1,10 @@ +=== +DML +=== + +.. toctree:: + :maxdepth: 1 + + DML/INSERT + DML/UPDATE + DML/DELETE diff --git a/docs/sphinx/source/reference/sql_commands/DML/DELETE.diagram b/docs/sphinx/source/reference/sql_commands/DML/DELETE.diagram new file mode 100644 index 0000000000..816d538b1e --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DML/DELETE.diagram @@ -0,0 +1,11 @@ +Diagram( + Terminal('DELETE'), + Terminal('FROM'), + NonTerminal('tableName'), + Optional( + Sequence( + Terminal('WHERE'), + NonTerminal('predicate') + ) + ) +) \ No newline at end of file diff --git a/docs/sphinx/source/reference/sql_commands/DML/DELETE.rst b/docs/sphinx/source/reference/sql_commands/DML/DELETE.rst new file mode 100644 index 0000000000..f71c4e4dd4 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DML/DELETE.rst @@ -0,0 +1,111 @@ +====== +DELETE +====== + +SQL command to delete rows from a table. + +Syntax +====== + +.. raw:: html + :file: DELETE.diagram.svg + +Parameters +========== + +``tableName`` + The name of the target table + +``predicate`` + A predicate which should evaluate to true for the given row to be updated + + +Examples +======== + +Delete all rows +--------------- + +.. code-block:: sql + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`1.0` + * - :json:`2` + - :json:`"two"` + - :json:`2.0` + * - :json:`3` + - :json:`"three"` + - :sql:`NULL` + * - :json:`4` + - :json:`"four"` + - :sql:`NULL` + +.. code-block:: sql + + DELETE FROM T; + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - + - + - + +Update all rows matching a give predicate +----------------------------------------- + +.. code-block:: sql + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`1.0` + * - :json:`2` + - :json:`"two"` + - :json:`2.0` + * - :json:`3` + - :json:`"three"` + - :sql:`NULL` + * - :json:`4` + - :json:`"four"` + - :sql:`NULL` + +.. code-block:: sql + + DELETE FROM T WHERE C IS NOT NULL; + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`3` + - :json:`"three"` + - :sql:`NULL` + * - :json:`4` + - :json:`"four"` + - :sql:`NULL` diff --git a/docs/sphinx/source/reference/sql_commands/DML/INSERT.diagram b/docs/sphinx/source/reference/sql_commands/DML/INSERT.diagram new file mode 100644 index 0000000000..80d4716b7c --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DML/INSERT.diagram @@ -0,0 +1,38 @@ +Diagram( + Stack( + Sequence( + Terminal('INSERT'), + Terminal('INTO'), + NonTerminal('tableName'), + Optional( + Sequence( + Terminal('('), + OneOrMore( + NonTerminal('columnName'), + Terminal(',') + ), + Terminal(')') + ) + ), + ), + Sequence( + Choice(0, + Sequence( + Terminal('VALUES'), + OneOrMore( + Sequence( + Terminal('('), + OneOrMore( + NonTerminal('literal'), + Terminal(',') + ), + Terminal(')'), + ), + Terminal(',') + ), + ), + NonTerminal('query') + ) + ) + ) +) \ No newline at end of file diff --git a/docs/sphinx/source/reference/sql_commands/DML/INSERT.rst b/docs/sphinx/source/reference/sql_commands/DML/INSERT.rst new file mode 100644 index 0000000000..3bb505d888 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DML/INSERT.rst @@ -0,0 +1,302 @@ +====== +INSERT +====== + +SQL command to insert rows in a given table. Rows to be inserted can be expressed via a :sql:`VALUES` clause or via a dedicated query. + +Syntax +====== + +.. raw:: html + :file: INSERT.diagram.svg + + +Parameters +========== + +``tableName`` + The name of the target table + +``columnName`` + The name of one of the target column in the target table + +``literal`` + A literal whose type must be compatible with the target column + +``query`` + A query whose result can be used to insert into the target table + +Examples +======== + +Insert a single row +------------------- + +.. code-block:: sql + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`1.0` + * - :json:`2` + - :json:`"two"` + - :json:`2.0` + + +.. code-block:: sql + + INSERT INTO T VALUES (3, 'three', 3.0); + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`1.0` + * - :json:`2` + - :json:`"two"` + - :json:`2.0` + * - :json:`3` + - :json:`"three"` + - :json:`3.0` + + +Insert multiple rows +-------------------- + +.. code-block:: sql + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`1.0` + * - :json:`2` + - :json:`"two"` + - :json:`2.0` + +.. code-block:: sql + + INSERT INTO T VALUES (3, 'three', 3.0), (4, 'four', 4.0); + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`1.0` + * - :json:`2` + - :json:`"two"` + - :json:`2.0` + * - :json:`3` + - :json:`"three"` + - :json:`3.0` + * - :json:`4` + - :json:`"four"` + - :json:`4.0` + + +Insert new rows without specifying all columns +---------------------------------------------- + +.. code-block:: sql + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`1.0` + * - :json:`2` + - :json:`"two"` + - :json:`2.0` + + +.. code-block:: sql + + INSERT INTO T(A, B) VALUES (3, 'three'), (4, 'four'); + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`1.0` + * - :json:`2` + - :json:`"two"` + - :json:`2.0` + * - :json:`3` + - :json:`"three"` + - :sql:`NULL` + * - :json:`4` + - :json:`"four"` + - :sql:`NULL` + + +Insert rows in a table with a :sql:`STRUCT` column +-------------------------------------------------- + +.. code-block:: sql + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`{ "S1": 10 , "S2": 100 }` + * - :json:`2` + - :json:`"two"` + - :json:`{ "S1": 20 , "S2": 200 }` + +.. code-block:: sql + + INSERT INTO T VALUES (3, 'three', (30, 300)), (4, 'four', (40, 400)); + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`{ "S1": 10 , "S2": 100 }` + * - :json:`2` + - :json:`"two"` + - :json:`{ "S1": 20 , "S2": 200 }` + * - :json:`3` + - :json:`"three"` + - :json:`{ "S1": 30 , "S2": 300 }` + * - :json:`4` + - :json:`"four"` + - :json:`{ "S1": 40 , "S2": 400 }` + + +Insert rows in a table with an ARRAY column +------------------------------------------- + +.. code-block:: sql + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`[ 1 ]` + * - :json:`2` + - :json:`"two"` + - :json:`[2, 20]` + + +.. code-block:: sql + + INSERT INTO T VALUES (3, 'three', [30, 300, 3000]), (4, 'four', [40, 400, 4000, 40000]); + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`[ 1 ]` + * - :json:`2` + - :json:`"two"` + - :json:`[2, 20]` + * - :json:`3` + - :json:`"three"` + - :json:`[3, 30, 300]` + * - :json:`4` + - :json:`"four"` + - :json:`[4, 40, 400, 4000]` + + +Insert from query +----------------- + +.. code-block:: sql + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`{ "S1": 10 , "S2": 100 }` + * - :json:`2` + - :json:`"two"` + - :json:`{ "S1": 20 , "S2": 200 }` + +.. code-block:: sql + + INSERT INTO T SELECT 3, B, C FROM T WHERE C.S1 = 20 + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`{ "S1": 10 , "S2": 100 }` + * - :json:`2` + - :json:`"two"` + - :json:`{ "S1": 20 , "S2": 200 }` + * - :json:`3` + - :json:`"two"` + - :json:`{ "S1": 20 , "S2": 200 }` diff --git a/docs/sphinx/source/reference/sql_commands/DML/UPDATE.diagram b/docs/sphinx/source/reference/sql_commands/DML/UPDATE.diagram new file mode 100644 index 0000000000..6f1e7aa6e9 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DML/UPDATE.diagram @@ -0,0 +1,28 @@ +Diagram( + Stack( + Sequence( + Terminal('UPDATE'), + NonTerminal('tableName'), + Terminal('SET'), + OneOrMore( + Sequence( + NonTerminal('columnName'), + Terminal('='), + Choice(0, + NonTerminal('literal'), + NonTerminal('identifier') + ) + ), + Terminal(',') + ), + ), + Sequence( + Optional( + Sequence( + Terminal('WHERE'), + NonTerminal('predicate') + ) + ) + ) + ) +) diff --git a/docs/sphinx/source/reference/sql_commands/DML/UPDATE.rst b/docs/sphinx/source/reference/sql_commands/DML/UPDATE.rst new file mode 100644 index 0000000000..09d122bbb8 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DML/UPDATE.rst @@ -0,0 +1,270 @@ +====== +UPDATE +====== + +SQL command to update rows in a table. + +Syntax +====== + +.. raw:: html + :file: UPDATE.diagram.svg + +Parameters +========== + +``tableName`` + The name of the target table + +``columnName`` + The name of one of the target column in the target table + +``literal`` + A literal whose type must be compatible with the target column + +``identifier`` + A database identifier whose type must be compatible with the target column + +``predicate`` + A predicate which should evaluate to true for the given row to be updated + +Examples +======== + +Update a column +--------------- + +.. code-block:: sql + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`1.0` + * - :json:`2` + - :json:`"two"` + - :json:`2.0` + * - :json:`3` + - :json:`"three"` + - :sql:`NULL` + * - :json:`4` + - :json:`"four"` + - :sql:`NULL` + +.. code-block:: sql + + UPDATE T SET C = 20; + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`20.0` + * - :json:`2` + - :json:`"two"` + - :json:`20.0` + * - :json:`3` + - :json:`"three"` + - :json:`20.0` + * - :json:`4` + - :json:`"four"` + - :json:`20.0` + + +Update a column for rows that match a certain predicate +------------------------------------------------------- + +.. code-block:: sql + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`1.0` + * - :json:`2` + - :json:`"two"` + - :json:`2.0` + * - :json:`3` + - :json:`"three"` + - :sql:`NULL` + * - :json:`4` + - :json:`"four"` + - :sql:`NULL` + + +.. code-block:: sql + + UPDATE T SET C = NULL WHERE C IS NOT NULL; + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :sql:`NULL` + * - :json:`2` + - :json:`"two"` + - :sql:`NULL` + * - :json:`3` + - :json:`"three"` + - :sql:`NULL` + * - :json:`4` + - :json:`"four"` + - :sql:`NULL` + + +Update multiple columns +----------------------- + +.. code-block:: sql + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`1.0` + * - :json:`2` + - :json:`"two"` + - :json:`2.0` + * - :json:`3` + - :json:`"three"` + - :sql:`NULL` + * - :json:`4` + - :json:`"four"` + - :sql:`NULL` + + +.. code-block:: sql + + UPDATE T SET B = 'zero', C = 0.0 WHERE C IS NULL; + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`1.0` + * - :json:`2` + - :json:`"two"` + - :json:`2.0` + * - :json:`3` + - :json:`"zero"` + - :json:`0.0` + * - :json:`4` + - :json:`"zero"` + - :json:`0.0` + +Update field inside a STRUCT +---------------------------- + +.. code-block:: sql + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`{ "S1": 10 , "S2": 100 }` + * - :json:`2` + - :json:`"two"` + - :json:`{ "S1": 20 , "S2": 200 }` + +.. code-block:: sql + + UPDATE T SET C.S1 = 45 WHERE C.S2 = 200; + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`{ "S1": 10 , "S2": 100 }` + * - :json:`2` + - :json:`"two"` + - :json:`{ "S1": 45 , "S2": 200 }` + + +Update STRUCT field +----------------------- + +.. code-block:: sql + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`{ "S1": 10 , "S2": 100 }` + * - :json:`2` + - :json:`"two"` + - :json:`{ "S1": 20 , "S2": 200 }` + + +.. code-block:: sql + + UPDATE T SET C = (0, 0) WHERE A = 2; + + SELECT * FROM T; + +.. list-table:: + :header-rows: 1 + + * - :sql:`A` + - :sql:`B` + - :sql:`C` + * - :json:`1` + - :json:`"one"` + - :json:`{ "S1": 10 , "S2": 100 }` + * - :json:`2` + - :json:`"two"` + - :json:`{ "S1": 0 , "S2": 0 }` + diff --git a/docs/sphinx/source/reference/sql_commands/DQL.diagram b/docs/sphinx/source/reference/sql_commands/DQL.diagram new file mode 100644 index 0000000000..27943324f6 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DQL.diagram @@ -0,0 +1,17 @@ +Diagram( + Stack( + Optional(Sequence(Terminal('WITH'), NonTerminal('withClause'))), + Sequence(Terminal('SELECT'), NonTerminal('selectClause')), + Optional(Sequence(Terminal('FROM'), NonTerminal('fromClause'))), + Optional(Sequence(Terminal('WHERE'), NonTerminal('whereClause'))), + Optional( + Sequence( + Terminal('GROUP BY'), NonTerminal('groupByClause'), + Optional( Sequence(Terminal('HAVING'), NonTerminal('havingClause')) ), + ) + ), + Optional( Sequence(Terminal('ORDER BY'), NonTerminal('orderByClause')) ), + Optional( Sequence(Terminal('WITH'), Terminal('CONTINUATION'), NonTerminal('continuation')) ) + ), +) + diff --git a/docs/sphinx/source/reference/sql_commands/DQL.rst b/docs/sphinx/source/reference/sql_commands/DQL.rst new file mode 100644 index 0000000000..4ba3a70675 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DQL.rst @@ -0,0 +1,29 @@ +=== +DQL +=== + +The Relational Layer supports a full query engine, complete with query planning and index selection. We generally recommend +using the query planner whenever possible over using the :doc:`../Direct_Access_Api` as it is more powerful and more capable, and +it is designed to make intelligent choices regarding query planning decisions like appropriate index selection. + +The language is heavily inspired by SQL. However, the Relational Layer *does* make some important changes to SQL to accommodate +the nature of its data model (particularly with regard to :ref:`struct <struct_types>` and :ref:`array <array_types>` data types). + +Syntax +###### + +.. raw:: html + :file: DQL.diagram.svg + +.. toctree:: + :maxdepth: 2 + + DQL/SELECT + DQL/WITH + DQL/WHERE + DQL/EXPLAIN + +.. toctree:: + :maxdepth: 2 + + DQL/Operators diff --git a/docs/sphinx/source/reference/sql_commands/DQL/EXPLAIN.diagram b/docs/sphinx/source/reference/sql_commands/DQL/EXPLAIN.diagram new file mode 100644 index 0000000000..3788ec243b --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DQL/EXPLAIN.diagram @@ -0,0 +1,4 @@ +Diagram( + Terminal('EXPLAIN'), + NonTerminal('query') +) \ No newline at end of file diff --git a/docs/sphinx/source/reference/sql_commands/DQL/EXPLAIN.rst b/docs/sphinx/source/reference/sql_commands/DQL/EXPLAIN.rst new file mode 100644 index 0000000000..c5b89c3577 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DQL/EXPLAIN.rst @@ -0,0 +1,121 @@ +======= +EXPLAIN +======= + +You can prepend any :sql:`SELECT` statement with the keyword :sql:`EXPLAIN` to display the physical plan that is used during the execution. + +Syntax +###### + +.. raw:: html + :file: EXPLAIN.diagram.svg + + +Parameters +########## + +``query`` + The query to explain. + +Returns +####### + +* :sql:`PLAN` + A textual representation of the plan +* :sql:`PLAN_HASH` + A hash that allows to quickly disambiguate between two plans +* :sql:`PLAN_DOT` + A dot representation of the plan +* :sql:`PLAN_GML` + A Graph Modelling Language (GML) representation of the plan +* :sql:`PLAN_CONTINUATION` + A byte array that contains information that is essential in the context of plan serialization + +Examples +######## + +TODO change examples to render dot graph and put actual plan hash + +Table scan +********** + +.. code-block:: sql + + EXPLAIN SELECT name FROM restaurant WHERE rest_no >= 44; + +.. list-table:: + :header-rows: 1 + :widths: 50 25 50 50 50 + + * - :sql:`PLAN` + - :sql:`PLAN_HASH` + - :sql:`PLAN_DOT` + - :sql:`PLAN_GML` + - :sql:`PLAN_CONTINUATION` + * - .. code-block:: + + SCAN(<,>) + | TFILTER RestaurantComplexRecord + | FILTER _.rest_no GREATER_THAN_OR_EQUALS 42 + | MAP (_.name AS name) + - :sql:`-1635569052` + - dot_graph + - gml_graph + - continuation + + +Index scan +********** + +.. code-block:: sql + + EXPLAIN + SELECT name + FROM RestaurantComplexRecord USE INDEX (record_name_idx) + WHERE rest_no > 10 + + +.. list-table:: + :header-rows: 1 + :widths: 50 25 50 50 50 + + * - :sql:`PLAN` + - :sql:`PLAN_HASH` + - :sql:`PLAN_DOT` + - :sql:`PLAN_GML` + - :sql:`PLAN_CONTINUATION` + * - .. code-block:: + + COVERING(record_name_idx <,> -> [name: KEY[1], rest_no: KEY[2]]) + | FILTER _.rest_no GREATER_THAN 10 + | MAP (_.name AS name) + - :sql:`-1543467542` + - dot_graph + - gml_graph + - continuation + + +.. code-block:: sql + + EXPLAIN + SELECT * + FROM RestaurantComplexRecord AS R + WHERE EXISTS (SELECT * FROM R.reviews AS RE WHERE RE.rating >= 9) + +.. list-table:: + :header-rows: 1 + :widths: 50 25 50 50 50 + + * - :sql:`PLAN` + - :sql:`PLAN_HASH` + - :sql:`PLAN_DOT` + - :sql:`PLAN_GML` + - :sql:`PLAN_CONTINUATION` + * - .. code-block:: + + ISCAN(mv1 [[9],>) + | MAP (_.0 AS rest_no, _.1 AS name, _.2 AS location, _.3 AS reviews, _.4 AS tags, _.5 AS customer, _.6 AS encoded_bytes) + - :sql:`-8677659052` + - dot_graph + - gml_graph + - continuation diff --git a/docs/sphinx/source/reference/sql_commands/DQL/Operators.rst b/docs/sphinx/source/reference/sql_commands/DQL/Operators.rst new file mode 100644 index 0000000000..59cd5c7878 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DQL/Operators.rst @@ -0,0 +1,21 @@ +========= +Operators +========= + +.. toctree:: + :hidden: + + Operators/Logical + Operators/Comparison + Operators/IS + +.. list-table:: + + * - Arithmetic Operators + - `+`, `-`, `*`, `/`, `%` + * - :ref:`Logical Operators <logical-operators>` + - `AND`, `OR`, `NOT` + * - :ref:`Comparison Operators <comparison-operators>` + - `<`, `>`, `<=`, `>=`, `=`, `!=` + * - :ref:`IS Operator <is-operators>` + - `IS`, `IS NOT` diff --git a/docs/sphinx/source/reference/sql_commands/DQL/Operators/Comparison.rst b/docs/sphinx/source/reference/sql_commands/DQL/Operators/Comparison.rst new file mode 100644 index 0000000000..cd90833873 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DQL/Operators/Comparison.rst @@ -0,0 +1,12 @@ +==================== +Comparison Operators +==================== + + +.. _comparison-operators: + +TODO: flesh out doc + +We support comparison operators: :sql:`>`, :sql:`>=`, :sql:`=`, :sql:`<`, :sql:`<=`, :sql:`!=` + +If the left or right operand is :sql:`NULL`, the comparison result is :sql:`NULL`. diff --git a/docs/sphinx/source/reference/sql_commands/DQL/Operators/IS.rst b/docs/sphinx/source/reference/sql_commands/DQL/Operators/IS.rst new file mode 100644 index 0000000000..cdeabf6994 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DQL/Operators/IS.rst @@ -0,0 +1,15 @@ +=========== +IS Operator +=========== + +.. _is-operators: + +TODO: flesh out doc + +For predicates on nullable fields, we support :sql:`IS [NOT] NULL`, :sql:`IS [NOT] UNKNOWN` operator, and +also :sql:`IS [NOT] true/false`. :sql:`NULL` and :sql:`UNKNOWN` are synonyms in these clauses. For Boolean +value fields, :sql:`IS [NOT] true/false` can be used to assume default values for unset fields during Boolean +predicates. For instance, +:sql:`x IS NOT true` is equivalent to :sql:`x IS NULL OR x = FALSE` (effectively treating unset fields as equivalent +to :sql:`false`). + diff --git a/docs/sphinx/source/reference/sql_commands/DQL/Operators/Logical.rst b/docs/sphinx/source/reference/sql_commands/DQL/Operators/Logical.rst new file mode 100644 index 0000000000..492dad3d25 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DQL/Operators/Logical.rst @@ -0,0 +1,45 @@ +================= +Logical Operators +================= + +.. _logical-operators: + +.. toctree:: + + Logical/EXISTS + +TODO: flesh out doc + +.. list-table:: + :header-rows: 1 + + * - Left Operand + - Right Operand + - OR + - AND + - XOR + * - TRUE + - NULL + - TRUE + - NULL + - NULL + * - FALSE + - NULL + - NULL + - FALSE + - NULL + * - NULL + - TRUE + - TRUE + - NULL + - NULL + * - NULL + - FALSE + - NULL + - FALSE + - NULL + * - NULL + - NULL + - NULL + - NULL + - NULL diff --git a/docs/sphinx/source/reference/sql_commands/DQL/Operators/Logical/EXISTS.diagram b/docs/sphinx/source/reference/sql_commands/DQL/Operators/Logical/EXISTS.diagram new file mode 100644 index 0000000000..25cb87b487 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DQL/Operators/Logical/EXISTS.diagram @@ -0,0 +1,4 @@ +Diagram( + Terminal('EXISTS'), + NonTerminal('subquery') +) diff --git a/docs/sphinx/source/reference/sql_commands/DQL/Operators/Logical/EXISTS.rst b/docs/sphinx/source/reference/sql_commands/DQL/Operators/Logical/EXISTS.rst new file mode 100644 index 0000000000..3e5d9abdd7 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DQL/Operators/Logical/EXISTS.rst @@ -0,0 +1,89 @@ +====== +EXISTS +====== + +Syntax +###### + +.. raw:: html + :file: EXISTS.diagram.svg + +Parameters +########## + +* :sql:`subquery` + A correlated subquery + +Returns +####### + +A Boolean that is :sql:`true` if and only if the the correlated subquery contains any row. + +Examples +######## + +Existence of an element within an array +--------------------------------------- + +This supports queries with existential predicates that could be used, for example, to check whether a certain repeated +field matches certain criteria. + +Let's assume the following table: + +.. code-block:: sql + + CREATE TYPE AS STRUCT restaurant_review (reviewer STRING, rating INT64); + CREATE TABLE restaurant ( + rest_no INT64, + name STRING, + reviews restaurant_review ARRAY, + PRIMARY KEY(rest_no)); + +and the following records: + +.. code-block:: json + + {"REST_NO": 42, "name": "Restaurant1", "reviews": + [{"reviewer": "Reviewer11", "rating": 2}, + {"reviewer": "Reviewer22", "rating": 5}, + {"reviewer": "Reviewer21", "rating": 1}]} + {"REST_NO": 43, "name": "Restaurant2", "reviews": + [{"reviewer": "Reviewer19", "rating": 1}]} + {"REST_NO": 44, "name": "Restaurant3", "reviews": + [{"reviewer": "Reviewer41", "rating": 3}, + {"reviewer": "Reviewer55", "rating": 5}]} + {"REST_NO": 45, "name": "Restaurant4", "reviews": + [{"reviewer": "Reviewer14", "rating": 3}, + {"reviewer": "Reviewer55", "rating": 2}]} + +For example, suppose we want to return a list of all restaurants with *at least* one +review with a (good) rating, that is, larger or equal to 5. Since :sql:`reviews` is a repeated field in :sql:`RestaurantRecord`, +it is not possible to query it via a simple predicate. However, we can use an existential predicate to iterate over +its values using a correlated alias to the parent table and apply, for each review, the predicate on its :sql:`rating`: + +.. code-block:: sql + + select * from restaurant as R where exists (select * from R.reviews where rating >= 5); + +For the data above, this query will return: + +.. list-table:: + :header-rows: 1 + + * - :sql:`rest_no` + - :sql:`name` + - :sql:`reviews` + * - :json:`42` + - :json:`"Restaurant1"` + - .. code-block:: json + + [{"reviewer": "Reviewer11","rating": "2"}, + {"reviewer": "Reviewer22","rating": "5"}, + {"reviewer": "Reviewer21","rating": "1"}] + * - :json:`44` + - :json:`"Restaurant3"` + - .. code-block:: json + + [{"reviewer": "Reviewer41","rating": "3"}, + {"reviewer": "Reviewer55","rating": "5"}] + diff --git a/docs/sphinx/source/reference/sql_commands/DQL/SELECT.diagram b/docs/sphinx/source/reference/sql_commands/DQL/SELECT.diagram new file mode 100644 index 0000000000..9ec47641a0 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DQL/SELECT.diagram @@ -0,0 +1,17 @@ +Diagram( + Terminal('SELECT'), + OneOrMore( + Sequence( + NonTerminal('selectExpression'), + Optional( + Sequence( + Terminal('AS'), + NonTerminal('label'), + ), + 'skip' + ), + ), + Terminal(','), + ), + NonTerminal('...'), +) \ No newline at end of file diff --git a/docs/sphinx/source/reference/sql_commands/DQL/SELECT.rst b/docs/sphinx/source/reference/sql_commands/DQL/SELECT.rst new file mode 100644 index 0000000000..c6540d013d --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DQL/SELECT.rst @@ -0,0 +1,127 @@ +====== +SELECT +====== + +Syntax +###### + +.. raw:: html + :file: SELECT.diagram.svg + +Parameters +########## + +* :sql:`selectExpression` + This can be a column, a column inside a :sql:`STRUCT`, or an expression + +* :sql:`label` + Label to use for the projection expression. The label will be the only way to access this expression at higher scopes + +Examples +######## + +To start, suppose we have the following table: + +.. code-block:: sql + + CREATE TYPE AS STRUCT restaurant_review (reviewer STRING, rating INT64); + CREATE TABLE restaurant ( + rest_no INT64, + name STRING, + reviews restaurant_review ARRAY, + PRIMARY KEY(rest_no)); + +and the following records: + +.. code-block:: json + + {"REST_NO": 42, "name": "Restaurant1", "reviews": + [{"reviewer": "Reviewer11", "rating": 2}, + {"reviewer": "Reviewer22", "rating": 5}, + {"reviewer": "Reviewer21", "rating": 1}]} + {"REST_NO": 43, "name": "Restaurant2", "reviews": + [{"reviewer": "Reviewer19", "rating": 1}]} + {"REST_NO": 44, "name": "Restaurant3", "reviews": + [{"reviewer": "Reviewer41", "rating": 3}, + {"reviewer": "Reviewer55", "rating": 5}]} + {"REST_NO": 45, "name": "Restaurant4", "reviews": + [{"reviewer": "Reviewer14", "rating": 3}, + {"reviewer": "Reviewer55", "rating": 2}]} + +Projecting all columns +---------------------- + +It is possible to query a table using normal :sql:`select` with optional predicates. For example: + +.. code-block:: sql + + SELECT * FROM restaurant; + +.. list-table:: + :header-rows: 1 + + * - :sql:`rest_no` + - :sql:`name` + - :sql:`reviews` + * - :json:`42` + - :json:`"Restaurant1"` + - .. code-block:: json + + [{"reviewer": "Reviewer11","rating": "2"}, + {"reviewer": "Reviewer22","rating": "5"}, + {"reviewer": "Reviewer21","rating": "1"}] + * - :json:`43` + - :json:`"Restaurant2"` + - .. code-block:: json + + [{"reviewer": "Reviewer19","rating": "1"}] + * - :json:`44` + - :json:`"Restaurant3"` + - .. code-block:: json + + [{"reviewer": "Reviewer41","rating": "3"}, + {"reviewer": "Reviewer55","rating": "5"}] + * - :json:`45` + - :json:`"Restaurant4"` + - .. code-block:: json + + [{"reviewer": "Reviewer14","rating": "3"}, + {"reviewer": "Reviewer55","rating": "2"}] + + +Note how :sql:`*` resolves all the attributes (top-level fields) of table :sql:`restaurant` and returns them as individual columns in the +result set. + +Projecting one column +--------------------- + +It is also possible to project individual columns from a table, for example, suppose we want to only project restaurant names: + +.. code-block:: sql + + select name from restaurant + +.. list-table:: + :header-rows: 1 + + * - :sql:`name` + * - :json:`"Restaurant1"` + * - :json:`"Restaurant2"` + * - :json:`"Restaurant3"` + * - :json:`"Restaurant4"` + + +Projecting column inside nested a nested :sql:`STRUCT` +------------------------------------------------------ + +It is also possible to project a nested field provided non of its parents is repeated: + +.. code-block:: sql + + CREATE TYPE AS STRUCT A ( b B ); + CREATE TYPE AS STRUCT B ( c C ); + CREATE TYPE AS STRUCT C ( d D ); + CREATE TYPE AS STRUCT D ( e E ); + CREATE TYPE AS STRUCT E ( f int64 ); + CREATE TABLE tbl1 (id int64, c C, a A, PRIMARY KEY(id)); + SELECT a.b.c.d.e.f FROM tbl1; diff --git a/docs/sphinx/source/reference/sql_commands/DQL/WHERE.diagram b/docs/sphinx/source/reference/sql_commands/DQL/WHERE.diagram new file mode 100644 index 0000000000..37d2fa8ab2 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DQL/WHERE.diagram @@ -0,0 +1,4 @@ +Diagram( + Terminal('WHERE'), + NonTerminal('booleanExpression') +) \ No newline at end of file diff --git a/docs/sphinx/source/reference/sql_commands/DQL/WHERE.rst b/docs/sphinx/source/reference/sql_commands/DQL/WHERE.rst new file mode 100644 index 0000000000..8fdabc6c75 --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DQL/WHERE.rst @@ -0,0 +1,62 @@ +##### +WHERE +##### + +The query languages allows defining simple filters as well as existential filters. A simple filter is a filter that compares +a single field to a constant expression, or a constant expression to another (in that case, the expression is evaluated immediately +and actual query execution will not happen if, e.g., the predicate evaluates to :sql:`false`). + +Syntax +###### + +.. raw:: html + :file: WHERE.diagram.svg + +Parameter +######### + +* :sql:`booleanExpression` + A Boolean expression. If :sql:`true`, the row should be included. If :sql:`false` or :sql:`NULL`, the row should not be included. + +Examples +######## + +Single predicate +---------------- + +Suppose that we want to filter all restaurants from the example above whose :sql:`rest_no` is larger or equal to 44: + +.. code-block:: sql + + select rest_no, name from restaurant where rest_no >= 44; + +will return: + +.. list-table:: + :header-rows: 1 + + * - :sql:`rest_no` + - :sql:`name` + * - 44 + - Restaurant3 + * - 45 + - Restaurant4 + +Multiple predicates +------------------- + +It is also possible to have a conjunction of predicates, say we want to view restaurants whose :sql:`rest_no` is between 43 and 45 + +.. code-block:: sql + + select rest_no, name from restaurant where rest_no > 43 and rest_no < 45; + +will return + +.. list-table:: + :header-rows: 1 + + * - :sql:`rest_no` + - :sql:`name` + * - 44 + - Restaurant3 diff --git a/docs/sphinx/source/reference/sql_commands/DQL/WITH.diagram b/docs/sphinx/source/reference/sql_commands/DQL/WITH.diagram new file mode 100644 index 0000000000..210d5e8b1d --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DQL/WITH.diagram @@ -0,0 +1,29 @@ +Diagram( + Stack( + Terminal('WITH'), + OneOrMore( + Sequence( + NonTerminal('cteName'), + Optional( + Sequence( + Terminal('('), + OneOrMore( + NonTerminal('columnAlias'), + Terminal(','), + ), + Terminal(')'), + ), + 'skip', + ), + Optional( + Terminal('AS'), + ), + Terminal('('), + NonTerminal('cteQuery'), + Terminal(')'), + ), + Terminal(','), + ), + NonTerminal('query') + ) +) \ No newline at end of file diff --git a/docs/sphinx/source/reference/sql_commands/DQL/WITH.rst b/docs/sphinx/source/reference/sql_commands/DQL/WITH.rst new file mode 100644 index 0000000000..12f058c54a --- /dev/null +++ b/docs/sphinx/source/reference/sql_commands/DQL/WITH.rst @@ -0,0 +1,92 @@ +##### +WITH +##### + +A common table expression (CTE) defines a named query that can referenced multiple times within the scope of a SQL statement. +CTEs are visible within the query block in which they are defined, and they can nested. + +Only non-recursive CTEs are supported at the moment. + +Syntax +###### + +.. raw:: html + :file: WITH.diagram.svg + +Parameter +######### + +* :sql:`cteName` + The name of the common table expression. +* :sql:`columnAlias` + The alias of the corresponding column of the CTE query. +* :sql:`cteQuery` + The common table expression query. +* :sql:`query` + The query the can optionally reference the CTE query via its given `cteName`. + +Example +####### + +Simple CTE +---------- + +Suppose we have the following tables representing a simple employee / department model: + +.. code-block:: sql + + create table employees(id bigint, name string, dept bigint, primary key(id)) + insert into employees values (1, 'Dylan', 42), + (2, 'Mark', 42), + (3, 'Helly', 42), + (4, 'Irving', 42), + (5, 'Seth', 1), + (50, 'Burt', 44) + +Here is a simple CTE query: + +.. code-block:: sql + + with macro_data_refinement_employees(name) as (select name from employees where dept = 42) + select * from macro_data_refinement_employees + +we get the following result: + +.. list-table:: + :header-rows: 1 + + * - :sql:`name` + * - :json:`"Dylan"` + * - :json:`"Mark"` + * - :json:`"Irving"` + * - :json:`"Helly"` + +Another CTE example +------------------- + +Suppose we want to find out which restaurant has an average rating higher than 3, we can leverage CTE to break the query +into two logical parts, one part for calculating the restaurant names and their averages: + +.. code-block:: sql + + select name, ar + from restaurant as a, (select avg(rating) as ar from a.reviews) as x + +The second part will operate on the result set of the first query, and filter out all the columns less or equal to 3, to +do this, we wrap the first part in a CTE and use it in the filtering query: + +.. code-block:: sql + + with ratings(name, avgRating) + as (select name, ar from restaurant as a, (select avg(rating) as ar from a.reviews) as x) + select name, avgRating from ratings where avgRating > 3.0f + +We get the desired result: + +.. list-table:: + :header-rows: 1 + + * - :sql:`name` + - :sql:`avgRating` + * - :json:`"Restaurant44"` + - :json:`4.0` diff --git a/docs/sphinx/source/reference/sql_types.rst b/docs/sphinx/source/reference/sql_types.rst new file mode 100644 index 0000000000..07dcf3944f --- /dev/null +++ b/docs/sphinx/source/reference/sql_types.rst @@ -0,0 +1,85 @@ +============== +SQL Data Types +============== + +.. _sql-types: + + +Primitive Types +############### + +As with most relational databases, the FDB Relational Layer supports the expected types--referred to as "primitive types" in Relational Layer parlance: + +* Strings (STRING) +* Scalars (INTEGER/BIGINT) +* Floating Point (FLOAT/DOUBLE) +* Booleans (BOOLEAN) +* byte arrays (BYTES) + +In addition, the Relational Layer supports two distinct user-definable types that are core to how it operates: `struct types`_ and `array types`_. + + +.. _struct_types: + +Struct Types +############ + +You can define a *struct type* (often interchangeably referred to as a *nested type*). A struct is a tuple of columns that allow the same types as a table does, but does _not_ have a primary key. Struct types are "nested" within another owning type, and are stored in the same location as their owning record. For example, a table :sql:`foo` can have the following layout (using the :doc:`DDL <sql_commands/DDL>` syntax): + + +.. code-block:: sql + + CREATE TYPE AS STRUCT nested_type (d INT64, e STRING); + CREATE TABLE foo (a STRING, b DOUBLE, c nested_type, PRIMARY KEY(a)); + +In this example, :sql:`nested_type` is a struct within the table `foo`, and its full contents are materialized alongside the full record for each entry in the `foo` table. + +Struct types can have columns which are themselves struct types. Thus, this example is fully allowed: + +.. code-block:: sql + + CREATE TYPE AS STRUCT nested_nested_type (f STRING, g STRING); + CREATE TYPE AS STRUCT nested_type (d INT64, e STRING, f nested_nested_type); + CREATE TABLE foo (a STRING, b DOUBLE, c nested_type, PRIMARY KEY(a)); + +In this example, :sql:`nested_type` is a struct within the table :sql:`foo`, and :sql:`nested_nested_type` is a struct within the type :sql:`nested_type`. + +The Relational Layer makes no direct limitations on how many structs can be nested within a single type, nor does it limit how deeply nested struct types can be. It is probably worth noting that there are practical limits, like the JVM stack size, that should discourage an adopter from designing types which are nested thousands deep. The general expectation is that nesting depth is probably within the 10s, not 1000s. Note that such "soft" limits may be restructured into hard limits (such as throwing an error if structs are nested more than X level deep) in the future. + +.. _array_types: + +Array types +########### + +The Relational DDL also supports the definition of *array types*. An array is a (finite) collection of records which share the same layout, and which are contained by an owning type. The elements of an array have their own layout, and can be one or more of any accepted column types. + +Thus, an array could be of a single primitive type: + +.. code-block:: sql + + CREATE TABLE foo (a STRING, b STRING ARRAY); + +In this example, `b` is an array with a single :sql:`STRING` column. + +Arrays can also be created with struct columns: + +.. code-block:: sql + + CREATE TYPE AS STRUCT nested_struct (b STRING, d STRING); + CREATE TABLE structArray (a STRING, c nested_struct array); + +In this example, `c` is an array, and each record within the array is a struct of type :sql:`nested_struct`. You can generally treat an array as a "nested `ResultSet`"--that is to say, you can just pull up a `ResultSet` of an array type, and interrogate it as if it were the output of its own query. + +It is possible to nest arrays within structs, and structs within arrays, to an arbitrary depth (limited by the JVM's stack size, currently). + +NULL Semantics +############## + +For any unset primitive type or struct type fields, queries will return NULL for the column, unless default values are defined. + +For array type fields: + +* If the whole array is unset, query returns NULL. +* If the array is set to empty, query returns empty list. +* All elements in the array should be set, arrays like :sql:`[1, NULL, 2, NULL]` are not supported. + diff --git a/docs/sphinx/source/reference/understanding_bitmap.rst b/docs/sphinx/source/reference/understanding_bitmap.rst new file mode 100644 index 0000000000..9a560495ca --- /dev/null +++ b/docs/sphinx/source/reference/understanding_bitmap.rst @@ -0,0 +1,69 @@ +=================================================== +Understanding How Bitmaps Identify Distinct Values +=================================================== + +.. _understanding-bitmaps: + +A bitmap is effectively a byte array in which each bit (zero-based) represents existence (bit = 1) or absence (bit = 0) +of an integer in a table. For example, + +.. code-block:: sql + + SELECT bitmap_construct_agg(val) AS bitmap FROM bitmap_test_values; + +Returns a byte array ``bitmap`` where if ``bitmap[i][j] = 1``, it means that ``value = 8 * i + j`` exists in the table. + +The data can be sparse and very big. To be more efficient, we group the data into "buckets". A distinct value is represented +by the combination of a "bucket number" and a bit that is set in that bitmap. To identify the bucket and the bit that +represents a specific value, use the following functions: + +* :ref:`BITMAP_BUCKET_NUMBER <bitmap-bucket-number>` to find the bucket. + +* :ref:`BITMAP_BIT_POSITION <bitmap-bit-position>` to find the zero-based position of the bit. + +In the FDB Relational Layer, the fixed bitmap bucket size is 10,000 bits (1250 bytes). +For example: + +.. list-table:: + :header-rows: 1 + + * - :sql:`value` + - :sql:`BITMAP_BUCKET_NUMBER` + - :sql:`BITMAP_BIT_POSITION` + * - :json:`1` + - :json:`0` + - :json:`1` + * - :json:`9999` + - :json:`0` + - :json:`9999` + * - :json:`10000` + - :json:`10000` + - :json:`0` + +If we create and populate the table ``bitmap_test_values``: + +.. code-block:: sql + + CREATE TABLE bitmap_test_values (val INT); + insert into bitmap_test_values values (1), (20003); + +Then the following query: + +.. code-block:: sql + + select bitmap_bucket_number(val) as offset, + bitmap_construct_agg(bitmap_bit_position(val)) as bitmap + from bitmap_test_values + group by offset; + +Returns: + +.. list-table:: + :header-rows: 1 + + * - :sql:`OFFSET` + - :sql:`BITMAP` + * - :json:`0` + - :json:`[b'00000010, 0, ...0]` + * - :json:`2` + - :json:`[b'00001000, 0, ...0]` diff --git a/gradle/sphinx.gradle b/gradle/sphinx.gradle new file mode 100644 index 0000000000..9e412e7694 --- /dev/null +++ b/gradle/sphinx.gradle @@ -0,0 +1,143 @@ +/* + * sphinx.gradle + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var sphinxRoot = "${rootDir}/docs/sphinx" +var venvDir = "${sphinxRoot}/.venv" +var sphinxBuildDir = "${sphinxRoot}/.out" + +var documented_subprojects = [ + "fdb-java-annotations", + "fdb-extensions", + "fdb-record-layer-core", + "fdb-record-layer-icu", + "fdb-record-layer-lucene", + "fdb-record-layer-spatial", + "fdb-relational-api", + "fdb-relational-core", + "fdb-relational-cli", + "fdb-relational-grpc", + "fdb-relational-jdbc", +] + +task sphinxEnv { + inputs.file("${sphinxRoot}/requirements.txt") + outputs.dir(venvDir) + + doLast { + exec { + commandLine 'python3', '-m', 'venv', venvDir + } + exec { + commandLine "${venvDir}/bin/pip", 'install', '-r', "${sphinxRoot}/requirements.txt" + } + } +} + +// Create svg files for railroad diagrams from specification files. These images are included +// in the docs +task generateRailroadDiagrams(type: Exec) { + inputs.file("${sphinxRoot}/generate_railroad_svg.py") + fileTree("${sphinxRoot}/source") { + include("**/*.diagram") + }.forEach { + inputs.file(it) + outputs.file(it.getPath() + ".svg") + } + + dependsOn sphinxEnv + commandLine "${venvDir}/bin/python", "${sphinxRoot}/generate_railroad_svg.py" +} + +// To include the API Javadocs in the final build, we need to have a place for the sphinx build +// to link to. Create temporary placeholders for each one. Later, we will override the location +// with the generated javadoc. Also, generates a table of contents so that we have all of the +// subprojects listed +task createSphinxApiPlaceholders { + var apiTemplatePath = "${sphinxRoot}/source/api/api.md.template" + inputs.file(apiTemplatePath) + var inputPath = "${sphinxRoot}/source/api/index.md.template" + inputs.file(inputPath) + var outputPath = "${sphinxRoot}/source/api/index.md" + outputs.file(outputPath) + + String baseInput = new File(inputPath).getText('UTF-8') + String outputText = baseInput + "\n```{toctree}\n\n" + + documented_subprojects.forEach { + outputText += "${it} <${it}/index>\n" + File placeholderDir = new File("${sphinxRoot}/source/api/${it}") + if (!placeholderDir.exists()) { + placeholderDir.mkdirs() + } + outputs.dir(placeholderDir) + } + outputText += "```\n" + + doLast { + new File(outputPath).text = outputText + documented_subprojects.forEach { + File placeholderDir = new File("${sphinxRoot}/source/api/${it}") + File placeholderIndex = new File(placeholderDir.getPath() + "/index.md") + + String placeholderText = "# ${it}\n\n" + new File(apiTemplatePath).text + placeholderIndex.text = placeholderText + } + } +} + +// Build the sphinx documentation +task sphinxDocs(type: Exec) { + inputs.dir(venvDir) + inputs.dir("${sphinxRoot}/source") + outputs.dir("${sphinxBuildDir}/html") + + dependsOn sphinxEnv + dependsOn generateRailroadDiagrams + dependsOn createSphinxApiPlaceholders + commandLine "${venvDir}/bin/sphinx-build", '-M', 'html', "${sphinxRoot}/source", sphinxBuildDir +} + +// Compile the final documentation site by copying the generated javadoc into the appropriate +// location in the build +task documentationSite(type: Copy) { + dependsOn(tasks.javadoc) + dependsOn(tasks.sphinxDocs) + + into("${sphinxBuildDir}/html/api") + documented_subprojects.forEach { proj -> + from(project(":${proj}").javadoc) { + into("${proj}") + } + } +} + +// Clean the sphinx build +task sphinxClean(type: Delete) { + delete(sphinxBuildDir) + delete fileTree("${sphinxRoot}/source") { + include "**/*.diagram.svg" + } + file("${sphinxRoot}/source/api").listFiles().each { + if (!it.getPath().endsWith(".md.template")) { + delete(it) + } + } +} +clean.dependsOn(sphinxClean)