diff --git a/.github/release.yml b/.github/release.yml deleted file mode 100644 index 2589eb57b1..0000000000 --- a/.github/release.yml +++ /dev/null @@ -1,24 +0,0 @@ -changelog: - # TODO exclude commits from release process - categories: - - title: Breaking Changes - labels: - - "breaking change" - - title: New Features - labels: - - enhancement - - title: Documentation improvements - labels: - - documentation - - title: Bug Fixes - labels: - - "bug fix" - - title: Performance Improvements - labels: - - performance - - title: Dependency Updates - labels: - - dependencies - - title: Other Changes - labels: - - "*" diff --git a/.github/workflows/patch_release.yml b/.github/workflows/patch_release.yml index 74990fd2e6..df08bcea17 100644 --- a/.github/workflows/patch_release.yml +++ b/.github/workflows/patch_release.yml @@ -23,28 +23,25 @@ jobs: with: ref: ${{ inputs.branch }} ssh-key: ${{ secrets.DEPLOY_KEY }} + fetch-tags: true + # fetch all the history to make sure that we have the last release + # I tried fetching part of the history, but I just couldn't get it to work, and fetching all still takes like 5s + fetch-depth: 0 - name: Setup Base Environment uses: ./actions/setup-base-env - name: Setup FDB uses: ./actions/setup-fdb - - # Push a version bump back to main. There are failure scenarios that can result - # in published artifacts but an erroneous build, so it's safer to bump the version - # at the beginning - name: Configure Git run: | git config --global user.name 'FoundationDB CI' git config --global user.email 'foundationdb_ci@apple.com' - - name: Increment Version - run: python build/versionutils.py gradle.properties -u PATCH --increment --commit - - name: Push Version Update - run: git push - name: Build and publish release uses: ./actions/release-build-publish with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }} + update_type: PATCH env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} MAVEN_USER: ${{ secrets.MAVEN_USER }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 096e9014e6..9ed62da8d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,29 +16,26 @@ jobs: uses: actions/checkout@v4.2.2 with: ssh-key: ${{ secrets.DEPLOY_KEY }} + fetch-tags: true + # fetch all the history to make sure that we have the last release + # I tried fetching part of the history, but I just couldn't get it to work, and fetching all still takes like 5s + fetch-depth: 0 - name: Setup Base Environment id: setup-base uses: ./actions/setup-base-env - name: Setup FDB uses: ./actions/setup-fdb - - # Push a version bump back to main. There are failure scenarios that can result - # in published artifacts but an erroneous build, so it's safer to bump the version - # at the beginning - name: Configure git run: | git config --global user.name 'FoundationDB CI' git config --global user.email 'foundationdb_ci@apple.com' - - name: Increment version - run: python build/versionutils.py gradle.properties --increment --commit - - name: Push version increment - run: git push - name: Build and publish uses: ./actions/release-build-publish with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }} + update_type: BUILD env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} MAVEN_USER: ${{ secrets.MAVEN_USER }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c3aa9be660..d0fbdf6abc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,7 +71,7 @@ importantly, it keeps the community open and communicative. ### Opening a Pull Request -When opening a pull request (PR), please ensure that the title of the pull request +When opening a pull request (PR), please ensure that the description of the pull request or the commit message indicates the issue that it is addressing (see [Closing issues with keywords](https://help.github.com/articles/closing-issues-using-keywords/)). For example: @@ -82,12 +82,28 @@ This will automatically create an association between the PR and the issue that it is addressing and, upon merging of the PR into the main code line, will automatically mark the issue as resolved. -If your pull request results in a user-visible change to the Record Layer, you should -also update the [release notes](docs/sphinx/source/ReleaseNotes.md). For most changes, it -is sufficient to fill in one of the bullets in the "next release" section of that -document. You should include a short description of the change as well as filling in -the issue number. The "next release" section is commented out, so the change won't -be visible in our documentation until the next time a release is cut. +PRs should have titles that clearly indicate what the change is accomplishing +as we use these to generate release notes. You can glance at +[release notes](docs/sphinx/source/ReleaseNotes.md) for inspiration. + +They should also have one of the following labels: +- `breaking change`: For any breaking change +- `enhancement`: For any new feature or enhancement +- `bug fix`: For bug fixes +- `performance`: For performance improvements +- `dependencies`: For updates to dependency versions +- `build improvement`: For updates to our build system that shouldn't have user visible impacts +- `testing improvement`: For new test coverage, or improvements to testing infrastructure +- `documentation`: For improvements to our documentation + +(Note: `build improvement`/`testing improvement`/`documentation` are combined in one +grouping in the release notes, that is collapsed). + +[release-notes-config.json](build/release-notes-config.json) describes how labels are +converted into categories in the release notes. +If a PR has multiple labels, it will appear in the first grouping as listed above / +in [release-notes-config.json](build/release-notes-config.json) (i.e., if it is has +`enhancement` and `dependencies`, it will only appear under `enhancement`). ### Reporting issues diff --git a/actions/release-build-publish/action.yml b/actions/release-build-publish/action.yml index 7756d78332..a0a795691f 100644 --- a/actions/release-build-publish/action.yml +++ b/actions/release-build-publish/action.yml @@ -7,15 +7,58 @@ inputs: gpg_passphrase: description: 'GPG passphrase for artifact signing' required: true + update_type: + description: 'One of MAJOR, MINOR, BUILD, PATCH' + required: true runs: using: "composite" steps: - - name: Get version - id: get_version + - name: Get old version + id: get_old_version + shell: bash + run: | + echo "version=$(python build/versionutils.py gradle.properties)" >> "$GITHUB_OUTPUT" + # Push a version bump back to main. There are failure scenarios that can result + # in published artifacts but an erroneous build, so it's safer to bump the version + # at the beginning + - name: Increment version + shell: bash + run: python build/versionutils.py gradle.properties --increment --commit -u ${{ inputs.update_type }} + - name: Get new version + id: get_new_version shell: bash run: | echo "version=$(python build/versionutils.py gradle.properties)" >> "$GITHUB_OUTPUT" + # We also want to push the tag, because that will be used for the next release's release notes + - name: Create tag + shell: bash + run: git tag -m "Release ${{ steps.get_new_version.outputs.version }}" -f "${{ steps.get_new_version.outputs.version }}" + + # We want to do this before anything else, because if the later steps fail, we want to make sure that the full + # change log includes all changes, even if they reference a release that was never actually published. + - name: Update release notes + shell: bash + run: | + python ./build/create_release_notes.py \ + --config ./build/release-notes-config.json \ + --release-notes-md docs/sphinx/source/ReleaseNotes.md \ + --skip-commit $(git log -n 1 --format=%H HEAD) \ + --repository ${{ github.repository }} \ + --commit \ + ${{ steps.get_old_version.outputs.version }} ${{ steps.get_new_version.outputs.version }} + # We move the tag to after the release notes are updated so that later steps (i.e. sphinx) will pick up the udpated + # release notes + - name: Move tag to HEAD + shell: bash + run: git tag -m "Release ${{ steps.get_new_version.outputs.version }}" -f "${{ steps.get_new_version.outputs.version }}" + + # push the changes to gradle.properties, the release notes, and the tag as one operation, so if it fails, + # it will be as if the release never did anything + - name: Push Version Update + shell: bash + run: git push origin HEAD "${{ steps.get_new_version.outputs.version }}" + - name: Run Gradle Test uses: ./actions/gradle-test with: @@ -29,24 +72,16 @@ runs: ORG_GRADLE_PROJECT_signingPassword: ${{ inputs.gpg_passphrase }} # Post release: Update various files which reference version - - name: Update release notes - shell: bash - run: ARTIFACT_VERSION="${{ steps.get_version.outputs.version }}" ./build/update_release_notes.bash + # Updating the yaml files has to be done after the tests complete, or it will mark tests as failing that aren't + # supported by the previous version. - name: Update YAML test file versions uses: ./actions/run-gradle with: gradle_command: updateYamsql -PreleaseBuild=true - name: Commit YAML updates shell: bash - run: python ./build/commit_yamsql_updates.py "${{ steps.get_version.outputs.version }}" + run: python ./build/commit_yamsql_updates.py "${{ steps.get_new_version.outputs.version }}" - # Create and push the tag - - name: Create tag - shell: bash - run: git tag -m "Release ${{ steps.get_version.outputs.version }}" -f "${{ steps.get_version.outputs.version }}" - - name: Push tag - shell: bash - run: git push origin "${{ steps.get_version.outputs.version }}" - name: Push Updates id: push_updates shell: bash @@ -62,12 +97,12 @@ runs: with: branch: release-build branch-suffix: timestamp - title: "Updates for ${{ steps.get_version.outputs.version }} release" + title: "Updates for ${{ steps.get_new_version.outputs.version }} release" sign-commits: true body: | - Updates from release for version ${{ steps.get_version.outputs.version }}. Conflicts during the build prevented automatic updating. Please resolve conflicts by checking out the current branch, merging, and then deleting this branch. + Updates from release for version ${{ steps.get_new_version.outputs.version }}. Conflicts during the build prevented automatic updating. Please resolve conflicts by checking out the current branch, merging, and then deleting this branch. # Creating the PR can change the current branch. Explicitly check out the tag here for downstream builds - name: Revert to tag shell: bash - run: git checkout "${{ steps.get_version.outputs.version }}" + run: git checkout "${{ steps.get_new_version.outputs.version }}" diff --git a/build/create_release_notes.py b/build/create_release_notes.py new file mode 100755 index 0000000000..adf4b92bab --- /dev/null +++ b/build/create_release_notes.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 + +# +# create_release_notes.py +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2015-2025 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import argparse +from collections import defaultdict +import json +import re +import subprocess +import sys + +def run(command): + ''' Run a command, returning its output + If the command fails, the output will be dumped, and an exception thrown + ''' + try: + process = subprocess.run(command, check=True, capture_output=True, text=True) + return process.stdout + except subprocess.CalledProcessError as e: + print("Failed: " + str(e.cmd)) + print(e.stdout) + print(e.stderr) + raise Exception(f"Failed to run command ({e.returncode}): {e.cmd}") + +def get_commits(old_version, new_version, skip_commits): + '''Return a list of tuples for each commit between the two versions + The tuples will be (hash, commit subject) + ''' + raw_log = run(['git', 'log', '--pretty=%H %s', old_version + ".." + new_version]) + commit_pairs = [commit.split(' ', maxsplit=1) for commit in raw_log.splitlines()] + return [commit for commit in commit_pairs if not commit[0] in skip_commits] + +def get_pr(commit_hash, pr_cache, repository): + '''Get the PRs associated with the given commit hash. + pr_cache: A path to a directory to cache PR results, or None. + This will return the raw pr info from github associated with the commit, + parsed into python objects. + ''' + if (pr_cache is not None): + try: + with open(pr_cache + "/" + commit_hash + ".json", 'r') as fin: + return json.load(fin) + except: + pass + raw_info = run(['gh', 'api', f'/repos/{repository}/commits/{commit_hash}/pulls']) + info = json.loads(raw_info) + if (pr_cache is not None): + with open(pr_cache + "/" + commit_hash + ".json", 'w') as fout: + json.dump(info, fout) + return info + +def get_prs(commits, pr_cache, repository): + ''' Return all the prs for the given commits, optionally using the cache. + pr_cache: A path to a directory to cache PR results or None. + Returns a list of tuple pairs, the first being the PR info, and the second + being the commit. + ''' + return [(get_pr(commit[0], pr_cache, repository), commit) for commit in commits] + +def dedup_prs(prs): + ''' Remove any duplicate prs from the list based on urls. + The prs should be the output of get_prs. + Returns a list with the first pair that references a given + pr. + ''' + found = set() + dedupped = [] + for pr, commit in prs: + if pr is None or len(pr) == 0: + print("No PR: " + str(commit)) + else: + if len(pr) != 1: + print("Too many PRs " + str(pr) + " " + str(commit)) + else: + if not pr[0]['url'] in found: + found.add(pr[0]['url']) + dedupped.append((pr, commit)) + return dedupped + +def get_category(pr, label_config, commit): + ''' Get the appropriate category based on the labels in the given pr.''' + main_label = None + label_names = [label['name'] for label in pr['labels']] + for category in label_config['categories']: + for label in category['labels']: + if label in label_names: + return category['title'] + print(f"No label: {pr['html_url']} {str(label_names)} {str(commit)}") + return label_config['catch_all'] + +def generate_note(prs, commit, label_config): + ''' Generate a release note for a single category, returning a pair of + the category for the note, and the text as markdown ''' + if len(prs) == 0: + return ("Direct Commit", commit[1]) + if len(prs) > 1: + print("Too many PRs?") + pr = prs[0] + category = get_category(pr, label_config, commit) + text = f'* {pr["title"]} - ' + \ + f'[PR #{pr["number"]}]({pr["html_url"]})' + return (category, text) + +def format_notes(notes, label_config, old_version, new_version, repository): + ''' Format a list of notes for the changes between the two given versions ''' + grouping = defaultdict(list) + for (category, line) in notes: + grouping[category].append(line) + text = '' + categories = label_config['categories'] + [{"title": label_config['catch_all']}] + for category in categories: + title = category['title'] + if title in grouping: + notes = grouping[title] + put_in_summary = category.get('collapsed', False) + # We use

rather than #### because #### confuses sphinx + if put_in_summary: + text += f"\n
\n\n\n

{title} (click to expand)

\n\n\n\n" + else: + text += f'

{title}

\n\n' + for note in notes: + text += f"{note}\n" + if put_in_summary: + text += '\n
\n' + text += f"\n\n**[Full Changelog ({old_version}...{new_version})](https://github.com/{repository}/compare/{old_version}...{new_version})**" + text += f"\n\n\n" + return text + +def replace_note(lines, note): + ''' Insert the given formatted release notes in to ReleaseNotes.md + at the appropriate location. + lines: The contents of ReleaseNotes.md split into lines + note: The release notes for a given version + Returns a new lits of lines, with the new notes inserted''' + print(f"Inserting note {note.old_version} -> {note.new_version}") + new_lines = [] + added = False + for line in lines: + if not added and line == note.old_version_header(): + new_lines.append(note.new_version_header()) + new_lines.append('') + new_lines.append(note.notes) + new_lines.append('') + new_lines.append(line) + added = True + elif not added and note.greater_than_minor_version(line): + new_lines.append(note.new_minor_version_header()) + new_lines.append('') + new_lines.append(note.new_version_header()) + new_lines.append('') + new_lines.append(note.notes) + new_lines.append('') + new_lines.append(line) + added = True + else: + new_lines.append(line) + if not added: + raise Exception(f"Could not find note for {note.old_version} -> {note.new_version}") + return new_lines + +def replace_notes(notes, filename): + ''' Insert all the given notes into the given file ''' + with open(filename, 'r') as fin: + lines = fin.read().split('\n') + for note in notes: + lines = replace_note(lines, note) + with open(filename, 'w') as fout: + fout.write('\n'.join(lines)) + print(f'Updated {filename} with new release notes from {notes[0].old_version} to {notes[-1].new_version}') + +def commit_release_notes(filename, new_versions): + ''' Commit the updates to the release notes ''' + message = f"Updating release notes for {new_versions[-1]}" + if len(new_versions) > 1: + message += f"\n\nAlso including versions {' '.join(new_versions[:-1])}" + subprocess.run(['git', 'commit', '-m', message, filename], check=True) + +def get_minor_version(version): + return '.'.join(version.split('.')[:2]) + +version_header = re.compile(r'^#+ (\d+(?:\.\d+)+)$') # match all version headers, including major or minor headers +class Note: + ''' The release notes for a single version bump ''' + def __init__(self, old_version, new_version, notes): + self.old_version = old_version + self.new_version = new_version + self.new_version_split = [int(part) for part in self.new_version.split('.')] + self.notes = notes + def old_minor_version_header(self): + return f'## {get_minor_version(self.old_version)}' + def new_minor_version_header(self): + return f'## {get_minor_version(self.new_version)}' + def changes_minor_version(self): + return get_minor_version(self.old_version) != get_minor_version(self.new_version) + def old_version_header(self): + return f'### {self.old_version}' + def new_version_header(self): + return f'### {self.new_version}' + def greater_than_minor_version(self, header): + ''' Check if the new version is greater than the version in the header. + This will also return false if the header is not a header for a version ''' + result = version_header.match(header) + if result: + version = [int(part) for part in result[1].split('.')] + for (s, h) in zip(self.new_version_split, version): + if s > h: + print(f'"{header}" IS LESS THAN {self.new_version} ({self.old_version}) because {s} > {h}') + return True + if s < h: + return False + return False + +def main(argv): + '''Replace placeholder release notes with the final release notes for a version.''' + parser = argparse.ArgumentParser() + parser.add_argument('--pr-cache', help='dump associated prs to json, or read from them (used in testing)') + parser.add_argument('--config', required=True, help="path to json configuration for release notes") + parser.add_argument('--release-notes-md', + help="path to ReleaseNotes.md to update, will just print if not provided " + + "(printing is only intended for testing)") + parser.add_argument('--commit', action='store_true', default=False, help="Commit the updates to the release notes") + # The below option is necessary because during the release we will have changed the version, and committed + # that, but not pushed it + parser.add_argument('--skip-commit', action='append', default=[], + help="If provided, skip this commit (can be repeated)") + parser.add_argument('--repository', default='FoundationDB/fdb-record-layer', help="Repository") + parser.add_argument('old_version', help='Old version to use when generating release notes') + parser.add_argument('new_version', nargs='+', + help='New version to use when generating release notes.\n' + + 'If multiple values are provided, release notes will be generated for all versions') + args = parser.parse_args(argv) + + with open(args.config, 'r') as fin: + label_config = json.load(fin) + + old_version = args.old_version + release_notes = [] + for new_version in args.new_version: + print(f"Generating release notes for {new_version}") + commits = get_commits(old_version, new_version, args.skip_commit) + prs = get_prs(commits, args.pr_cache, args.repository) + prs = dedup_prs(prs) + new_notes = [generate_note(pr[0], pr[1], label_config) for pr in prs] + release_notes.append(Note(old_version, new_version, format_notes(new_notes, label_config, old_version, new_version, args.repository))) + old_version = new_version + print("\n\n------------------------------\n\n") + if args.release_notes_md is None: + release_notes.reverse() + for notes in release_notes: + print(notes[1]) + else: + replace_notes(release_notes, args.release_notes_md) + if args.commit: + commit_release_notes(args.release_notes_md, args.new_version) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/docs/sphinx/source/ReleaseNotes.md b/docs/sphinx/source/ReleaseNotes.md index dc95f9cf7e..1beceefab0 100644 --- a/docs/sphinx/source/ReleaseNotes.md +++ b/docs/sphinx/source/ReleaseNotes.md @@ -15,39 +15,35 @@ Builds and releases have been moved to a new CI system. This includes the resump Users performing online updates are encouraged to update from [4.0.559.4](#405594). The continuations of some queries have changed in ways that may break if continued on other 4.0 builds. See: [Issue #3093](https://github.com/FoundationDB/fdb-record-layer/issues/3093), [PR #3092](https://github.com/FoundationDB/fdb-record-layer/pull/3092) fixing the issue, and [PR #3108](https://github.com/FoundationDB/fdb-record-layer/issues/3108) preparing 4.0.559.4 to accept newer continuations. - - ### 4.1.8.0 -* **Bug fix** The relational component now specifies `Locale.ROOT` in all usages of `String.format` which may change number formatting if the default locale does not align [(Issue #3121)](https://github.com/FoundationDB/fdb-record-layer/issues/3121) -* **Feature** FRL now respects PLAN_CACHE_*_MAX_ENTRIES options [(Issue #3155)](https://github.com/FoundationDB/fdb-record-layer/issues/3155) +

New Features

+ +* FRL respects PLAN_CACHE_*_MAX_ENTRIES - [PR #3156](https://github.com/FoundationDB/fdb-record-layer/pull/3156) +

Bug Fixes

+ +* Use `Locale.ROOT` in `String.format`, fixing messages if JVM has non-US default locale - [PR #3117](https://github.com/FoundationDB/fdb-record-layer/pull/3117) + +
+ + +

Build/Test/Documentation Improvements (click to expand)

+ + + +* Run mixed mode tests in a workflow, and publish results to release notes - [PR #3166](https://github.com/FoundationDB/fdb-record-layer/pull/3166) +* Add support for running YAML tests against a server built from local branch - [PR #3161](https://github.com/FoundationDB/fdb-record-layer/pull/3161) +* Kill abandoned external servers started during Yaml Tests - [PR #3152](https://github.com/FoundationDB/fdb-record-layer/pull/3152) +* Commit automatic updates from 4.1.6.0 - [PR #3163](https://github.com/FoundationDB/fdb-record-layer/pull/3163) +* Fix typo in action output reference - [PR #3165](https://github.com/FoundationDB/fdb-record-layer/pull/3165) +* Do not try to add coverage comment in pull request itself - [PR #3162](https://github.com/FoundationDB/fdb-record-layer/pull/3162) +* Restrict repository of nightly workflow, to avoid running it on forks - [PR #3160](https://github.com/FoundationDB/fdb-record-layer/pull/3160) + +
+**[Full Changelog (4.1.6.0...4.1.8.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.1.6.0...4.1.8.0)** + #### Mixed Mode Test Results Mixed mode testing run against the following previous versions: @@ -58,20 +54,96 @@ Mixed mode testing run against the following previous versions: ### 4.1.6.0 -* **Bug fix** Plan hashes for queries involving recursive CTEs are now stable across JVM invocations [(Issue #3139)](https://github.com/FoundationDB/fdb-record-layer/issues/3139) -* **Bug fix** Insert statement does not fully validate the column names with supplied values [(Issue #3069)](https://github.com/FoundationDB/fdb-record-layer/issues/3069) -* **Feature** Add enum column support to relational server [(Issue #3073)](https://github.com/FoundationDB/fdb-record-layer/issues/3073) -* **Feature** Allow scrubbing of indexes in READABLE_UNIQUE_PENDING state [(Issue #3135)](https://github.com/FoundationDB/fdb-record-layer/issues/3135) -* **Feature** Support Lucene index scrubbing [(Issue #3008)](https://github.com/FoundationDB/fdb-record-layer/issues/3008) +

New Features

+ +* Add support for prepared statements in JDBC - [PR #3116](https://github.com/FoundationDB/fdb-record-layer/pull/3116) +* Automatically publish to maven central - [PR #3144](https://github.com/FoundationDB/fdb-record-layer/pull/3144) +* Allow scrubbing of indexes in READABLE_UNIQUE_PENDING state - [PR #3137](https://github.com/FoundationDB/fdb-record-layer/pull/3137) +* Support Lucene index scrubbing of missing entries - [PR #3009](https://github.com/FoundationDB/fdb-record-layer/pull/3009) +* Add enum column support to relational server - [PR #3074](https://github.com/FoundationDB/fdb-record-layer/pull/3074) +* Add validation to Insert statement parsing to match the column names with supplied values - [PR #3070](https://github.com/FoundationDB/fdb-record-layer/pull/3070) +

Bug Fixes

+ +* Plan Hash Instability in recursive union query - [PR #3142](https://github.com/FoundationDB/fdb-record-layer/pull/3142) +

Dependency Updates

+ +* Update grpc-commonProtos, assertj, and junit dependencies - [PR #3115](https://github.com/FoundationDB/fdb-record-layer/pull/3115) +* Bump spotbugs gradle plugin from 4.6.1 to 6.1.3 - [PR #3104](https://github.com/FoundationDB/fdb-record-layer/pull/3104) + +
+ + +

Build/Test/Documentation Improvements (click to expand)

+ + + +* Commit automatic updates from 4.1.6.0 - [PR #3163](https://github.com/FoundationDB/fdb-record-layer/pull/3163) +* Fix signing configuration location - [PR #3159](https://github.com/FoundationDB/fdb-record-layer/pull/3159) +* Another try at getting pull-request number through from a forked PR - [PR #3158](https://github.com/FoundationDB/fdb-record-layer/pull/3158) +* Don't include empty :destructiveTest in Test Summary - [PR #3148](https://github.com/FoundationDB/fdb-record-layer/pull/3148) +* Publish a consolidated JaCoCo report from multiple subprojects and split test jobs - [PR #3147](https://github.com/FoundationDB/fdb-record-layer/pull/3147) +* Support multiple server versions in multi-version tests - [PR #3138](https://github.com/FoundationDB/fdb-record-layer/pull/3138) +* Bump supported_version in recursive cte tests due to plan hash change - [PR #3143](https://github.com/FoundationDB/fdb-record-layer/pull/3143) +* Initial attempt at automatic GitHub release notes - [PR #3136](https://github.com/FoundationDB/fdb-record-layer/pull/3136) +* Change context options to a type-safe option with generics - [PR #3132](https://github.com/FoundationDB/fdb-record-layer/pull/3132) +* Remove caching settings when downloading old server - [PR #3141](https://github.com/FoundationDB/fdb-record-layer/pull/3141) +* Remove test-summary action from the nightly workflow - [PR #3140](https://github.com/FoundationDB/fdb-record-layer/pull/3140) +* Split out the PR coverage commenting workflow so that it can run in the target (default) branch context. - [PR #3120](https://github.com/FoundationDB/fdb-record-layer/pull/3120) +* Split PRB into three jobs that can run in parallel - [PR #3122](https://github.com/FoundationDB/fdb-record-layer/pull/3122) +* Use fixed SHA for create-pull-request action - [PR #3127](https://github.com/FoundationDB/fdb-record-layer/pull/3127) +* Replace test-summary action with custom action code - [PR #3119](https://github.com/FoundationDB/fdb-record-layer/pull/3119) +* Fix rendering of plans so they display properly when calling `show` during tests and investigations - [PR #3134](https://github.com/FoundationDB/fdb-record-layer/pull/3134) + +
+ + +**[Full Changelog (4.1.4.0...4.1.6.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.1.4.0...4.1.6.0)** + ### 4.1.4.0 -* **Bug fix** Ungrouped GROUP BY queries result in infinite continuations when maxRows is 1 [(Issue #3093)](https://github.com/FoundationDB/fdb-record-layer/issues/3093) -* **Bug fix** Skips and limits are now enforced on CTE queries [(Issue #3100)](https://github.com/FoundationDB/fdb-record-layer/issues/3100) -* **Feature** Add verifications to yaml-tests that assert on planner metrics regressions [(Issue #3113)](https://github.com/FoundationDB/fdb-record-layer/issues/3113) -* **Breaking change** Changes to supprt CTE skips and limits can result in plan hash mismatches during online upgrades [(Issue #3100)](https://github.com/FoundationDB/fdb-record-layer/issues/3100) +

New Features

+ +* Plan generator now uses Cascades's `FunctionCatalog` - [PR #3061](https://github.com/FoundationDB/fdb-record-layer/pull/3061) +

Bug Fixes

+ +* Exclude transitive maven information in shaded jar - [PR #3129](https://github.com/FoundationDB/fdb-record-layer/pull/3129) +* Fix the publication of `fdb-record-layer-core-shaded` jars so that only the correct artifacts get uploaded - [PR #3125](https://github.com/FoundationDB/fdb-record-layer/pull/3125) +* Set skip and limit for the recursive union cursor correctly - [PR #3111](https://github.com/FoundationDB/fdb-record-layer/pull/3111) +* Fix infinite continuations when NULL_ON_EMPTY is in the plan - [PR #3092](https://github.com/FoundationDB/fdb-record-layer/pull/3092) +

Dependency Updates

+ +* Upgrade the versions gradle plugin from 0.38.0 to 0.52.0 - [PR #3106](https://github.com/FoundationDB/fdb-record-layer/pull/3106) +* Upgrade de.undercouch.download gradle plugin from 4.1.1 to 5.6.0 - [PR #3088](https://github.com/FoundationDB/fdb-record-layer/pull/3088) + +
+ + +

Build/Test/Documentation Improvements (click to expand)

+ + + +* Make build/update_release_notes.bash executable - [PR #3131](https://github.com/FoundationDB/fdb-record-layer/pull/3131) +* Address more comments related to metrics pr - [PR #3126](https://github.com/FoundationDB/fdb-record-layer/pull/3126) +* Address previous comments in metrics in explain - [PR #3124](https://github.com/FoundationDB/fdb-record-layer/pull/3124) +* Use deploy key to allow release build to write back to repo - [PR #3123](https://github.com/FoundationDB/fdb-record-layer/pull/3123) +* Assert on planner metrics when running queries in YAML tests - [PR #3085](https://github.com/FoundationDB/fdb-record-layer/pull/3085) +* Increment version in gradle.properties automatically - [PR #3082](https://github.com/FoundationDB/fdb-record-layer/pull/3082) +* Fixes for Gradle tasks for Antlr - [PR #3079](https://github.com/FoundationDB/fdb-record-layer/pull/3079) +* Rework YamlIntegrationTests to not use inheritance - [PR #3083](https://github.com/FoundationDB/fdb-record-layer/pull/3083) +* Don't rerun PRBs on edit, just when the commits change - [PR #3110](https://github.com/FoundationDB/fdb-record-layer/pull/3110) +* Yaml test option to force continuations - [PR #3075](https://github.com/FoundationDB/fdb-record-layer/pull/3075) +* Remove the sonarqube plugin from our gradle plugins - [PR #3103](https://github.com/FoundationDB/fdb-record-layer/pull/3103) + +
+ + +**[Full Changelog (4.0.575.0...4.1.4.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.0.575.0...4.1.4.0)** + + + ## 4.0 @@ -85,27 +157,221 @@ Our API stability annotations have been updated to reflect greater API instabili ### 4.0.575.0 +

Breaking Changes

+ +* Only allow continuation to be returned after a result set has been exhausted - [PR #3038](https://github.com/FoundationDB/fdb-record-layer/pull/3038) +

Dependency Updates

+ +* Upgrade git-version plugin to the latest version: 3.1.0 - [PR #3089](https://github.com/FoundationDB/fdb-record-layer/pull/3089) +* Upgrade jmh plugin to latest version: 0.7.2 - [PR #3090](https://github.com/FoundationDB/fdb-record-layer/pull/3090) + +
+ + +

Build/Test/Documentation Improvements (click to expand)

+ + + +* Use git diff instead of git status to determine if there are changes to yamsql files during release - [PR #3091](https://github.com/FoundationDB/fdb-record-layer/pull/3091) +* Implement global (within factory) state for alternating connections - [PR #3081](https://github.com/FoundationDB/fdb-record-layer/pull/3081) +* Remove fancy checking of status when committing yamsql updates - [PR #3087](https://github.com/FoundationDB/fdb-record-layer/pull/3087) +* In commit_yamsql_updates, switch away from text parameter - [PR #3086](https://github.com/FoundationDB/fdb-record-layer/pull/3086) +* In commit_yamsql_updates, switch from using capture_output to stdout=subprocess.PIPE - [PR #3084](https://github.com/FoundationDB/fdb-record-layer/pull/3084) +* Add support to LuceneIndexTestDataModel for saving old records, updating synthetic records and sampling `recordsUnderTest`; remove the `start` parameter - [PR #3076](https://github.com/FoundationDB/fdb-record-layer/pull/3076) +* Require FileOptions to be first block of yamsql - [PR #3046](https://github.com/FoundationDB/fdb-record-layer/pull/3046) +* Add some GitHub workflow actions to begin CI/CD transition there - [PR #3071](https://github.com/FoundationDB/fdb-record-layer/pull/3071) +* Auto update supported_version in yamsql files - [PR #3066](https://github.com/FoundationDB/fdb-record-layer/pull/3066) + +
+ + +**[Full Changelog (4.0.567.0...4.0.575.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.0.567.0...4.0.575.0)** + + + ### 4.0.567.0 -* **Bug fix** Relational insert statement in does not work with an array value [(Issue #3041)](https://github.com/FoundationDB/fdb-record-layer/issues/3041) -* **Feature** make EXPLAIN return a column for planner metrics [(Issue #3063)](https://github.com/FoundationDB/fdb-record-layer/issues/3063) +

New Features

+ +* Make explain return planner metrics as separate column - [PR #3064](https://github.com/FoundationDB/fdb-record-layer/pull/3064) + + +**[Full Changelog (4.0.566.0...4.0.567.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.0.566.0...4.0.567.0)** + + + + +### 4.0.566.0 + +

Bug Fixes

+ +* Support for Array and Enums in insert statement - [PR #3042](https://github.com/FoundationDB/fdb-record-layer/pull/3042) + + +**[Full Changelog (4.0.565.0...4.0.566.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.0.565.0...4.0.566.0)** + + + ### 4.0.565.0 -* **Bug fix** Break out a helper class from GenerateVisitor annotation processor [(Issue #3060)](https://github.com/FoundationDB/fdb-record-layer/issues/3060) -* **Feature** Support enums in rel ops [(Issue #3011)](https://github.com/FoundationDB/fdb-record-layer/issues/3011) +

New Features

+ +* Support enums in relational operations - [PR #3012](https://github.com/FoundationDB/fdb-record-layer/pull/3012) +

Bug Fixes

+ +* Break out a helper class from GenerateVisitor annotation processor - [PR #3060](https://github.com/FoundationDB/fdb-record-layer/pull/3060) + +
+ + +

Build/Test/Documentation Improvements (click to expand)

+ + + +* Enable multi version tests - [PR #3047](https://github.com/FoundationDB/fdb-record-layer/pull/3047) +* Skip explain when running in mixed mode - [PR #3055](https://github.com/FoundationDB/fdb-record-layer/pull/3055) +* yamsql: Add supported_version to blocks and queries, and support for actual versions - [PR #3049](https://github.com/FoundationDB/fdb-record-layer/pull/3049) + +
+ + +**[Full Changelog (4.0.564.0...4.0.565.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.0.564.0...4.0.565.0)** + + + ### 4.0.564.0 +

New Features

+ +* Improve aggregate index matching algorithm to match queries with different grouping order - [PR #2943](https://github.com/FoundationDB/fdb-record-layer/pull/2943) + + +**[Full Changelog (4.0.562.0...4.0.564.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.0.562.0...4.0.564.0)** + + + ### 4.0.562.0 -* **Bug fix** Merge META-INFO/services in relational-server-all.jar [(Issue #3052)](https://github.com/FoundationDB/fdb-record-layer/issues/3052) +

Bug Fixes

+ +* Merge service descriptor files in fdb-relational-server's shadow-jar - [PR #3053](https://github.com/FoundationDB/fdb-record-layer/pull/3053) + +
+ + +

Build/Test/Documentation Improvements (click to expand)

+ + + +* Use `EXECUTE CONTINUATION ?continuation` when continuing in yaml tests - [PR #3051](https://github.com/FoundationDB/fdb-record-layer/pull/3051) +* Add a class for comparing versions of the record layer - [PR #3036](https://github.com/FoundationDB/fdb-record-layer/pull/3036) +* Enable load schema in JDBC tests - [PR #3033](https://github.com/FoundationDB/fdb-record-layer/pull/3033) +* Ability to prevent yamsql tests from running against old versions - [PR #3031](https://github.com/FoundationDB/fdb-record-layer/pull/3031) + +
+ + +**[Full Changelog (4.0.561.0...4.0.562.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.0.561.0...4.0.562.0)** + + + ### 4.0.561.0 -* **Feature** Introduce SQL support to Recursive CTEs [(Issue #3034)](https://github.com/FoundationDB/fdb-record-layer/issues/3034) +

New Features

+ +* Add support for continuations to the relational server - [PR #3030](https://github.com/FoundationDB/fdb-record-layer/pull/3030) +* Introduce SQL support to Recursive CTEs - [PR #3035](https://github.com/FoundationDB/fdb-record-layer/pull/3035) +* Align aggregate query behaviour on empty tables more towards SQL standard - [PR #3029](https://github.com/FoundationDB/fdb-record-layer/pull/3029) +

Dependency Updates

+ +* Bump black from 22.10.0 to 24.3.0 in /scripts - [PR #3024](https://github.com/FoundationDB/fdb-record-layer/pull/3024) + +
+ + +

Build/Test/Documentation Improvements (click to expand)

+ + + +* Use example.com domain for email addresses in tests - [PR #3023](https://github.com/FoundationDB/fdb-record-layer/pull/3023) +* Change yaml-tests to download latest released server - [PR #3032](https://github.com/FoundationDB/fdb-record-layer/pull/3032) +* Update gradle dependency management to use a version platform - [PR #3027](https://github.com/FoundationDB/fdb-record-layer/pull/3027) + +
+ + +**[Full Changelog (4.0.559.0...4.0.561.0)](https://github.com/FoundationDB/fdb-record-layer/compare/4.0.559.0...4.0.561.0)** + + + + +### 4.0.559.6 + +

New Features

+ +* Backport prepared statement changes to the patch release - [PR #3157](https://github.com/FoundationDB/fdb-record-layer/pull/3157) + + +**[Full Changelog (4.0.559.4...4.0.559.6)](https://github.com/FoundationDB/fdb-record-layer/compare/4.0.559.4...4.0.559.6)** + + + + +### 4.0.559.4 + +

Bug Fixes

+ +* Prepare for up-level DEFAULTS_ON_EMPTY changes - [PR #3108](https://github.com/FoundationDB/fdb-record-layer/pull/3108) + +
+ + +

Build/Test/Documentation Improvements (click to expand)

+ + + +* Backport PRB workflow to 4.0.559 branch - [PR #3109](https://github.com/FoundationDB/fdb-record-layer/pull/3109) + +
+ + +**[Full Changelog (4.0.559.3...4.0.559.4)](https://github.com/FoundationDB/fdb-record-layer/compare/4.0.559.3...4.0.559.4)** + + + + +### 4.0.559.3 + +

Bug Fixes

+ +* Break out a helper class from GenerateVisitor annotation processor (backport PR #3060) - [PR #3062](https://github.com/FoundationDB/fdb-record-layer/pull/3062) + + +**[Full Changelog (4.0.559.1...4.0.559.3)](https://github.com/FoundationDB/fdb-record-layer/compare/4.0.559.1...4.0.559.3)** + + + + +### 4.0.559.1 + +

New Features

+ +* 4.0.559 release branch and backport PRs #3030, #3033 and #3053 - [PR #3058](https://github.com/FoundationDB/fdb-record-layer/pull/3058) + * This includes 3 initial changes for the start of the 4.0.559 branch: + * Backport of: Add support for continuations to the relational server - [PR #3030](https://github.com/FoundationDB/fdb-record-layer/pull/3030) + * Backport of: Enable load schema in JDBC tests - [PR #3033](https://github.com/FoundationDB/fdb-record-layer/pull/3033) + * Backport of: Merge service descriptor files in fdb-relational-server's shadow-jar - [PR #3053](https://github.com/FoundationDB/fdb-record-layer/pull/3053) + + +**[Full Changelog (4.0.559.0...4.0.559.1)](https://github.com/FoundationDB/fdb-record-layer/compare/4.0.559.0...4.0.559.1)** + + + ### 4.0.559.0