From 3aab09b16f9ef3215ba3d20448de7150c21c4193 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Fri, 2 Feb 2024 21:56:15 +0100 Subject: [PATCH 1/3] .github/workflows: Update actions The v4 versions don't use the deprecated Node.js version 16 [1]. [1]: https://github.blog/changelog/2023-09-22-github-actions-transitioning-from-node-16-to-node-20/ --- .github/workflows/build-pr.yml | 24 ++++++++++++------------ .github/workflows/build-release.yml | 18 +++++++++--------- .github/workflows/test-igor-workflow.yml | 6 +++--- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index 83d2d2f79d..f4c7cd0eb3 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -48,7 +48,7 @@ jobs: runs-on: [ self-hosted, Windows, Certificate ] steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive fetch-depth: 0 # load all commits @@ -59,7 +59,7 @@ jobs: - name: Sign installer run: tools/sign-installer.sh -p "${{ secrets.GHA_MIES_CERTIFICATE_PIN }}" - name: upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: name: BuildInstaller-${{ matrix.kind }}-assets @@ -72,13 +72,13 @@ jobs: runs-on: [ self-hosted, Linux, Docker ] steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Code Checks run: tools/check-code.sh # straight from the documentation, see https://pre-commit.com/#github-actions-example - name: set PY run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - - uses: actions/cache@v3 + - uses: actions/cache@v4 with: path: ~/.cache/pre-commit key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} @@ -115,7 +115,7 @@ jobs: runs-on: [ self-hosted, Linux, Docker ] steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive fetch-depth: 0 # load all commits @@ -124,7 +124,7 @@ jobs: - name: Build documentation run: tools/documentation/run.sh - name: upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: name: Documentation-assets @@ -230,21 +230,21 @@ jobs: - TestNI steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Initial repo config run: tools/initial-repo-config.sh - name: Download ITC18-USB artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: test-itc18-assets - name: Download ITC1600 artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: test-itc1600-assets - name: Download NI artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: test-ni-assets - name: Validate and read NWBv2 files @@ -265,12 +265,12 @@ jobs: - TestITC1600 steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Initial repo config run: tools/initial-repo-config.sh - name: Download all artifacts id: download - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: artifacts - name: Flatten artifact structure diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index a39c18c51e..26a1f9db31 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -27,14 +27,14 @@ jobs: - CallPR steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive fetch-depth: 0 # load all commits - name: Initial repo config run: tools/initial-repo-config.sh - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: Documentation-assets - name: Deploy documentation to github pages @@ -48,14 +48,14 @@ jobs: - CallPR steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive fetch-depth: 0 # load all commits - name: Initial repo config run: tools/initial-repo-config.sh - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: BuildInstaller-user-assets - name: Deploy release assets to github @@ -69,12 +69,12 @@ jobs: - CallPR steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Initial repo config run: tools/initial-repo-config.sh - name: Download all artifacts id: download - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: artifacts - name: 📥 Download report cache from FTP @@ -112,7 +112,7 @@ jobs: -d "${{ steps.gen.outputs.history }}" \ -t "cache/coverage-history" - name: upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: report-artifacts @@ -130,12 +130,12 @@ jobs: - GenerateReport steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Initial repo config run: tools/initial-repo-config.sh - name: Download all artifacts id: download - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: artifacts - name: Flatten artifact structure diff --git a/.github/workflows/test-igor-workflow.yml b/.github/workflows/test-igor-workflow.yml index 0d519e0c83..e8674c6cc5 100644 --- a/.github/workflows/test-igor-workflow.yml +++ b/.github/workflows/test-igor-workflow.yml @@ -64,14 +64,14 @@ jobs: CI_INSTRUMENT_TESTS: ${{ inputs.instrument_tests }} steps: - name: Check out repository code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Initial repo config run: tools/initial-repo-config.sh - name: Download artifacts if: inputs.installer_artifact_name != '' - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 continue-on-error: false with: name: ${{ inputs.installer_artifact_name }} @@ -83,7 +83,7 @@ jobs: if: always() run: tools/gather-logfiles-and-crashdumps.sh - name: upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ always() }} with: name: ${{ inputs.artifact_name }} From c79186e49526f1f7c88cd760fdbef3552b851ac5 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Wed, 20 Dec 2023 16:34:46 +0100 Subject: [PATCH 2/3] .github/workflows: Move compression code to a script This avoids code duplication. --- .github/workflows/build-pr.yml | 7 +------ .github/workflows/build-release.yml | 7 +------ tools/ftp-upload/compress-nwb-files.sh | 13 +++++++++++++ 3 files changed, 15 insertions(+), 12 deletions(-) create mode 100755 tools/ftp-upload/compress-nwb-files.sh diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index f4c7cd0eb3..d10f8d4aec 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -278,12 +278,7 @@ jobs: find "${{ steps.download.outputs.download-path }}" -mindepth 1 -maxdepth 1 -type d |\ xargs -I {} tools/ftp-upload/flatten-files.sh "{}" - name: Compress NWB artifacts - run: | - for dir in test-itc18-assets test-itc1600-assets test-ni-assets - do - tar --remove-files --use-compress-program=zstd -cvf $dir/NWB.tar.zst $dir/*nwb - done - working-directory: ${{ steps.download.outputs.download-path }} + run: tools/ftp-upload/compress-nwb-files.sh "${{ steps.download.outputs.download-path }}" - name: Upload artifacts using FTP run: | tools/ftp-upload/upload-files.sh \ diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index 26a1f9db31..a00a913c74 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -143,12 +143,7 @@ jobs: find "${{ steps.download.outputs.download-path }}" -mindepth 1 -maxdepth 1 -type d |\ xargs -I {} tools/ftp-upload/flatten-files.sh "{}" - name: Compress NWB artifacts - run: | - for dir in test-itc18-assets test-itc1600-assets test-ni-assets - do - tar --remove-files --use-compress-program=zstd -cvf $dir/NWB.tar.zst $dir/*nwb - done - working-directory: ${{ steps.download.outputs.download-path }} + run: tools/ftp-upload/compress-nwb-files.sh "${{ steps.download.outputs.download-path }}" - name: Upload artifacts using FTP run: | tools/ftp-upload/upload-files.sh \ diff --git a/tools/ftp-upload/compress-nwb-files.sh b/tools/ftp-upload/compress-nwb-files.sh new file mode 100755 index 0000000000..3de3edc3fe --- /dev/null +++ b/tools/ftp-upload/compress-nwb-files.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +cd $1 + +for dir in test-itc18-assets test-itc1600-assets test-ni-assets +do + if [ -n "$(find "$dir" -maxdepth 0 -type d -empty)" ] + then + continue + fi + + tar --remove-files --use-compress-program=zstd -cvf $dir/NWB.tar.zst $dir/*nwb +done From 92b3dca162fafa039ad0511fb78429a487761830 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Tue, 19 Dec 2023 21:37:00 +0100 Subject: [PATCH 3/3] .github/workflows: Make assets names unique This is required for v4 of the artifact upload job [1] as the artifact name must now be unique. [1]: https://github.com/actions/upload-artifact/blob/main/README.md#breaking-changes --- .github/workflows/build-pr.yml | 14 +++++++------- tools/ftp-upload/compress-nwb-files.sh | 7 +++++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index d10f8d4aec..be618bed7d 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -151,7 +151,7 @@ jobs: job_name: 🧪 Test ${{ matrix.name }} overwrite_job_name: ${{ inputs.is_called_workflow || false }} experiment: ${{ matrix.experiment }} - artifact_name: test-without-hw-assets + artifact_name: TestWithoutHardware-${{ matrix.name }}-assets expensive_checks: "1" instrument_tests: ${{ fromJson('["0", "1"]')[inputs.do_instrumentation] }} @@ -173,7 +173,7 @@ jobs: overwrite_job_name: ${{ inputs.is_called_workflow || false }} experiment: ${{ matrix.experiment }} target: "[ 'self-hosted', 'Windows', 'IgorPro', 'NI' ]" - artifact_name: test-ni-assets + artifact_name: TestNI-${{ matrix.name }}-assets expensive_checks: "1" instrument_tests: ${{ fromJson('["0", "1"]')[inputs.do_instrumentation] }} @@ -195,7 +195,7 @@ jobs: overwrite_job_name: ${{ inputs.is_called_workflow || false }} experiment: ${{ matrix.experiment }} target: "[ 'self-hosted', 'Windows', 'IgorPro', 'ITC' ]" - artifact_name: test-itc18-assets + artifact_name: TestITC18-${{ matrix.name }}-assets expensive_checks: "1" instrument_tests: ${{ fromJson('["0", "1"]')[inputs.do_instrumentation] }} @@ -217,7 +217,7 @@ jobs: overwrite_job_name: ${{ inputs.is_called_workflow || false }} experiment: ${{ matrix.experiment }} target: "[ 'self-hosted', 'Windows', 'IgorPro', 'ITC1600' ]" - artifact_name: test-itc1600-assets + artifact_name: TestITC1600-${{ matrix.name }}-assets expensive_checks: "1" instrument_tests: ${{ fromJson('["0", "1"]')[inputs.do_instrumentation] }} @@ -238,15 +238,15 @@ jobs: - name: Download ITC18-USB artifacts uses: actions/download-artifact@v4 with: - name: test-itc18-assets + pattern: TestITC18-* - name: Download ITC1600 artifacts uses: actions/download-artifact@v4 with: - name: test-itc1600-assets + pattern: TestITC1600-* - name: Download NI artifacts uses: actions/download-artifact@v4 with: - name: test-ni-assets + pattern: TestNI-* - name: Validate and read NWBv2 files run: tools/nwb-read-tests/run.sh diff --git a/tools/ftp-upload/compress-nwb-files.sh b/tools/ftp-upload/compress-nwb-files.sh index 3de3edc3fe..6c61f6c1a7 100755 --- a/tools/ftp-upload/compress-nwb-files.sh +++ b/tools/ftp-upload/compress-nwb-files.sh @@ -2,9 +2,12 @@ cd $1 -for dir in test-itc18-assets test-itc1600-assets test-ni-assets +for dir in TestNI-* TestITC18-* TestITC1600-* do - if [ -n "$(find "$dir" -maxdepth 0 -type d -empty)" ] + if [ ! -d $dir ] + then + continue + elif [ -n "$(find "$dir" -maxdepth 0 -type d -empty)" ] then continue fi