diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3fc04e5..84da22b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,8 +8,11 @@ name: ci # ref) - it's hygiene, keeping the repo's cache budget from being consumed # by entries on refs nothing else can restore from. Applies to fmt, clippy, # test (its postgres:17 leg only), cargo-publish-dry-run, -# third-party-license, smoke (one sqlite leg per os) and release below -# (release also skips the cache entirely on tags). +# third-party-license, smoke (one sqlite leg per os), release and macos below +# (release and macos also skip the cache entirely on tags). The two compat +# jobs only restore test's entry, through the `dev` shared key, and never +# save. build-storage and build-storage-lake use no cache: they run `cargo +# check`, which reuses little of a full build's artifacts. # # Two tiers. The split was made on GitHub's Free plan (20 concurrent jobs), # where a full run of this workflow (~60 jobs, 31 of them with no `needs`) on @@ -22,8 +25,17 @@ name: ci # suite, `test` on postgres:17 only (compose.yaml's default), and ci-gate. # - Tier B runs once per merge-queue entry (`merge_group`) and on push: # the full build-storage/build-storage-lake matrix, `test` on pg16/pg18, -# the experimental compat legs, cargo-publish-dry-run, src, package, -# smoke, smoke-oldest-client, smoke-report, release. +# the experimental compat legs, cargo-publish-dry-run, src, release, +# macos, smoke, smoke-oldest-client, smoke-report. +# +# Only `typos` runs ahead of `release`, and only `release` runs ahead of +# `smoke`, so the release -> smoke chain starts with the rest of the run. +# `macos` is a compile-only check except on a `v*` tag, where it builds the +# release tarball (tier-c.yml builds the macOS release nightly). smoke builds its image locally from release's binaries +# and pushes nothing. `publish` (push only) needs every job ci-gate does and +# is the only job that pushes an image, so no image reaches the registry +# untested. `publish-release` attaches the tarballs on a tag, after publish. +# Both are outside ci-gate, see their comments. # # On top of the tiers, the `changes` job skips every Rust job on a # pull_request that only touches CI config this workflow doesn't read, or @@ -32,8 +44,9 @@ name: ci # `changes` too, so a failed classification fails the gate. # # Tier B jobs carry `if: github.event_name != 'pull_request' || -# (vars.MERGE_QUEUE != 'on' && )`, and the trimmed matrices pick the smaller list -# under the same condition, so a leg's check-run name +# (vars.MERGE_QUEUE != 'on' && )`, written once as the `tier_b` +# anchor on build-storage and aliased by the others. The trimmed matrices pick +# the smaller list under the inverse condition, so a leg's check-run name # (`test (ubuntu-latest, postgres:17)`) means the same thing whichever tier # produced it. The `MERGE_QUEUE` repository variable is the rollback switch: # the split only makes sense while a merge queue runs Tier B before merge, @@ -57,9 +70,9 @@ name: ci # is scoped to `main`, so the queue branch itself doesn't also trigger a # push run. The queue fast-forwards main to the exact commit it tested, so # the post-merge push run does repeat Tier B on the same tree; that is what -# refreshes the caches and publishes the `main` image, and it is the same -# cost as today's post-merge run, not an addition. Trimming it is a -# follow-up. +# refreshes the caches and, through `publish`, tags the `main` image, and it +# is the same cost as today's post-merge run, not an addition. Trimming it is +# a follow-up. permissions: contents: read on: @@ -137,7 +150,7 @@ jobs: # merge_group/push. build-storage/build-storage-lake are Tier B, so they # can't be `needs` here (see the header). if: needs.changes.outputs.rust == 'true' - needs: [changes, fmt, clippy] + needs: [changes] runs-on: ${{matrix.os}} timeout-minutes: 45 strategy: @@ -161,8 +174,6 @@ jobs: NEXTEST_PROFILE: ci POSTGRES_IMAGE: ${{matrix.pg}} PROMETHEUS_LISTENER_URL: tcp://[::]:9100 - RUST_BACKTRACE: 1 - RUST_LOG: debug,nisshi_sans_io=warn,nisshi_model=warn SCHEMA_REGISTRY: file://./etc/schema STORAGE_ENGINE: s3://nisshi/ NISSHI_IMAGE: ghcr.io/nisshi-io/nisshi @@ -182,15 +193,23 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 with: components: clippy, rustfmt - # rust-cache keys on the job, not the matrix, so every pg leg - # shares one entry and only one of them should save it: pg17, - # the leg that runs in both tiers. + # Every pg leg shares one entry, and only one of them should save + # it: pg17, the leg that runs in both tiers. The compat jobs restore + # this entry under the same key and never save it. cache-save-if: ${{ github.ref == 'refs/heads/main' && matrix.pg == 'postgres:17' }} + cache-shared-key: dev - uses: taiki-e/install-action@83ac0ad63c0167e6f06796fab0fce28db1bf3db0 # v2.87.22 with: tool: cargo-nextest@0.9.146 - run: df -h + # Set on the step, not the job: rust-cache hashes every variable that + # starts with RUST into its key, and the jobs that restore this entry + # don't set these, so a job-level value would give every job a key of + # its own. - run: just test + env: + RUST_BACKTRACE: 1 + RUST_LOG: debug,nisshi_sans_io=warn,nisshi_model=warn - run: df -h if: always() - run: ls -l @@ -252,7 +271,8 @@ jobs: # this job instead of in a job of its own. - run: just doc build-storage: - if: needs.changes.outputs.rust == 'true' && (github.event_name != 'pull_request' || (vars.MERGE_QUEUE != 'on' && github.event.pull_request.head.repo.full_name == github.repository)) + # Every Tier B job reuses this condition as `*tier_b`. + if: &tier_b needs.changes.outputs.rust == 'true' && (github.event_name != 'pull_request' || (vars.MERGE_QUEUE != 'on' && github.event.pull_request.head.repo.full_name == github.repository)) needs: [changes] runs-on: ${{matrix.os}} timeout-minutes: 30 @@ -281,7 +301,7 @@ jobs: # and linking. - run: cargo check --bin nisshi --no-default-features --features ${{matrix.storage}} build-storage-lake: - if: needs.changes.outputs.rust == 'true' && (github.event_name != 'pull_request' || (vars.MERGE_QUEUE != 'on' && github.event.pull_request.head.repo.full_name == github.repository)) + if: *tier_b needs: [changes] runs-on: ${{matrix.os}} timeout-minutes: 30 @@ -323,8 +343,8 @@ jobs: os: - ubuntu-latest # Tier A runs the one non-experimental leg (memory); merge_group/push - # run all four. Keep this list identical to compat-franz-go's. - storage: >- + # run all four. compat-franz-go reuses this list as `*compat_storage`. + storage: &compat_storage >- ${{ fromJSON((github.event_name == 'pull_request' && (vars.MERGE_QUEUE == 'on' || github.event.pull_request.head.repo.full_name != github.repository)) && '[{"name": "memory", "engine": "memory://", "features": "dynostore", "experimental": false}]' || '[{"name": "memory", "engine": "memory://", "features": "dynostore", "experimental": false}, @@ -341,8 +361,6 @@ jobs: CLUSTER_ID: librdkafka-integration LAKEKEEPER_IMAGE: unused RESULTS_FILE: results-${{matrix.storage.name}}.csv - RUST_BACKTRACE: 1 - RUST_LOG: warn STORAGE_ENGINE: ${{matrix.storage.engine}} NISSHI_IMAGE: hello-world steps: @@ -356,9 +374,14 @@ jobs: - run: sudo apt-get install -y libclang-dev libkrb5-dev - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 with: - cache: false + cache-shared-key: dev + cache-save-if: false - run: just compat-infra-up + # RUST_* on the step, not the job, for the cache key (see `test`). - run: just compat-librdkafka '${{matrix.storage.engine}}' '${{matrix.storage.features}}' + env: + RUST_BACKTRACE: 1 + RUST_LOG: warn - if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -406,14 +429,7 @@ jobs: matrix: os: - ubuntu-latest - # Same tiering as compat-librdkafka; keep the two lists identical. - storage: >- - ${{ fromJSON((github.event_name == 'pull_request' && (vars.MERGE_QUEUE == 'on' || github.event.pull_request.head.repo.full_name != github.repository)) - && '[{"name": "memory", "engine": "memory://", "features": "dynostore", "experimental": false}]' - || '[{"name": "memory", "engine": "memory://", "features": "dynostore", "experimental": false}, - {"name": "s3", "engine": "s3://nisshi/", "features": "dynostore", "experimental": true}, - {"name": "postgres", "engine": "postgres://postgres:postgres@localhost", "features": "postgres", "experimental": true}, - {"name": "sqlite", "engine": "sqlite://nisshi.db", "features": "libsql", "experimental": true}]') }} + storage: *compat_storage env: ADVERTISED_LISTENER_URL: tcp://localhost:9092 AWS_ACCESS_KEY_ID: minioadmin @@ -424,8 +440,6 @@ jobs: CLUSTER_ID: librdkafka-integration LAKEKEEPER_IMAGE: unused RESULTS_FILE: results-${{matrix.storage.name}}.csv - RUST_BACKTRACE: 1 - RUST_LOG: warn STORAGE_ENGINE: ${{matrix.storage.engine}} NISSHI_IMAGE: hello-world steps: @@ -437,12 +451,16 @@ jobs: just-version: 1.58.0 - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 with: - cache: false + cache-shared-key: dev + cache-save-if: false - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: "1.27.1" - run: just compat-infra-up - run: just compat-franz-go '${{matrix.storage.engine}}' '${{matrix.storage.features}}' + env: + RUST_BACKTRACE: 1 + RUST_LOG: warn - if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -480,7 +498,7 @@ jobs: - run: | "${DUCKDB_CLI}/duckdb" -markdown -f reports/.github/workflows/scripts/pivot-integration-report.sql >> "${GITHUB_STEP_SUMMARY}" cargo-publish-dry-run: - if: needs.changes.outputs.rust == 'true' && (github.event_name != 'pull_request' || (vars.MERGE_QUEUE != 'on' && github.event.pull_request.head.repo.full_name == github.repository)) + if: *tier_b needs: [changes] runs-on: ${{matrix.os}} timeout-minutes: 30 @@ -531,7 +549,7 @@ jobs: path: THIRD-PARTY-LICENSE.html if-no-files-found: error src: - if: needs.changes.outputs.rust == 'true' && (github.event_name != 'pull_request' || (vars.MERGE_QUEUE != 'on' && github.event.pull_request.head.repo.full_name == github.repository)) + if: *tier_b needs: [changes] runs-on: ubuntu-latest timeout-minutes: 10 @@ -546,60 +564,11 @@ jobs: name: src path: ${{runner.temp}}/src.tar.gz if-no-files-found: error - package: - if: needs.changes.outputs.rust == 'true' && (github.event_name != 'pull_request' || (vars.MERGE_QUEUE != 'on' && github.event.pull_request.head.repo.full_name == github.repository)) - needs: [changes, test, cargo-publish-dry-run, typos, release] - runs-on: ${{matrix.os}} - timeout-minutes: 15 - permissions: - packages: write - strategy: - fail-fast: false - matrix: - os: - - ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - # release's static musl binaries, as dist/linux//nisshi. - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: image-* - merge-multiple: true - path: dist - - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 - id: meta - with: - images: ${{env.REGISTRY}}/${{github.repository}} - tags: | - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - with: - buildkitd-flags: --debug - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ${{env.REGISTRY}} - username: ${{github.actor}} - password: ${{secrets.GITHUB_TOKEN}} - - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 - with: - context: . - platforms: linux/amd64,linux/arm64 - # Push on repo-owned refs (push, merge_group) and same-repo PRs. - # Tier B's `if:` already skips this job on fork PRs; the repo check - # stays as a guard, since a fork's token is read-only. - push: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} # Runs the nisshi-smoke-test suite (`just smoke `): the Kafka CLI # tools from each Kafka image against the packaged image on each storage engine. smoke: - if: needs.changes.outputs.rust == 'true' && (github.event_name != 'pull_request' || (vars.MERGE_QUEUE != 'on' && github.event.pull_request.head.repo.full_name == github.repository)) - needs: [changes, package] + if: *tier_b + needs: [changes, release] strategy: fail-fast: false matrix: @@ -628,15 +597,6 @@ jobs: - run: | kafka=${KAFKA_IMAGE#*:} echo "NISSHI_SMOKE_LEG=${STORAGE}_kafka${kafka//./}_${RUNNER_ARCH,,}" >> "$GITHUB_ENV" - - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 - id: meta - with: - images: ${{env.REGISTRY}}/${{github.repository}} - tags: | - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -657,16 +617,21 @@ jobs: - uses: taiki-e/install-action@83ac0ad63c0167e6f06796fab0fce28db1bf3db0 # v2.87.22 with: tool: cargo-nextest@0.9.146 - - env: - IMAGE: ${{env.REGISTRY}}/${{github.repository}}:${{steps.meta.outputs.version}} + # The image is built here from release's binaries and never pushed; + # only publish pushes an image. The packaged image crashes at startup + # on sqlite:// (#796), so the sqlite legs build the broker from source + # instead, and say nothing about the image on sqlite. The fix for #796 + # removes this condition and the sqlite-only setup steps above. + - if: matrix.storage != 'sqlite' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: image-* + merge-multiple: true + path: dist + - if: matrix.storage != 'sqlite' run: | - # The packaged image crashes at startup on sqlite:// (#796), so - # the sqlite legs build the broker from source instead, and say - # nothing about the image on sqlite. The fix for #796 removes this - # and the sqlite-only setup steps above. - if [[ "${STORAGE}" != sqlite ]]; then - echo "NISSHI_SMOKE_IMAGE=${IMAGE}" >> "$GITHUB_ENV" - fi + docker build --tag nisshi:smoke . + echo "NISSHI_SMOKE_IMAGE=nisshi:smoke" >> "$GITHUB_ENV" # The step stops well before the job's timeout, so the steps that # upload the results and logs still run when the suite hangs. - run: just smoke "${STORAGE}" @@ -698,8 +663,8 @@ jobs: # Runs the smoke suite with the Kafka CLI tools from the oldest Kafka release that CI tests, so # CI checks that an older Java client still negotiates API versions with the broker. smoke-oldest-client: - if: github.event_name != 'pull_request' || (vars.MERGE_QUEUE != 'on' && github.event.pull_request.head.repo.full_name == github.repository) - needs: [package] + if: *tier_b + needs: [changes, release] strategy: fail-fast: false # The job runs on every storage engine, because an older client can call other APIs than a @@ -725,15 +690,6 @@ jobs: - run: | kafka=${KAFKA_IMAGE#*:} echo "NISSHI_SMOKE_LEG=${STORAGE}_kafka${kafka//./}_${RUNNER_ARCH,,}" >> "$GITHUB_ENV" - - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 - id: meta - with: - images: ${{env.REGISTRY}}/${{github.repository}} - tags: | - type=ref,event=branch - type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -751,14 +707,21 @@ jobs: - uses: taiki-e/install-action@83ac0ad63c0167e6f06796fab0fce28db1bf3db0 # v2.87.22 with: tool: cargo-nextest@0.9.146 - - env: - IMAGE: ${{env.REGISTRY}}/${{github.repository}}:${{steps.meta.outputs.version}} + # The image is built here from release's binaries and never pushed; + # only publish pushes an image. The packaged image crashes at startup + # on sqlite:// (#796), so the sqlite leg builds the broker from source + # instead. The fix for #796 removes this condition and the sqlite-only + # setup steps above. + - if: matrix.storage != 'sqlite' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: image-* + merge-multiple: true + path: dist + - if: matrix.storage != 'sqlite' run: | - # The packaged image crashes at startup on sqlite:// (#796), so - # the sqlite leg builds the broker from source instead. - if [[ "${STORAGE}" != sqlite ]]; then - echo "NISSHI_SMOKE_IMAGE=${IMAGE}" >> "$GITHUB_ENV" - fi + docker build --tag nisshi:smoke . + echo "NISSHI_SMOKE_IMAGE=nisshi:smoke" >> "$GITHUB_ENV" # The step stops well before the job's timeout, so the steps that # upload the results and logs still run when the suite hangs. - run: just smoke "${STORAGE}" @@ -817,15 +780,16 @@ jobs: - run: echo "## Smoke report card" >> "${GITHUB_STEP_SUMMARY}" - run: | "${DUCKDB_CLI}/duckdb" -markdown -f reports/.github/workflows/scripts/pivot-integration-report.sql >> "${GITHUB_STEP_SUMMARY}" + # The static Linux binaries: smoke and publish build the image from them. + # macos is the same build for macOS, in a job of its own so smoke needn't + # wait for it. release: - if: needs.changes.outputs.rust == 'true' && (github.event_name != 'pull_request' || (vars.MERGE_QUEUE != 'on' && github.event.pull_request.head.repo.full_name == github.repository)) - # build-storage/build-storage-lake are here (not on `test`, which is Tier - # A) so a feature combination that doesn't compile stops a queue entry - # before the release -> package -> smoke chain, and never publishes on a - # `v*` tag. - needs: [changes, test, src, typos, build-storage, build-storage-lake] - permissions: - contents: write + if: *tier_b + # test, src and the build matrices are not here, so this leg starts with + # the rest of the run. publish needs them, so a failure there still + # blocks every image and tag. typos stays: it takes seconds and stops a + # run before the expensive jobs. + needs: [changes, typos] runs-on: ${{matrix.os}} timeout-minutes: 45 strategy: @@ -838,8 +802,6 @@ jobs: - target: aarch64-unknown-linux-musl os: ubuntu-24.04-arm platform: linux/arm64 - - target: aarch64-apple-darwin - os: macos-latest env: # cc-rs otherwise looks for aarch64-linux-musl-gcc, which Ubuntu doesn't # ship, and falls back to glibc gcc for the C deps (aws-lc-sys, ring, @@ -864,15 +826,10 @@ jobs: cache-key: ${{ matrix.target }} cache-save-if: ${{ github.ref == 'refs/heads/main' }} - run: rustc -vV - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - path: artifacts - merge-multiple: true - env: TARGET: ${{matrix.target}} run: rustup target add "$TARGET" - - if: ${{runner.os == 'Linux'}} - run: | + - run: | sudo apt-get update sudo apt-get install -y musl-tools # Embeds the crate list so scanners can read the binary's dependencies. @@ -883,8 +840,7 @@ jobs: - env: TARGET: ${{matrix.target}} run: cargo auditable build --release --bin nisshi --all-features --target "$TARGET" - - if: ${{runner.os == 'Linux'}} - env: + - env: TARGET: ${{matrix.target}} run: | bin=target/"${TARGET}"/release/nisshi @@ -899,30 +855,77 @@ jobs: cp compose.yaml example.env LICENSE README.md "$dist" cp -r etc "$dist" tar --create --verbose --gzip --file=nisshi-"${TARGET}".tar.gz -C "$dist" . - - if: ${{github.ref_type != 'tag'}} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + # publish attaches the tarballs to the GitHub release, on a tag. + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: nisshi-${{matrix.target}} path: nisshi-${{matrix.target}}.tar.gz if-no-files-found: error - - if: ${{github.ref_type == 'tag'}} - uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 - with: - files: nisshi-${{matrix.target}}.tar.gz - # package's Dockerfile copies dist//nisshi into the image. - - if: ${{runner.os == 'Linux'}} - env: + # The Dockerfile copies dist//nisshi into the image, in smoke + # and in publish. + - env: PLATFORM: ${{matrix.platform}} TARGET: ${{matrix.target}} run: | mkdir -p dist/"${PLATFORM}" cp target/"${TARGET}"/release/nisshi dist/"${PLATFORM}"/nisshi - - if: ${{runner.os == 'Linux'}} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: image-${{matrix.target}} path: dist/ - retention-days: 1 + # publish reads this on a push, and a re-run of it can come days + # later. Other runs only need it for smoke. + retention-days: ${{ github.event_name == 'push' && 3 || 1 }} + if-no-files-found: error + # The full release build (about 20 minutes) runs on a `v*` tag only. Every + # other run compiles the binary without optimizing or linking it, which + # keeps the queue's critical path and the `main` image short. The nightly + # macos-release job in tier-c.yml builds the release on main's tip, so a + # link or release-profile break shows up before a tag. + macos: + if: *tier_b + needs: [changes, typos] + runs-on: macos-latest + timeout-minutes: 45 + env: + TARGET: aarch64-apple-darwin + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + with: + # The same settings as release, for the same reasons. + rustflags: "" + build-warnings: "" + cache: ${{ !startsWith(github.ref, 'refs/tags/') }} + # Not release's key: `check` writes different artifacts. + cache-key: ${{ env.TARGET }}-check + cache-save-if: ${{ github.ref == 'refs/heads/main' }} + - run: rustc -vV + - run: rustup target add "$TARGET" + - if: ${{github.ref_type != 'tag'}} + run: cargo check --bin nisshi --all-features --target "$TARGET" + - if: ${{github.ref_type == 'tag'}} + uses: taiki-e/install-action@83ac0ad63c0167e6f06796fab0fce28db1bf3db0 # v2.87.22 + with: + tool: cargo-auditable@0.7.6 + fallback: none + - if: ${{github.ref_type == 'tag'}} + run: cargo auditable build --release --bin nisshi --all-features --target "$TARGET" + - if: ${{github.ref_type == 'tag'}} + run: | + dist=$(mktemp -d -p ${{runner.temp}}) + mkdir "$dist"/bin + cp target/"${TARGET}"/release/nisshi "$dist"/bin + cp compose.yaml example.env LICENSE README.md "$dist" + cp -r etc "$dist" + tar --create --verbose --gzip --file=nisshi-"${TARGET}".tar.gz -C "$dist" . + - if: ${{github.ref_type == 'tag'}} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: nisshi-${{env.TARGET}} + path: nisshi-${{env.TARGET}}.tar.gz if-no-files-found: error # Single stable required-check context for branch protection. Every other @@ -936,7 +939,8 @@ jobs: # `needs.*.result` reads each listed job's own outcome, not the masked one. ci-gate: if: always() - needs: + # publish reuses this list as `*gate_needs`. + needs: &gate_needs - changes - fmt - clippy @@ -949,10 +953,10 @@ jobs: - typos - third-party-license - src - - package - smoke - smoke-oldest-client - release + - macos runs-on: ubuntu-latest timeout-minutes: 10 env: @@ -989,7 +993,101 @@ jobs: - name: Tier A ran on pull_request if: github.event_name == 'pull_request' && needs.changes.outputs.rust == 'true' env: - TIER_B: build-storage build-storage-lake cargo-publish-dry-run src package smoke smoke-oldest-client release + TIER_B: build-storage build-storage-lake cargo-publish-dry-run src smoke smoke-oldest-client release macos run: | skipped=$(jq -r --arg tier_b "$TIER_B" '($tier_b | split(" ")) as $exempt | to_entries[] | select(.value.result == "skipped") | select(.key as $k | $exempt | index($k) | not) | .key' <<<"$NEEDS") [ -z "$skipped" ] || { echo "::error::Tier A job skipped on a pull_request run, check for a needs edge onto a Tier B job: $(tr '\n' ' ' <<<"$skipped")"; exit 1; } + # The only job that pushes an image, and it holds only `packages: write`. It + # needs every job ci-gate does, so a red or skipped job anywhere stops it. It is not in + # ci-gate's `needs`: it runs on push only, and ci-gate's "Nothing skipped + # off pull_request" step would fail every merge_group run. A `v*` tag isn't + # covered by the main ruleset, so this `needs` list is the only gate a tag + # gets. On a tag, macos has also run the full release build by now. + publish: + if: github.event_name == 'push' + needs: *gate_needs + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + packages: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # Each push run has a concurrency group of its own, so an older commit's + # run can finish after a newer one. On main, push only while this commit + # is still the branch tip, so the `main` image never moves backwards. If + # the tip's own run then fails, re-run that run to publish. + - id: tip + if: github.ref_type == 'branch' + run: | + set -euo pipefail + tip=$(git ls-remote origin "$GITHUB_REF" | cut -f1) + if [ "$tip" = "$GITHUB_SHA" ]; then + echo "stale=false" >> "$GITHUB_OUTPUT" + else + echo "::notice::${GITHUB_REF} is at ${tip}, not ${GITHUB_SHA}; the newer run publishes the image" + echo "stale=true" >> "$GITHUB_OUTPUT" + fi + # release's static musl binaries, as dist/linux//nisshi. + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: image-* + merge-multiple: true + path: dist + - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 + id: meta + with: + images: ${{env.REGISTRY}}/${{github.repository}} + # `latest` comes with the semver tags, for a tag that isn't a + # prerelease. + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + with: + buildkitd-flags: --debug + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ${{env.REGISTRY}} + username: ${{github.actor}} + password: ${{secrets.GITHUB_TOKEN}} + - if: steps.tip.outputs.stale != 'true' + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + # Attaches the tarballs to the GitHub release. It is a job of its own so + # that `contents: write`, which can create tags the main ruleset doesn't + # cover, is held only on a tag run, and the `release` environment's + # deployment policy (`v*` tags) is a second, server-side check on the ref. + # It needs publish, so it never runs unless every gate passed. The image is + # pushed first, so a failed upload leaves an image without a release page + # until this job is re-run. The tarballs it reads keep the default artifact + # retention. + # Release first would put a `needs` on a job that is skipped on main pushes, + # which skips publish there. + publish-release: + if: github.event_name == 'push' && github.ref_type == 'tag' + needs: [publish] + environment: release + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + # Two hyphens: the tarballs are nisshi-, and docker + # build-push-action's build record is `nisshi-io~nisshi~.dockerbuild`. + pattern: nisshi-*-* + merge-multiple: true + path: tarballs + - uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 + with: + files: tarballs/*.tar.gz + fail_on_unmatched_files: true diff --git a/.github/workflows/tier-c.yml b/.github/workflows/tier-c.yml index 075c17fb..3eaef7ef 100644 --- a/.github/workflows/tier-c.yml +++ b/.github/workflows/tier-c.yml @@ -3,8 +3,9 @@ name: tier-c # Tier C is the optional nightly tier. It runs the checks that are too slow # or too unstable to gate a merge: the compat suites on the S3 engine, the -# `test` suite on more Postgres versions, including one beta, and the build -# on Windows. +# `test` suite on more Postgres versions, including one beta, the build on +# Windows, and the full macOS release build (ci.yml only checks macOS, except +# on a `v*` tag). # # Tier C does not gate anything. It is not in the `needs` of ci-gate and it is # not a required check. A red run opens or updates one GitHub issue with the @@ -69,8 +70,6 @@ jobs: NEXTEST_PROFILE: ci POSTGRES_IMAGE: ${{matrix.pg}} PROMETHEUS_LISTENER_URL: tcp://[::]:9100 - RUST_BACKTRACE: 1 - RUST_LOG: debug,nisshi_sans_io=warn,nisshi_model=warn SCHEMA_REGISTRY: file://./etc/schema STORAGE_ENGINE: s3://nisshi/ NISSHI_IMAGE: ghcr.io/nisshi-io/nisshi @@ -90,16 +89,21 @@ jobs: with: components: clippy, rustfmt # These legs read the cache that the `test` job in ci.yml saves on - # main. That needs the same job id, `test`, and the same values of - # every `RUST*` and `CARGO*` variable as that job, because the + # main. That needs the same shared key, `dev`, and the same values + # of every `RUST*` and `CARGO*` variable as that job, because the # cache key hashes them. A mismatch gives a cold build. This # workflow never saves, because a save here would replace the cache # that ci.yml restores on every pull request. + cache-shared-key: dev cache-save-if: false - uses: taiki-e/install-action@83ac0ad63c0167e6f06796fab0fce28db1bf3db0 # v2.87.22 with: tool: cargo-nextest@0.9.146 + # Set on the step, as in ci.yml: the cache key hashes `RUST*` variables. - run: just test + env: + RUST_BACKTRACE: 1 + RUST_LOG: debug,nisshi_sans_io=warn,nisshi_model=warn - run: df -h if: always() - run: tar --create --verbose --gzip --file="logs-${PG_IMAGE_DASH}.tar.gz" logs @@ -264,11 +268,35 @@ jobs: with: cache: false - run: just build dev + # The release profile, the linker and cargo-auditable, which the `macos` + # job in ci.yml skips off a tag. It uploads nothing. + macos-release: + if: github.ref == 'refs/heads/main' + runs-on: macos-latest + timeout-minutes: 45 + env: + TARGET: aarch64-apple-darwin + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f # v2.0.0 + with: + # The same settings as the release build in ci.yml. + rustflags: "" + build-warnings: "" + cache: false + - run: rustup target add "$TARGET" + - uses: taiki-e/install-action@83ac0ad63c0167e6f06796fab0fce28db1bf3db0 # v2.87.22 + with: + tool: cargo-auditable@0.7.6 + fallback: none + - run: cargo auditable build --release --bin nisshi --all-features --target "$TARGET" alert: # This job reads the result of each job above. A beta leg that fails does # not change the result of `test`, so it does not raise an alert. if: ${{ !cancelled() && github.ref == 'refs/heads/main' }} - needs: [test, compat-librdkafka, compat-franz-go, build-windows] + needs: [test, compat-librdkafka, compat-franz-go, build-windows, macos-release] runs-on: ubuntu-latest timeout-minutes: 5 # The job writes to issues and nothing else. It has no checkout, so diff --git a/CLAUDE.md b/CLAUDE.md index 30998d7e..7431a65d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -147,7 +147,9 @@ Lake features: `parquet`, `iceberg`, `delta` - enable writing schema-backed topi GitHub Actions (`.github/workflows/ci.yml`) runs in two tiers, gated by `ci-gate`, the single required check that fans in every other job: - **Tier A, every pull_request push:** `fmt`, `clippy` (which also runs `just doc`), `typos`, `third-party-license`, `test` (postgres:17 only), one non-experimental leg each of `compat-librdkafka` / `compat-franz-go`. -- **Tier B, once per merge-queue entry (`merge_group`) and on push to `main`:** the `build-storage` matrix (one build per storage engine, `turso` included) and `build-storage-lake` (one build per lake format on `dynostore`), `test` on postgres:16/17/18, the experimental compat legs, `cargo-publish-dry-run`, `src`, `release`, `package`, `smoke` (the `nisshi-smoke-test` suite: Kafka CLI tools from Kafka 3.9 and 4.3 against the packaged image on postgres, memory and s3, and against a source build on sqlite until #796 is fixed, on x86 and arm), `smoke-oldest-client` (the same suite with the Kafka 3.7 CLI tools, the oldest client CI tests, on every engine on x86 only); `smoke-report` puts every leg of both in one pass/fail grid in the step summary. +- **Tier B, once per merge-queue entry (`merge_group`) and on push to `main`:** the `build-storage` matrix (one build per storage engine, `turso` included) and `build-storage-lake` (one build per lake format on `dynostore`), `test` on postgres:16/17/18, the experimental compat legs, `cargo-publish-dry-run`, `src`, `release` (the static Linux binaries), `macos` (`aarch64-apple-darwin`: a compile-only `cargo check`, and the full release build only on a `v*` tag; the nightly Tier C workflow builds the macOS release on main), `smoke` (the `nisshi-smoke-test` suite: Kafka CLI tools from Kafka 3.9 and 4.3 against an image built in the job from `release`'s binaries on postgres, memory and s3, and against a source build on sqlite until #796 is fixed, on x86 and arm), `smoke-oldest-client` (the same suite with the Kafka 3.7 CLI tools, the oldest client CI tests, on every engine on x86 only); `smoke-report` puts every leg of both in one pass/fail grid in the step summary. +- **Publishing, on push to `main` and `v*` tags only:** `publish` needs every job `ci-gate` does, and is not in `ci-gate`. It is the only job that pushes an image (multi-arch, built from `release`'s binaries; `main` on main, semver and `latest` on tags) and holds only `packages: write`. On main it pushes only while its commit is still the branch tip, so a slower run for an older commit can't move the `main` image backwards. On a tag, `publish-release` (needs `publish`, holds `contents: write`, runs in the `release` environment) uploads the tarballs to the GitHub release. No run of any other kind pushes anything, and a failed leg can't leave a partial release. `release` needs only `typos` (and `changes`), and `smoke` needs only `release`, so `test`, `macos` and the build matrices run beside the `release` -> `smoke` chain instead of ahead of it. `test` doesn't wait for `fmt` or `clippy`. +- **Cache:** `test` saves the Rust cache under the `dev` shared key on main; the compat jobs restore it and never save. `build-storage` and `build-storage-lake` use no cache, because `cargo check` reuses little of a full build. rust-cache hashes every `CARGO*`, `CC*`, `CFLAGS*`, `CXX*`, `CMAKE*` and `RUST*` variable into the key, so set `RUST_BACKTRACE` and `RUST_LOG` on the step that needs them, not on the job. Merging goes through a merge queue: "Merge when ready" queues the PR, the queue re-runs CI on it against the current tip of `main`, and squash-merges it if everything is green: the PR title becomes the commit's subject and the PR description its body. Tier B is skipped on same-repo PRs only while the `MERGE_QUEUE` repository variable is `on`; with it unset, they run everything. Fork PRs can't read the variable, so Tier B is always skipped on them and runs in the queue. The other required checks come from `codeql.yml`, `workflow-lint.yml`, `dependencies.yml` and `pr-title.yml` (the PR title must follow Conventional Commits; see `CONTRIBUTING.md`).