diff --git a/.github/actions/cache-cargo-build-sbf/action.yaml b/.github/actions/cache-cargo-build-sbf/action.yaml new file mode 100644 index 00000000..8e25af3e --- /dev/null +++ b/.github/actions/cache-cargo-build-sbf/action.yaml @@ -0,0 +1,22 @@ +# Required input env vars +# - SOLANAVERS + +name: "Cache cargo-build-sbf" +description: "caches cargo-build-sbf output in target folder" + +runs: + using: "composite" + steps: + - name: cargo-build-sbf Cache + uses: actions/cache@v4 + # Make sure vars match those in restore-cache-cargo-build-sbf/action.yaml + with: + path: | + target/CACHEDIR.TAG + target/deploy + target/release + target/.rustc_info.json + target/sbpf-solana-solana + key: ${{ runner.os }}-sbf-${{ env.SOLANAVERS }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-sbf- diff --git a/.github/actions/cache-cargo-fetch/action.yaml b/.github/actions/cache-cargo-fetch/action.yaml new file mode 100644 index 00000000..96b1d107 --- /dev/null +++ b/.github/actions/cache-cargo-fetch/action.yaml @@ -0,0 +1,15 @@ +name: "Cache cargo fetch" +description: "caches cargo fetch output" + +runs: + using: "composite" + steps: + - name: cargo fetch Cache + uses: actions/cache@v4 + with: + # Make sure vars match those in restore-cache-cargo-fetch/action.yaml + path: | + ${{ env.CARGO_HOME }} + key: ${{ runner.os }}-rustfetch-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-rustfetch- diff --git a/.github/actions/cache-cargo-test-sbf/action.yaml b/.github/actions/cache-cargo-test-sbf/action.yaml new file mode 100644 index 00000000..6149b8af --- /dev/null +++ b/.github/actions/cache-cargo-test-sbf/action.yaml @@ -0,0 +1,18 @@ +# Required input env vars +# - SOLANAVERS + +name: "Cache cargo-test-sbf" +description: "caches cargo-test-sbf output in target folder" + +runs: + using: "composite" + steps: + - name: cargo-test-sbf Cache + uses: actions/cache@v4 + with: + path: | + target/CACHEDIR.TAG + target/debug + key: ${{ runner.os }}-sbftest-${{ env.SOLANAVERS }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-sbftest- diff --git a/.github/actions/cache-cargo-test/action.yaml b/.github/actions/cache-cargo-test/action.yaml new file mode 100644 index 00000000..c1cd9d5e --- /dev/null +++ b/.github/actions/cache-cargo-test/action.yaml @@ -0,0 +1,18 @@ +# Required input env vars +# - RUSTVERS + +name: "Cache cargo test" +description: "caches cargo test output in target folder" + +runs: + using: "composite" + steps: + - name: cargo test Cache + uses: actions/cache@v4 + with: + path: | + target/CACHEDIR.TAG + target/debug + key: ${{ runner.os }}-rusttest-${{ env.RUSTVERS }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-rusttest- diff --git a/.github/actions/cache-clippy/action.yaml b/.github/actions/cache-clippy/action.yaml new file mode 100644 index 00000000..957a3a2c --- /dev/null +++ b/.github/actions/cache-clippy/action.yaml @@ -0,0 +1,17 @@ +# Required input env vars +# - RUSTVERS + +name: "Cache clippy" +description: "caches cargo clippy output in target folder" + +runs: + using: "composite" + steps: + - name: clippy Cache + uses: actions/cache@v4 + with: + path: | + target + key: ${{ runner.os }}-rustlint-${{ env.RUSTVERS }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-rustlint- diff --git a/.github/actions/cache-solana-platform-tools/action.yaml b/.github/actions/cache-solana-platform-tools/action.yaml new file mode 100644 index 00000000..b6422f50 --- /dev/null +++ b/.github/actions/cache-solana-platform-tools/action.yaml @@ -0,0 +1,22 @@ +# Required input env vars +# - SOLANAVERS + +name: "Cache solana platform tools" +description: "caches initial download of solana platform tools" + +runs: + using: "composite" + steps: + - name: Solana Platform Tools Cache + uses: actions/cache@v4 + with: + # 500MB thats downloaded on first invoke + # of cargo-build-sbf is the toolchain + # that goes into $HOME/.cache/solana + # This will be fixed by + # https://github.com/Ellipsis-Labs/solana-verifiable-build/pull/202 + # Make sure vars match those in restore-cache-solana-platform-tools/action.yaml + path: ~/.cache/solana + key: ${{ runner.os }}-solana-platform-tools-${{ env.SOLANAVERS }} + restore-keys: | + ${{ runner.os }}-solana-platform-tools- diff --git a/.github/actions/cache-wasm-build/action.yaml b/.github/actions/cache-wasm-build/action.yaml new file mode 100644 index 00000000..2acf1471 --- /dev/null +++ b/.github/actions/cache-wasm-build/action.yaml @@ -0,0 +1,20 @@ +# Required input env vars +# - RUSTVERS + +name: "Cache wasm Build" +description: "caches wasm (ts) rust build" + +runs: + using: "composite" + steps: + - name: wasm Cache + uses: actions/cache@v4 + with: + path: | + target/CACHEDIR.TAG + target/release + target/.rustc_info.json + target/wasm32-unknown-unknown + key: ${{ runner.os }}-wasmbuild-${{ env.RUSTVERS }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-wasmbuild- diff --git a/.github/actions/download-sbf-progs/action.yaml b/.github/actions/download-sbf-progs/action.yaml new file mode 100644 index 00000000..0ad46f64 --- /dev/null +++ b/.github/actions/download-sbf-progs/action.yaml @@ -0,0 +1,12 @@ +name: "Download SBF Programs Artifact" + +runs: + using: "composite" + steps: + - name: SBF Programs Artifact Download + uses: actions/download-artifact@v4 + with: + # Make sure vars match those in upload-sbf-progs/action.yaml + name: sbfprogs + path: | + target/deploy diff --git a/.github/actions/download-wasm-pkg/action.yaml b/.github/actions/download-wasm-pkg/action.yaml new file mode 100644 index 00000000..7d41d03f --- /dev/null +++ b/.github/actions/download-wasm-pkg/action.yaml @@ -0,0 +1,12 @@ +name: "Download wasm Package Artifact" + +runs: + using: "composite" + steps: + - name: wasm Package Artifact Download + uses: actions/download-artifact@v4 + with: + # Make sure vars match those in upload-wasm-pkg/action.yaml + name: wasmpkg + path: | + ts/sdk/pkg diff --git a/.github/actions/restore-cache-cargo-build-sbf/action.yaml b/.github/actions/restore-cache-cargo-build-sbf/action.yaml new file mode 100644 index 00000000..7558a80b --- /dev/null +++ b/.github/actions/restore-cache-cargo-build-sbf/action.yaml @@ -0,0 +1,28 @@ +# Required input env vars +# - SOLANAVERS + +name: "Restore cargo-build-sbf cache" +description: "restores cached cargo-build-sbf output in target folder" +inputs: + fail-on-cache-miss: + description: "fwd to actions/cache/restore" + required: false + default: false + +runs: + using: "composite" + steps: + - name: cargo-build-sbf Cache Restore + uses: actions/cache/restore@v4 + with: + fail-on-cache-miss: ${{ inputs.fail-on-cache-miss }} + # Make sure vars match those in cache-cargo-build-sbf/action.yaml + path: | + target/CACHEDIR.TAG + target/deploy + target/release + target/.rustc_info.json + target/sbpf-solana-solana + key: ${{ runner.os }}-sbf-${{ env.SOLANAVERS }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-sbf- diff --git a/.github/actions/restore-cache-cargo-fetch/action.yaml b/.github/actions/restore-cache-cargo-fetch/action.yaml new file mode 100644 index 00000000..eff11b67 --- /dev/null +++ b/.github/actions/restore-cache-cargo-fetch/action.yaml @@ -0,0 +1,15 @@ +name: "Restore Cache cargo fetch" +description: "restores cache from cache cargo fetch action" + +runs: + using: "composite" + steps: + - name: cargo fetch Cache Restore + uses: actions/cache/restore@v4 + with: + # Make sure vars match those in cache-cargo-fetch/action.yaml + path: | + ${{ env.CARGO_HOME }} + key: ${{ runner.os }}-rustfetch-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-rustfetch- diff --git a/.github/actions/restore-cache-solana-platform-tools/action.yaml b/.github/actions/restore-cache-solana-platform-tools/action.yaml new file mode 100644 index 00000000..2acff5b0 --- /dev/null +++ b/.github/actions/restore-cache-solana-platform-tools/action.yaml @@ -0,0 +1,17 @@ +# Required input env vars +# - SOLANAVERS + +name: "Restore Cache solana platform tools" +description: "restores cache of solana platform tools" + +runs: + using: "composite" + steps: + - name: Solana Platform Tools Cache Restore + uses: actions/cache/restore@v4 + with: + # Make sure vars match those in cache-solana-platform-tools/action.yaml + path: ~/.cache/solana + key: ${{ runner.os }}-solana-platform-tools-${{ env.SOLANAVERS }} + restore-keys: | + ${{ runner.os }}-solana-platform-tools- diff --git a/.github/actions/upload-sbf-progs/action.yaml b/.github/actions/upload-sbf-progs/action.yaml new file mode 100644 index 00000000..ad67b727 --- /dev/null +++ b/.github/actions/upload-sbf-progs/action.yaml @@ -0,0 +1,12 @@ +name: "Upload SBF Programs Artifact" + +runs: + using: "composite" + steps: + - name: SBF Programs Artifact Upload + uses: actions/upload-artifact@v4 + with: + # Make sure vars match those in download-sbf-progs/action.yaml + name: sbfprogs + path: | + target/deploy diff --git a/.github/actions/upload-wasm-pkg/action.yaml b/.github/actions/upload-wasm-pkg/action.yaml new file mode 100644 index 00000000..aad33181 --- /dev/null +++ b/.github/actions/upload-wasm-pkg/action.yaml @@ -0,0 +1,14 @@ +# NB: the uploaded package is not read for use since postbuild has not been ran on it + +name: "Upload wasm Package Artifact" + +runs: + using: "composite" + steps: + - name: wasm Package Artifact Upload + uses: actions/upload-artifact@v4 + with: + # Make sure vars match those in download-wasm-pkg/action.yaml + name: wasmpkg + path: | + ts/sdk/pkg diff --git a/.github/workflows/rust.yaml b/.github/workflows/rust.yaml index e65dae38..e58e2a7e 100644 --- a/.github/workflows/rust.yaml +++ b/.github/workflows/rust.yaml @@ -40,9 +40,24 @@ jobs: - name: Format run: cargo fmt --all -- --check - lint: + fetch: runs-on: ubuntu-latest needs: output-rust-build-image-name + container: + image: ${{ needs.output-rust-build-image-name.outputs.image }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: cargo fetch Cache + uses: ./.github/actions/cache-cargo-fetch + - name: Fetch + run: cargo fetch + + lint: + runs-on: ubuntu-latest + needs: + - output-rust-build-image-name + - fetch container: image: ${{ needs.output-rust-build-image-name.outputs.image }} steps: @@ -50,16 +65,11 @@ jobs: uses: actions/checkout@v4 - name: Install clippy run: rustup component add clippy - - name: Rust build cache - uses: actions/cache@v4 - with: - path: | - ${{ env.CARGO_HOME }} - target - key: ${{ runner.os }}-rustlint-${{ env.RUSTVERS }}-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-rustlint- - - name: Clippy + - name: cargo fetch Cache Restore + uses: ./.github/actions/restore-cache-cargo-fetch + - name: clippy cache + uses: ./.github/actions/cache-clippy + - name: clippy run: cargo clippy --all-features --tests -- -D clippy::all # hax from https://github.com/orgs/community/discussions/26324#discussioncomment-3251460 @@ -72,41 +82,134 @@ jobs: - id: set_image run: echo "image=solanafoundation/solana-verifiable-build:${{ env.SOLANAVERS }}" >> $GITHUB_OUTPUT - build-and-test-sbf: + build-sbf: runs-on: ubuntu-latest - needs: output-solana-build-image-name + needs: + - output-solana-build-image-name + - fetch container: image: ${{ needs.output-solana-build-image-name.outputs.image }} steps: - name: Checkout uses: actions/checkout@v4 + - name: cargo fetch Cache Restore + uses: ./.github/actions/restore-cache-cargo-fetch - name: Solana Platform Tools Cache - uses: actions/cache@v4 - with: - # 500MB thats downloaded on first invoke - # of cargo-build-sbf is the toolchain - # that goes into $HOME/.cache/solana - # This will be fixed by - # https://github.com/Ellipsis-Labs/solana-verifiable-build/pull/202 - path: ~/.cache/solana - key: ${{ runner.os }}-solana-platform-tools-${{ env.SOLANAVERS }} - restore-keys: | - ${{ runner.os }}-solana-platform-tools- - - name: Rust build cache - uses: actions/cache@v4 - with: - path: | - ${{ env.CARGO_HOME }} - target - key: ${{ runner.os }}-sbf-${{ env.SOLANAVERS }}-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-sbf- + uses: ./.github/actions/cache-solana-platform-tools + - name: cargo-build-sbf Cache + uses: ./.github/actions/cache-cargo-build-sbf - name: Build run: cargo-build-sbf + - name: Upload + uses: ./.github/actions/upload-sbf-progs + + test-sbf: + runs-on: ubuntu-latest + needs: + - output-solana-build-image-name + - fetch + - build-sbf + container: + image: ${{ needs.output-solana-build-image-name.outputs.image }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Solana Platform Tools Cache Restore + uses: ./.github/actions/restore-cache-solana-platform-tools + - name: cargo fetch Cache Restore + uses: ./.github/actions/restore-cache-cargo-fetch + - name: cargo-build-sbf Cache Restore + uses: ./.github/actions/restore-cache-cargo-build-sbf + - name: cargo-test-sbf Cache + # use cache instead of artifact because + # cargo-test-sbf runs cargo-build-sbf again + uses: ./.github/actions/cache-cargo-test-sbf - name: Test SBF run: cargo-test-sbf - # Running `cargo test` in the same job because - # - some non SBF tests depend on programs being built - # - make use of shared build cache (even though its not 100% overlap) - - name: Test Non-SBF + + test: + runs-on: ubuntu-latest + needs: + - output-rust-build-image-name + - fetch + - build-sbf + container: + image: ${{ needs.output-rust-build-image-name.outputs.image }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Download SBF Programs + uses: ./.github/actions/download-sbf-progs + - name: cargo fetch Cache Restore + uses: ./.github/actions/restore-cache-cargo-fetch + - name: cargo test Cache + uses: ./.github/actions/cache-cargo-test + - name: Test run: cargo test + + # hax from https://github.com/orgs/community/discussions/26324#discussioncomment-3251460 + # so that we can interpolate env vars in docker image name + output-wasm-build-image-name: + runs-on: ubuntu-latest + outputs: + image: ${{ steps.set_image.outputs.image }} + steps: + - id: set_image + run: echo "image=dylanowen/wasm-pack:${{ env.RUSTVERS }}" >> $GITHUB_OUTPUT + + build-wasm: + runs-on: ubuntu-latest + needs: + - output-wasm-build-image-name + - fetch + container: + image: ${{ needs.output-wasm-build-image-name.outputs.image }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: cargo fetch Cache Restore + uses: ./.github/actions/restore-cache-cargo-fetch + - name: wasm Cache + uses: ./.github/actions/cache-wasm-build + - name: Delete Outdated System wasm-opt + # causes wasm-opt to fail. Delete to force to + # use the one bundled in wasm-pack instead + run: rm -rf /usr/local/cargo/bin/wasm-opt + - name: Build wasm + working-directory: ts/sdk + # wasm build image does not have nodejs required to run full `make`, + # so we only run until prod target + run: make prod + - name: Upload + uses: ./.github/actions/upload-wasm-pkg + + test-ts: + runs-on: ubuntu-latest + needs: + - build-sbf + - build-wasm + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Download wasm pkg + uses: ./.github/actions/download-wasm-pkg + - name: Download SBF Programs + uses: ./.github/actions/download-sbf-progs + - name: wasm Postbuild + # Run postbuild step that was omitted in build-wasm + working-directory: ts/sdk/postbuild + run: node postbuild.mjs + - name: pnpm Setup + uses: pnpm/action-setup@v4 + with: + version: 10 + # false because we need to run it in working-directory + run_install: false + - name: Install pnpm Deps + # TODO: cache pnpm install when required. + # Currently takes ~3s to download all deps, so prolly not worth it + working-directory: ts/tests + run: pnpm install + - name: Test TS + working-directory: ts/tests + run: pnpm test diff --git a/docker-compose-local-validator.yml b/docker-compose-local-validator.yml index 39da0f9a..19c99754 100644 --- a/docker-compose-local-validator.yml +++ b/docker-compose-local-validator.yml @@ -1,6 +1,9 @@ services: validator: - image: docker.io/lifeofpavs/solana-test-validator:latest + image: docker.io/sanctumso/solana-test-validator:3.1.1 + security_opt: + # insecure, needed for io_uring, else validator will fail with no stdout + - seccomp:unconfined command: [ "solana-test-validator", "-r", # Reset diff --git a/ts/tests/README.md b/ts/tests/README.md index a511eb19..63a74296 100644 --- a/ts/tests/README.md +++ b/ts/tests/README.md @@ -18,7 +18,17 @@ Before running the tests: ``` - rebuild the onchain programs if they have changed. The compiled `.so` files **MUST** be in `target/deploy/` -Then, start the local test validator with: +Then, run tests with + +```sh +pnpm test +``` + +## Run With Independent Validator Process + +Having a long-running test validator in the background that doesn't shutdown on test completion can be useful for debugging with explorer and other tools. + +Start the local test validator with: ```sh pnpm start:infra @@ -27,7 +37,7 @@ pnpm start:infra Then, run the test script with: ```sh -pnpm test +pnpm vitest run ``` After tests complete, teardown the local test validator with: diff --git a/ts/tests/package.json b/ts/tests/package.json index d58e7ba8..4313703e 100644 --- a/ts/tests/package.json +++ b/ts/tests/package.json @@ -3,10 +3,10 @@ "type": "module", "private": true, "scripts": { - "test": "vitest run", + "test": "./scripts/run-test.sh", "script:eli": "tsx scripts/enable-lst-input.ts", "script:gclus": "tsx scripts/generic-calc-last-update-slot.ts", - "start:infra": "docker compose -f ../../docker-compose-local-validator.yml up -d", + "start:infra": "docker compose -f ../../docker-compose-local-validator.yml up -d --wait", "stop:infra": "docker compose -f ../../docker-compose-local-validator.yml down -v" }, "devDependencies": { diff --git a/ts/tests/scripts/run-test.sh b/ts/tests/scripts/run-test.sh new file mode 100755 index 00000000..1561d3e0 --- /dev/null +++ b/ts/tests/scripts/run-test.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +REPO_DIR="$(dirname "$0")/../../.." +DOCKER_COMPOSE_LOCAL_VALIDATOR="$REPO_DIR/docker-compose-local-validator.yml" + +# Ensure cleanup happens on script exit +cleanup() { + echo "validator logs:" + echo "" + docker compose -f $DOCKER_COMPOSE_LOCAL_VALIDATOR logs + + echo "" + + # Clean up solana-test-validator container + echo "Cleaning up validator..." + docker compose -f $DOCKER_COMPOSE_LOCAL_VALIDATOR down -v +} + +trap cleanup EXIT + +docker compose -f $DOCKER_COMPOSE_LOCAL_VALIDATOR up -d --wait + +vitest run "$@"