diff --git a/.github/.patch_files b/.github/.patch_files index 7a34e1cb..dd65bdc8 100644 --- a/.github/.patch_files +++ b/.github/.patch_files @@ -1,22 +1,33 @@ -.github/.patch_files -.github/.syncignore -.github/CODEOWNERS -.github/dependabot.yml -.github/labels.yml .github/workflows/approve-bot-pr.yml +.github/workflows/codeql-analysis.yml +.github/workflows/create-draft-release.yml .github/workflows/label-pr.yml .github/workflows/lint-yaml.yml .github/workflows/lint.yml +.github/workflows/publish-releases.yml +.github/workflows/push-buildpackage.yml .github/workflows/synchronize-labels.yml .github/workflows/test-pull-request.yml .github/workflows/update-buildpack-toml.yml .github/workflows/update-github-config.yml +.github/workflows/update-go-mod-version.yml +.github/.patch_files +.github/.syncignore +.github/CODEOWNERS +.github/dependabot.yml +.github/labels.yml .gitignore LICENSE NOTICE README.md +go.mod +go.sum scripts/.util/builders.sh scripts/.util/print.sh scripts/.util/tools.json scripts/.util/tools.sh +scripts/.syncignore scripts/integration.sh +scripts/options.json +scripts/package.sh +scripts/publish.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml index bf99f0e3..dbe82d23 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -17,7 +17,7 @@ updates: - "minor" - "patch" exclude-patterns: - - "github.com/anchore/stereoscope" - - "github.com/testcontainers/testcontainers-go" - - "github.com/docker/docker" - - "github.com/containerd/containerd" + # go-diskfs 1.8 broke API wrt 1.7 which is in use by syft + # at the time of writing + # keep until https://github.com/anchore/syft/pull/4719 is merged or issue resolved. + - "github.com/diskfs/go-diskfs" diff --git a/.github/workflows/approve-bot-pr.yml b/.github/workflows/approve-bot-pr.yml index c798e647..00e6345a 100644 --- a/.github/workflows/approve-bot-pr.yml +++ b/.github/workflows/approve-bot-pr.yml @@ -10,7 +10,7 @@ jobs: download: name: Download PR Artifact if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' }} - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: pr-author: ${{ steps.pr-data.outputs.author }} pr-number: ${{ steps.pr-data.outputs.number }} @@ -32,7 +32,7 @@ jobs: name: Approve Bot PRs needs: download if: ${{ needs.download.outputs.pr-author == 'paketo-bot' || needs.download.outputs.pr-author == 'dependabot[bot]' }} - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Check Commit Verification id: unverified-commits @@ -52,7 +52,7 @@ jobs: - name: Checkout if: steps.human-commits.outputs.human_commits == 'false' && steps.unverified-commits.outputs.unverified_commits == 'false' - uses: actions/checkout@v3 + uses: actions/checkout@v6 - name: Approve if: steps.human-commits.outputs.human_commits == 'false' && steps.unverified-commits.outputs.unverified_commits == 'false' diff --git a/.github/workflows/create-draft-release.yml b/.github/workflows/create-draft-release.yml index 4e897f39..295cbd2a 100644 --- a/.github/workflows/create-draft-release.yml +++ b/.github/workflows/create-draft-release.yml @@ -17,12 +17,12 @@ concurrency: release jobs: builders: name: Get Builders for Testing - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: builders: ${{ steps.builders.outputs.builders }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 - name: Get builders from integration.json id: builders run: | @@ -33,31 +33,39 @@ jobs: printf "builders=%s\n" "${builders}" >> "$GITHUB_OUTPUT" integration: name: Integration Tests - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [builders] strategy: matrix: builder: ${{ fromJSON(needs.builders.outputs.builders) }} fail-fast: false # don't cancel all test jobs when one fails steps: + - name: Checkout + uses: actions/checkout@v6 - name: Setup Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v6 with: - go-version: 'stable' - - name: Checkout - uses: actions/checkout@v3 + go-version-file: go.mod + + # Causes errors with integration tests + - name: Disable containerd snapshotter + run: | + echo '{"features": {"containerd-snapshotter": false}}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + - name: Run Integration Tests env: TMPDIR: "${{ runner.temp }}" + GIT_TOKEN: ${{ github.token }} run: ./scripts/integration.sh --builder ${{ matrix.builder }} release: name: Release - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: integration steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: fetch-tags: true - name: Reset Draft Release @@ -106,12 +114,17 @@ jobs: "path": "build/buildpackage.cnb", "name": "${{ github.event.repository.name }}-${{ steps.tag.outputs.tag }}.cnb", "content_type": "application/x-tar" + }, + { + "path": "build/buildpack-release-artifact.tgz", + "name": "${{ github.event.repository.name }}-${{ steps.tag.outputs.tag }}.tgz", + "content_type": "application/gzip" } ] failure: name: Alert on Failure - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [ integration, release ] if: ${{ always() && needs.integration.result == 'failure' || needs.release.result == 'failure' }} steps: diff --git a/.github/workflows/label-pr.yml b/.github/workflows/label-pr.yml index b09cdf09..55f68506 100644 --- a/.github/workflows/label-pr.yml +++ b/.github/workflows/label-pr.yml @@ -15,10 +15,10 @@ concurrency: pr_labels_${{ github.event.number }} jobs: autolabel: name: Ensure Minimal Semver Labels - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Check Minimal Semver Labels - uses: mheap/github-action-required-labels@v3 + uses: mheap/github-action-required-labels@v5 with: count: 1 labels: semver:major, semver:minor, semver:patch diff --git a/.github/workflows/lint-yaml.yml b/.github/workflows/lint-yaml.yml index e6f4a8b1..69d60672 100644 --- a/.github/workflows/lint-yaml.yml +++ b/.github/workflows/lint-yaml.yml @@ -8,20 +8,20 @@ on: jobs: lintYaml: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: Checkout github-config - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: repository: paketo-buildpacks/github-config path: github-config - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.14 - name: Install yamllint run: pip install yamllint diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d1008181..469c029d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,9 +1,6 @@ name: Lint on: - push: - branches: - - main pull_request: branches: - main @@ -11,18 +8,18 @@ on: jobs: golangci: name: lint - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Setup Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v6 with: - go-version: 'stable' - - - name: Checkout - uses: actions/checkout@v3 + go-version-file: go.mod - name: golangci-lint - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@v9 with: version: latest args: --timeout 3m0s diff --git a/.github/workflows/publish-releases.yml b/.github/workflows/publish-releases.yml index 4654c388..44abfb3f 100644 --- a/.github/workflows/publish-releases.yml +++ b/.github/workflows/publish-releases.yml @@ -11,7 +11,7 @@ concurrency: jobs: publish: name: Publish - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Publish Draft Release With Highest Semantic Version id: drafts @@ -23,7 +23,7 @@ jobs: failure: name: Alert on Failure - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [ publish ] if: ${{ always() && needs.publish.result == 'failure' }} steps: diff --git a/.github/workflows/push-buildpackage.yml b/.github/workflows/push-buildpackage.yml index 7ba5bd76..9e705fb5 100644 --- a/.github/workflows/push-buildpackage.yml +++ b/.github/workflows/push-buildpackage.yml @@ -4,13 +4,24 @@ on: release: types: - published +env: + REGISTRIES_FILENAME: "registries.json" + GCR_REGISTRY: "gcr.io" + GCR_PASSWORD: ${{ secrets.GCR_PUSH_BOT_JSON_KEY }} + GCR_USERNAME: "_json_key" + DOCKERHUB_REGISTRY: docker.io + DOCKERHUB_USERNAME: ${{ secrets.PAKETO_BUILDPACKS_DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.PAKETO_BUILDPACKS_DOCKERHUB_PASSWORD }} jobs: push: name: Push - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Parse Event id: event run: | @@ -20,63 +31,103 @@ jobs: echo "tag_full=${FULL_VERSION}" >> "$GITHUB_OUTPUT" echo "tag_minor=${MINOR_VERSION}" >> "$GITHUB_OUTPUT" echo "tag_major=${MAJOR_VERSION}" >> "$GITHUB_OUTPUT" - echo "download_url=$(jq -r '.release.assets[] | select(.name | endswith(".cnb")) | .url' "${GITHUB_EVENT_PATH}")" >> "$GITHUB_OUTPUT" + echo "download_tgz_file_url=$(jq -r '.release.assets[] | select(.name | endswith(".tgz")) | .url' "${GITHUB_EVENT_PATH}")" >> "$GITHUB_OUTPUT" - - name: Download + - name: Download .tgz buildpack release artifact uses: paketo-buildpacks/github-config/actions/release/download-asset@main with: - url: ${{ steps.event.outputs.download_url }} - output: "/github/workspace/buildpackage.cnb" + url: ${{ steps.event.outputs.download_tgz_file_url }} + output: "/github/workspace/buildpack-release-artifact.tgz" token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} + - name: Parse Configs + id: parse_configs + run: | + registries_filename="${{ env.REGISTRIES_FILENAME }}" + + push_to_dockerhub=true + push_to_gcr=false + + if [[ -f $registries_filename ]]; then + if jq 'has("dockerhub")' $registries_filename > /dev/null; then + push_to_dockerhub=$(jq '.dockerhub' $registries_filename) + fi + if jq 'has("GCR")' $registries_filename > /dev/null; then + push_to_gcr=$(jq '.GCR' $registries_filename) + fi + fi + + echo "push_to_dockerhub=${push_to_dockerhub}" >> "$GITHUB_OUTPUT" + echo "push_to_gcr=${push_to_gcr}" >> "$GITHUB_OUTPUT" + + - name: Install yj and crane + uses: buildpacks/github-actions/setup-tools@v5.11.0 + - name: Validate version run: | - buidpackTomlVersion=$(sudo skopeo inspect "oci-archive:${GITHUB_WORKSPACE}/buildpackage.cnb" | jq -r '.Labels."io.buildpacks.buildpackage.metadata" | fromjson | .version') + buidpackTomlVersion=$(tar -xzf buildpack-release-artifact.tgz --to-stdout buildpack.toml | yj -tj | jq -r .buildpack.version) githubReleaseVersion="${{ steps.event.outputs.tag_full }}" if [[ "$buidpackTomlVersion" != "$githubReleaseVersion" ]]; then echo "Version in buildpack.toml ($buidpackTomlVersion) and github release ($githubReleaseVersion) are not identical" exit 1 fi + - name: Docker login docker.io + uses: docker/login-action@v4 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_PASSWORD }} + registry: ${{ env.DOCKERHUB_REGISTRY }} + + - name: Docker login gcr.io + uses: docker/login-action@v4 + if: ${{ steps.parse_configs.outputs.push_to_gcr == 'true' }} + with: + username: ${{ env.GCR_USERNAME }} + password: ${{ env.GCR_PASSWORD }} + registry: ${{ env.GCR_REGISTRY }} + - name: Push to GCR - env: - GITHUB_TOKEN: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} - GCR_PUSH_BOT_JSON_KEY: ${{ secrets.GCR_PUSH_BOT_JSON_KEY }} + if: ${{ steps.parse_configs.outputs.push_to_gcr == 'true' }} run: | - echo "${GCR_PUSH_BOT_JSON_KEY}" | sudo skopeo login --username _json_key --password-stdin gcr.io - sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/buildpackage.cnb" "docker://gcr.io/${{ github.repository }}:${{ steps.event.outputs.tag_full }}" - sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/buildpackage.cnb" "docker://gcr.io/${{ github.repository }}:${{ steps.event.outputs.tag_minor }}" - sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/buildpackage.cnb" "docker://gcr.io/${{ github.repository }}:${{ steps.event.outputs.tag_major }}" - sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/buildpackage.cnb" "docker://gcr.io/${{ github.repository }}:latest" + ./scripts/publish.sh \ + --archive-path buildpack-release-artifact.tgz \ + --image-ref "gcr.io/${{ github.repository }}:${{ steps.event.outputs.tag_full }}" + + crane copy "gcr.io/${{ github.repository }}:${{ steps.event.outputs.tag_full }}" "gcr.io/${{ github.repository }}:${{ steps.event.outputs.tag_minor }}" + crane copy "gcr.io/${{ github.repository }}:${{ steps.event.outputs.tag_full }}" "gcr.io/${{ github.repository }}:${{ steps.event.outputs.tag_major }}" + crane copy "gcr.io/${{ github.repository }}:${{ steps.event.outputs.tag_full }}" "gcr.io/${{ github.repository }}:latest" - name: Push to DockerHub + if: ${{ steps.parse_configs.outputs.push_to_dockerhub == 'true' }} id: push - env: - DOCKERHUB_USERNAME: ${{ secrets.PAKETO_BUILDPACKS_DOCKERHUB_USERNAME }} - DOCKERHUB_PASSWORD: ${{ secrets.PAKETO_BUILDPACKS_DOCKERHUB_PASSWORD }} - GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} run: | - REPOSITORY="${GITHUB_REPOSITORY_OWNER/-/}/${GITHUB_REPOSITORY#${GITHUB_REPOSITORY_OWNER}/}" # translates 'paketo-buildpacks/bundle-install' to 'paketobuildpacks/bundle-install' - IMAGE="index.docker.io/${REPOSITORY}" - echo "${DOCKERHUB_PASSWORD}" | sudo skopeo login --username "${DOCKERHUB_USERNAME}" --password-stdin index.docker.io - sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/buildpackage.cnb" "docker://${IMAGE}:${{ steps.event.outputs.tag_full }}" - sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/buildpackage.cnb" "docker://${IMAGE}:${{ steps.event.outputs.tag_minor }}" - sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/buildpackage.cnb" "docker://${IMAGE}:${{ steps.event.outputs.tag_major }}" - sudo skopeo copy "oci-archive:${GITHUB_WORKSPACE}/buildpackage.cnb" "docker://${IMAGE}:latest" + IMAGE="${GITHUB_REPOSITORY_OWNER/-/}/${GITHUB_REPOSITORY#${GITHUB_REPOSITORY_OWNER}/}" # translates 'paketo-buildpacks/bundle-install' to 'paketobuildpacks/bundle-install' + + ./scripts/publish.sh \ + --archive-path buildpack-release-artifact.tgz \ + --image-ref "${DOCKERHUB_REGISTRY}/${IMAGE}:${{ steps.event.outputs.tag_full }}" + + pushed_image_index_digest=$(crane digest "${DOCKERHUB_REGISTRY}/${IMAGE}:${{ steps.event.outputs.tag_full }}" | xargs) + + crane copy "${DOCKERHUB_REGISTRY}/${IMAGE}:${{ steps.event.outputs.tag_full }}" "${DOCKERHUB_REGISTRY}/${IMAGE}:${{ steps.event.outputs.tag_minor }}" + crane copy "${DOCKERHUB_REGISTRY}/${IMAGE}:${{ steps.event.outputs.tag_full }}" "${DOCKERHUB_REGISTRY}/${IMAGE}:${{ steps.event.outputs.tag_major }}" + crane copy "${DOCKERHUB_REGISTRY}/${IMAGE}:${{ steps.event.outputs.tag_full }}" "${DOCKERHUB_REGISTRY}/${IMAGE}:latest" + echo "image=${IMAGE}" >> "$GITHUB_OUTPUT" - echo "digest=$(sudo skopeo inspect "oci-archive:${GITHUB_WORKSPACE}/buildpackage.cnb" | jq -r .Digest)" >> "$GITHUB_OUTPUT" + echo "digest=$pushed_image_index_digest" >> "$GITHUB_OUTPUT" - name: Register with CNB Registry uses: docker://ghcr.io/buildpacks/actions/registry/request-add-entry:main with: id: ${{ github.repository }} version: ${{ steps.event.outputs.tag_full }} - address: ${{ steps.push.outputs.image }}@${{ steps.push.outputs.digest }} + address: index.docker.io/${{ steps.push.outputs.image }}@${{ steps.push.outputs.digest }} token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} failure: name: Alert on Failure - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [push] if: ${{ always() && needs.push.result == 'failure' }} steps: diff --git a/.github/workflows/synchronize-labels.yml b/.github/workflows/synchronize-labels.yml index 3aaa0177..09db8880 100644 --- a/.github/workflows/synchronize-labels.yml +++ b/.github/workflows/synchronize-labels.yml @@ -10,9 +10,9 @@ jobs: synchronize: name: Synchronize Labels runs-on: - - ubuntu-22.04 + - ubuntu-24.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - uses: micnncim/action-label-syncer@v1 env: GITHUB_TOKEN: ${{ github.token }} diff --git a/.github/workflows/test-pull-request.yml b/.github/workflows/test-pull-request.yml index 8d861459..1bfa8c37 100644 --- a/.github/workflows/test-pull-request.yml +++ b/.github/workflows/test-pull-request.yml @@ -13,12 +13,12 @@ concurrency: jobs: builders: name: Get Builders for Testing - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: builders: ${{ steps.builders.outputs.builders }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 - name: Get builders from integration.json id: builders run: | @@ -30,30 +30,37 @@ jobs: integration: name: Integration Tests with Builders - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [builders] strategy: matrix: builder: ${{ fromJSON(needs.builders.outputs.builders) }} fail-fast: false # don't cancel all test jobs when one fails steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Setup Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v6 with: - go-version: 'stable' + go-version-file: go.mod - - name: Checkout - uses: actions/checkout@v3 + # Causes errors with integration tests + - name: Disable containerd snapshotter + run: | + echo '{"features": {"containerd-snapshotter": false}}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker - name: Run Integration Tests env: TMPDIR: "${{ runner.temp }}" + GIT_TOKEN: ${{ github.token }} run: ./scripts/integration.sh --builder ${{ matrix.builder }} roundup: name: Integration Tests if: ${{ always() }} - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: integration steps: - run: | @@ -68,10 +75,10 @@ jobs: upload: name: Upload Workflow Event Payload - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Upload Artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v6 with: name: event-payload path: ${{ github.event_path }} diff --git a/.github/workflows/update-buildpack-toml.yml b/.github/workflows/update-buildpack-toml.yml index ecf1d703..7b544f51 100644 --- a/.github/workflows/update-buildpack-toml.yml +++ b/.github/workflows/update-buildpack-toml.yml @@ -2,19 +2,19 @@ name: Update buildpack.toml on: schedule: - - cron: '1 6 * * *' # daily at 06:01 UTC + - cron: '1 6 * * *' # daily at 06:01 UTC workflow_dispatch: {} concurrency: buildpack_update jobs: update-buildpack-toml: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 name: Update buildpack.toml steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 - name: Checkout Branch uses: paketo-buildpacks/github-config/actions/pull-request/checkout-branch@main @@ -59,7 +59,7 @@ jobs: failure: name: Alert on Failure - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [update-buildpack-toml] if: ${{ always() && needs.update-buildpack-toml.result == 'failure' }} steps: diff --git a/.github/workflows/update-github-config.yml b/.github/workflows/update-github-config.yml index d4c6a3d5..3a4aa67a 100644 --- a/.github/workflows/update-github-config.yml +++ b/.github/workflows/update-github-config.yml @@ -2,7 +2,7 @@ name: Update shared github-config on: schedule: - - cron: '20 17 * * *' # daily at 17:20 UTC + - cron: '20 17 * * *' # daily at 17:20 UTC workflow_dispatch: {} concurrency: github_config_update @@ -10,16 +10,16 @@ concurrency: github_config_update jobs: build: name: Create PR to update shared files - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} - name: Checkout github-config - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: repository: paketo-buildpacks/github-config path: github-config @@ -63,7 +63,7 @@ jobs: failure: name: Alert on Failure - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [build] if: ${{ always() && needs.build.result == 'failure' }} steps: diff --git a/.github/workflows/update-go-mod-version.yml b/.github/workflows/update-go-mod-version.yml index d02d04af..41cc6f9b 100644 --- a/.github/workflows/update-go-mod-version.yml +++ b/.github/workflows/update-go-mod-version.yml @@ -13,16 +13,17 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Checkout PR Branch uses: paketo-buildpacks/github-config/actions/pull-request/checkout-branch@main with: branch: automation/go-mod-update/update-main - name: Setup Go id: setup-go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 + # Fetching the latest stable Go version with: - go-version: 'stable' + go-version: stable - name: Get current go toolchain version id: current-go-version uses: paketo-buildpacks/github-config/actions/update-go-mod-version@main @@ -75,7 +76,7 @@ jobs: failure: name: Alert on Failure - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: [update-go] if: ${{ always() && needs.update-go.result == 'failure' }} steps: diff --git a/scripts/.util/tools.json b/scripts/.util/tools.json index ebaf6117..121f3326 100644 --- a/scripts/.util/tools.json +++ b/scripts/.util/tools.json @@ -1,4 +1,7 @@ { - "jam": "v2.10.1", - "pack": "v0.35.1" + "createpackage": "v1.73.0", + "jam": "v2.17.3", + "libpaktools": "v0.5.0", + "pack": "v0.40.3", + "yj": "v5.1.0" } diff --git a/scripts/.util/tools.sh b/scripts/.util/tools.sh index acca6135..be9a7a54 100644 --- a/scripts/.util/tools.sh +++ b/scripts/.util/tools.sh @@ -31,6 +31,10 @@ function util::tools::arch() { amd64|x86_64) if [[ "${1:-}" == "--blank-amd64" ]]; then echo "" + elif [[ "${1:-}" == "--format-amd64-x86_64" ]]; then + echo "x86_64" + elif [[ "${1:-}" == "--format-amd64-x86-64" ]]; then + echo "x86-64" else echo "amd64" fi @@ -135,12 +139,17 @@ function util::tools::pack::install() { version="$(jq -r .pack "$(dirname "${BASH_SOURCE[0]}")/tools.json")" - tmp_location="/tmp/pack.tgz" + local pack_config_enable_experimental + if [ -f "$(dirname "${BASH_SOURCE[0]}")/../options.json" ]; then + pack_config_enable_experimental="$(jq -r .pack_config_enable_experimental "$(dirname "${BASH_SOURCE[0]}")/../options.json")" + else + pack_config_enable_experimental="false" + fi + curl_args=( "--fail" "--silent" "--location" - "--output" "${tmp_location}" ) if [[ "${token}" != "" ]]; then @@ -153,17 +162,183 @@ function util::tools::pack::install() { arch=$(util::tools::arch --blank-amd64) curl "https://github.com/buildpacks/pack/releases/download/${version}/pack-${version}-${os}${arch:+-$arch}.tgz" \ - "${curl_args[@]}" - - tar xzf "${tmp_location}" -C "${dir}" + "${curl_args[@]}" | \ + tar xzf - -C "${dir}" chmod +x "${dir}/pack" - rm "${tmp_location}" + if [[ "${pack_config_enable_experimental}" == "true" ]]; then + "${dir}"/pack config experimental true + fi + else util::print::info "Using pack $("${dir}"/pack version)" fi } +function util::tools::yj::install() { + local dir token + token="" + + while [[ "${#}" != 0 ]]; do + case "${1}" in + --directory) + dir="${2}" + shift 2 + ;; + + --token) + token="${2}" + shift 2 + ;; + + *) + util::print::error "unknown argument \"${1}\"" + esac + done + + mkdir -p "${dir}" + util::tools::path::export "${dir}" + + if [[ ! -f "${dir}/yj" ]]; then + local version curl_args os arch + + version="$(jq -r .yj "$(dirname "${BASH_SOURCE[0]}")/tools.json")" + + curl_args=( + "--fail" + "--silent" + "--location" + "--output" "${dir}/yj" + ) + + if [[ "${token}" != "" ]]; then + curl_args+=("--header" "Authorization: Token ${token}") + fi + + util::print::title "Installing yj ${version}" + + os=$(util::tools::os macos) + arch=$(util::tools::arch) + + curl "https://github.com/sclevine/yj/releases/download/${version}/yj-${os}-${arch}" \ + "${curl_args[@]}" + + chmod +x "${dir}/yj" + else + util::print::info "Using yj $("${dir}"/yj -v)" + fi +} + +function util::tools::packager::install () { + local dir + while [[ "${#}" != 0 ]]; do + case "${1}" in + --directory) + dir="${2}" + shift 2 + ;; + + *) + util::print::error "unknown argument \"${1}\"" + ;; + + esac + done + + mkdir -p "${dir}" + util::tools::path::export "${dir}" + + if [[ ! -f "${dir}/packager" ]]; then + util::print::title "Installing packager" + GOBIN="${dir}" go install github.com/cloudfoundry/libcfbuildpack/packager@latest + fi +} + +function util::tools::libpak-tools::install () { + local dir token + token="" + + while [[ "${#}" != 0 ]]; do + case "${1}" in + --directory) + dir="${2}" + shift 2 + ;; + + --token) + token="${2}" + shift 2 + ;; + + *) + util::print::error "unknown argument \"${1}\"" + esac + done + + mkdir -p "${dir}" + util::tools::path::export "${dir}" + + + if [[ ! -f "${dir}/libpak-tools" ]]; then + local version curl_args os arch + + version="$(jq -r .libpaktools "$(dirname "${BASH_SOURCE[0]}")/tools.json")" + + curl_args=( + "--fail" + "--silent" + "--location" + "--output" "${dir}/libpak-tools.tar.gz" + ) + + if [[ "${token}" != "" ]]; then + curl_args+=("--header" "Authorization: Token ${token}") + fi + + util::print::title "Installing libpak-tools ${version}" + + os=$(util::tools::os) + arch=$(util::tools::arch --format-amd64-x86_64) + + curl "https://github.com/paketo-buildpacks/libpak-tools/releases/download/${version}/libpak-tools_${os}_${arch}.tar.gz" \ + "${curl_args[@]}" + + tar -xzf "${dir}/libpak-tools.tar.gz" -C $dir + rm "${dir}/libpak-tools.tar.gz" + + chmod +x "${dir}/libpak-tools" + else + util::print::info "Using libpak-tools" + fi +} + +function util::tools::create-package::install () { + local dir version + while [[ "${#}" != 0 ]]; do + case "${1}" in + --directory) + dir="${2}" + shift 2 + ;; + + *) + util::print::error "unknown argument \"${1}\"" + ;; + + esac + done + + version="$(jq -r .createpackage "$(dirname "${BASH_SOURCE[0]}")/tools.json")" + + mkdir -p "${dir}" + util::tools::path::export "${dir}" + + if [[ ! -f "${dir}/create-package" ]]; then + util::print::title "Installing create-package" + GOBIN="${dir}" go install -ldflags="-s -w" "github.com/paketo-buildpacks/libpak/cmd/create-package@${version}" + fi +} + function util::tools::tests::checkfocus() { testout="${1}" if grep -q 'Focused: [1-9]' "${testout}"; then diff --git a/scripts/integration.sh b/scripts/integration.sh index f1e1f11f..3257ad9f 100755 --- a/scripts/integration.sh +++ b/scripts/integration.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash + set -eu set -o pipefail @@ -32,7 +33,6 @@ function main() { shift 2 ;; - --token|-t) token="${2}" shift 2 @@ -49,7 +49,7 @@ function main() { done if [[ ! -d "${BUILDPACKDIR}/integration" ]]; then - util::print::warn "** WARNING No Integration tests **" + util::print::warn "** WARNING No Integration tests **" fi tools::install "${token}" @@ -109,6 +109,17 @@ function tools::install() { util::tools::jam::install \ --directory "${BUILDPACKDIR}/.bin" \ --token "${token}" + + util::tools::libpak-tools::install \ + --directory "${BUILDPACKDIR}/.bin" + + util::tools::create-package::install \ + --directory "${BUILDPACKDIR}/.bin" + + if [[ -f "${BUILDPACKDIR}/.libbuildpack" ]]; then + util::tools::packager::install \ + --directory "${BUILDPACKDIR}/.bin" + fi } function builder_images::pull() { @@ -123,6 +134,7 @@ function builder_images::pull() { pack inspect-builder "${builder}" --output json \ | jq -r '.remote_info.run_images[0].name' )" + lifecycle_image="index.docker.io/buildpacksio/lifecycle:$( pack inspect-builder "${builder}" --output json \ | jq -r '.remote_info.lifecycle.version' @@ -141,7 +153,6 @@ function tests::run() { export CGO_ENABLED=0 pushd "${BUILDPACKDIR}" > /dev/null - #shellcheck disable=SC2068 if GOMAXPROCS="${GOMAXPROCS:-4}" go test -count=1 -timeout 0 ./integration/... -v -run Integration | tee "${2}"; then util::print::info "** GO Test Succeeded with ${1}**" else diff --git a/scripts/package.sh b/scripts/package.sh index 34e9a29c..aa55b40e 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -71,6 +71,7 @@ function main { tools::install "${token}" buildpack::archive "${version}" + buildpack::release::archive buildpackage::create "${output}" "${flags[@]}" } @@ -110,6 +111,10 @@ function tools::install() { util::tools::pack::install \ --directory "${BIN_DIR}" \ --token "${token}" + + util::tools::yj::install \ + --directory "${BIN_DIR}" \ + --token "${token}" } function buildpack::archive() { @@ -125,24 +130,100 @@ function buildpack::archive() { --output "${BUILD_DIR}/buildpack.tgz" } +function buildpack::release::archive() { + local tmp_dir + + util::print::title "Packaging buildpack into ${BUILD_DIR}/buildpack-release-artifact.tgz..." + + tmp_dir=$(mktemp -d -p $BUILD_DIR) + + cat <<'README_EOF' > $tmp_dir/README.md +# Composite buildpack release artifact + +This is a buildpack release artifact that contains everything needed to package and publish a composite buildpack. Composite buildpacks are a logic grouping of other buildpacks. + +It contains the following files: + +* `buildpack.toml` - this is needed because it contains the buildpacks and ordering information for the composite buildpack +* `package.toml` - this is needed because it contains the dependencies (and URIs) that let pack know where to find the buildpacks referenced in `buildpack.toml`. + * `package.toml` can contain targets (platforms) for multi-arch support +* `build/buildpack.tgz` - this is added because it is referenced in `package.toml` by some buildpacks + +## package locally + +To package this buildpack to local .cnb file(s) run the following. + +``` +pack buildpack package mybuildpack.cnb --format file --config package.toml +``` + +## package and publish to a registry + +To package this buildpack and publish it to a registry run the following. + +* Note that as of pack v0.38.2 at least one target is required in package.toml or on the command line when publishing to a registry with `--publish`. + +* replace SOME-REGISTRY with your registry (e.g. index.docker.io/yourdockerhubusername) +* replace SOME-VERSION with the version you want to publish (e.g. 0.0.1) + +``` +pack buildpack package SOME-REGISTRY/mybuildpack:SOME-VERSION --format image --config package.toml --publish +``` +README_EOF + + mkdir -p $tmp_dir/build + cp ${BUILD_DIR}/buildpack.tgz $tmp_dir/build + cp ${ROOT_DIR}/package.toml $tmp_dir/ + # add the buildpack.toml from the tgz file because it has the version populated + tar -xzf ${BUILD_DIR}/buildpack.tgz -C $tmp_dir/ buildpack.toml + + tar -czf ${BUILD_DIR}/buildpack-release-artifact.tgz -C $tmp_dir $(ls $tmp_dir) + rm -rf $tmp_dir +} + function buildpackage::create() { - local output flags + local output flags release_archive_path tmp_dir output="${1}" flags=("${@:2}") + release_archive_path="${BUILD_DIR}/buildpack-release-artifact.tgz" util::print::title "Packaging buildpack..." + util::print::info "Extracting release archive..." + tmp_dir=$(mktemp -d -p $BUILD_DIR) + tar -xvf $release_archive_path -C $tmp_dir + + current_dir=$(pwd) + cd $tmp_dir + args=( - --config "${ROOT_DIR}/package.toml" + --config package.toml --format file ) - args+=("${flags[@]}") + # Use the local architecture to support running locally and in CI, which will be linux/amd64 by default. + arch=$(util::tools::arch) + + # If package.toml has no targets we must specify one on the command line, otherwise pack will complain. + # This is here for backward compatibility but eventually all package.toml files should have targets defined. + if cat package.toml | yj -tj | jq -r .targets | grep -q null; then + echo "package.toml has no targets so --target linux/${arch} will be passed to pack" + args+=("--target linux/${arch}") + fi + pack \ buildpack package "${output}" \ - "${args[@]}" + ${args[@]} + + if [[ -e "${BUILD_DIR}/buildpackage-linux-${arch}.cnb" ]]; then + echo "Copying linux-${arch} buildpackage to buildpackage.cnb" + cp "${BUILD_DIR}/buildpackage-linux-${arch}.cnb" "${BUILD_DIR}/buildpackage.cnb" + fi + + cd $current_dir + rm -rf $tmp_dir } main "${@:-}" diff --git a/scripts/publish.sh b/scripts/publish.sh new file mode 100755 index 00000000..f330ba7c --- /dev/null +++ b/scripts/publish.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +readonly ROOT_DIR="$(cd "$(dirname "${0}")/.." && pwd)" +readonly BIN_DIR="${ROOT_DIR}/.bin" + +# shellcheck source=SCRIPTDIR/.util/tools.sh +source "${ROOT_DIR}/scripts/.util/tools.sh" + +# shellcheck source=SCRIPTDIR/.util/print.sh +source "${ROOT_DIR}/scripts/.util/print.sh" + +function main { + local archive_path image_ref token + token="" + + while [[ "${#}" != 0 ]]; do + case "${1}" in + --archive-path | -a) + archive_path="${2}" + shift 2 + ;; + + --image-ref | -i) + image_ref="${2}" + shift 2 + ;; + + --token | -t) + token="${2}" + shift 2 + ;; + + --help | -h) + shift 1 + usage + exit 0 + ;; + + "") + # skip if the argument is empty + shift 1 + ;; + + *) + util::print::error "unknown argument \"${1}\"" + ;; + esac + done + + if [[ -z "${image_ref:-}" ]]; then + usage + util::print::error "--image-ref is required" + fi + + if [[ -z "${archive_path:-}" ]]; then + util::print::info "Using default archive path: ${ROOT_DIR}/build/buildpack-release-artifact.tgz" + archive_path="${ROOT_DIR}/build/buildpack-release-artifact.tgz" + else + archive_path="${archive_path}" + fi + + repo::prepare + + tools::install "${token}" + + buildpack::publish "${image_ref}" "${archive_path}" +} + +function usage() { + cat <<-USAGE +Publishes a composite buildpack to a registry. + +OPTIONS + -a, --archive-path Path to the buildpack release artifact (default: ${ROOT_DIR}/build/buildpack-release-artifact.tgz) (optional) + -h, --help Prints the command usage + -i, --image-ref List of image reference to publish to (required) + -t, --token Token used to download assets from GitHub (e.g. jam, pack, etc) (optional) + +USAGE +} + +function repo::prepare() { + util::print::title "Preparing repo..." + + mkdir -p "${BIN_DIR}" + + export PATH="${BIN_DIR}:${PATH}" +} + +function tools::install() { + local token + token="${1}" + + util::tools::pack::install \ + --directory "${BIN_DIR}" \ + --token "${token}" + + util::tools::yj::install \ + --directory "${BIN_DIR}" \ + --token "${token}" +} + +function buildpack::publish() { + local image_ref archive_path tmp_dir + image_ref="${1}" + archive_path="${2}" + + util::print::title "Publishing composite buildpack..." + + util::print::info "Extracting archive..." + tmp_dir=$(mktemp -d -p $ROOT_DIR) + tar -xvf $archive_path -C $tmp_dir + + util::print::info "Publishing buildpack to ${image_ref}" + + current_dir=$(pwd) + cd $tmp_dir + + # If package.toml has no targets we must specify one on the command line, otherwise pack will complain. + # This is here for backward compatibility but eventually all package.toml files should have targets defined. + targets="" + if cat package.toml | yj -tj | jq -r .targets | grep -q null; then + # Use the local architecture to support running locally and in CI, which will be linux/amd64 by default. + arch=$(util::tools::arch) + targets="--target linux/${arch}" + echo "package.toml has no targets so ${targets} will be used" + fi + + pack \ + buildpack package "${image_ref}" \ + --config package.toml \ + --format image \ + --publish \ + ${targets} + + cd $current_dir + rm -rf $tmp_dir +} + +main "${@:-}"