diff --git a/.github/workflows/ami-release-nix-single.yml b/.github/workflows/ami-release-nix-single.yml index 500bf26d5..dd579df30 100644 --- a/.github/workflows/ami-release-nix-single.yml +++ b/.github/workflows/ami-release-nix-single.yml @@ -40,7 +40,13 @@ jobs: run: | echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - uses: DeterminateSystems/nix-installer-action@main + - name: Install nix + uses: cachix/install-nix-action@v27 + with: + install_url: https://releases.nixos.org/nix/nix-2.29.1/install + extra_nix_config: | + substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com + trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ github.event.inputs.postgres_version }}" >> $GITHUB_ENV @@ -57,18 +63,18 @@ jobs: env: POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} run: | - packer init amazon-arm64-nix.pkr.hcl GIT_SHA=${{ steps.get_sha.outputs.sha }} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl + nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl + nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 env: POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} run: | - packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{ steps.get_sha.outputs.sha }} + nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }} - packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl + nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl - name: Grab release version id: process_release_version diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 0c97d8374..df0bf096e 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -24,7 +24,13 @@ jobs: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD - - uses: DeterminateSystems/nix-installer-action@main + - name: Install nix + uses: cachix/install-nix-action@v27 + with: + install_url: https://releases.nixos.org/nix/nix-2.29.1/install + extra_nix_config: | + substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com + trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - name: Set PostgreSQL versions id: set-versions @@ -52,7 +58,14 @@ jobs: aws-region: "us-east-1" output-credentials: true role-duration-seconds: 7200 - - uses: DeterminateSystems/nix-installer-action@main + + - name: Install nix + uses: cachix/install-nix-action@v27 + with: + install_url: https://releases.nixos.org/nix/nix-2.29.1/install + extra_nix_config: | + substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com + trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - name: Run checks if triggered manually if: ${{ github.event_name == 'workflow_dispatch' }} @@ -78,19 +91,19 @@ jobs: env: POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} run: | - packer init amazon-arm64-nix.pkr.hcl GIT_SHA=${{github.sha}} + nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl # why is postgresql_major defined here instead of where the _three_ other postgresql_* variables are defined? - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl + nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 env: POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} run: | - packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} + nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }} - packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl + nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl - name: Grab release version id: process_release_version diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 362d3ff5e..058d9087b 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -118,6 +118,8 @@ jobs: needs: build-run-image if: ${{ success() }} uses: ./.github/workflows/testinfra-ami-build.yml + secrets: + DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} run-tests: needs: build-run-image diff --git a/.github/workflows/testinfra-ami-build.yml b/.github/workflows/testinfra-ami-build.yml index 5636a34c3..892b54b74 100644 --- a/.github/workflows/testinfra-ami-build.yml +++ b/.github/workflows/testinfra-ami-build.yml @@ -3,9 +3,13 @@ name: Testinfra Integration Tests Nix on: workflow_dispatch: workflow_call: + secrets: + DEV_AWS_ROLE: + description: 'AWS role for dev environment' + required: true permissions: - contents: read + contents: write id-token: write jobs: @@ -17,7 +21,13 @@ jobs: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD - - uses: DeterminateSystems/nix-installer-action@main + - name: Install nix + uses: cachix/install-nix-action@v27 + with: + install_url: https://releases.nixos.org/nix/nix-2.29.1/install + extra_nix_config: | + substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com + trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - name: Set PostgreSQL versions id: set-versions @@ -32,22 +42,44 @@ jobs: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} include: - - runner: arm-runner + - runner: large-linux-arm arch: arm64 ubuntu_release: noble ubuntu_version: 24.04 mcpu: neoverse-n1 runs-on: ${{ matrix.runner }} timeout-minutes: 150 - permissions: - contents: write - packages: write - id-token: write steps: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD + - name: Debug AWS role secret + run: | + echo "Checking DEV_AWS_ROLE secret availability..." + if [ -z "${{ secrets.DEV_AWS_ROLE }}" ]; then + echo "❌ DEV_AWS_ROLE is empty or not available" + else + echo "✅ DEV_AWS_ROLE is available" + fi + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.DEV_AWS_ROLE }} + aws-region: "us-east-1" + output-credentials: true + role-duration-seconds: 7200 + + - name: Install nix + uses: cachix/install-nix-action@v27 + with: + install_url: https://releases.nixos.org/nix/nix-2.29.1/install + extra_nix_config: | + substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com + trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= + + - id: args uses: mikefarah/yq@master with: @@ -68,7 +100,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo "$PG_VERSION" | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl # Ensure there's a newline at the end of the file @@ -76,15 +108,15 @@ jobs: - name: Build AMI stage 1 run: | - packer init amazon-arm64-nix.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl + nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl + nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 run: | - packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl + nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl + nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl - name: Run tests timeout-minutes: 10 diff --git a/flake.lock b/flake.lock index 8b6d868bf..9d2865e1d 100644 --- a/flake.lock +++ b/flake.lock @@ -208,6 +208,22 @@ "type": "github" } }, + "nixpkgs-go124": { + "locked": { + "lastModified": 1754085309, + "narHash": "sha256-3RTSdhnqTcxS5wjKNEBpbt0hiSKfBZiQPlWHn90N1qQ=", + "owner": "Nixos", + "repo": "nixpkgs", + "rev": "d2ac4dfa61fba987a84a0a81555da57ae0b9a2b0", + "type": "github" + }, + "original": { + "owner": "Nixos", + "repo": "nixpkgs", + "rev": "d2ac4dfa61fba987a84a0a81555da57ae0b9a2b0", + "type": "github" + } + }, "nixpkgs-lib": { "locked": { "lastModified": 1750555020, @@ -295,6 +311,7 @@ "nix-fast-build": "nix-fast-build", "nix2container": "nix2container", "nixpkgs": "nixpkgs_4", + "nixpkgs-go124": "nixpkgs-go124", "rust-overlay": "rust-overlay", "treefmt-nix": "treefmt-nix_2" } diff --git a/flake.nix b/flake.nix index 3aa651b09..db14dac9a 100644 --- a/flake.nix +++ b/flake.nix @@ -13,6 +13,7 @@ treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; git-hooks.url = "github:cachix/git-hooks.nix"; git-hooks.inputs.nixpkgs.follows = "nixpkgs"; + nixpkgs-go124.url = "github:Nixos/nixpkgs/d2ac4dfa61fba987a84a0a81555da57ae0b9a2b0"; }; outputs = diff --git a/nix/checks.nix b/nix/checks.nix index 13be10f23..2dc51797a 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -294,6 +294,7 @@ wal-g-2 wal-g-3 dbmate-tool + packer pg_regress ; } diff --git a/nix/devShells.nix b/nix/devShells.nix index 7ff4d0720..03768a770 100644 --- a/nix/devShells.nix +++ b/nix/devShells.nix @@ -45,9 +45,7 @@ shellcheck ansible ansible-lint - (packer.overrideAttrs (_oldAttrs: { - version = "1.7.8"; - })) + self'.packages.packer self'.packages.start-server self'.packages.start-client diff --git a/nix/docs/updating-dependencies.md b/nix/docs/updating-dependencies.md new file mode 100644 index 000000000..73c5cf79a --- /dev/null +++ b/nix/docs/updating-dependencies.md @@ -0,0 +1,68 @@ +# Updating Dependencies + +This document explains how to update various dependencies used in the nix configuration. + +## Updating Packer + +Packer is used for creating machine images and is defined in `nix/packages/packer.nix`. + +### Steps to update Packer version: + +1. Create a branch off of `develop` +2. Navigate to `nix/packages/packer.nix` +3. Update the version field: + ```nix + version = "1.15.0"; # Update to desired version + ``` +4. Update the git revision to match the new version: + ```nix + rev = "v${version}"; + ``` +5. Temporarily clear the hash to get the new SHA256: + ```nix + hash = ""; # Clear this temporarily + ``` +6. Save the file and run: + ```bash + nix build .#packer + ``` +7. Nix will fail and output the correct SHA256 hash. Copy this hash and update the file: + ```nix + hash = "sha256-NEWHASHHEREFROMBUILDOUTPUT"; + ``` +8. Update the vendorHash if needed. If the build fails due to vendor hash mismatch, temporarily set: + ```nix + vendorHash = ""; # Clear this temporarily + ``` +9. Run `nix build .#packer` again to get the correct vendorHash, then update: + ```nix + vendorHash = "sha256-NEWVENDORHASHHEREFROMBUILDOUTPUT"; + ``` +10. Verify the build works: + ```bash + nix build .#packer + ``` +11. Test the packer binary: + ```bash + ./result/bin/packer version + ``` +12. Run the full test suite to ensure nothing is broken: + ```bash + nix flake check -L + ``` +13. Commit your changes and create a PR for review +14. Update any CI/CD workflows or documentation that reference the old Packer version + +### Notes: +- Always check the [Packer changelog](https://github.com/hashicorp/packer/releases) for breaking changes +- Packer uses Go, so ensure compatibility with the Go version specified in the flake inputs +- The current Go version is specified in `flake.nix` under `nixpkgs-go124` input +- If updating to a major version, test all packer templates (`.pkr.hcl` files) in the repository + +## Updating Other Dependencies + +Similar patterns can be followed for other dependencies defined in the nix packages. Always: +1. Check for breaking changes in changelogs +2. Update version numbers and hashes +3. Run local tests +4. Verify functionality before creating PR \ No newline at end of file diff --git a/nix/packages/default.nix b/nix/packages/default.nix index cd6d6a1d0..342763961 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -1,4 +1,4 @@ -{ self, ... }: +{ self, inputs, ... }: { imports = [ ./postgres.nix ]; perSystem = @@ -37,6 +37,7 @@ supabase-groonga = pkgs.callPackage ./groonga { }; local-infra-bootstrap = pkgs.callPackage ./local-infra-bootstrap.nix { }; migrate-tool = pkgs.callPackage ./migrate-tool.nix { psql_15 = self'.packages."psql_15/bin"; }; + packer = pkgs.callPackage ./packer.nix { inherit inputs; }; pg-restore = pkgs.callPackage ./pg-restore.nix { psql_15 = self'.packages."psql_15/bin"; }; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; pg_regress = makePgRegress activeVersion; diff --git a/nix/packages/packer.nix b/nix/packages/packer.nix new file mode 100644 index 000000000..7f0063c30 --- /dev/null +++ b/nix/packages/packer.nix @@ -0,0 +1,50 @@ +{ + pkgs, + inputs, + lib, + fetchFromGitHub, + installShellFiles, + ... +}: +let + go124 = inputs.nixpkgs-go124.legacyPackages.${pkgs.system}.go_1_24; + buildGoModule = pkgs.buildGoModule.override { go = go124; }; +in +buildGoModule rec { + pname = "packer"; + version = "1.14.1"; + + src = fetchFromGitHub { + owner = "hashicorp"; + repo = "packer"; + rev = "v${version}"; + hash = "sha256-3g9hsmrfLzGhjcGvUza/L9PMGUFw+KLbg2pIK0CxlQI="; + }; + + vendorHash = "sha256-F6hn+pXPyPe70UTK8EF24lk7ArYz7ygUyVVsatW6+hI="; + + subPackages = [ "." ]; + + ldflags = [ + "-s" + "-w" + ]; + + nativeBuildInputs = [ installShellFiles ]; + + buildInputs = lib.optionals pkgs.stdenv.isDarwin [ + pkgs.darwin.apple_sdk.frameworks.IOKit + pkgs.darwin.apple_sdk.frameworks.Security + ]; + + postInstall = '' + installShellCompletion --zsh contrib/zsh-completion/_packer + ''; + + meta = { + description = "Tool for creating identical machine images for multiple platforms from a single source configuration"; + homepage = "https://www.packer.io"; + license = lib.licenses.bsl11; + changelog = "https://github.com/hashicorp/packer/blob/v${version}/CHANGELOG.md"; + }; +}