diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index 763c5f27ee6..00000000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,17 +0,0 @@ -# Pull requests concerning the listed files will automatically invite the respective maintainers as reviewers. -# This file is not used for denoting any kind of ownership, but is merely a tool for handling notifications. -# -# Merge permissions are required for maintaining an entry in this file. -# For documentation on this mechanism, see https://help.github.com/articles/about-codeowners/ - -# Default reviewers if nothing else matches -* @edolstra - -# This file -.github/CODEOWNERS @edolstra - -# Documentation of built-in functions -src/libexpr/primops.cc @roberth - -# Libstore layer -/src/libstore @ericson2314 diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index a5005f8a002..58ef1690feb 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,10 +1,9 @@ --- name: Bug report about: Report unexpected or incorrect behaviour -title: '' +title: "" labels: bug -assignees: '' - +assignees: "" --- ## Describe the bug @@ -32,7 +31,9 @@ assignees: '' ## Metadata - + + + ## Additional context @@ -42,13 +43,9 @@ assignees: '' -- [ ] checked [latest Nix manual] \([source]) +- [ ] checked [latest Determinate Nix manual] \([source]) - [ ] checked [open bug issues and pull requests] for possible duplicates -[latest Nix manual]: https://nixos.org/manual/nix/unstable/ -[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source -[open bug issues and pull requests]: https://github.com/NixOS/nix/labels/bug - ---- - -Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). +[latest Determinate Nix manual]: https://manual.determinate.systems/ +[source]: https://github.com/DeterminateSystems/nix-src/tree/detsys-main/doc/manual/source +[open bug issues and pull requests]: https://github.com/DeterminateSystems/nix-src/labels/bug diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index c75a4695170..345a05c533e 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,10 +1,9 @@ --- name: Feature request about: Suggest a new feature -title: '' +title: "" labels: feature -assignees: '' - +assignees: "" --- ## Is your feature request related to a problem? @@ -27,13 +26,9 @@ assignees: '' -- [ ] checked [latest Nix manual] \([source]) -- [ ] checked [open feature issues and pull requests] for possible duplicates - -[latest Nix manual]: https://nixos.org/manual/nix/unstable/ -[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source -[open feature issues and pull requests]: https://github.com/NixOS/nix/labels/feature - ---- +- [ ] checked [latest Determinate Nix manual] \([source]) +- [ ] checked [open bug issues and pull requests] for possible duplicates -Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). +[latest Determinate Nix manual]: https://manual.determinate.systems/ +[source]: https://github.com/DeterminateSystems/nix-src/tree/detsys-main/doc/manual/source +[open bug issues and pull requests]: https://github.com/DeterminateSystems/nix-src/labels/bug diff --git a/.github/ISSUE_TEMPLATE/installer.md b/.github/ISSUE_TEMPLATE/installer.md index ed5e1ce87b9..9bf6541c78e 100644 --- a/.github/ISSUE_TEMPLATE/installer.md +++ b/.github/ISSUE_TEMPLATE/installer.md @@ -1,18 +1,17 @@ --- name: Installer issue about: Report problems with installation -title: '' +title: "" labels: installer -assignees: '' - +assignees: "" --- ## Platform - + -- [ ] Linux: - [ ] macOS +- [ ] Linux: - [ ] WSL ## Additional information @@ -35,13 +34,9 @@ assignees: '' -- [ ] checked [latest Nix manual] \([source]) -- [ ] checked [open installer issues and pull requests] for possible duplicates - -[latest Nix manual]: https://nixos.org/manual/nix/unstable/ -[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source -[open installer issues and pull requests]: https://github.com/NixOS/nix/labels/installer - ---- +- [ ] checked [latest Determinate Nix manual] \([source]) +- [ ] checked [open bug issues and pull requests] for possible duplicates -Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). +[latest Determinate Nix manual]: https://manual.determinate.systems/ +[source]: https://github.com/DeterminateSystems/nix-src/tree/detsys-main/doc/manual/source +[open bug issues and pull requests]: https://github.com/DeterminateSystems/nix-src/labels/bug diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md index 6c334b72206..eaa6b11709a 100644 --- a/.github/ISSUE_TEMPLATE/missing_documentation.md +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -1,10 +1,9 @@ --- name: Missing or incorrect documentation about: Help us improve the reference manual -title: '' +title: "" labels: documentation -assignees: '' - +assignees: "" --- ## Problem @@ -19,13 +18,9 @@ assignees: '' -- [ ] checked [latest Nix manual] \([source]) -- [ ] checked [open documentation issues and pull requests] for possible duplicates - -[latest Nix manual]: https://nixos.org/manual/nix/unstable/ -[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source -[open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation - ---- +- [ ] checked [latest Determinate Nix manual] \([source]) +- [ ] checked [open bug issues and pull requests] for possible duplicates -Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). +[latest Determinate Nix manual]: https://manual.determinate.systems/ +[source]: https://github.com/DeterminateSystems/nix-src/tree/detsys-main/doc/manual/source +[open bug issues and pull requests]: https://github.com/DeterminateSystems/nix-src/labels/bug diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c6843d86fa7..d3e1f817736 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,22 +1,3 @@ - - ## Motivation @@ -30,9 +11,3 @@ so you understand the process and the expectations. - ---- - -Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc). - -The Nix maintainer team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19) to [schedule and track reviews](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol). diff --git a/.github/STALE-BOT.md b/.github/STALE-BOT.md index bc0005413f1..281d0f79a8b 100644 --- a/.github/STALE-BOT.md +++ b/.github/STALE-BOT.md @@ -2,34 +2,21 @@ - Thanks for your contribution! - To remove the stale label, just leave a new comment. -- _How to find the right people to ping?_ → [`git blame`](https://git-scm.com/docs/git-blame) to the rescue! (or GitHub's history and blame buttons.) -- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on [Matrix - #users:nixos.org](https://matrix.to/#/#users:nixos.org). +- You can always ask for help on [Discord](https://determinate.systems/discord). ## Suggestions for PRs -1. GitHub sometimes doesn't notify people who commented / reviewed a PR previously, when you (force) push commits. If you have addressed the reviews you can [officially ask for a review](https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/requesting-a-pull-request-review) from those who commented to you or anyone else. -2. If it is unfinished but you plan to finish it, please mark it as a draft. -3. If you don't expect to work on it any time soon, closing it with a short comment may encourage someone else to pick up your work. -4. To get things rolling again, rebase the PR against the target branch and address valid comments. -5. If you need a review to move forward, ask in [the Discourse thread for PRs that need help](https://discourse.nixos.org/t/prs-in-distress/3604). -6. If all you need is a merge, check the git history to find and [request reviews](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/requesting-a-pull-request-review) from people who usually merge related contributions. +1. If it is unfinished but you plan to finish it, please mark it as a draft. +1. If you don't expect to work on it any time soon, closing it with a short comment may encourage someone else to pick up your work. +1. To get things rolling again, rebase the PR against the target branch and address valid comments. +1. If you need a review to move forward, ask in [Discord](https://determinate.systems/discord). ## Suggestions for issues 1. If it is resolved (either for you personally, or in general), please consider closing it. 2. If this might still be an issue, but you are not interested in promoting its resolution, please consider closing it while encouraging others to take over and reopen an issue if they care enough. -3. If you still have interest in resolving it, try to ping somebody who you believe might have an interest in the topic. Consider discussing the problem in [our Discourse Forum](https://discourse.nixos.org/). -4. As with all open source projects, your best option is to submit a Pull Request that addresses this issue. We :heart: this attitude! +3. If you still have interest in resolving it, try to ping somebody who you believe might have an interest in the topic. Consider discussing the problem in [Discord](https://determinate.systems/discord). **Memorandum on closing issues** Don't be afraid to close an issue that holds valuable information. Closed issues stay in the system for people to search, read, cross-reference, or even reopen--nothing is lost! Closing obsolete issues is an important way to help maintainers focus their time and effort. - -## Useful GitHub search queries - -- [Open PRs with any stale-bot interaction](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+) -- [Open PRs with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22) -- [Open PRs with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+) -- [Open Issues with any stale-bot interaction](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+) -- [Open Issues with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22+) -- [Open Issues with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+) diff --git a/.github/release-notes.sh b/.github/release-notes.sh new file mode 100755 index 00000000000..19836116126 --- /dev/null +++ b/.github/release-notes.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +# SC2002 disables "useless cat" warnings. +# I prefer pipelines that start with an explicit input, and go from there. +# Overly fussy. +# shellcheck disable=SC2002 + +scratch=$(mktemp -d -t tmp.XXXXXXXXXX) +finish() { + rm -rf "$scratch" +} +trap finish EXIT + +DATE=$(date +%Y-%m-%d) +DETERMINATE_NIX_VERSION=$(cat .version-determinate) +TAG_NAME="v${DETERMINATE_NIX_VERSION}" +NIX_VERSION=$(cat .version) +NIX_VERSION_MAJOR_MINOR=$(echo "$NIX_VERSION" | cut -d. -f1,2) +GITHUB_REPOSITORY="${GITHUB_REPOSITORY:-DeterminateSystems/nix-src}" + +gh api "/repos/${GITHUB_REPOSITORY}/releases/generate-notes" \ + -f "tag_name=${TAG_NAME}" > "$scratch/notes.json" + +trim_trailing_newlines() { + local text + text="$(cat)" + echo -n "${text}" +} + +linkify_gh() { + sed \ + -e 's!\(https://github.com/DeterminateSystems/nix-src/\(pull\|issue\)/\([[:digit:]]\+\)\)![DeterminateSystems/nix-src#\3](\1)!' \ + -e 's#\(https://github.com/DeterminateSystems/nix-src/compare/\([^ ]\+\)\)#[\2](\1)#' +} + +( + cat doc/manual/source/release-notes-determinate/changes.md \ + | sed 's/^.*\(\)$/This section lists the differences between upstream Nix '"$NIX_VERSION_MAJOR_MINOR"' and Determinate Nix '"$DETERMINATE_NIX_VERSION"'.\1/' \ + + printf "\n\n" "$DETERMINATE_NIX_VERSION" + cat "$scratch/notes.json" \ + | jq -r .body \ + | grep -v '^#' \ + | grep -v "Full Changelog" \ + | trim_trailing_newlines \ + | sed -e 's/^\* /\n* /' \ + | linkify_gh + echo "" # final newline +) > "$scratch/changes.md" + +( + printf "# Release %s (%s)\n\n" \ + "$DETERMINATE_NIX_VERSION" \ + "$DATE" + printf "* Based on [upstream Nix %s](../release-notes/rl-%s.md).\n\n" \ + "$NIX_VERSION" \ + "$NIX_VERSION_MAJOR_MINOR" + + cat "$scratch/notes.json" | jq -r .body | linkify_gh +) > "$scratch/rl.md" + +( + cat doc/manual/source/SUMMARY.md.in \ + | sed 's/\(\)$/\1\n - [Release '"$DETERMINATE_NIX_VERSION"' ('"$DATE"')](release-notes-determinate\/rl-'"$DETERMINATE_NIX_VERSION"'.md)/' +) > "$scratch/summary.md" + +mv "$scratch/changes.md" doc/manual/source/release-notes-determinate/changes.md +mv "$scratch/rl.md" "doc/manual/source/release-notes-determinate/rl-${DETERMINATE_NIX_VERSION}.md" +mv "$scratch/summary.md" doc/manual/source/SUMMARY.md.in diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000000..e34a03bd0b2 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,229 @@ +on: + workflow_call: + inputs: + system: + required: true + type: string + runner: + required: true + type: string + runner_for_virt: + required: true + type: string + runner_small: + required: true + type: string + if: + required: false + default: true + type: boolean + run_tests: + required: false + default: true + type: boolean + run_vm_tests: + required: false + default: false + type: boolean + run_regression_tests: + required: false + default: false + type: boolean + publish_manual: + required: false + default: false + type: boolean + secrets: + manual_netlify_auth_token: + required: false + manual_netlify_site_id: + required: false + +jobs: + build: + if: ${{ inputs.if }} + strategy: + fail-fast: false + runs-on: ${{ inputs.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - run: nix build .#packages.${{ inputs.system }}.default .#packages.${{ inputs.system }}.binaryTarball --no-link -L + - run: nix build .#packages.${{ inputs.system }}.binaryTarball --out-link tarball + - uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.system }} + path: ./tarball/*.xz + + test: + if: ${{ inputs.if && inputs.run_tests}} + needs: build + strategy: + fail-fast: false + runs-on: ${{ inputs.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - run: nix flake check -L --system ${{ inputs.system }} + + vm_tests_smoke: + if: inputs.run_vm_tests && github.event_name != 'merge_group' + needs: build + runs-on: ${{ inputs.runner_for_virt }} + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - run: | + nix build -L \ + .#hydraJobs.tests.functional_user \ + .#hydraJobs.tests.githubFlakes \ + .#hydraJobs.tests.nix-docker \ + .#hydraJobs.tests.tarballFlakes \ + ; + + vm_tests_all: + if: inputs.run_vm_tests && github.event_name == 'merge_group' + needs: build + runs-on: ${{ inputs.runner_for_virt }} + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - run: | + cmd() { + nix build -L --keep-going --timeout 600 \ + $(nix flake show --json \ + | jq -r ' + .hydraJobs.tests + | with_entries(select(.value.type == "derivation")) + | keys[] + | ".#hydraJobs.tests." + .') + } + + if ! cmd; then + echo "failed, retrying once ..." + printf "\n\n\n\n\n\n\n\n" + cmd + fi + + flake_regressions: + if: | + (inputs.run_regression_tests && github.event_name == 'merge_group') + || ( + inputs.run_regression_tests + && github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' + && ( + (github.event.action == 'labeled' && github.event.label.name == 'flake-regression-test') + || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'flake-regression-test')) + ) + ) + needs: build + runs-on: ${{ inputs.runner }} + strategy: + matrix: + nix_config: + - "lazy-trees = true" + - "lazy-trees = false" + glob: + - "[0-d]*" + - "[e-l]*" + - "[m]*" + - "[n-r]*" + - "[s-z]*" + + steps: + - name: Checkout nix + uses: actions/checkout@v4 + - name: Checkout flake-regressions + uses: actions/checkout@v4 + with: + repository: DeterminateSystems/flake-regressions + path: flake-regressions + - name: Checkout flake-regressions-data + uses: actions/checkout@v4 + with: + repository: DeterminateSystems/flake-regressions-data + path: flake-regressions/tests + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - env: + PARALLEL: "-P 50%" + FLAKE_REGRESSION_GLOB: ${{ matrix.glob }} + NIX_CONFIG: ${{ matrix.nix_config }} + run: | + set -x + if [ ! -z "${NSC_CACHE_PATH:-}" ]; then + mkdir -p "${NSC_CACHE_PATH}/nix/xdg-cache" + export XDG_CACHE_HOME="${NSC_CACHE_PATH}/nix/xdg-cache" + fi + nix build -L --out-link ./new-nix + export PATH=$(pwd)/new-nix/bin:$PATH + + if ! flake-regressions/eval-all.sh; then + echo "Some failed, trying again" + printf "\n\n\n\n\n\n\n\n" + flake-regressions/eval-all.sh + fi + + manual: + if: github.event_name != 'merge_group' + needs: build + runs-on: ${{ inputs.runner_small }} + permissions: + id-token: "write" + contents: "read" + pull-requests: "write" + statuses: "write" + deployments: "write" + steps: + - name: Checkout nix + uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - name: Build manual + if: inputs.system == 'x86_64-linux' + run: nix build .#hydraJobs.manual + - uses: nwtgck/actions-netlify@v3.0 + if: inputs.publish_manual && inputs.system == 'x86_64-linux' + with: + publish-dir: "./result/share/doc/nix/manual" + production-branch: detsys-main + github-token: ${{ secrets.GITHUB_TOKEN }} + deploy-message: "Deploy from GitHub Actions" + # NOTE(cole-h): We have a perpetual PR displaying our changes against upstream open, but + # its conversation is locked, so this PR comment can never be posted. + # https://github.com/DeterminateSystems/nix-src/pull/4 + enable-pull-request-comment: ${{ github.event.pull_request.number != 4 }} + enable-commit-comment: true + enable-commit-status: true + overwrites-pull-request-comment: true + env: + NETLIFY_AUTH_TOKEN: ${{ secrets.manual_netlify_auth_token }} + NETLIFY_SITE_ID: ${{ secrets.manual_netlify_site_id }} + + success: + needs: + - build + - test + - vm_tests_smoke + - vm_tests_all + - flake_regressions + - manual + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - run: "true" + - run: | + echo "A dependent in the build matrix failed:" + echo "$needs" + exit 1 + env: + needs: ${{ toJSON(needs) }} + if: | + contains(needs.*.result, 'failure') || + contains(needs.*.result, 'cancelled') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29cb33f56af..cca05418392 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,204 +3,144 @@ name: "CI" on: pull_request: push: + branches: + # NOTE: make sure any branches here are also valid directory names, + # otherwise creating the directory and uploading to s3 will fail + - detsys-main + - main + - master + merge_group: + release: + types: + - published -permissions: read-all +permissions: + id-token: "write" + contents: "read" + pull-requests: "write" + statuses: "write" + deployments: "write" jobs: eval: - runs-on: ubuntu-24.04 + runs-on: UbuntuLatest32Cores128G steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: cachix/install-nix-action@v31 - - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: DeterminateSystems/determinate-nix-action@main + - run: nix flake show --all-systems --json - tests: - strategy: - fail-fast: false - matrix: - include: - - scenario: on ubuntu - runs-on: ubuntu-24.04 - os: linux - - scenario: on macos - runs-on: macos-14 - os: darwin - name: tests ${{ matrix.scenario }} - runs-on: ${{ matrix.runs-on }} - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: cachix/install-nix-action@v31 - with: - # The sandbox would otherwise be disabled by default on Darwin - extra_nix_config: | - sandbox = true - max-jobs = 1 - - uses: DeterminateSystems/magic-nix-cache-action@main - # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: - # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - if: matrix.os == 'linux' - - run: scripts/build-checks - - run: scripts/prepare-installer-for-github-actions - - name: Upload installer tarball - uses: actions/upload-artifact@v4 - with: - name: installer-${{matrix.os}} - path: out/* + build_x86_64-linux: + uses: ./.github/workflows/build.yml + with: + system: x86_64-linux + runner: namespace-profile-linuxamd32c64g-cache + runner_for_virt: UbuntuLatest32Cores128G + runner_small: ubuntu-latest + run_tests: true + run_vm_tests: true + run_regression_tests: true + publish_manual: true + secrets: + manual_netlify_auth_token: ${{ secrets.NETLIFY_AUTH_TOKEN }} + manual_netlify_site_id: ${{ secrets.NETLIFY_SITE_ID }} - installer_test: - needs: [tests] - strategy: - fail-fast: false - matrix: - include: - - scenario: on ubuntu - runs-on: ubuntu-24.04 - os: linux - - scenario: on macos - runs-on: macos-14 - os: darwin - name: installer test ${{ matrix.scenario }} - runs-on: ${{ matrix.runs-on }} - steps: - - uses: actions/checkout@v4 - - name: Download installer tarball - uses: actions/download-artifact@v4 - with: - name: installer-${{matrix.os}} - path: out - - name: Serving installer - id: serving_installer - run: ./scripts/serve-installer-for-github-actions - - uses: cachix/install-nix-action@v31 - with: - install_url: 'http://localhost:8126/install' - install_options: "--tarball-url-prefix http://localhost:8126/" - - run: sudo apt install fish zsh - if: matrix.os == 'linux' - - run: brew install fish - if: matrix.os == 'darwin' - - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval" - - run: exec sh -c "nix-instantiate -E 'builtins.currentTime' --eval" - - run: exec zsh -c "nix-instantiate -E 'builtins.currentTime' --eval" - - run: exec fish -c "nix-instantiate -E 'builtins.currentTime' --eval" - - run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs" - - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello" + build_aarch64-linux: + uses: ./.github/workflows/build.yml + with: + if: ${{ github.event_name != 'pull_request' }} + system: aarch64-linux + runner: UbuntuLatest32Cores128GArm + runner_for_virt: UbuntuLatest32Cores128GArm + runner_small: UbuntuLatest32Cores128GArm - # Steps to test CI automation in your own fork. - # 1. Sign-up for https://hub.docker.com/ - # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions) - # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork - check_secrets: - permissions: - contents: none - name: Check Docker secrets present for installer tests - runs-on: ubuntu-24.04 - outputs: - docker: ${{ steps.secret.outputs.docker }} + build_x86_64-darwin: + uses: ./.github/workflows/build.yml + with: + if: ${{ github.event_name != 'pull_request' }} + system: x86_64-darwin + runner: macos-latest-large + runner_for_virt: macos-latest-large + runner_small: macos-latest-large + run_tests: false + + build_aarch64-darwin: + uses: ./.github/workflows/build.yml + with: + system: aarch64-darwin + runner: namespace-profile-mac-m2-12c28g + runner_for_virt: namespace-profile-mac-m2-12c28g + runner_small: macos-latest-xlarge + + success: + runs-on: ubuntu-latest + needs: + - eval + - build_x86_64-linux + - build_aarch64-linux + - build_x86_64-darwin + - build_aarch64-darwin + if: ${{ always() }} steps: - - name: Check for secrets - id: secret + - run: "true" + - run: | + echo "A dependent in the build matrix failed:" + echo "$needs" + exit 1 env: - _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} + needs: ${{ toJSON(needs) }} + if: | + contains(needs.*.result, 'failure') || + contains(needs.*.result, 'cancelled') + + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + + - name: Create artifacts directory + run: mkdir -p ./artifacts + + - name: Fetch artifacts + uses: actions/download-artifact@v4 + with: + path: downloaded + - name: Move downloaded artifacts to artifacts directory run: | - echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" + for dir in ./downloaded/*; do + arch="$(basename "$dir")" + mv "$dir"/*.xz ./artifacts/"${arch}" + done - docker_push_image: - needs: [tests, vm_tests, check_secrets] - permissions: - contents: read - packages: write - if: >- - needs.check_secrets.outputs.docker == 'true' && - github.event_name == 'push' && - github.ref_name == 'master' - runs-on: ubuntu-24.04 - steps: - - name: Check for secrets - id: secret - env: - _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} - run: | - echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: cachix/install-nix-action@v31 - with: - install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - uses: DeterminateSystems/magic-nix-cache-action@main - - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV - - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L - - run: docker load -i ./result/image.tar.gz - - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION - - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:master - # We'll deploy the newly built image to both Docker Hub and Github Container Registry. - # - # Push to Docker Hub first - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION - - run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:master - # Push to GitHub Container Registry as well - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Push image - run: | - IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix - # Change all uppercase to lowercase - IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + - name: Build fallback-paths.nix + if: ${{ github.event_name != 'pull_request' }} + run: | + nix build .#fallbackPathsNix --out-link fallback + cat fallback > ./artifacts/fallback-paths.nix - docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION - docker tag nix:$NIX_VERSION $IMAGE_ID:latest - docker push $IMAGE_ID:$NIX_VERSION - docker push $IMAGE_ID:latest - # deprecated 2024-02-24 - docker tag nix:$NIX_VERSION $IMAGE_ID:master - docker push $IMAGE_ID:master + - uses: DeterminateSystems/push-artifact-ids@main + with: + s3_upload_role: ${{ secrets.AWS_S3_UPLOAD_ROLE_ARN }} + bucket: ${{ secrets.AWS_S3_UPLOAD_BUCKET_NAME }} + directory: ./artifacts + ids_project_name: determinate-nix + ids_binary_prefix: determinate-nix + skip_acl: true + allowed_branches: '["detsys-main"]' - vm_tests: - runs-on: ubuntu-24.04 + publish: + needs: + - success + if: (!github.repository.fork && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || startsWith(github.ref, 'refs/tags/'))) + environment: ${{ github.event_name == 'release' && 'production' || '' }} + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - - run: | - nix build -L \ - .#hydraJobs.tests.functional_user \ - .#hydraJobs.tests.githubFlakes \ - .#hydraJobs.tests.nix-docker \ - .#hydraJobs.tests.tarballFlakes \ - ; - - flake_regressions: - needs: vm_tests - runs-on: ubuntu-24.04 - steps: - - name: Checkout nix - uses: actions/checkout@v4 - - name: Checkout flake-regressions - uses: actions/checkout@v4 - with: - repository: NixOS/flake-regressions - path: flake-regressions - - name: Checkout flake-regressions-data - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-push@main with: - repository: NixOS/flake-regressions-data - path: flake-regressions/tests - - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh + rolling: ${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + visibility: "public" + tag: "${{ github.ref_name }}" diff --git a/.github/workflows/propose-release.yml b/.github/workflows/propose-release.yml new file mode 100644 index 00000000000..ea01e4b7afe --- /dev/null +++ b/.github/workflows/propose-release.yml @@ -0,0 +1,32 @@ +on: + workflow_dispatch: + inputs: + reference-id: + type: string + required: true + version: + type: string + required: true + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + propose-release: + uses: DeterminateSystems/propose-release/.github/workflows/workflow.yml@main + permissions: + id-token: write + contents: write + pull-requests: write + with: + update-flake: false + reference-id: ${{ inputs.reference-id }} + version: ${{ inputs.version }} + extra-commands-early: | + echo ${{ inputs.version }} > .version-determinate + git add .version-determinate + git commit -m "Set .version-determinate to ${{ inputs.version }}" || true + ./.github/release-notes.sh + git add doc + git commit -m "Generate release notes for ${{ inputs.version }}" || true diff --git a/.version b/.version index 6a6900382e2..bcec02eeb96 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.30.0 +2.30.1 diff --git a/.version-determinate b/.version-determinate new file mode 100644 index 00000000000..269aa9c86de --- /dev/null +++ b/.version-determinate @@ -0,0 +1 @@ +3.8.3 diff --git a/README.md b/README.md index 02498944cdb..4e304b28bf8 100644 --- a/README.md +++ b/README.md @@ -1,38 +1,92 @@ -# Nix +

+ +

+

+  Discord  +  Bluesky  +  Mastodon  +  Twitter  +  LinkedIn  +

-[![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos) -[![CI](https://github.com/NixOS/nix/workflows/CI/badge.svg)](https://github.com/NixOS/nix/actions/workflows/ci.yml) +# Determinate Nix -Nix is a powerful package manager for Linux and other Unix systems that makes package -management reliable and reproducible. Please refer to the [Nix manual](https://nix.dev/reference/nix-manual) -for more details. +[![CI](https://github.com/DeterminateSystems/nix-src/workflows/CI/badge.svg)](https://github.com/DeterminateSystems/nix-src/actions/workflows/ci.yml) -## Installation and first steps +This repository houses the source for [**Determinate Nix**][det-nix], a downstream distribution of [Nix][upstream] created and maintained by [Determinate Systems][detsys]. +Nix is a powerful [language], [package manager][package-management], and [CLI] for [macOS](#macos), [Linux](linux), and other Unix systems that enables you to create fully reproducible [development environments][envs], to build [packages] in sandboxed environments, to build entire Linux systems using [NixOS], and much more. -Visit [nix.dev](https://nix.dev) for [installation instructions](https://nix.dev/tutorials/install-nix) and [beginner tutorials](https://nix.dev/tutorials/first-steps). +Determinate Nix is part of the [Determinate platform][determinate], which also includes [FlakeHub], a secure flake repository with features like [FlakeHub Cache][cache], [private flakes][private-flakes], and [semantic versioning][semver] (SemVer) for [flakes]. -Full reference documentation can be found in the [Nix manual](https://nix.dev/reference/nix-manual). +## Installing Determinate -## Building and developing +You can install Determinate on [macOS](#macos), non-NixOS [Linux](#linux) and WSL, and [NixOS](#nixos). -Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/development/building.html). +### macOS -## Contributing +On macOS, we recommend using the graphical installer from Determinate Systems. +Click [here][gui] to download and run it. + +### Linux + +On Linux, including Windows Subsystem for Linux (WSL), we recommend installing Determinate Nix using [Determinate Nix Installer][installer]: + +```shell +curl -fsSL https://install.determinate.systems/nix | sh -s -- install --determinate +``` + +### NixOS + +On [NixOS], we recommend following our [dedicated installation guide][nixos-install]. -Check the [contributing guide](./CONTRIBUTING.md) if you want to get involved with developing Nix. +## Other resources -## Additional resources +Nix was created by [Eelco Dolstra][eelco] and developed as the subject of his 2006 PhD thesis, [The Purely Functional Software Deployment Model][thesis]. +Today, a worldwide developer community contributes to Nix and the ecosystem that has grown around it. -Nix was created by Eelco Dolstra and developed as the subject of his PhD thesis [The Purely Functional Software Deployment Model](https://edolstra.github.io/pubs/phd-thesis.pdf), published 2006. -Today, a world-wide developer community contributes to Nix and the ecosystem that has grown around it. +- [Zero to Nix][z2n], Determinate Systems' guide to Nix and [flakes] for beginners +- [Nixpkgs], a collection of well over 100,000 software packages that you can build and manage using Nix +- [NixOS] is a Linux distribution that can be configured fully declaratively +- The Nix, Nixpkgs, and NixOS community on [nixos.org][website] -- [The Nix, Nixpkgs, NixOS Community on nixos.org](https://nixos.org/) -- [Official documentation on nix.dev](https://nix.dev) -- [Nixpkgs](https://github.com/NixOS/nixpkgs) is [the largest, most up-to-date free software repository in the world](https://repology.org/repositories/graphs) -- [NixOS](https://github.com/NixOS/nixpkgs/tree/master/nixos) is a Linux distribution that can be configured fully declaratively -- [Discourse](https://discourse.nixos.org/) -- Matrix: [#users:nixos.org](https://matrix.to/#/#users:nixos.org) for user support and [#nix-dev:nixos.org](https://matrix.to/#/#nix-dev:nixos.org) for development +## Reference + +The primary documentation for Determinate and Determinate Nix is available at [docs.determinate.systems][determinate]. +For deeply technical reference material, see the [Determinate Nix manual][manual] which is based on the upstream Nix manual. ## License -Nix is released under the [LGPL v2.1](./COPYING). +[Upstream Nix][upstream] is released under the [LGPL v2.1][license] license. +[Determinate Nix][det-nix] is also released under LGPL v2.1 in accordance with the terms of the upstream license. + +## Contributing + +Check the [contributing guide][contributing] if you want to get involved with developing Nix. + +[cache]: https://docs.determinate.systems/flakehub/cache +[cli]: https://manual.determinate.systems/command-ref/new-cli/nix.html +[contributing]: ./CONTRIBUTING.md +[det-nix]: https://docs.determinate.systems/determinate-nix +[determinate]: https://docs.determinate.systems +[detsys]: https://determinate.systems +[dnixd]: https://docs.determinate.systems/determinate-nix#determinate-nixd +[eelco]: https://determinate.systems/people/eelco-dolstra +[envs]: https://zero-to-nix.com/concepts/dev-env +[flakehub]: https://flakehub.com +[flakes]: https://zero-to-nix.com/concepts/flakes +[gui]: https://install.determinate.systems/determinate-pkg/stable/Universal +[installer]: https://github.com/DeterminateSystems/nix-installer +[language]: https://zero-to-nix.com/concepts/nix-language +[license]: ./COPYING +[manual]: https://manual.determinate.systems +[nixpkgs]: https://github.com/NixOS/nixpkgs +[nixos]: https://github.com/NixOS/nixpkgs/tree/master/nixos +[nixos-install]: https://docs.determinate.systems/guides/advanced-installation#nixos +[packages]: https://zero-to-nix.com/concepts/packages +[package-management]: https://zero-to-nix.com/concepts/package-management +[private-flakes]: https://docs.determinate.systems/flakehub/private-flakes +[semver]: https://docs.determinate.systems/flakehub/concepts/semver +[thesis]: https://edolstra.github.io/pubs/phd-thesis.pdf +[upstream]: https://github.com/NixOS/nix +[website]: https://nixos.org +[z2n]: https://zero-to-nix.com diff --git a/default.nix b/default.nix deleted file mode 100644 index 6466507b714..00000000000 --- a/default.nix +++ /dev/null @@ -1,9 +0,0 @@ -(import ( - let - lock = builtins.fromJSON (builtins.readFile ./flake.lock); - in - fetchTarball { - url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; - sha256 = lock.nodes.flake-compat.locked.narHash; - } -) { src = ./.; }).defaultNix diff --git a/doc/manual/book.toml.in b/doc/manual/book.toml.in index 34acf642edb..f3fd2722f3c 100644 --- a/doc/manual/book.toml.in +++ b/doc/manual/book.toml.in @@ -1,12 +1,12 @@ [book] -title = "Nix @version@ Reference Manual" +title = "Determinate Nix @version@ Reference Manual" src = "source" [output.html] additional-css = ["custom.css"] additional-js = ["redirects.js"] -edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}" -git-repository-url = "https://github.com/NixOS/nix" +edit-url-template = "https://github.com/DeterminateSystems/nix-src/tree/master/doc/manual/{path}" +git-repository-url = "https://github.com/DeterminateSystems/nix-src" # Handles replacing @docroot@ with a path to ./source relative to that markdown file, # {{#include handlebars}}, and the @generated@ syntax used within these. it mostly diff --git a/doc/manual/custom.css b/doc/manual/custom.css index 7af150be391..119c6d12543 100644 --- a/doc/manual/custom.css +++ b/doc/manual/custom.css @@ -1,5 +1,5 @@ :root { - --sidebar-width: 23em; + --sidebar-width: 23em; } h1.menu-title::before { @@ -7,11 +7,10 @@ h1.menu-title::before { background-image: url("./favicon.svg"); padding: 1.25em; background-position: center center; - background-size: 2em; + background-size: 1.5em; background-repeat: no-repeat; } - .menu-bar { padding: 0.5em 0em; } @@ -21,13 +20,13 @@ h1.menu-title::before { } h1:not(:first-of-type) { - margin-top: 1.3em; + margin-top: 1.3em; } h2 { - margin-top: 1em; + margin-top: 1em; } .hljs-meta { - user-select: none; + user-select: none; } diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 31e74e17d26..292cb283d3d 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -42,11 +42,6 @@ let let result = '' - > **Warning** \ - > This program is - > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) - > and its interface is subject to change. - # Name `${command}` - ${details.description} diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 6fe2374a764..c8a5282aab3 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -4,6 +4,8 @@ project('nix-manual', license : 'LGPL-2.1-or-later', ) +fs = import('fs') + nix = find_program('nix', native : true) mdbook = find_program('mdbook', native : true) @@ -20,7 +22,7 @@ nix_env_for_docs = { 'NIX_CONFIG': 'cores = 0', } -nix_for_docs = [nix, '--experimental-features', 'nix-command'] +nix_for_docs = [nix] nix_eval_for_docs_common = nix_for_docs + [ 'eval', '-I', 'nix=' + meson.current_source_dir(), @@ -93,7 +95,7 @@ manual = custom_target( python.full_path(), mdbook.full_path(), meson.current_build_dir(), - meson.project_version(), + fs.read('../../.version-determinate').strip(), ), ], input : [ diff --git a/doc/manual/package.nix b/doc/manual/package.nix index af6d46a2a00..3953cb861ce 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -24,7 +24,7 @@ let in mkMesonDerivation (finalAttrs: { - pname = "nix-manual"; + pname = "determinate-nix-manual"; inherit version; workDir = ./.; @@ -32,6 +32,7 @@ mkMesonDerivation (finalAttrs: { fileset.difference (fileset.unions [ ../../.version + ../../.version-determinate # Too many different types of files to filter for now ../../doc/manual ./. diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 9612438481f..b2295cf4fc5 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -271,14 +271,10 @@ const redirects = { "sect-multi-user-installation": "installation/installing-binary.html#multi-user-installation", "sect-nix-install-binary-tarball": "installation/installing-binary.html#installing-from-a-binary-tarball", "sect-nix-install-pinned-version-url": "installation/installing-binary.html#installing-a-pinned-nix-version-from-a-url", - "sect-single-user-installation": "installation/installing-binary.html#single-user-installation", "ch-installing-source": "installation/installing-source.html", "ssec-multi-user": "installation/multi-user.html", - "ch-nix-security": "installation/nix-security.html", "sec-obtaining-source": "installation/obtaining-source.html", "sec-prerequisites-source": "installation/prerequisites-source.html", - "sec-single-user": "installation/single-user.html", - "ch-supported-platforms": "installation/supported-platforms.html", "ch-upgrading-nix": "installation/upgrading.html", "ch-about-nix": "introduction.html", "chap-introduction": "introduction.html", diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index bfb921567c3..0f90b2c6ea1 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -3,17 +3,12 @@ - [Introduction](introduction.md) - [Quick Start](quick-start.md) - [Installation](installation/index.md) - - [Supported Platforms](installation/supported-platforms.md) - - [Installing a Binary Distribution](installation/installing-binary.md) - [Installing Nix from Source](installation/installing-source.md) - [Prerequisites](installation/prerequisites-source.md) - [Obtaining a Source Distribution](installation/obtaining-source.md) - [Building Nix from Source](installation/building-source.md) - [Using Nix within Docker](installation/installing-docker.md) - [Security](installation/nix-security.md) - - [Single-User Mode](installation/single-user.md) - - [Multi-User Mode](installation/multi-user.md) - - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) - [Uninstalling Nix](installation/uninstall.md) - [Nix Store](store/index.md) @@ -61,8 +56,11 @@ - [Command Reference](command-ref/index.md) - [Common Options](command-ref/opt-common.md) - [Common Environment Variables](command-ref/env-common.md) - - [Main Commands](command-ref/main-commands.md) + - [Subcommands](command-ref/subcommands.md) +{{#include ./command-ref/new-cli/SUMMARY.md}} + - [Deprecated Commands](command-ref/main-commands.md) - [nix-build](command-ref/nix-build.md) + - [nix-channel](command-ref/nix-channel.md) - [nix-shell](command-ref/nix-shell.md) - [nix-store](command-ref/nix-store.md) - [nix-store --add-fixed](command-ref/nix-store/add-fixed.md) @@ -98,22 +96,17 @@ - [nix-env --uninstall](command-ref/nix-env/uninstall.md) - [nix-env --upgrade](command-ref/nix-env/upgrade.md) - [Utilities](command-ref/utilities.md) - - [nix-channel](command-ref/nix-channel.md) - [nix-collect-garbage](command-ref/nix-collect-garbage.md) - [nix-copy-closure](command-ref/nix-copy-closure.md) - [nix-daemon](command-ref/nix-daemon.md) - [nix-hash](command-ref/nix-hash.md) - [nix-instantiate](command-ref/nix-instantiate.md) - [nix-prefetch-url](command-ref/nix-prefetch-url.md) - - [Experimental Commands](command-ref/experimental-commands.md) -{{#include ./command-ref/new-cli/SUMMARY.md}} - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) - [Profiles](command-ref/files/profiles.md) - [manifest.nix](command-ref/files/manifest.nix.md) - [manifest.json](command-ref/files/manifest.json.md) - - [Channels](command-ref/files/channels.md) - - [Default Nix expression](command-ref/files/default-nix-expression.md) - [Architecture and Design](architecture/architecture.md) - [Formats and Protocols](protocols/index.md) - [JSON Formats](protocols/json/index.md) @@ -135,67 +128,32 @@ - [C++ style guide](development/cxx.md) - [Experimental Features](development/experimental-features.md) - [Contributing](development/contributing.md) -- [Releases](release-notes/index.md) +- [Determinate Nix Release Notes](release-notes-determinate/index.md) + - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.8.3 (2025-07-18)](release-notes-determinate/rl-3.8.3.md) + - [Release 3.8.2 (2025-07-12)](release-notes-determinate/rl-3.8.2.md) + - [Release 3.8.1 (2025-07-11)](release-notes-determinate/rl-3.8.1.md) + - [Release 3.8.0 (2025-07-10)](release-notes-determinate/rl-3.8.0.md) + - [Release 3.7.0 (2025-07-03)](release-notes-determinate/rl-3.7.0.md) + - [Release 3.6.8 (2025-06-25)](release-notes-determinate/rl-3.6.8.md) + - [Release 3.6.7 (2025-06-24)](release-notes-determinate/rl-3.6.7.md) + - [Release 3.6.6 (2025-06-17)](release-notes-determinate/rl-3.6.6.md) + - [Release 3.6.5 (2025-06-16)](release-notes-determinate/rl-3.6.5.md) + - [Release 3.6.2 (2025-06-02)](release-notes-determinate/rl-3.6.2.md) + - [Release 3.6.1 (2025-05-24)](release-notes-determinate/rl-3.6.1.md) + - [Release 3.6.0 (2025-05-22)](release-notes-determinate/rl-3.6.0.md) + - [Release 3.5.2 (2025-05-12)](release-notes-determinate/rl-3.5.2.md) + - [Release 3.5.1 (2025-05-09)](release-notes-determinate/rl-3.5.1.md) + - [~~Release 3.5.0 (2025-05-09)~~](release-notes-determinate/rl-3.5.0.md) + - [Release 3.4.2 (2025-05-05)](release-notes-determinate/rl-3.4.2.md) + - [Release 3.4.0 (2025-04-25)](release-notes-determinate/rl-3.4.0.md) + - [Release 3.3.0 (2025-04-11)](release-notes-determinate/rl-3.3.0.md) + - [Release 3.1.0 (2025-03-27)](release-notes-determinate/rl-3.1.0.md) + - [Release 3.0.0 (2025-03-04)](release-notes-determinate/rl-3.0.0.md) +- [Nix Release Notes](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} - [Release 2.30 (2025-07-07)](release-notes/rl-2.30.md) - [Release 2.29 (2025-05-14)](release-notes/rl-2.29.md) - [Release 2.28 (2025-04-02)](release-notes/rl-2.28.md) - [Release 2.27 (2025-03-03)](release-notes/rl-2.27.md) - [Release 2.26 (2025-01-22)](release-notes/rl-2.26.md) - - [Release 2.25 (2024-11-07)](release-notes/rl-2.25.md) - - [Release 2.24 (2024-07-31)](release-notes/rl-2.24.md) - - [Release 2.23 (2024-06-03)](release-notes/rl-2.23.md) - - [Release 2.22 (2024-04-23)](release-notes/rl-2.22.md) - - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md) - - [Release 2.20 (2024-01-29)](release-notes/rl-2.20.md) - - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) - - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) - - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) - - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) - - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) - - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) - - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md) - - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) - - [Release 2.11 (2022-08-25)](release-notes/rl-2.11.md) - - [Release 2.10 (2022-07-11)](release-notes/rl-2.10.md) - - [Release 2.9 (2022-05-30)](release-notes/rl-2.9.md) - - [Release 2.8 (2022-04-19)](release-notes/rl-2.8.md) - - [Release 2.7 (2022-03-07)](release-notes/rl-2.7.md) - - [Release 2.6 (2022-01-24)](release-notes/rl-2.6.md) - - [Release 2.5 (2021-12-13)](release-notes/rl-2.5.md) - - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md) - - [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md) - - [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md) - - [Release 2.1 (2018-09-02)](release-notes/rl-2.1.md) - - [Release 2.0 (2018-02-22)](release-notes/rl-2.0.md) - - [Release 1.11.10 (2017-06-12)](release-notes/rl-1.11.10.md) - - [Release 1.11 (2016-01-19)](release-notes/rl-1.11.md) - - [Release 1.10 (2015-09-03)](release-notes/rl-1.10.md) - - [Release 1.9 (2015-06-12)](release-notes/rl-1.9.md) - - [Release 1.8 (2014-12-14)](release-notes/rl-1.8.md) - - [Release 1.7 (2014-04-11)](release-notes/rl-1.7.md) - - [Release 1.6.1 (2013-10-28)](release-notes/rl-1.6.1.md) - - [Release 1.6 (2013-09-10)](release-notes/rl-1.6.md) - - [Release 1.5.2 (2013-05-13)](release-notes/rl-1.5.2.md) - - [Release 1.5 (2013-02-27)](release-notes/rl-1.5.md) - - [Release 1.4 (2013-02-26)](release-notes/rl-1.4.md) - - [Release 1.3 (2013-01-04)](release-notes/rl-1.3.md) - - [Release 1.2 (2012-12-06)](release-notes/rl-1.2.md) - - [Release 1.1 (2012-07-18)](release-notes/rl-1.1.md) - - [Release 1.0 (2012-05-11)](release-notes/rl-1.0.md) - - [Release 0.16 (2010-08-17)](release-notes/rl-0.16.md) - - [Release 0.15 (2010-03-17)](release-notes/rl-0.15.md) - - [Release 0.14 (2010-02-04)](release-notes/rl-0.14.md) - - [Release 0.13 (2009-11-05)](release-notes/rl-0.13.md) - - [Release 0.12 (2008-11-20)](release-notes/rl-0.12.md) - - [Release 0.11 (2007-12-31)](release-notes/rl-0.11.md) - - [Release 0.10.1 (2006-10-11)](release-notes/rl-0.10.1.md) - - [Release 0.10 (2006-10-06)](release-notes/rl-0.10.md) - - [Release 0.9.2 (2005-09-21)](release-notes/rl-0.9.2.md) - - [Release 0.9.1 (2005-09-20)](release-notes/rl-0.9.1.md) - - [Release 0.9 (2005-09-16)](release-notes/rl-0.9.md) - - [Release 0.8.1 (2005-04-13)](release-notes/rl-0.8.1.md) - - [Release 0.8 (2005-04-11)](release-notes/rl-0.8.md) - - [Release 0.7 (2005-01-12)](release-notes/rl-0.7.md) - - [Release 0.6 (2004-11-14)](release-notes/rl-0.6.md) - - [Release 0.5 and earlier](release-notes/rl-0.5.md) diff --git a/doc/manual/source/command-ref/env-common.md b/doc/manual/source/command-ref/env-common.md index ee3995111e9..bd428a232eb 100644 --- a/doc/manual/source/command-ref/env-common.md +++ b/doc/manual/source/command-ref/env-common.md @@ -102,7 +102,7 @@ Most Nix commands interpret the following environment variables: This variable should be set to `daemon` if you want to use the Nix daemon to execute Nix operations. This is necessary in [multi-user - Nix installations](@docroot@/installation/multi-user.md). If the Nix + Nix installations](@docroot@/installation/nix-security.md#multi-user-model). If the Nix daemon's Unix socket is at some non-standard path, this variable should be set to `unix://path/to/socket`. Otherwise, it should be left unset. diff --git a/doc/manual/source/command-ref/experimental-commands.md b/doc/manual/source/command-ref/experimental-commands.md deleted file mode 100644 index 1190729a230..00000000000 --- a/doc/manual/source/command-ref/experimental-commands.md +++ /dev/null @@ -1,8 +0,0 @@ -# Experimental Commands - -This section lists [experimental commands](@docroot@/development/experimental-features.md#xp-feature-nix-command). - -> **Warning** -> -> These commands may be removed in the future, or their syntax may -> change in incompatible ways. diff --git a/doc/manual/source/command-ref/files/default-nix-expression.md b/doc/manual/source/command-ref/files/default-nix-expression.md index 2bd45ff5deb..e886e3ff499 100644 --- a/doc/manual/source/command-ref/files/default-nix-expression.md +++ b/doc/manual/source/command-ref/files/default-nix-expression.md @@ -31,12 +31,12 @@ Then, the resulting expression is interpreted like this: The file [`manifest.nix`](@docroot@/command-ref/files/manifest.nix.md) is always ignored. -The command [`nix-channel`] places a symlink to the current user's [channels] in this directory, the [user channel link](#user-channel-link). +The command [`nix-channel`] places a symlink to the current user's channels in this directory, the [user channel link](#user-channel-link). This makes all subscribed channels available as attributes in the default expression. ## User channel link -A symlink that ensures that [`nix-env`] can find the current user's [channels]: +A symlink that ensures that [`nix-env`] can find the current user's channels: - `~/.nix-defexpr/channels` - `$XDG_STATE_HOME/defexpr/channels` if [`use-xdg-base-directories`] is set to `true`. @@ -51,4 +51,3 @@ In a multi-user installation, you may also have `~/.nix-defexpr/channels_root`, [`nix-channel`]: @docroot@/command-ref/nix-channel.md [`nix-env`]: @docroot@/command-ref/nix-env.md [`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories -[channels]: @docroot@/command-ref/files/channels.md diff --git a/doc/manual/source/command-ref/files/profiles.md b/doc/manual/source/command-ref/files/profiles.md index b5c7378800f..e46e2418b4c 100644 --- a/doc/manual/source/command-ref/files/profiles.md +++ b/doc/manual/source/command-ref/files/profiles.md @@ -67,7 +67,7 @@ By default, this symlink points to: - `$NIX_STATE_DIR/profiles/per-user/root/profile` for `root` The `PATH` environment variable should include `/bin` subdirectory of the profile link (e.g. `~/.nix-profile/bin`) for the user environment to be visible to the user. -The [installer](@docroot@/installation/installing-binary.md) sets this up by default, unless you enable [`use-xdg-base-directories`]. +The installer sets this up by default, unless you enable [`use-xdg-base-directories`]. [`nix-env`]: @docroot@/command-ref/nix-env.md [`nix profile`]: @docroot@/command-ref/new-cli/nix3-profile.md diff --git a/doc/manual/source/command-ref/nix-channel.md b/doc/manual/source/command-ref/nix-channel.md index ed9cbb41fbf..a65ec97c558 100644 --- a/doc/manual/source/command-ref/nix-channel.md +++ b/doc/manual/source/command-ref/nix-channel.md @@ -8,6 +8,12 @@ # Description +> **Warning** +> +> nix-channel is deprecated in favor of flakes in Determinate Nix. +> For a guide on Nix flakes, see: . +> For details and to offer feedback on the deprecation process, see: . + Channels are a mechanism for referencing remote Nix expressions and conveniently retrieving their latest version. The moving parts of channels are: diff --git a/doc/manual/source/command-ref/nix-env.md b/doc/manual/source/command-ref/nix-env.md index bda02149ed0..d01caaf7f78 100644 --- a/doc/manual/source/command-ref/nix-env.md +++ b/doc/manual/source/command-ref/nix-env.md @@ -52,7 +52,7 @@ These pages can be viewed offline: `nix-env` can obtain packages from multiple sources: - An attribute set of derivations from: - - The [default Nix expression](@docroot@/command-ref/files/default-nix-expression.md) (by default) + - The default Nix expression (by default) - A Nix file, specified via `--file` - A [profile](@docroot@/command-ref/files/profiles.md), specified via `--from-profile` - A Nix expression that is a function which takes default expression as argument, specified via `--from-expression` diff --git a/doc/manual/source/command-ref/nix-env/install.md b/doc/manual/source/command-ref/nix-env/install.md index 527fd8f90d8..26a32aa6b6b 100644 --- a/doc/manual/source/command-ref/nix-env/install.md +++ b/doc/manual/source/command-ref/nix-env/install.md @@ -22,12 +22,11 @@ It is based on the current generation of the active [profile](@docroot@/command- The arguments *args* map to store paths in a number of possible ways: -- By default, *args* is a set of names denoting derivations in the [default Nix expression]. +- By default, *args* is a set of names denoting derivations in the default Nix expression. These are [realised], and the resulting output paths are installed. Currently installed derivations with a name equal to the name of a derivation being added are removed unless the option `--preserve-installed` is specified. [derivation expression]: @docroot@/glossary.md#gloss-derivation-expression - [default Nix expression]: @docroot@/command-ref/files/default-nix-expression.md [realised]: @docroot@/glossary.md#gloss-realise If there are multiple derivations matching a name in *args* that @@ -45,7 +44,7 @@ The arguments *args* map to store paths in a number of possible ways: gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will probably cause a user environment conflict\!). -- If [`--attr`](#opt-attr) / `-A` is specified, the arguments are *attribute paths* that select attributes from the [default Nix expression]. +- If [`--attr`](#opt-attr) / `-A` is specified, the arguments are *attribute paths* that select attributes from the default Nix expression. This is faster than using derivation names and unambiguous. Show the attribute paths of available packages with [`nix-env --query`](./query.md): @@ -58,7 +57,7 @@ The arguments *args* map to store paths in a number of possible ways: easy way to copy user environment elements from one profile to another. -- If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/syntax.md#functions) that are called with the [default Nix expression] as their single argument. +- If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/syntax.md#functions) that are called with the default Nix expression as their single argument. The derivations returned by those function calls are installed. This allows derivations to be specified in an unambiguous way, which is necessary if there are multiple derivations with the same name. diff --git a/doc/manual/source/command-ref/nix-store/query.md b/doc/manual/source/command-ref/nix-store/query.md index b5ba63adae2..94eee05b8a8 100644 --- a/doc/manual/source/command-ref/nix-store/query.md +++ b/doc/manual/source/command-ref/nix-store/query.md @@ -103,6 +103,13 @@ symlink. example when *paths* were substituted from a binary cache. Use `--valid-derivers` instead to obtain valid paths only. + > **Note** + > + > `nix-store --query --deriver` is replaced with the following `nix` command: + > + > nix path-info --json ... | jq -r '.[].deriver' + + [deriver]: @docroot@/glossary.md#gloss-deriver - `--valid-derivers` diff --git a/doc/manual/source/command-ref/subcommands.md b/doc/manual/source/command-ref/subcommands.md new file mode 100644 index 00000000000..6a26732338d --- /dev/null +++ b/doc/manual/source/command-ref/subcommands.md @@ -0,0 +1,3 @@ +# Subcommands + +This section lists all the subcommands of the `nix` CLI. diff --git a/doc/manual/source/development/building.md b/doc/manual/source/development/building.md index 16746383704..bf2f73c103e 100644 --- a/doc/manual/source/development/building.md +++ b/doc/manual/source/development/building.md @@ -1,73 +1,5 @@ # Building Nix -This section provides some notes on how to start hacking on Nix. -To get the latest version of Nix from GitHub: - -```console -$ git clone https://github.com/NixOS/nix.git -$ cd nix -``` - -> **Note** -> -> The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. -> If you don't have it installed, follow the [installation instructions](../installation/index.md). - - -To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To get a shell with one of the other [supported compilation environments](#compilation-environments): - -```console -$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages -``` - -> **Note** -> -> You can use `native-ccacheStdenv` to drastically improve rebuild time. -> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. - -To build Nix itself in this shell: - -```console -[nix-shell]$ mesonFlags+=" --prefix=$(pwd)/outputs/out" -[nix-shell]$ dontAddPrefix=1 configurePhase -[nix-shell]$ buildPhase -``` - -To test it: - -```console -[nix-shell]$ checkPhase -``` - -To install it in `$(pwd)/outputs`: - -```console -[nix-shell]$ installPhase -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 2.12 -``` - -To build a release version of Nix for the current operating system and CPU architecture: - -```console -$ nix-build -``` - -You can also build Nix for one of the [supported platforms](#platforms). - -## Building Nix with flakes - -This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. - -[`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command - To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: ```console @@ -145,12 +77,6 @@ platform. Common solutions include [remote build machines] and [binary format em Given such a setup, executing the build only requires selecting the respective attribute. For example, to compile for `aarch64-linux`: -```console -$ nix-build --attr packages.aarch64-linux.default -``` - -or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled: - ```console $ nix build .#packages.aarch64-linux.default ``` @@ -239,20 +165,12 @@ To build with one of those environments, you can use $ nix build .#nix-cli-ccacheStdenv ``` -for flake-enabled Nix, or - -```console -$ nix-build --attr nix-cli-ccacheStdenv -``` - -for classic Nix. - You can use any of the other supported environments in place of `nix-cli-ccacheStdenv`. ## Editor integration The `clangd` LSP server is installed by default on the `clang`-based `devShell`s. -See [supported compilation environments](#compilation-environments) and instructions how to set up a shell [with flakes](#nix-with-flakes) or in [classic Nix](#classic-nix). +See [supported compilation environments](#compilation-environments) and instructions how to [set up a shell with flakes](#nix-with-flakes). To use the LSP with your editor, you will want a `compile_commands.json` file telling `clangd` how we are compiling the code. Meson's configure always produces this inside the build directory. diff --git a/doc/manual/source/development/experimental-features.md b/doc/manual/source/development/experimental-features.md index ad5cffa91ee..56a45b23890 100644 --- a/doc/manual/source/development/experimental-features.md +++ b/doc/manual/source/development/experimental-features.md @@ -6,7 +6,7 @@ Experimental features are considered unstable, which means that they can be chan Users must explicitly enable them by toggling the associated [experimental feature flags](@docroot@/command-ref/conf-file.md#conf-experimental-features). This allows accessing unstable functionality without unwittingly relying on it. -Experimental feature flags were first introduced in [Nix 2.4](@docroot@/release-notes/rl-2.4.md). +Experimental feature flags were first introduced in [Nix 2.4](https://nix.dev/manual/nix/latest/release-notes/rl-2.4). Before that, Nix did have experimental features, but they were not guarded by flags and were merely documented as unstable. This was a source of confusion and controversy. diff --git a/doc/manual/source/favicon.png b/doc/manual/source/favicon.png deleted file mode 100644 index 1ed2b5fe0fd..00000000000 Binary files a/doc/manual/source/favicon.png and /dev/null differ diff --git a/doc/manual/source/favicon.svg b/doc/manual/source/favicon.svg index 1d2a6e835d5..55fb9479b06 100644 --- a/doc/manual/source/favicon.svg +++ b/doc/manual/source/favicon.svg @@ -1 +1,29 @@ - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index e6a294e7de7..9e76ad37b96 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -353,14 +353,6 @@ See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. -- [`∅`]{#gloss-empty-set} - - The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. - -- [`ε`]{#gloss-epsilon} - - The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. - - [package]{#package} A software package; files that belong together for a particular purpose, and metadata. diff --git a/doc/manual/source/installation/env-variables.md b/doc/manual/source/installation/env-variables.md deleted file mode 100644 index 0350904211a..00000000000 --- a/doc/manual/source/installation/env-variables.md +++ /dev/null @@ -1,62 +0,0 @@ -# Environment Variables - -To use Nix, some environment variables should be set. In particular, -`PATH` should contain the directories `prefix/bin` and -`~/.nix-profile/bin`. The first directory contains the Nix tools -themselves, while `~/.nix-profile` is a symbolic link to the current -*user environment* (an automatically generated package consisting of -symlinks to installed packages). The simplest way to set the required -environment variables is to include the file -`prefix/etc/profile.d/nix.sh` in your `~/.profile` (or similar), like -this: - -```bash -source prefix/etc/profile.d/nix.sh -``` - -# `NIX_SSL_CERT_FILE` - -If you need to specify a custom certificate bundle to account for an -HTTPS-intercepting man in the middle proxy, you must specify the path to -the certificate bundle in the environment variable `NIX_SSL_CERT_FILE`. - -If you don't specify a `NIX_SSL_CERT_FILE` manually, Nix will install -and use its own certificate bundle. - -Set the environment variable and install Nix - -```console -$ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt -$ curl -L https://nixos.org/nix/install | sh -``` - -In the shell profile and rc files (for example, `/etc/bashrc`, -`/etc/zshrc`), add the following line: - -```bash -export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt -``` - -> **Note** -> -> You must not add the export and then do the install, as the Nix -> installer will detect the presence of Nix configuration, and abort. - -If you use the Nix daemon, you should also add the following to -`/etc/nix/nix.conf`: - -``` -ssl-cert-file = /etc/ssl/my-certificate-bundle.crt -``` - -## Proxy Environment Variables - -The Nix installer has special handling for these proxy-related -environment variables: `http_proxy`, `https_proxy`, `ftp_proxy`, -`all_proxy`, `no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, -`ALL_PROXY`, `NO_PROXY`. - -If any of these variables are set when running the Nix installer, then -the installer will create an override file at -`/etc/systemd/system/nix-daemon.service.d/override.conf` so `nix-daemon` -will use them. diff --git a/doc/manual/source/installation/index.md b/doc/manual/source/installation/index.md index 3c09f103184..21aca146fd2 100644 --- a/doc/manual/source/installation/index.md +++ b/doc/manual/source/installation/index.md @@ -1,44 +1,11 @@ # Installation -This section describes how to install and configure Nix for first-time use. - -The current recommended option on Linux and MacOS is [multi-user](#multi-user). - -## Multi-user - -This installation offers better sharing, improved isolation, and more security -over a single user installation. - -This option requires either: - -* Linux running systemd, with SELinux disabled -* MacOS - -> **Updating to macOS 15 Sequoia** -> -> If you recently updated to macOS 15 Sequoia and are getting -> ```console -> error: the user '_nixbld1' in the group 'nixbld' does not exist -> ``` -> when running Nix commands, refer to GitHub issue [NixOS/nix#10892](https://github.com/NixOS/nix/issues/10892) for instructions to fix your installation without reinstalling. +We recommend that macOS users install Determinate Nix using our graphical installer, [Determinate.pkg][pkg]. +For Linux and Windows Subsystem for Linux (WSL) users: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --daemon -``` - -## Single-user - -> Single-user is not supported on Mac. - -> `warning: installing Nix as root is not supported by this script!` - -This installation has less requirements than the multi-user install, however it -cannot offer equivalent sharing, isolation, or security. - -This option is suitable for systems without systemd. - -```console -$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon +curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | \ + sh -s -- install --determinate ``` ## Distributions @@ -46,3 +13,5 @@ $ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon The Nix community maintains installers for several distributions. They can be found in the [`nix-community/nix-installers`](https://github.com/nix-community/nix-installers) repository. + +[pkg]: https://install.determinate.systems/determinate-pkg/stable/Universal diff --git a/doc/manual/source/installation/installing-binary.md b/doc/manual/source/installation/installing-binary.md deleted file mode 100644 index 21c15637437..00000000000 --- a/doc/manual/source/installation/installing-binary.md +++ /dev/null @@ -1,158 +0,0 @@ -# Installing a Binary Distribution - -> **Updating to macOS 15 Sequoia** -> -> If you recently updated to macOS 15 Sequoia and are getting -> ```console -> error: the user '_nixbld1' in the group 'nixbld' does not exist -> ``` -> when running Nix commands, refer to GitHub issue [NixOS/nix#10892](https://github.com/NixOS/nix/issues/10892) for instructions to fix your installation without reinstalling. - -To install the latest version Nix, run the following command: - -```console -$ curl -L https://nixos.org/nix/install | sh -``` - -This performs the default type of installation for your platform: - -- [Multi-user](#multi-user-installation): - - Linux with systemd and without SELinux - - macOS -- [Single-user](#single-user-installation): - - Linux without systemd - - Linux with SELinux - -We recommend the multi-user installation if it supports your platform and you can authenticate with `sudo`. - -The installer can be configured with various command line arguments and environment variables. -To show available command line flags: - -```console -$ curl -L https://nixos.org/nix/install | sh -s -- --help -``` - -To check what it does and how it can be customised further, [download and edit the second-stage installation script](#installing-from-a-binary-tarball). - -# Installing a pinned Nix version from a URL - -Version-specific installation URLs for all Nix versions since 1.11.16 can be found at [releases.nixos.org](https://releases.nixos.org/?prefix=nix/). -The directory for each version contains the corresponding SHA-256 hash. - -All installation scripts are invoked the same way: - -```console -$ export VERSION=2.19.2 -$ curl -L https://releases.nixos.org/nix/nix-$VERSION/install | sh -``` - -# Multi User Installation - -The multi-user Nix installation creates system users and a system service for the Nix daemon. - -Supported systems: - -- Linux running systemd, with SELinux disabled -- macOS - -To explicitly instruct the installer to perform a multi-user installation on your system: - -```console -$ bash <(curl -L https://nixos.org/nix/install) --daemon -``` - -You can run this under your usual user account or `root`. -The script will invoke `sudo` as needed. - -# Single User Installation - -To explicitly select a single-user installation on your system: - -```console -$ bash <(curl -L https://nixos.org/nix/install) --no-daemon -``` - -In a single-user installation, `/nix` is owned by the invoking user. -The script will invoke `sudo` to create `/nix` if it doesn’t already exist. -If you don’t have `sudo`, manually create `/nix` as `root`: - -```console -$ su root -# mkdir /nix -# chown alice /nix -``` - -# Installing from a binary tarball - -You can also download a binary tarball that contains Nix and all its dependencies: -- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) -- Download and unpack the tarball -- Run the installer - -> **Example** -> -> ```console -> $ pushd $(mktemp -d) -> $ export VERSION=2.19.2 -> $ export SYSTEM=x86_64-linux -> $ curl -LO https://releases.nixos.org/nix/nix-$VERSION/nix-$VERSION-$SYSTEM.tar.xz -> $ tar xfj nix-$VERSION-$SYSTEM.tar.xz -> $ cd nix-$VERSION-$SYSTEM -> $ ./install -> $ popd -> ``` - -The installer can be customised with the environment variables declared in the file named `install-multi-user`. - -## Native packages for Linux distributions - -The Nix community maintains installers for some Linux distributions in their native packaging format(https://nix-community.github.io/nix-installers/). - -# macOS Installation - - -[]{#sect-macos-installation-change-store-prefix}[]{#sect-macos-installation-encrypted-volume}[]{#sect-macos-installation-symlink}[]{#sect-macos-installation-recommended-notes} - -We believe we have ironed out how to cleanly support the read-only root file system -on modern macOS. New installs will do this automatically. - -This section previously detailed the situation, options, and trade-offs, -but it now only outlines what the installer does. You don't need to know -this to run the installer, but it may help if you run into trouble: - -- create a new APFS volume for your Nix store -- update `/etc/synthetic.conf` to direct macOS to create a "synthetic" - empty root directory to mount your volume -- specify mount options for the volume in `/etc/fstab` - - `rw`: read-write - - `noauto`: prevent the system from auto-mounting the volume (so the - LaunchDaemon mentioned below can control mounting it, and to avoid - masking problems with that mounting service). - - `nobrowse`: prevent the Nix Store volume from showing up on your - desktop; also keeps Spotlight from spending resources to index - this volume - -- if you have FileVault enabled - - generate an encryption password - - put it in your system Keychain - - use it to encrypt the volume -- create a system LaunchDaemon to mount this volume early enough in the - boot process to avoid problems loading or restoring any programs that - need access to your Nix store - diff --git a/doc/manual/source/installation/nix-security.md b/doc/manual/source/installation/nix-security.md index 1e9036b68b2..61cad24c2b3 100644 --- a/doc/manual/source/installation/nix-security.md +++ b/doc/manual/source/installation/nix-security.md @@ -1,15 +1,85 @@ # Security -Nix has two basic security models. First, it can be used in “single-user -mode”, which is similar to what most other package management tools do: -there is a single user (typically root) who performs all package -management operations. All other users can then use the installed -packages, but they cannot perform package management operations -themselves. - -Alternatively, you can configure Nix in “multi-user mode”. In this -model, all users can perform package management operations — for -instance, every user can install software without requiring root -privileges. Nix ensures that this is secure. For instance, it’s not -possible for one user to overwrite a package used by another user with a -Trojan horse. +Nix follows a [**multi-user**](#multi-user-model) security model in which all +users can perform package management operations. Every user can, for example, +install software without requiring root privileges, and Nix ensures that this +is secure. It's *not* possible for one user to, for example, overwrite a +package used by another user with a Trojan horse. + +## Multi-User model + +To allow a Nix store to be shared safely among multiple users, it is +important that users are not able to run builders that modify the Nix +store or database in arbitrary ways, or that interfere with builds +started by other users. If they could do so, they could install a Trojan +horse in some package and compromise the accounts of other users. + +To prevent this, the Nix store and database are owned by some privileged +user (usually `root`) and builders are executed under special user +accounts (usually named `nixbld1`, `nixbld2`, etc.). When a unprivileged +user runs a Nix command, actions that operate on the Nix store (such as +builds) are forwarded to a *Nix daemon* running under the owner of the +Nix store/database that performs the operation. + +> **Note** +> +> Multi-user mode has one important limitation: only root and a set of +> trusted users specified in `nix.conf` can specify arbitrary binary +> caches. So while unprivileged users may install packages from +> arbitrary Nix expressions, they may not get pre-built binaries. + +### Setting up the build users + +The *build users* are the special UIDs under which builds are performed. +They should all be members of the *build users group* `nixbld`. This +group should have no other members. The build users should not be +members of any other group. On Linux, you can create the group and users +as follows: + +```console +$ groupadd -r nixbld +$ for n in $(seq 1 10); do useradd -c "Nix build user $n" \ + -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(which nologin)" \ + nixbld$n; done +``` + +This creates 10 build users. There can never be more concurrent builds +than the number of build users, so you may want to increase this if you +expect to do many builds at the same time. + +### Running the daemon + +The [Nix daemon](../command-ref/nix-daemon.md) should be started as +follows (as `root`): + +```console +$ nix-daemon +``` + +You’ll want to put that line somewhere in your system’s boot scripts. + +To let unprivileged users use the daemon, they should set the +[`NIX_REMOTE` environment variable](../command-ref/env-common.md) to +`daemon`. So you should put a line like + +```console +export NIX_REMOTE=daemon +``` + +into the users’ login scripts. + +### Restricting access + +To limit which users can perform Nix operations, you can use the +permissions on the directory `/nix/var/nix/daemon-socket`. For instance, +if you want to restrict the use of Nix to the members of a group called +`nix-users`, do + +```console +$ chgrp nix-users /nix/var/nix/daemon-socket +$ chmod ug=rwx,o= /nix/var/nix/daemon-socket +``` + +This way, users who are not in the `nix-users` group cannot connect to +the Unix domain socket `/nix/var/nix/daemon-socket/socket`, so they +cannot perform Nix operations. diff --git a/doc/manual/source/installation/single-user.md b/doc/manual/source/installation/single-user.md deleted file mode 100644 index f9a3b26edf4..00000000000 --- a/doc/manual/source/installation/single-user.md +++ /dev/null @@ -1,9 +0,0 @@ -# Single-User Mode - -In single-user mode, all Nix operations that access the database in -`prefix/var/nix/db` or modify the Nix store in `prefix/store` must be -performed under the user ID that owns those directories. This is -typically root. (If you install from RPM packages, that’s in fact the -default ownership.) However, on single-user machines, it is often -convenient to `chown` those directories to your normal user account so -that you don’t have to `su` to root all the time. diff --git a/doc/manual/source/installation/supported-platforms.md b/doc/manual/source/installation/supported-platforms.md deleted file mode 100644 index 8ca3ce8d445..00000000000 --- a/doc/manual/source/installation/supported-platforms.md +++ /dev/null @@ -1,7 +0,0 @@ -# Supported Platforms - -Nix is currently supported on the following platforms: - - - Linux (i686, x86\_64, aarch64). - - - macOS (x86\_64, aarch64). diff --git a/doc/manual/source/installation/uninstall.md b/doc/manual/source/installation/uninstall.md index 8d45da6bba0..e95634c213a 100644 --- a/doc/manual/source/installation/uninstall.md +++ b/doc/manual/source/installation/uninstall.md @@ -1,165 +1,15 @@ # Uninstalling Nix -## Multi User - -Removing a [multi-user installation](./installing-binary.md#multi-user-installation) depends on the operating system. - -### Linux - -If you are on Linux with systemd: - -1. Remove the Nix daemon service: - - ```console - sudo systemctl stop nix-daemon.service - sudo systemctl disable nix-daemon.socket nix-daemon.service - sudo systemctl daemon-reload - ``` - -Remove files created by Nix: +To uninstall Determinate Nix, use the uninstallation utility built into the [Determinate Nix Installer][installer]: ```console -sudo rm -rf /etc/nix /etc/profile.d/nix.sh /etc/tmpfiles.d/nix-daemon.conf /nix ~root/.nix-channels ~root/.nix-defexpr ~root/.nix-profile ~root/.cache/nix +$ /nix/nix-installer uninstall ``` -Remove build users and their group: +If you're certain that you want to uninstall, you can skip the confirmation step: ```console -for i in $(seq 1 32); do - sudo userdel nixbld$i -done -sudo groupdel nixbld +$ /nix/nix-installer uninstall --no-confirm ``` -There may also be references to Nix in - -- `/etc/bash.bashrc` -- `/etc/bashrc` -- `/etc/profile` -- `/etc/zsh/zshrc` -- `/etc/zshrc` - -which you may remove. - -### macOS - -> **Updating to macOS 15 Sequoia** -> -> If you recently updated to macOS 15 Sequoia and are getting -> ```console -> error: the user '_nixbld1' in the group 'nixbld' does not exist -> ``` -> when running Nix commands, refer to GitHub issue [NixOS/nix#10892](https://github.com/NixOS/nix/issues/10892) for instructions to fix your installation without reinstalling. - -1. If system-wide shell initialisation files haven't been altered since installing Nix, use the backups made by the installer: - - ```console - sudo mv /etc/zshrc.backup-before-nix /etc/zshrc - sudo mv /etc/bashrc.backup-before-nix /etc/bashrc - sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc - ``` - - Otherwise, edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: - - ```bash - # Nix - if [ -e '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' ]; then - . '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' - fi - # End Nix - ``` - -2. Stop and remove the Nix daemon services: - - ```console - sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist - sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist - sudo launchctl unload /Library/LaunchDaemons/org.nixos.darwin-store.plist - sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist - ``` - - This stops the Nix daemon and prevents it from being started next time you boot the system. - -3. Remove the `nixbld` group and the `_nixbuildN` users: - - ```console - sudo dscl . -delete /Groups/nixbld - for u in $(sudo dscl . -list /Users | grep _nixbld); do sudo dscl . -delete /Users/$u; done - ``` - - This will remove all the build users that no longer serve a purpose. - -4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store volume on `/nix`, which looks like - - ``` - UUID= /nix apfs rw,noauto,nobrowse,suid,owners - ``` - or - - ``` - LABEL=Nix\040Store /nix apfs rw,nobrowse - ``` - - by setting the cursor on the respective line using the arrow keys, and pressing `dd`, and then `:wq` to save the file. - - This will prevent automatic mounting of the Nix Store volume. - -5. Edit `/etc/synthetic.conf` to remove the `nix` line. - If this is the only line in the file you can remove it entirely: - - ```bash - if [ -f /etc/synthetic.conf ]; then - if [ "$(cat /etc/synthetic.conf)" = "nix" ]; then - sudo rm /etc/synthetic.conf - else - sudo vi /etc/synthetic.conf - fi - fi - ``` - - This will prevent the creation of the empty `/nix` directory. - -6. Remove the files Nix added to your system, except for the store: - - ```console - sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels - ``` - - -7. Remove the Nix Store volume: - - ```console - sudo diskutil apfs deleteVolume /nix - ``` - - This will remove the Nix Store volume and everything that was added to the store. - - If the output indicates that the command couldn't remove the volume, you should make sure you don't have an _unmounted_ Nix Store volume. - Look for a "Nix Store" volume in the output of the following command: - - ```console - diskutil list - ``` - - If you _do_ find a "Nix Store" volume, delete it by running `diskutil apfs deleteVolume` with the store volume's `diskXsY` identifier. - - If you get an error that the volume is in use by the kernel, reboot and immediately delete the volume before starting any other process. - -> **Note** -> -> After you complete the steps here, you will still have an empty `/nix` directory. -> This is an expected sign of a successful uninstall. -> The empty `/nix` directory will disappear the next time you reboot. -> -> You do not have to reboot to finish uninstalling Nix. -> The uninstall is complete. -> macOS (Catalina+) directly controls root directories, and its read-only root will prevent you from manually deleting the empty `/nix` mountpoint. - -## Single User - -To remove a [single-user installation](./installing-binary.md#single-user-installation) of Nix, run: - -```console -rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile -``` -You might also want to manually remove references to Nix from your `~/.profile`. +[installer]: https://github.com/DeterminateSystems/nix-installer diff --git a/doc/manual/source/installation/upgrading.md b/doc/manual/source/installation/upgrading.md index a433f1d30e6..8fe342b09b7 100644 --- a/doc/manual/source/installation/upgrading.md +++ b/doc/manual/source/installation/upgrading.md @@ -1,40 +1,10 @@ # Upgrading Nix -> **Note** -> -> These upgrade instructions apply where Nix was installed following the [installation instructions in this manual](./index.md). - -Check which Nix version will be installed, for example from one of the [release channels](http://channels.nixos.org/) such as `nixpkgs-unstable`: - -```console -$ nix-shell -p nix -I nixpkgs=channel:nixpkgs-unstable --run "nix --version" -nix (Nix) 2.18.1 -``` - -> **Warning** -> -> Writing to the [local store](@docroot@/store/types/local-store.md) with a newer version of Nix, for example by building derivations with [`nix-build`](@docroot@/command-ref/nix-build.md) or [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md), may change the database schema! -> Reverting to an older version of Nix may therefore require purging the store database before it can be used. - -## Linux multi-user +You can upgrade Determinate Nix using Determinate Nixd: ```console -$ sudo su -# nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable -# systemctl daemon-reload -# systemctl restart nix-daemon +sudo determinate-nixd upgrade ``` -## macOS multi-user +Note that the `sudo` is necessary here and upgrading fails without it. -```console -$ sudo nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable -$ sudo launchctl remove org.nixos.nix-daemon -$ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist -``` - -## Single-user all platforms - -```console -$ nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable -``` diff --git a/doc/manual/source/introduction.md b/doc/manual/source/introduction.md index e70411c11f5..fedb5595a1d 100644 --- a/doc/manual/source/introduction.md +++ b/doc/manual/source/introduction.md @@ -1,4 +1,19 @@ -# Introduction +# Determinate Nix + +**Determinate Nix** is a downstream distribution of [Nix], a purely functional language, CLI tool, and package management system. +It's available on Linux, macOS, and Windows Subsystem for Linux (WSL). + +## Installing + +We recommend that macOS users install Determinate Nix using our graphical installer, [Determinate.pkg][pkg]. +For Linux and Windows Subsystem for Linux (WSL) users: + +```console +curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | \ + sh -s -- install --determinate +``` + +## How Nix works Nix is a _purely functional package manager_. This means that it treats packages like values in a purely functional programming language @@ -184,10 +199,14 @@ to build configuration files in `/etc`). This means, among other things, that it is easy to roll back the entire configuration of the system to an earlier state. Also, users can install software without root privileges. For more information and downloads, see the [NixOS -homepage](https://nixos.org/). +homepage][nix]. ## License Nix is released under the terms of the [GNU LGPLv2.1 or (at your option) any later -version](http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html). +version][license]. + +[license]: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html +[pkg]: https://install.determinate.systems/determinate-pkg/stable/Universal +[site]: https://nixos.org diff --git a/doc/manual/source/protocols/json/derivation.md b/doc/manual/source/protocols/json/derivation.md index 04881776abc..2fc018c33ff 100644 --- a/doc/manual/source/protocols/json/derivation.md +++ b/doc/manual/source/protocols/json/derivation.md @@ -1,11 +1,5 @@ # Derivation JSON Format -> **Warning** -> -> This JSON format is currently -> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) -> and subject to change. - The JSON serialization of a [derivations](@docroot@/glossary.md#gloss-store-derivation) is a JSON object with the following fields: diff --git a/doc/manual/source/protocols/json/store-object-info.md b/doc/manual/source/protocols/json/store-object-info.md index b7348538c35..4b029c40b5d 100644 --- a/doc/manual/source/protocols/json/store-object-info.md +++ b/doc/manual/source/protocols/json/store-object-info.md @@ -1,11 +1,5 @@ # Store object info JSON format -> **Warning** -> -> This JSON format is currently -> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) -> and subject to change. - Info about a [store object]. * `path`: diff --git a/doc/manual/source/quick-start.md b/doc/manual/source/quick-start.md index 9eb7a326590..ffb87aa725f 100644 --- a/doc/manual/source/quick-start.md +++ b/doc/manual/source/quick-start.md @@ -3,10 +3,13 @@ This chapter is for impatient people who don't like reading documentation. For more in-depth information you are kindly referred to subsequent chapters. -1. Install Nix: +1. Install Nix. + We recommend that macOS users install Determinate Nix using our graphical installer, [Determinate.pkg][pkg]. + For Linux and Windows Subsystem for Linux (WSL) users: ```console - $ curl -L https://nixos.org/nix/install | sh + $ curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | \ + sh -s -- install --determinate ``` The install script will use `sudo`, so make sure you have sufficient rights. @@ -41,3 +44,5 @@ For more in-depth information you are kindly referred to subsequent chapters. ```console $ nix-collect-garbage ``` + +[pkg]: https://install.determinate.systems/determinate-pkg/stable/Universal diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md new file mode 100644 index 00000000000..7273196ee7b --- /dev/null +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -0,0 +1,118 @@ +# Changes between Nix and Determinate Nix + +This section lists the differences between upstream Nix 2.30 and Determinate Nix 3.8.3. + +* In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. + +* In Determinate Nix, the new Nix CLI (i.e. the `nix` command) is stable. You no longer need to enable the `nix-command` experimental feature. + +* Determinate Nix has a setting [`json-log-path`](@docroot@/command-ref/conf-file.md#conf-json-log-path) to send a copy of all Nix log messages (in JSON format) to a file or Unix domain socket. + +* Determinate Nix has made `nix profile install` an alias to `nix profile add`, a more symmetrical antonym of `nix profile remove`. + +* `nix-channel` and `channel:` url syntax (like `channel:nixos-24.11`) is deprecated, see: https://github.com/DeterminateSystems/nix-src/issues/34 + +* Using indirect flake references and implicit inputs is deprecated, see: https://github.com/DeterminateSystems/nix-src/issues/37 + +* Warnings around "dirty trees" are updated to reduce "dirty" jargon, and now refers to "uncommitted changes". + + + + + + + +* `nix upgrade-nix` is now inert, and suggests using `determinate-nixd upgrade` -- [DeterminateSystems/nix-src#55](https://github.com/DeterminateSystems/nix-src/pull/55) + +* Initial Lazy Trees support has been merged, but remains off by default. ([DeterminateSystems/nix-src#27](https://github.com/DeterminateSystems/nix-src/pull/27), [DeterminateSystems/nix-src#56](https://github.com/DeterminateSystems/nix-src/pull/56)) + + + +* Tell users a source is corrupted ("cannot read file from tarball: Truncated tar archive detected while reading data"), improving over the previous 'cannot read file from tarball' error by @edolstra in [DeterminateSystems/nix-src#64](https://github.com/DeterminateSystems/nix-src/pull/64) + + +* Emit warnings when using import-from-derivation by setting the `trace-import-from-derivation` option to `true` by @gustavderdrache in [DeterminateSystems/nix-src#70](https://github.com/DeterminateSystems/nix-src/pull/70) + + + + +* Faster `nix store copy-sigs` by @edolstra in [DeterminateSystems/nix-src#80](https://github.com/DeterminateSystems/nix-src/pull/80) + +* Document how to replicate nix-store --query --deriver with the nix cli by @grahamc in [DeterminateSystems/nix-src#82](https://github.com/DeterminateSystems/nix-src/pull/82) + +* Garbage collector: Keep going even when encountering an undeletable file by @edolstra in [DeterminateSystems/nix-src#83](https://github.com/DeterminateSystems/nix-src/pull/83) + +* nix profile: Replace ε and ∅ with descriptive English words by @grahamc in [DeterminateSystems/nix-src#81](https://github.com/DeterminateSystems/nix-src/pull/81) + +* Call out that `--keep-failed` with remote builders will keep the failed build directory on that builder by @cole-h in [DeterminateSystems/nix-src#85](https://github.com/DeterminateSystems/nix-src/pull/85) + + + + + + +* When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) + +* Indicate that sandbox-paths specifies a missing file in the corresponding error message. by @cole-h in [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) + +* Use 'published' release type to avoid double uploads by @gustavderdrache in [DeterminateSystems/nix-src#90](https://github.com/DeterminateSystems/nix-src/pull/90) + +* Render lazy tree paths in messages withouth the/nix/store/hash... prefix in substituted source trees by @edolstra in [DeterminateSystems/nix-src#91](https://github.com/DeterminateSystems/nix-src/pull/91) + +* Use FlakeHub inputs by @lucperkins in [DeterminateSystems/nix-src#89](https://github.com/DeterminateSystems/nix-src/pull/89) + +* Proactively cache more flake inputs and fetches by @edolstra in [DeterminateSystems/nix-src#93](https://github.com/DeterminateSystems/nix-src/pull/93) + +* Fix: register extra builtins just once by @edolstra in [DeterminateSystems/nix-src#97](https://github.com/DeterminateSystems/nix-src/pull/97) + +* Fix: Make the S3 test more robust by @gustavderdrache in [DeterminateSystems/nix-src#101](https://github.com/DeterminateSystems/nix-src/pull/101) + +* Fix the link to `builders-use-substitutes` documentation for `builders` by @lucperkins in [DeterminateSystems/nix-src#102](https://github.com/DeterminateSystems/nix-src/pull/102) + +* Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) + +* Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) + + + + + + + +* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) + +* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) + +* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124) + +* Release v3.6.7 by @github-actions in [DeterminateSystems/nix-src#126](https://github.com/DeterminateSystems/nix-src/pull/126) + + + +* Overriding deeply-nested transitive flake inputs now works, by @edolstra in [DeterminateSystems/nix-src#108](https://github.com/DeterminateSystems/nix-src/pull/108) + +* `nix store delete` now explains why deletion fails by @edolstra in [DeterminateSystems/nix-src#130](https://github.com/DeterminateSystems/nix-src/pull/130) + +* New command: `nix flake prefetch-inputs` for improved CI performance, by @edolstra in [DeterminateSystems/nix-src#127](https://github.com/DeterminateSystems/nix-src/pull/127) + + + +* nix flake check: Skip substitutable derivations by @edolstra in [DeterminateSystems/nix-src#134](https://github.com/DeterminateSystems/nix-src/pull/134) + +* lockFlake(): When updating a lock, respect the input's lock file by @edolstra in [DeterminateSystems/nix-src#137](https://github.com/DeterminateSystems/nix-src/pull/137) + + + +* Address ifdef problem with macOS/BSD sandboxing by @gustavderdrache in [DeterminateSystems/nix-src#142](https://github.com/DeterminateSystems/nix-src/pull/142) + + + +* ci: don't run the full test suite for x86_64-darwin by @grahamc in [DeterminateSystems/nix-src#144](https://github.com/DeterminateSystems/nix-src/pull/144) + +* Try publishing the manual again by @grahamc in [DeterminateSystems/nix-src#145](https://github.com/DeterminateSystems/nix-src/pull/145) + + + +* Add an `external-builders` experimental feature by @cole-h in [DeterminateSystems/nix-src#141](https://github.com/DeterminateSystems/nix-src/pull/141) + +* Add support for external builders by @edolstra in [DeterminateSystems/nix-src#78](https://github.com/DeterminateSystems/nix-src/pull/78) diff --git a/doc/manual/source/release-notes-determinate/index.md b/doc/manual/source/release-notes-determinate/index.md new file mode 100644 index 00000000000..bba33084424 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/index.md @@ -0,0 +1,3 @@ +# Determinate Nix Release Notes + +This chapter lists the differences between Nix and Determinate Nix, as well as the release history of Determinate Nix. diff --git a/doc/manual/source/release-notes-determinate/rl-3.0.0.md b/doc/manual/source/release-notes-determinate/rl-3.0.0.md new file mode 100644 index 00000000000..d60786e9a72 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.0.0.md @@ -0,0 +1,5 @@ +# Release 3.0.0 (2025-03-04) + +* Initial release of Determinate Nix. + +* Based on [upstream Nix 2.26.2](../release-notes/rl-2.26.md). diff --git a/doc/manual/source/release-notes-determinate/rl-3.1.0.md b/doc/manual/source/release-notes-determinate/rl-3.1.0.md new file mode 100644 index 00000000000..96b7819d08d --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.1.0.md @@ -0,0 +1,5 @@ +# Release 3.1.0 (2025-03-27) + +* Based on [upstream Nix 2.27.1](../release-notes/rl-2.27.md). + +* New setting `json-log-path` that sends a copy of all Nix log messages (in JSON format) to a file or Unix domain socket. diff --git a/doc/manual/source/release-notes-determinate/rl-3.3.0.md b/doc/manual/source/release-notes-determinate/rl-3.3.0.md new file mode 100644 index 00000000000..badf96415df --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.3.0.md @@ -0,0 +1,5 @@ +# Release 3.3.0 (2025-04-11) + +* Based on [upstream Nix 2.28.1](../release-notes/rl-2.28.md). + +* The `nix profile install` command is now an alias to `nix profile add`, a more symmetrical antonym of `nix profile remove`. diff --git a/doc/manual/source/release-notes-determinate/rl-3.4.0.md b/doc/manual/source/release-notes-determinate/rl-3.4.0.md new file mode 100644 index 00000000000..24ae03ca554 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.4.0.md @@ -0,0 +1,50 @@ +# Release 3.4.0 (2025-04-25) + +* Based on [upstream Nix 2.28.2](../release-notes/rl-2.28.md). + +* **Warn users that `nix-channel` is deprecated.** + +This is the first change accomplishing our roadmap item of deprecating Nix channels: https://github.com/DeterminateSystems/nix-src/issues/34 + +This is due to user confusion and surprising behavior of channels, especially in the context of user vs. root channels. + +The goal of this change is to make the user experience of Nix more predictable. +In particular, these changes are to support users with lower levels of experience who are following guides that focus on channels as the mechanism of distribution. + +Users will now see this message: + +> nix-channel is deprecated in favor of flakes in Determinate Nix. For a guide on Nix flakes, see: https://zero-to-nix.com/. or details and to offer feedback on the deprecation process, see: https://github.com/DeterminateSystems/nix-src/issues/34. + + +* **Warn users that `channel:` URLs are deprecated.** + +This is the second change regarding our deprecation of Nix channels. +Using a `channel:` URL (like `channel:nixos-24.11`) will yield a warning like this: + +> Channels are deprecated in favor of flakes in Determinate Nix. Instead of 'channel:nixos-24.11', use 'https://nixos.org/channels/nixos-24.11/nixexprs.tar.xz'. For a guide on Nix flakes, see: https://zero-to-nix.com/. For details and to offer feedback on the deprecation process, see: https://github.com/DeterminateSystems/nix-src/issues/34. + +* **Warn users against indirect flake references in `flake.nix` inputs** + +This is the first change accomplishing our roadmap item of deprecating implicit and indirect flake inputs: https://github.com/DeterminateSystems/nix-src/issues/37 + +The flake registry provides an important UX affordance for using Nix flakes and remote sources in command line uses. +For that reason, the registry is not being deprecated entirely and will still be used for command-line incantations, like nix run. + +This move will eliminate user confusion and surprising behavior around global and local registries during flake input resolution. + +The goal of this change is to make the user experience of Nix more predictable. +We have seen a pattern of confusion when using automatic flake inputs and local registries. +Specifically, users' flake inputs resolving and locking inconsistently depending on the configuration of the host system. + +Users will now see the following warning if their flake.nix uses an implicit or indirect Flake reference input: + +> Flake input 'nixpkgs' uses the flake registry. Using the registry in flake inputs is deprecated in Determinate Nix. To make your flake future-proof, add the following to 'xxx/flake.nix': +> +> inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; +> +> For more information, see: https://github.com/DeterminateSystems/nix-src/issues/37 + + +### Other updates: +* Improve the "dirty tree" message. Determinate Nix will now say `Git tree '...' has uncommitted changes` instead of `Git tree '...' is dirty` +* Stop warning about uncommitted changes in a Git repository when using `nix develop` diff --git a/doc/manual/source/release-notes-determinate/rl-3.4.2.md b/doc/manual/source/release-notes-determinate/rl-3.4.2.md new file mode 100644 index 00000000000..8acabd4425f --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.4.2.md @@ -0,0 +1,4 @@ +# Release 3.4.2 (2025-05-05) + +* Based on [upstream Nix 2.28.3](../release-notes/rl-2.28.md). + diff --git a/doc/manual/source/release-notes-determinate/rl-3.5.0.md b/doc/manual/source/release-notes-determinate/rl-3.5.0.md new file mode 100644 index 00000000000..d5b26b9419e --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.5.0.md @@ -0,0 +1,4 @@ +# Release 3.5.0 (2025-05-09) + +* Based on [upstream Nix 2.28.3](../release-notes/rl-2.28.md). + diff --git a/doc/manual/source/release-notes-determinate/rl-3.5.1.md b/doc/manual/source/release-notes-determinate/rl-3.5.1.md new file mode 100644 index 00000000000..b0813ca59c9 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.5.1.md @@ -0,0 +1,57 @@ +# Release 3.5.1 (2025-05-09) + +* Based on [upstream Nix 2.28.3](../release-notes/rl-2.28.md). + +## What's Changed + +Most notably, Lazy Trees has merged in to Determinate Nix and is in Feature Preview status, but remains disabled by default. +Lazy trees massively improves performance in virtually all scenarios because it enables Nix to avoid making unnecessary copies of files into the Nix store. +In testing, we saw iteration times on Nixpkgs **drop from over 12 seconds to 3.5 seconds**. + +After upgrading to Determinate Nix 3.5.1 with `sudo determinate-nixd upgrade`, enable lazy trees by adding this to `/etc/nix/nix.custom.conf`: + +``` +lazy-trees = true +``` + +Please note that our full flake regression test suite passes with no changes with lazy trees, and please report compatibility issues. + +Read [this GitHub comment](https://github.com/DeterminateSystems/nix-src/pull/27#pullrequestreview-2822153088) for further details and next steps. +We'll be publishing an update on the [Determinate Systems blog](https://determinate.systems/posts/) in the next few days with more information as well. + +Relevant PRs: +* Lazy trees v2 by @edolstra in [DeterminateSystems/nix-src#27](https://github.com/DeterminateSystems/nix-src/pull/27) +* Improve lazy trees backward compatibility by @edolstra in [DeterminateSystems/nix-src#56](https://github.com/DeterminateSystems/nix-src/pull/56) + + +### Additional changes in this release: +* Bug fix: Flake input URLs are canonicalized before checking flake.lock file staleness, avoiding needlessly regenerating flake.lock files with `dir` in URL-style flakerefs by @edolstra in [DeterminateSystems/nix-src#57](https://github.com/DeterminateSystems/nix-src/pull/57) +* `nix upgrade-nix` is deprecated in favor of `determinate-nixd upgrade`, by @gustavderdrache in [DeterminateSystems/nix-src#55](https://github.com/DeterminateSystems/nix-src/pull/55) +* UX: Improved build failure and dependency failure error messages to include needed output paths by @edolstra in [DeterminateSystems/nix-src#58](https://github.com/DeterminateSystems/nix-src/pull/58). + +Previously: + +``` +error: builder for '/nix/store/[...]-nested-failure-bottom.drv' failed with exit code 1 +error: 1 dependencies of derivation '/nix/store/[...]-nested-failure-middle.drv' failed to build +error: 1 dependencies of derivation '/nix/store/[...]-nested-failure-top.drv' failed to build +``` + +Now: + +``` +error: Cannot build '/nix/store/w37gflm9wz9dcnsgy3sfrmnlvm8qigaj-nested-failure-bottom.drv'. + Reason: builder failed with exit code 1. + Output paths: + /nix/store/yzybs8kp35dfipbzdlqcc6lxz62hax04-nested-failure-bottom +error: Cannot build '/nix/store/00gr5hlxfc03x2675w6nn3pwfrz2fr62-nested-failure-middle.drv'. + Reason: 1 dependency failed. + Output paths: + /nix/store/h781j5h4bdchmb4c2lvy8qzh8733azhz-nested-failure-middle +error: Cannot build '/nix/store/8am0ng1gyx8sbzyr0yx6jd5ix3yy5szc-nested-failure-top.drv'. + Reason: 1 dependency failed. + Output paths: + /nix/store/fh12637kgvp906s9yhi9w2dc7ghfwxs1-nested-failure-top +``` + +**Full Changelog**: [v3.4.2...v3.5.1](https://github.com/DeterminateSystems/nix-src/compare/v3.4.2...v3.5.1) diff --git a/doc/manual/source/release-notes-determinate/rl-3.5.2.md b/doc/manual/source/release-notes-determinate/rl-3.5.2.md new file mode 100644 index 00000000000..bc5396c255b --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.5.2.md @@ -0,0 +1,11 @@ +# Release 3.5.2 (2025-05-12) + +* Based on [upstream Nix 2.28.3](../release-notes/rl-2.28.md). + +## What's Changed +* Fix a regression where narHash was not added to lock files when lazy trees were disabled by @edolstra in [DeterminateSystems/nix-src#63](https://github.com/DeterminateSystems/nix-src/pull/63) + +* Tell users a source is corrupted ("cannot read file from tarball: Truncated tar archive detected while reading data"), improving over the previous 'cannot read file from tarball' error by @edolstra in [DeterminateSystems/nix-src#64](https://github.com/DeterminateSystems/nix-src/pull/64) + + +**Full Changelog**: [v3.5.1...v3.5.2](https://github.com/DeterminateSystems/nix-src/compare/v3.5.1...v3.5.2) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.0.md b/doc/manual/source/release-notes-determinate/rl-3.6.0.md new file mode 100644 index 00000000000..453ab6c301d --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.0.md @@ -0,0 +1,11 @@ +# Release 3.6.0 (2025-05-22) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* Install 'nix profile add' manpage by @edolstra in [DeterminateSystems/nix-src#69](https://github.com/DeterminateSystems/nix-src/pull/69) +* Sync with upstream 2.29.0 by @edolstra in [DeterminateSystems/nix-src#67](https://github.com/DeterminateSystems/nix-src/pull/67) +* Emit warnings when using import-from-derivation by setting the `trace-import-from-derivation` option to `true` by @gustavderdrache in [DeterminateSystems/nix-src#70](https://github.com/DeterminateSystems/nix-src/pull/70) + + +**Full Changelog**: [v3.5.2...v3.6.0](https://github.com/DeterminateSystems/nix-src/compare/v3.5.2...v3.6.0) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.1.md b/doc/manual/source/release-notes-determinate/rl-3.6.1.md new file mode 100644 index 00000000000..12505afee27 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.1.md @@ -0,0 +1,9 @@ +# Release 3.6.1 (2025-05-24) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* Fix nlohmann error in fromStructuredAttrs() by @edolstra in [DeterminateSystems/nix-src#73](https://github.com/DeterminateSystems/nix-src/pull/73) + + +**Full Changelog**: [v3.6.0...v3.6.1](https://github.com/DeterminateSystems/nix-src/compare/v3.6.0...v3.6.1) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.2.md b/doc/manual/source/release-notes-determinate/rl-3.6.2.md new file mode 100644 index 00000000000..882c142f00c --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.2.md @@ -0,0 +1,15 @@ +# Release 3.6.2 (2025-06-02) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* Dramatically improve the performance of nix store copy-sigs: Use http-connections setting to control parallelism by @edolstra in [DeterminateSystems/nix-src#80](https://github.com/DeterminateSystems/nix-src/pull/80) +* Document how to replicate nix-store --query --deriver with the nix cli by @grahamc in [DeterminateSystems/nix-src#82](https://github.com/DeterminateSystems/nix-src/pull/82) +* The garbage collector no longer gives up if it encounters an undeletable file, by @edolstra in [DeterminateSystems/nix-src#83](https://github.com/DeterminateSystems/nix-src/pull/83) +* nix profile: Replace ε and ∅ with descriptive English words by @grahamc in [DeterminateSystems/nix-src#81](https://github.com/DeterminateSystems/nix-src/pull/81) +* Rework README to clarify that this distribution is our distribution, by @lucperkins in [DeterminateSystems/nix-src#84](https://github.com/DeterminateSystems/nix-src/pull/84) +* Include the source location when warning about inefficient double copies by @edolstra in [DeterminateSystems/nix-src#79](https://github.com/DeterminateSystems/nix-src/pull/79) +* Call out that `--keep-failed` with remote builders will keep the failed build directory on that builder by @cole-h in [DeterminateSystems/nix-src#85](https://github.com/DeterminateSystems/nix-src/pull/85) + + +**Full Changelog**: [v3.6.1...v3.6.2](https://github.com/DeterminateSystems/nix-src/compare/v3.6.1...v3.6.2) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.5.md b/doc/manual/source/release-notes-determinate/rl-3.6.5.md new file mode 100644 index 00000000000..8ef5be0fd0d --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.5.md @@ -0,0 +1,19 @@ +# Release 3.6.5 (2025-06-12) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) +* Indicate that sandbox-paths specifies a missing file in the corresponding error message. by @cole-h in [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) +* Render lazy tree paths in messages withouth the/nix/store/hash... prefix in substituted source trees by @edolstra in [DeterminateSystems/nix-src#91](https://github.com/DeterminateSystems/nix-src/pull/91) +* Use FlakeHub inputs by @lucperkins in [DeterminateSystems/nix-src#89](https://github.com/DeterminateSystems/nix-src/pull/89) +* Proactively cache more flake inputs and fetches by @edolstra in [DeterminateSystems/nix-src#93](https://github.com/DeterminateSystems/nix-src/pull/93) +* Fix: register extra builtins just once by @edolstra in [DeterminateSystems/nix-src#97](https://github.com/DeterminateSystems/nix-src/pull/97) +* Fix the link to `builders-use-substitutes` documentation for `builders` by @lucperkins in [DeterminateSystems/nix-src#102](https://github.com/DeterminateSystems/nix-src/pull/102) +* Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) +* Make the `nix repl` test more stable by @edolstra in [DeterminateSystems/nix-src#103](https://github.com/DeterminateSystems/nix-src/pull/103) +* Run nixpkgsLibTests against lazy trees by @edolstra in [DeterminateSystems/nix-src#100](https://github.com/DeterminateSystems/nix-src/pull/100) +* Run the Nix test suite against lazy trees by @edolstra in [DeterminateSystems/nix-src#105](https://github.com/DeterminateSystems/nix-src/pull/105) +* Improve caching of inputs by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98), [DeterminateSystems/nix-src#110](https://github.com/DeterminateSystems/nix-src/pull/110), and [DeterminateSystems/nix-src#115](https://github.com/DeterminateSystems/nix-src/pull/115) + +**Full Changelog**: [v3.6.2...v3.6.5](https://github.com/DeterminateSystems/nix-src/compare/v3.6.2...v3.6.4) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.6.md b/doc/manual/source/release-notes-determinate/rl-3.6.6.md new file mode 100644 index 00000000000..bf4e3690afa --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.6.md @@ -0,0 +1,7 @@ +# Release 3.6.6 (2025-06-17) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed + +* No-op release on the nix-src side, due to a regression on nix-darwin in determinate-nixd. diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.7.md b/doc/manual/source/release-notes-determinate/rl-3.6.7.md new file mode 100644 index 00000000000..197587f1b3a --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.7.md @@ -0,0 +1,17 @@ +# Release 3.6.7 (2025-06-24) + +* Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md). + +## What's Changed + +### Security contents + +* Patched against GHSA-g948-229j-48j3 + +### Lazy trees: + +* Lazy trees now produces `flake.lock` files with NAR hashes unless `lazy-locks` is set to `true` by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) +* Improved caching with lazy-trees when using --impure, with enhanced testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) + + +**Full Changelog**: [v3.6.6...v3.6.7](https://github.com/DeterminateSystems/nix-src/compare/v3.6.6...v3.6.7) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.8.md b/doc/manual/source/release-notes-determinate/rl-3.6.8.md new file mode 100644 index 00000000000..c4b4b96c9e7 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.8.md @@ -0,0 +1,12 @@ +# Release 3.6.8 (2025-06-25) + +* Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md). + +## What's Changed +* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) +* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) +* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124) +* Release v3.6.7 by @github-actions in [DeterminateSystems/nix-src#126](https://github.com/DeterminateSystems/nix-src/pull/126) + + +**Full Changelog**: [v3.6.6...v3.6.8](https://github.com/DeterminateSystems/nix-src/compare/v3.6.6...v3.6.8) diff --git a/doc/manual/source/release-notes-determinate/rl-3.7.0.md b/doc/manual/source/release-notes-determinate/rl-3.7.0.md new file mode 100644 index 00000000000..8e5fc9ca6a1 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.7.0.md @@ -0,0 +1,63 @@ +# Release 3.7.0 (2025-07-03) + +- Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md). + +## What's Changed + +### Prefetch flake inputs in parallel + +By @edolstra in [DeterminateSystems/nix-src#127](https://github.com/DeterminateSystems/nix-src/pull/127) + +This release brings the command `nix flake prefetch-inputs`. + +Flake inputs are typically fetched "just in time." +That means Nix fetches a flake input when the evaluator needs it, and not before. +When the evaluator needs an input, evaluation is paused until the source is available. + +This causes a significant slow-down on projects with lots of flake inputs. + +The new command `nix flake prefetch-inputs` fetches all flake inputs in parallel. +We expect running this new command before building will dramatically improve evaluation performance for most projects, especially in CI. +Note that projects which with many unused flake inputs may not benefit from this change, since the new command fetches every input whether they're used or not. + +### Deep flake input overrides now work as expected + +By @edolstra in [DeterminateSystems/nix-src#108](https://github.com/DeterminateSystems/nix-src/pull/108) + +An override like: + +``` +inputs.foo.inputs.bar.inputs.nixpkgs.follows = "nixpkgs"; +``` + +implicitly set `inputs.foo.inputs.bar` to `flake:bar`, which led to an unexpected error like: + +``` +error: cannot find flake 'flake:bar' in the flake registries +``` + +We now no longer create a parent override (like for `foo.bar` in the example above) if it doesn't set an explicit ref or follows attribute. +We only recursively apply its child overrides. + +### `nix store delete` now shows you why deletion was not possible + +By @edolstra in [DeterminateSystems/nix-src#130](https://github.com/DeterminateSystems/nix-src/pull/130) + +For example: + +``` +error: Cannot delete path '/nix/store/6fcrjgfjip2ww3sx51rrmmghfsf60jvi-patchelf-0.14.3' + because it's referenced by the GC root '/home/eelco/Dev/nix-master/build/result'. + +error: Cannot delete path '/nix/store/rn0qyn3kmky26xgpr2n10vr787g57lff-cowsay-3.8.4' + because it's referenced by the GC root '/proc/3600568/environ'. + +error: Cannot delete path '/nix/store/klyng5rpdkwi5kbxkncy4gjwb490dlhb-foo.drv' + because it's in use by '{nix-process:3605324}'. +``` + +### Lazy-tree improvements + +- Improved lazy-tree evaluation caching for flakes accessed with a `path` flakeref by @edolstra in [DeterminateSystems/nix-src#131](https://github.com/DeterminateSystems/nix-src/pull/131) + +**Full Changelog**: [v3.6.8...v3.7.0](https://github.com/DeterminateSystems/nix-src/compare/v3.6.8...v3.7.0) diff --git a/doc/manual/source/release-notes-determinate/rl-3.8.0.md b/doc/manual/source/release-notes-determinate/rl-3.8.0.md new file mode 100644 index 00000000000..4103d6df94e --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.8.0.md @@ -0,0 +1,29 @@ +# Release 3.8.0 (2025-07-10) + +* Based on [upstream Nix 2.30.0](../release-notes/rl-2.30.md). + +## What's Changed + +### Faster CI with `nix flake check` + +`nix flake check` no longer downloads flake outputs if no building is necessary. + +This command is intended to validate that a flake can fully evaluate and all outputs can build. +If the outputs are available in a binary cache then both properties are confirmed to be true. +Notably, downloading the output from the binary cache is not strictly necessary for the validation. + +Previously, `nix flake check` would download a flake output if the full build is available in a binary cache. + +Some users will find this change significantly reduces costly bandwidth and CI workflow time. + +PR: [DeterminateSystems/nix-src#134](https://github.com/DeterminateSystems/nix-src/pull/134) + +### Improved flake locking of transitive dependencies + +Determinate Nix now re-locks all transitive dependencies when changing a flake input's source URL. + +This fixes an issue where in some scenarios Nix would not re-lock those inputs and incorrectly use the old inputs' dependencies. + +PR: [DeterminateSystems/nix-src#137](https://github.com/DeterminateSystems/nix-src/pull/137) + +**Full Changelog**: [v3.7.0...v3.8.0](https://github.com/DeterminateSystems/nix-src/compare/v3.7.0...v3.8.0) diff --git a/doc/manual/source/release-notes-determinate/rl-3.8.1.md b/doc/manual/source/release-notes-determinate/rl-3.8.1.md new file mode 100644 index 00000000000..90dc328f6ec --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.8.1.md @@ -0,0 +1,9 @@ +# Release 3.8.1 (2025-07-11) + +* Based on [upstream Nix 2.30.0](../release-notes/rl-2.30.md). + +## What's Changed +* Address ifdef problem with macOS/BSD sandboxing by @gustavderdrache in [DeterminateSystems/nix-src#142](https://github.com/DeterminateSystems/nix-src/pull/142) + + +**Full Changelog**: [v3.8.0...v3.8.1](https://github.com/DeterminateSystems/nix-src/compare/v3.8.0...v3.8.1) diff --git a/doc/manual/source/release-notes-determinate/rl-3.8.2.md b/doc/manual/source/release-notes-determinate/rl-3.8.2.md new file mode 100644 index 00000000000..638d90f6841 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.8.2.md @@ -0,0 +1,10 @@ +# Release 3.8.2 (2025-07-12) + +* Based on [upstream Nix 2.30.0](../release-notes/rl-2.30.md). + +## What's Changed +* ci: don't run the full test suite for x86_64-darwin by @grahamc in [DeterminateSystems/nix-src#144](https://github.com/DeterminateSystems/nix-src/pull/144) +* Try publishing the manual again by @grahamc in [DeterminateSystems/nix-src#145](https://github.com/DeterminateSystems/nix-src/pull/145) + + +**Full Changelog**: [v3.8.1...v3.8.2](https://github.com/DeterminateSystems/nix-src/compare/v3.8.1...v3.8.2) diff --git a/doc/manual/source/release-notes-determinate/rl-3.8.3.md b/doc/manual/source/release-notes-determinate/rl-3.8.3.md new file mode 100644 index 00000000000..d3eb02bc7ea --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.8.3.md @@ -0,0 +1,26 @@ +# Release 3.8.3 (2025-07-18) + +* Based on [upstream Nix 2.30.1](../release-notes/rl-2.30.md). + +## What's Changed + +### Non-blocking evaluation caching + +Users reported evaluation would occasionally block other evaluation processes. + +The evaluation cache database is now opened in write-ahead mode to prevent delaying evaluations. + +PR: [DeterminateSystems/nix-src#150](https://github.com/DeterminateSystems/nix-src/pull/150) + +### New experimental feature: `external-builders` + +This experimental feature allows Nix to call an external program for the build environment. + +The interface and behavior of this feature may change at any moment without a correspondingly major semver version change. + +PRs: +- [DeterminateSystems/nix-src#141](https://github.com/DeterminateSystems/nix-src/pull/141) +- [DeterminateSystems/nix-src#152](https://github.com/DeterminateSystems/nix-src/pull/152) +- [DeterminateSystems/nix-src#78](https://github.com/DeterminateSystems/nix-src/pull/78) + +**Full Changelog**: [v3.8.2...v3.8.3](https://github.com/DeterminateSystems/nix-src/compare/v3.8.2...v3.8.3) diff --git a/doc/manual/source/release-notes/rl-2.19.md b/doc/manual/source/release-notes/rl-2.19.md index 06c704324dd..47a0dd3db99 100644 --- a/doc/manual/source/release-notes/rl-2.19.md +++ b/doc/manual/source/release-notes/rl-2.19.md @@ -69,7 +69,7 @@ This makes it match `nix derivation show`, which also maps store paths to information. -- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish) +- When Nix is installed using the binary installer, in supported shells (Bash, Zsh, Fish) [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile. This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). diff --git a/doc/manual/source/release-notes/rl-2.24.md b/doc/manual/source/release-notes/rl-2.24.md index d4af3cb5174..33fc0db03f9 100644 --- a/doc/manual/source/release-notes/rl-2.24.md +++ b/doc/manual/source/release-notes/rl-2.24.md @@ -268,6 +268,21 @@ be configured using the `warn-large-path-threshold` setting, e.g. `--warn-large-path-threshold 100M`. +- Wrap filesystem exceptions more correctly [#11378](https://github.com/NixOS/nix/pull/11378) + + With the switch to `std::filesystem` in different places, Nix started to throw `std::filesystem::filesystem_error` in many places instead of its own exceptions. + + This led to no longer generating error traces, for example when listing a non-existing directory. + + This version catches these types of exception correctly and wraps them into Nix's own exeception type. + + Author: [**@Mic92**](https://github.com/Mic92) + +- `` uses TLS verification [#11585](https://github.com/NixOS/nix/pull/11585) + + Previously `` did not do TLS verification. This was because the Nix sandbox in the past did not have access to TLS certificates, and Nix checks the hash of the fetched file anyway. However, this can expose authentication data from `netrc` and URLs to man-in-the-middle attackers. In addition, Nix now in some cases (such as when using impure derivations) does *not* check the hash. Therefore we have now enabled TLS verification. This means that downloads by `` will now fail if you're fetching from a HTTPS server that does not have a valid certificate. + + `` is also known as the builtin derivation builder `builtin:fetchurl`. It's not to be confused with the evaluation-time function `builtins.fetchurl`, which was not affected by this issue. ## Contributors diff --git a/docker.nix b/docker.nix index c6e8e478e7e..ca7a81c1f67 100644 --- a/docker.nix +++ b/docker.nix @@ -184,11 +184,14 @@ let } " = "; }; - nixConfContents = toConf { - sandbox = false; - build-users-group = "nixbld"; - trusted-public-keys = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ]; - }; + nixConfContents = toConf ( + { + sandbox = false; + build-users-group = "nixbld"; + trusted-public-keys = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ]; + } + // nixConf + ); userHome = if uid == 0 then "/root" else "/home/${uname}"; @@ -330,7 +333,7 @@ let globalFlakeRegistryPath="$nixCacheDir/flake-registry.json" ln -s ${flake-registry-path} $out$globalFlakeRegistryPath mkdir -p $out/nix/var/nix/gcroots/auto - rootName=$(${lib.getExe' nix "nix"} --extra-experimental-features nix-command hash file --type sha1 --base32 <(echo -n $globalFlakeRegistryPath)) + rootName=$(${lib.getExe' nix "nix"} hash file --type sha1 --base32 <(echo -n $globalFlakeRegistryPath)) ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName '') ); diff --git a/flake.lock b/flake.lock index 3075eabc233..a9639166b05 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1733328505, - "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -23,58 +23,51 @@ ] }, "locked": { - "lastModified": 1733312601, - "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9", - "type": "github" + "lastModified": 1748821116, + "narHash": "sha256-F82+gS044J1APL0n4hH50GYdPRv/5JWm34oCJYmVKdE=", + "rev": "49f0870db23e8c1ca0b5259734a02cd9e1e371a1", + "revCount": 377, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/hercules-ci/flake-parts/0.1.377%2Brev-49f0870db23e8c1ca0b5259734a02cd9e1e371a1/01972f28-554a-73f8-91f4-d488cc502f08/source.tar.gz" }, "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" + "type": "tarball", + "url": "https://flakehub.com/f/hercules-ci/flake-parts/0.1" } }, "git-hooks-nix": { "inputs": { - "flake-compat": [], + "flake-compat": "flake-compat", "gitignore": [], "nixpkgs": [ "nixpkgs" - ], - "nixpkgs-stable": [ - "nixpkgs" ] }, "locked": { - "lastModified": 1734279981, - "narHash": "sha256-NdaCraHPp8iYMWzdXAt5Nv6sA3MUzlCiGiR586TCwo0=", - "owner": "cachix", - "repo": "git-hooks.nix", - "rev": "aa9f40c906904ebd83da78e7f328cd8aeaeae785", - "type": "github" + "lastModified": 1747372754, + "narHash": "sha256-2Y53NGIX2vxfie1rOW0Qb86vjRZ7ngizoo+bnXU9D9k=", + "rev": "80479b6ec16fefd9c1db3ea13aeb038c60530f46", + "revCount": 1026, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/cachix/git-hooks.nix/0.1.1026%2Brev-80479b6ec16fefd9c1db3ea13aeb038c60530f46/0196d79a-1b35-7b8e-a021-c894fb62163d/source.tar.gz" }, "original": { - "owner": "cachix", - "repo": "git-hooks.nix", - "type": "github" + "type": "tarball", + "url": "https://flakehub.com/f/cachix/git-hooks.nix/0.1.941" } }, "nixpkgs": { "locked": { "lastModified": 1747179050, "narHash": "sha256-qhFMmDkeJX9KJwr5H32f1r7Prs7XbQWtO0h3V0a0rFY=", - "owner": "NixOS", - "repo": "nixpkgs", "rev": "adaa24fbf46737f3f1b5497bf64bae750f82942e", - "type": "github" + "revCount": 799423, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.1.799423%2Brev-adaa24fbf46737f3f1b5497bf64bae750f82942e/0196d1c3-1974-7bf1-bcf6-06620ac40c8c/source.tar.gz" }, "original": { - "owner": "NixOS", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" + "type": "tarball", + "url": "https://flakehub.com/f/NixOS/nixpkgs/%3D0.1.799423" } }, "nixpkgs-23-11": { @@ -111,7 +104,6 @@ }, "root": { "inputs": { - "flake-compat": "flake-compat", "flake-parts": "flake-parts", "git-hooks-nix": "git-hooks-nix", "nixpkgs": "nixpkgs", diff --git a/flake.nix b/flake.nix index c884fb0ff5b..6d9906b9f25 100644 --- a/flake.nix +++ b/flake.nix @@ -1,24 +1,18 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + inputs.nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/=0.1.799423"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; - inputs.flake-compat = { - url = "github:edolstra/flake-compat"; - flake = false; - }; # dev tooling - inputs.flake-parts.url = "github:hercules-ci/flake-parts"; - inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; + inputs.flake-parts.url = "https://flakehub.com/f/hercules-ci/flake-parts/0.1"; + inputs.git-hooks-nix.url = "https://flakehub.com/f/cachix/git-hooks.nix/0.1.941"; # work around https://github.com/NixOS/nix/issues/7730 inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; - inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; # work around 7730 and https://github.com/NixOS/nix/issues/7807 - inputs.git-hooks-nix.inputs.flake-compat.follows = ""; inputs.git-hooks-nix.inputs.gitignore.follows = ""; outputs = @@ -34,7 +28,7 @@ officialRelease = true; - linux32BitSystems = [ "i686-linux" ]; + linux32BitSystems = [ ]; linux64BitSystems = [ "x86_64-linux" "aarch64-linux" @@ -47,13 +41,12 @@ systems = linuxSystems ++ darwinSystems; crossSystems = [ - "armv6l-unknown-linux-gnueabihf" - "armv7l-unknown-linux-gnueabihf" - "riscv64-unknown-linux-gnu" + #"armv6l-unknown-linux-gnueabihf" + #"armv7l-unknown-linux-gnueabihf" + #"riscv64-unknown-linux-gnu" # Disabled because of https://github.com/NixOS/nixpkgs/issues/344423 # "x86_64-unknown-netbsd" - "x86_64-unknown-freebsd" - "x86_64-w64-mingw32" + #"x86_64-unknown-freebsd" ]; stdenvs = [ @@ -204,7 +197,6 @@ system: { installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system}; - installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = let @@ -215,6 +207,11 @@ ''; repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }; + lazyTrees = nixpkgsFor.${system}.native.nixComponents2.nix-functional-tests.override { + pname = "nix-lazy-trees-tests"; + lazyTrees = true; + }; + /** Checks for our packaging expressions. This shouldn't build anything significant; just check that things @@ -323,6 +320,40 @@ nix-manual = nixpkgsFor.${system}.native.nixComponents2.nix-manual; nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents2.nix-internal-api-docs; nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents2.nix-external-api-docs; + + fallbackPathsNix = + let + pkgs = nixpkgsFor.${system}.native; + + closures = forAllSystems (system: self.packages.${system}.default.outPath); + + closures_json = + pkgs.runCommand "versions.json" + { + buildInputs = [ pkgs.jq ]; + passAsFile = [ "json" ]; + json = builtins.toJSON closures; + } + '' + cat "$jsonPath" | jq . > $out + ''; + + closures_nix = + pkgs.runCommand "versions.nix" + { + buildInputs = [ pkgs.jq ]; + passAsFile = [ "template" ]; + jsonPath = closures_json; + template = '' + builtins.fromJSON('''@closures@''') + ''; + } + '' + export closures=$(cat "$jsonPath"); + substituteAll "$templatePath" "$out" + ''; + in + closures_nix; } # We need to flatten recursive attribute sets of derivations to pass `flake check`. // @@ -377,8 +408,6 @@ { # These attributes go right into `packages.`. "${pkgName}" = nixpkgsFor.${system}.native.nixComponents2.${pkgName}; - "${pkgName}-static" = nixpkgsFor.${system}.native.pkgsStatic.nixComponents2.${pkgName}; - "${pkgName}-llvm" = nixpkgsFor.${system}.native.pkgsLLVM.nixComponents2.${pkgName}; } // lib.optionalAttrs supportsCross ( flatMapAttrs (lib.genAttrs crossSystems (_: { })) ( @@ -434,32 +463,6 @@ } ) ) - // lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) ( - prefixAttrs "static" ( - forAllStdenvs ( - stdenvName: - makeShell { - pkgs = nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.pkgsStatic; - } - ) - ) - // prefixAttrs "llvm" ( - forAllStdenvs ( - stdenvName: - makeShell { - pkgs = nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.pkgsLLVM; - } - ) - ) - // prefixAttrs "cross" ( - forAllCrossSystems ( - crossSystem: - makeShell { - pkgs = nixpkgsFor.${system}.cross.${crossSystem}; - } - ) - ) - ) // { native = self.devShells.${system}.native-stdenv; default = self.devShells.${system}.native; diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json index 48e8685e6d9..bf00b69bc23 100644 --- a/maintainers/data/release-credits-email-to-handle.json +++ b/maintainers/data/release-credits-email-to-handle.json @@ -186,4 +186,4 @@ "hey@ewen.works": "gwennlbh", "matt@sturgeon.me.uk": "MattSturgeon", "pbsds@hotmail.com": "pbsds" -} \ No newline at end of file +} diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json index a6352c44b22..40258300b23 100644 --- a/maintainers/data/release-credits-handle-to-name.json +++ b/maintainers/data/release-credits-handle-to-name.json @@ -163,4 +163,4 @@ "egorkonovalov": "Egor Konovalov", "jayeshv": "jayeshv", "vcunat": "Vladim\u00edr \u010cun\u00e1t" -} \ No newline at end of file +} diff --git a/maintainers/link-headers b/maintainers/link-headers new file mode 100755 index 00000000000..2457a2dc829 --- /dev/null +++ b/maintainers/link-headers @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +# This script must be run from the root of the Nix repository. +# +# For include path hygiene, we need to put headers in a separate +# directory than sources. But during development, it is nice to paths +# that are similar for headers and source files, e.g. +# `foo/bar/baz.{cc,hh}`, e.g. for less typing when opening one file, and +# then opening the other file. +# +# This script symlinks the headers next to the source files to +# facilitate such a development workflows. It also updates +# `.git/info/exclude` so that the symlinks are not accidentally committed +# by mistake. + +from pathlib import Path +import subprocess +import os + + +def main() -> None: + # Path to the source directory + GIT_TOPLEVEL = Path( + subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + text=True, + stdout=subprocess.PIPE, + check=True, + ).stdout.strip() + ) + + # Get header files from git + result = subprocess.run( + ["git", "-C", str(GIT_TOPLEVEL), "ls-files", "*/include/nix/**.hh"], + text=True, + stdout=subprocess.PIPE, + check=True, + ) + header_files = result.stdout.strip().split("\n") + header_files.sort() + + links = [] + for file_str in header_files: + project_str, header_str = file_str.split("/include/nix/", 1) + project = Path(project_str) + header = Path(header_str) + + # Reconstruct the full path (relative to SRC_DIR) to the header file. + file = project / "include" / "nix" / header + + # The symlink should be created at "project/header", i.e. next to the project's sources. + link = project / header + + # Compute a relative path from the symlink's parent directory to the actual header file. + relative_source = os.path.relpath( + GIT_TOPLEVEL / file, GIT_TOPLEVEL / link.parent + ) + + # Create the symbolic link. + full_link_path = GIT_TOPLEVEL / link + full_link_path.parent.mkdir(parents=True, exist_ok=True) + if full_link_path.is_symlink(): + full_link_path.unlink() + full_link_path.symlink_to(relative_source) + links.append(link) + + # Generate .gitignore file + gitignore_path = GIT_TOPLEVEL / ".git" / "info" / "exclude" + gitignore_path.parent.mkdir(parents=True, exist_ok=True) + with gitignore_path.open("w") as gitignore: + gitignore.write("# DO NOT EDIT! Autogenerated\n") + gitignore.write( + "# Symlinks for headers to be next to sources for development\n" + ) + gitignore.write('# Run "maintainers/link-headers" to regenerate\n\n') + gitignore.write('# Run "maintainers/link-headers" to regenerate\n\n') + + for link in links: + gitignore.write(f"/{link}\n") + + +if __name__ == "__main__": + main() diff --git a/packaging/components.nix b/packaging/components.nix index b40bd45b0e4..89272200eb2 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -27,7 +27,7 @@ let pkg-config ; - baseVersion = lib.fileContents ../.version; + baseVersion = lib.fileContents ../.version-determinate; versionSuffix = lib.optionalString (!officialRelease) "pre"; @@ -51,15 +51,6 @@ let exts: userFn: stdenv.mkDerivation (lib.extends (lib.composeManyExtensions exts) userFn); setVersionLayer = finalAttrs: prevAttrs: { - preConfigure = - prevAttrs.preConfigure or "" - + - # Update the repo-global .version file. - # Symlink ./.version points there, but by default only workDir is writable. - '' - chmod u+w ./.version - echo ${finalAttrs.version} > ./.version - ''; }; localSourceLayer = diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index 8d3fa38527a..2b4615c17c6 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -26,7 +26,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( pname = "shell-for-" + attrs.pname; # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop - version = lib.fileContents ../.version; + version = lib.fileContents ../.version-determinate; name = attrs.pname; installFlags = "sysconfdir=$(out)/etc"; diff --git a/packaging/everything.nix b/packaging/everything.nix index 5bf57f95a26..c6229271f44 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -75,7 +75,7 @@ let }; devdoc = buildEnv { - name = "nix-${nix-cli.version}-devdoc"; + name = "determinate-nix-${nix-cli.version}-devdoc"; paths = [ nix-internal-api-docs nix-external-api-docs @@ -84,7 +84,7 @@ let in stdenv.mkDerivation (finalAttrs: { - pname = "nix"; + pname = "determinate-nix"; version = nix-cli.version; /** diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 27c09d9c9d4..6df8782393d 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -119,65 +119,6 @@ in system: self.devShells.${system}.default.inputDerivation )) [ "i686-linux" ]; - buildStatic = forAllPackages ( - pkgName: - lib.genAttrs linux64BitSystems ( - system: nixpkgsFor.${system}.native.pkgsStatic.nixComponents2.${pkgName} - ) - ); - - buildCross = forAllPackages ( - pkgName: - # Hack to avoid non-evaling package - ( - if pkgName == "nix-functional-tests" then - lib.flip builtins.removeAttrs [ "x86_64-w64-mingw32" ] - else - lib.id - ) - ( - forAllCrossSystems ( - crossSystem: - lib.genAttrs [ "x86_64-linux" ] ( - system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents2.${pkgName} - ) - ) - ) - ); - - buildNoGc = - let - components = forAllSystems ( - system: - nixpkgsFor.${system}.native.nixComponents2.overrideScope ( - self: super: { - nix-expr = super.nix-expr.override { enableGC = false; }; - } - ) - ); - in - forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName})); - - buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-cli); - - # Toggles some settings for better coverage. Windows needs these - # library combinations, and Debian build Nix with GNU readline too. - buildReadlineNoMarkdown = - let - components = forAllSystems ( - system: - nixpkgsFor.${system}.native.nixComponents2.overrideScope ( - self: super: { - nix-cmd = super.nix-cmd.override { - enableMarkdown = false; - readlineFlavor = "readline"; - }; - } - ) - ); - in - forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName})); - # Perl bindings for various platforms. perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-perl-bindings); @@ -188,30 +129,6 @@ in system: nixpkgsFor.${system}.native.callPackage ./binary-tarball.nix { } ); - binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] ( - system: - forAllCrossSystems ( - crossSystem: nixpkgsFor.${system}.cross.${crossSystem}.callPackage ./binary-tarball.nix { } - ) - ); - - # The first half of the installation script. This is uploaded - # to https://nixos.org/nix/install. It downloads the binary - # tarball for the user's system and calls the second half of the - # installation script. - installerScript = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."i686-linux" - self.hydraJobs.binaryTarball."aarch64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" - ]; - installerScriptForGHA = forAllSystems ( system: nixpkgsFor.${system}.native.callPackage ./installer { @@ -270,31 +187,23 @@ in pkgs = nixpkgsFor.${system}.native; } ); + + nixpkgsLibTestsLazy = forAllSystems ( + system: + lib.overrideDerivation + (import (nixpkgs + "/lib/tests/test-with-nix.nix") { + lib = nixpkgsFor.${system}.native.lib; + nix = self.packages.${system}.nix-cli; + pkgs = nixpkgsFor.${system}.native; + }) + (_: { + "NIX_CONFIG" = "lazy-trees = true"; + }) + ); }; metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { pkgs = nixpkgsFor.x86_64-linux.native; nixpkgs = nixpkgs-regression; }; - - installTests = forAllSystems ( - system: - let - pkgs = nixpkgsFor.${system}.native; - in - pkgs.runCommand "install-tests" { - againstSelf = testNixVersions pkgs pkgs.nix; - againstCurrentLatest = - # FIXME: temporarily disable this on macOS because of #3605. - if system == "x86_64-linux" then testNixVersions pkgs pkgs.nixVersions.latest else null; - # Disabled because the latest stable version doesn't handle - # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work - # againstLatestStable = testNixVersions pkgs pkgs.nixStable; - } "touch $out" - ); - - installerTests = import ../tests/installer { - binaryTarballs = self.hydraJobs.binaryTarball; - inherit nixpkgsFor; - }; } diff --git a/packaging/installer/default.nix b/packaging/installer/default.nix index e171f36f99f..a8e344b496c 100644 --- a/packaging/installer/default.nix +++ b/packaging/installer/default.nix @@ -32,7 +32,7 @@ runCommand "installer-script" in '' \ - --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ + --replace '@tarballHash_${system}@' $(nix hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \ '' ) tarballs diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index f051ccc46b9..e9ddfc0140d 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -834,8 +834,13 @@ install_from_extracted_nix() { ( cd "$EXTRACTED_NIX_PATH" - _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \ - cp -RPp ./store/* "$NIX_ROOT/store/" + if is_os_darwin; then + _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \ + cp -RPp ./store/* "$NIX_ROOT/store/" + else + _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \ + cp -RP --preserve=ownership,timestamps ./store/* "$NIX_ROOT/store/" + fi _sudo "to make the new store non-writable at $NIX_ROOT/store" \ chmod -R ugo-w "$NIX_ROOT/store/" diff --git a/scripts/install-nix-from-tarball.sh b/scripts/install-nix-from-tarball.sh index 8d127a9c52c..ec326479323 100644 --- a/scripts/install-nix-from-tarball.sh +++ b/scripts/install-nix-from-tarball.sh @@ -167,7 +167,11 @@ for i in $(cd "$self/store" >/dev/null && echo ./*); do rm -rf "$i_tmp" fi if ! [ -e "$dest/store/$i" ]; then - cp -RPp "$self/store/$i" "$i_tmp" + if [ "$(uname -s)" = "Darwin" ]; then + cp -RPp "$self/store/$i" "$i_tmp" + else + cp -RP --preserve=ownership,timestamps "$self/store/$i" "$i_tmp" + fi chmod -R a-w "$i_tmp" chmod +w "$i_tmp" mv "$i_tmp" "$dest/store/$i" diff --git a/shell.nix b/shell.nix deleted file mode 100644 index 918f4bbd9e9..00000000000 --- a/shell.nix +++ /dev/null @@ -1,3 +0,0 @@ -(import (fetchTarball "https://github.com/edolstra/flake-compat/archive/master.tar.gz") { - src = ./.; -}).shellNix diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix index b194e16d460..28cde8c09e6 100644 --- a/src/external-api-docs/package.nix +++ b/src/external-api-docs/package.nix @@ -14,7 +14,7 @@ let in mkMesonDerivation (finalAttrs: { - pname = "nix-external-api-docs"; + pname = "determinate-nix-external-api-docs"; inherit version; workDir = ./.; diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix index 6c4f354aee5..636c19653ea 100644 --- a/src/internal-api-docs/package.nix +++ b/src/internal-api-docs/package.nix @@ -14,7 +14,7 @@ let in mkMesonDerivation (finalAttrs: { - pname = "nix-internal-api-docs"; + pname = "determinate-nix-internal-api-docs"; inherit version; workDir = ./.; diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index d275beb12c3..a183e6f0e4f 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -29,7 +29,6 @@ EvalSettings evalSettings { { "flake", [](EvalState & state, std::string_view rest) { - experimentalFeatureSettings.require(Xp::Flakes); // FIXME `parseFlakeRef` should take a `std::string_view`. auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); debug("fetching flake search path element '%s''", rest); @@ -191,7 +190,6 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas } else if (hasPrefix(s, "flake:")) { - experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store); auto storePath = nix::fetchToStore( diff --git a/src/libcmd/include/nix/cmd/command.hh b/src/libcmd/include/nix/cmd/command.hh index 20cd1abc1c4..0455a1d3c85 100644 --- a/src/libcmd/include/nix/cmd/command.hh +++ b/src/libcmd/include/nix/cmd/command.hh @@ -214,6 +214,8 @@ struct InstallableCommand : virtual Args, SourceExprCommand { InstallableCommand(); + virtual void preRun(ref store); + virtual void run(ref store, ref installable) = 0; void run(ref store) override; diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index e92496347e0..f5a129205c8 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -57,7 +57,8 @@ std::optional InstallableValue::trySinglePathToDerivedPaths else if (v.type() == nString) { return {{ .path = DerivedPath::fromSingle( - state->coerceToSingleDerivedPath(pos, v, errorCtx)), + state->devirtualize( + state->coerceToSingleDerivedPath(pos, v, errorCtx))), .info = make_ref(), }}; } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 49ffd82e1a3..713fe2f929b 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -399,9 +399,6 @@ void completeFlakeRefWithFragment( void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix) { - if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) - return; - if (prefix == "") completions.add("."); @@ -908,8 +905,13 @@ InstallableCommand::InstallableCommand() }); } +void InstallableCommand::preRun(ref store) +{ +} + void InstallableCommand::run(ref store) { + preRun(store); auto installable = parseInstallable(store, _installable); run(store, std::move(installable)); } diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index be5054f6403..5c9040e8b17 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -35,7 +35,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-cmd"; + pname = "determinate-nix-cmd"; inherit version; workDir = ./.; diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index 694fbc1fe78..ec92ecce105 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -15,7 +15,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-expr-c"; + pname = "determinate-nix-expr-c"; inherit version; workDir = ./.; diff --git a/src/libexpr-test-support/include/nix/expr/tests/value/context.hh b/src/libexpr-test-support/include/nix/expr/tests/value/context.hh index a6a851d3ac7..a473f6f12f8 100644 --- a/src/libexpr-test-support/include/nix/expr/tests/value/context.hh +++ b/src/libexpr-test-support/include/nix/expr/tests/value/context.hh @@ -23,6 +23,11 @@ struct Arbitrary { static Gen arbitrary(); }; +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + template<> struct Arbitrary { static Gen arbitrary(); diff --git a/src/libexpr-test-support/package.nix b/src/libexpr-test-support/package.nix index 5cb4adaa8c4..1879a571608 100644 --- a/src/libexpr-test-support/package.nix +++ b/src/libexpr-test-support/package.nix @@ -18,7 +18,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-util-test-support"; + pname = "determinate-nix-util-test-support"; inherit version; workDir = ./.; diff --git a/src/libexpr-test-support/tests/value/context.cc b/src/libexpr-test-support/tests/value/context.cc index 51ff1b2ae61..9a27f87309d 100644 --- a/src/libexpr-test-support/tests/value/context.cc +++ b/src/libexpr-test-support/tests/value/context.cc @@ -15,6 +15,15 @@ Gen Arbitrary::arb }); } +Gen Arbitrary::arbitrary() +{ + return gen::map(gen::arbitrary(), [](StorePath storePath) { + return NixStringContextElem::Path{ + .storePath = storePath, + }; + }); +} + Gen Arbitrary::arbitrary() { return gen::mapcat( @@ -30,6 +39,9 @@ Gen Arbitrary::arbitrary() case 2: return gen::map( gen::arbitrary(), [](NixStringContextElem a) { return a; }); + case 3: + return gen::map( + gen::arbitrary(), [](NixStringContextElem a) { return a; }); default: assert(false); } diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 27d60d6ef49..39c1b827dff 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -618,18 +618,21 @@ string_t AttrCursor::getStringWithContext() if (auto s = std::get_if(&cachedValue->second)) { bool valid = true; for (auto & c : s->second) { - const StorePath & path = std::visit(overloaded { - [&](const NixStringContextElem::DrvDeep & d) -> const StorePath & { - return d.drvPath; + const StorePath * path = std::visit(overloaded { + [&](const NixStringContextElem::DrvDeep & d) -> const StorePath * { + return &d.drvPath; }, - [&](const NixStringContextElem::Built & b) -> const StorePath & { - return b.drvPath->getBaseStorePath(); + [&](const NixStringContextElem::Built & b) -> const StorePath * { + return &b.drvPath->getBaseStorePath(); }, - [&](const NixStringContextElem::Opaque & o) -> const StorePath & { - return o.path; + [&](const NixStringContextElem::Opaque & o) -> const StorePath * { + return &o.path; + }, + [&](const NixStringContextElem::Path & p) -> const StorePath * { + return nullptr; }, }, c.raw); - if (!root->state.store->isValidPath(path)) { + if (!path || !root->state.store->isValidPath(*path)) { valid = false; break; } diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 659c01a9e63..dd498fdf2dd 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -84,9 +84,17 @@ bool EvalSettings::isPseudoUrl(std::string_view s) std::string EvalSettings::resolvePseudoUrl(std::string_view url) { - if (hasPrefix(url, "channel:")) - return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; - else + if (hasPrefix(url, "channel:")) { + auto realUrl = "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; + static bool haveWarned = false; + warnOnce(haveWarned, + "Channels are deprecated in favor of flakes in Determinate Nix. " + "Instead of '%s', use '%s'. " + "See https://zero-to-nix.com for a guide to Nix flakes. " + "For details and to offer feedback on the deprecation process, see: https://github.com/DeterminateSystems/nix-src/issues/34.", + url, realUrl); + return realUrl; + } else return std::string(url); } @@ -103,4 +111,4 @@ Path getNixDefExpr() : getHome() + "/.nix-defexpr"; } -} // namespace nix \ No newline at end of file +} // namespace nix diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1321e00a5a5..2baed9bcafb 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -16,6 +16,7 @@ #include "nix/expr/print.hh" #include "nix/fetchers/filtering-source-accessor.hh" #include "nix/util/memory-source-accessor.hh" +#include "nix/util/mounted-source-accessor.hh" #include "nix/expr/gc-small-vector.hh" #include "nix/util/url.hh" #include "nix/fetchers/fetch-to-store.hh" @@ -266,7 +267,7 @@ EvalState::EvalState( exception, and make union source accessor catch it, so we don't need to do this hack. */ - {CanonPath(store->storeDir), store->getFSAccessor(settings.pureEval)}, + {CanonPath(store->storeDir), makeFSSourceAccessor(dirOf(store->toRealPath(StorePath::dummy)))} })) , rootFS( ({ @@ -281,12 +282,9 @@ EvalState::EvalState( /nix/store while using a chroot store. */ auto accessor = getFSSourceAccessor(); - auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy)); - if (settings.pureEval || store->storeDir != realStoreDir) { - accessor = settings.pureEval - ? storeFS - : makeUnionSourceAccessor({accessor, storeFS}); - } + accessor = settings.pureEval + ? storeFS.cast() + : makeUnionSourceAccessor({accessor, storeFS}); /* Apply access control if needed. */ if (settings.restrictEval || settings.pureEval) @@ -979,7 +977,16 @@ void EvalState::mkPos(Value & v, PosIdx p) auto origin = positions.originOf(p); if (auto path = std::get_if(&origin)) { auto attrs = buildBindings(3); - attrs.alloc(sFile).mkString(path->path.abs()); + if (path->accessor == rootFS && store->isInStore(path->path.abs())) + // FIXME: only do this for virtual store paths? + attrs.alloc(sFile).mkString(path->path.abs(), + { + NixStringContextElem::Path{ + .storePath = store->toStorePath(path->path.abs()).first + } + }); + else + attrs.alloc(sFile).mkString(path->path.abs()); makePositionThunks(*this, p, attrs.alloc(sLine), attrs.alloc(sColumn)); v.mkAttrs(attrs); } else @@ -1602,7 +1609,7 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, symbols[i.name]) .atPos(lambda.pos) .withTrace(pos, "from call site") - .withFrame(*fun.lambda().env, lambda) + .withFrame(*vCur.lambda().env, lambda) .debugThrow(); } env2.values[displ++] = i.def->maybeThunk(*this, env2); @@ -1629,7 +1636,7 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, .atPos(lambda.pos) .withTrace(pos, "from call site") .withSuggestions(suggestions) - .withFrame(*fun.lambda().env, lambda) + .withFrame(*vCur.lambda().env, lambda) .debugThrow(); } unreachable(); @@ -2106,7 +2113,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) else if (firstType == nFloat) v.mkFloat(nf); else if (firstType == nPath) { - if (!context.empty()) + if (hasContext(context)) state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow(); v.mkPath(state.rootPath(CanonPath(str()))); } else @@ -2315,7 +2322,10 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s { auto s = forceString(v, pos, errorCtx); if (v.context()) { - error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow(); + NixStringContext context; + copyContext(v, context); + if (hasContext(context)) + error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow(); } return s; } @@ -2364,14 +2374,26 @@ BackedStringView EvalState::coerceToString( } if (v.type() == nPath) { + // FIXME: instead of copying the path to the store, we could + // return a virtual store path that lazily copies the path to + // the store in devirtualize(). return !canonicalizePath && !copyToStore ? // FIXME: hack to preserve path literals that end in a // slash, as in /foo/${x}. v.pathStr() : copyToStore - ? store->printStorePath(copyPathToStore(context, v.path())) - : std::string(v.path().path.abs()); + ? store->printStorePath(copyPathToStore(context, v.path(), v.determinePos(pos))) + : ({ + auto path = v.path(); + if (path.accessor == rootFS && store->isInStore(path.path.abs())) { + context.insert( + NixStringContextElem::Path{ + .storePath = store->toStorePath(path.path.abs()).first + }); + } + std::string(path.path.abs()); + }); } if (v.type() == nAttrs) { @@ -2440,7 +2462,7 @@ BackedStringView EvalState::coerceToString( } -StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePath & path) +StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePath & path, PosIdx pos) { if (nix::isDerivation(path.path.abs())) error("file names are not allowed to end in '%1%'", drvExtension).debugThrow(); @@ -2455,7 +2477,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat *store, path.resolveSymlinks(SymlinkResolution::Ancestors), settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, - path.baseName(), + computeBaseName(path, pos), ContentAddressMethod::Raw::NixArchive, nullptr, repair); @@ -2510,7 +2532,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (auto storePath = store->maybeParseStorePath(path)) return *storePath; - error("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow(); + error("cannot coerce '%s' to a store path because it is not a subpath of the Nix store", path).withTrace(pos, errorCtx).debugThrow(); } @@ -2536,6 +2558,11 @@ std::pair EvalState::coerceToSingleDerivedP [&](NixStringContextElem::Built && b) -> SingleDerivedPath { return std::move(b); }, + [&](NixStringContextElem::Path && p) -> SingleDerivedPath { + error( + "string '%s' has no context", + s).withTrace(pos, errorCtx).debugThrow(); + }, }, ((NixStringContextElem &&) *context.begin()).raw); return { std::move(derivedPath), @@ -3119,6 +3146,11 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ auto res = (r / CanonPath(suffix)).resolveSymlinks(); if (res.pathExists()) return res; + + // Backward compatibility hack: throw an exception if access + // to this path is not allowed. + if (auto accessor = res.accessor.dynamic_pointer_cast()) + accessor->checkAccess(res.path); } if (hasPrefix(path, "nix/")) @@ -3189,6 +3221,11 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pat if (path.resolveSymlinks().pathExists()) return finish(std::move(path)); else { + // Backward compatibility hack: throw an exception if access + // to this path is not allowed. + if (auto accessor = path.accessor.dynamic_pointer_cast()) + accessor->checkAccess(path.path); + logWarning({ .msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value) }); diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index eee3b0f0e76..7fa3f96be68 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -89,7 +89,7 @@ struct EvalSettings : Config - `$HOME/.nix-defexpr/channels` - The [user channel link](@docroot@/command-ref/files/default-nix-expression.md#user-channel-link), pointing to the current state of [channels](@docroot@/command-ref/files/channels.md) for the current user. + The user channel link pointing to the current state of channels for the current user. - `nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs` @@ -99,7 +99,7 @@ struct EvalSettings : Config The current state of all channels for the `root` user. - These files are set up by the [Nix installer](@docroot@/installation/installing-binary.md). + These files are set up by the Nix installer. See [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) for details on the environment variable. > **Note** @@ -134,7 +134,7 @@ struct EvalSettings : Config R"( If set to `true`, the Nix evaluator doesn't allow access to any files outside of - [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath), + [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) or to URIs outside of [`allowed-uris`](@docroot@/command-ref/conf-file.md#conf-allowed-uris). )"}; @@ -237,7 +237,7 @@ struct EvalSettings : Config Setting ignoreExceptionsDuringTry{this, false, "ignore-try", R"( If set to true, ignore exceptions inside 'tryEval' calls when evaluating Nix expressions in - debug mode (using the --debugger flag). By default the debugger pauses on all exceptions. + debug mode (using the --debugger flag). By default, the debugger pauses on all exceptions. )"}; Setting traceVerbose{this, false, "trace-verbose", @@ -249,7 +249,7 @@ struct EvalSettings : Config Setting builtinsTraceDebugger{this, false, "debugger-on-trace", R"( If set to true and the `--debugger` flag is given, the following functions - enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break): + enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break). * [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace) * [`builtins.traceVerbose`](@docroot@/language/builtins.md#builtins-traceVerbose) @@ -262,7 +262,7 @@ struct EvalSettings : Config Setting builtinsDebuggerOnWarn{this, false, "debugger-on-warn", R"( If set to true and the `--debugger` flag is given, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) - will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break). + enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break). This is useful for debugging warnings in third-party Nix code. @@ -273,7 +273,7 @@ struct EvalSettings : Config R"( If set to true, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) throws an error when logging a warning. - This will give you a stack trace that leads to the location of the warning. + This gives you a stack trace that leads to the location of the warning. This is useful for finding information about warnings in third-party Nix code when you can not start the interactive debugger, such as when Nix is called from a non-interactive script. See [`debugger-on-warn`](#conf-debugger-on-warn). @@ -281,6 +281,24 @@ struct EvalSettings : Config This option can be enabled by setting `NIX_ABORT_ON_WARN=1` in the environment. )"}; + + Setting lazyTrees{this, false, "lazy-trees", + R"( + If set to true, flakes and trees fetched by [`builtins.fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) are only copied to the Nix store when they're used as a dependency of a derivation. This avoids copying (potentially large) source trees unnecessarily. + )"}; + + // FIXME: this setting should really be in libflake, but it's + // currently needed in mountInput(). + Setting lazyLocks{ + this, + false, + "lazy-locks", + R"( + If enabled, Nix only includes NAR hashes in lock file entries if they're necessary to lock the input (i.e. when there is no other attribute that allows the content to be verified, like a Git revision). + This is not backward compatible with older versions of Nix. + If disabled, lock file entries always contain a NAR hash. + )" + }; }; /** diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index 27294d11403..763ce184c90 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -37,6 +37,7 @@ class Store; namespace fetchers { struct Settings; struct InputCache; +struct Input; } struct EvalSettings; class EvalState; @@ -44,6 +45,7 @@ class StorePath; struct SingleDerivedPath; enum RepairFlag : bool; struct MemorySourceAccessor; +struct MountedSourceAccessor; namespace eval_cache { class EvalCache; } @@ -272,7 +274,7 @@ public: /** * The accessor corresponding to `store`. */ - const ref storeFS; + const ref storeFS; /** * The accessor for the root filesystem. @@ -450,6 +452,15 @@ public: void checkURI(const std::string & uri); + /** + * Mount an input on the Nix store. + */ + StorePath mountInput( + fetchers::Input & input, + const fetchers::Input & originalInput, + ref accessor, + bool requireLockable); + /** * Parse a Nix expression from the specified file. */ @@ -564,6 +575,18 @@ public: std::optional tryAttrsToString(const PosIdx pos, Value & v, NixStringContext & context, bool coerceMore = false, bool copyToStore = true); + StorePath devirtualize( + const StorePath & path, + StringMap * rewrites = nullptr); + + SingleDerivedPath devirtualize( + const SingleDerivedPath & path, + StringMap * rewrites = nullptr); + + std::string devirtualize( + std::string_view s, + const NixStringContext & context); + /** * String coercion. * @@ -577,7 +600,20 @@ public: bool coerceMore = false, bool copyToStore = true, bool canonicalizePath = true); - StorePath copyPathToStore(NixStringContext & context, const SourcePath & path); + StorePath copyPathToStore(NixStringContext & context, const SourcePath & path, PosIdx pos); + + + /** + * Compute the base name for a `SourcePath`. For non-store paths, + * this is just `SourcePath::baseName()`. But for store paths, for + * backwards compatibility, it needs to be `-source`, + * i.e. as if the path were copied to the Nix store. This results + * in a "double-copied" store path like + * `/nix/store/--source`. We don't need to + * materialize /nix/store/-source though. Still, this + * requires reading/hashing the path twice. + */ + std::string computeBaseName(const SourcePath & path, PosIdx pos); /** * Path coercion. diff --git a/src/libexpr/include/nix/expr/print-ambiguous.hh b/src/libexpr/include/nix/expr/print-ambiguous.hh index 9e5a27e6d6e..d4ecea0bf3e 100644 --- a/src/libexpr/include/nix/expr/print-ambiguous.hh +++ b/src/libexpr/include/nix/expr/print-ambiguous.hh @@ -16,10 +16,10 @@ namespace nix { * See: https://github.com/NixOS/nix/issues/9730 */ void printAmbiguous( - Value &v, - const SymbolTable &symbols, - std::ostream &str, - std::set *seen, + EvalState & state, + Value & v, + std::ostream & str, + std::set * seen, int depth); } diff --git a/src/libexpr/include/nix/expr/value/context.hh b/src/libexpr/include/nix/expr/value/context.hh index f2de184ea1f..f53c9b99762 100644 --- a/src/libexpr/include/nix/expr/value/context.hh +++ b/src/libexpr/include/nix/expr/value/context.hh @@ -54,10 +54,35 @@ struct NixStringContextElem { */ using Built = SingleDerivedPath::Built; + /** + * A store path that will not result in a store reference when + * used in a derivation or toFile. + * + * When you apply `builtins.toString` to a path value representing + * a path in the Nix store (as is the case with flake inputs), + * historically you got a string without context + * (e.g. `/nix/store/...-source`). This is broken, since it allows + * you to pass a store path to a derivation/toFile without a + * proper store reference. This is especially a problem with lazy + * trees, since the store path is a virtual path that doesn't + * exist. + * + * For backwards compatibility, and to warn users about this + * unsafe use of `toString`, we keep track of such strings as a + * special type of context. + */ + struct Path + { + StorePath storePath; + + GENERATE_CMP(Path, me->storePath); + }; + using Raw = std::variant< Opaque, DrvDeep, - Built + Built, + Path >; Raw raw; @@ -82,4 +107,10 @@ struct NixStringContextElem { typedef std::set NixStringContext; +/** + * Returns false if `context` has no elements other than + * `NixStringContextElem::Path`. + */ +bool hasContext(const NixStringContext & context); + } diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index 50161c58ba2..7e00416170c 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -36,7 +36,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-expr"; + pname = "determinate-nix-expr"; inherit version; workDir = ./.; diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index c5107de3a5e..64b6f80d48d 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -1,5 +1,7 @@ #include "nix/store/store-api.hh" #include "nix/expr/eval.hh" +#include "nix/util/mounted-source-accessor.hh" +#include "nix/fetchers/fetch-to-store.hh" namespace nix { @@ -18,4 +20,93 @@ SourcePath EvalState::storePath(const StorePath & path) return {rootFS, CanonPath{store->printStorePath(path)}}; } +StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites) +{ + if (auto mount = storeFS->getMount(CanonPath(store->printStorePath(path)))) { + auto storePath = fetchToStore( + fetchSettings, + *store, + SourcePath{ref(mount)}, + settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, + path.name()); + assert(storePath.name() == path.name()); + if (rewrites) + rewrites->emplace(path.hashPart(), storePath.hashPart()); + return storePath; + } else + return path; +} + +SingleDerivedPath EvalState::devirtualize(const SingleDerivedPath & path, StringMap * rewrites) +{ + if (auto o = std::get_if(&path.raw())) + return SingleDerivedPath::Opaque{devirtualize(o->path, rewrites)}; + else + return path; +} + +std::string EvalState::devirtualize(std::string_view s, const NixStringContext & context) +{ + StringMap rewrites; + + for (auto & c : context) + if (auto o = std::get_if(&c.raw)) + devirtualize(o->path, &rewrites); + + return rewriteStrings(std::string(s), rewrites); +} + +std::string EvalState::computeBaseName(const SourcePath & path, PosIdx pos) +{ + if (path.accessor == rootFS) { + if (auto storePath = store->maybeParseStorePath(path.path.abs())) { + debug( + "Copying '%s' to the store again.\n" + "You can make Nix evaluate faster and copy fewer files by replacing `./.` with the `self` flake input, " + "or `builtins.path { path = ./.; name = \"source\"; }`.\n", + path); + return std::string( + fetchToStore(fetchSettings, *store, path, FetchMode::DryRun, storePath->name()).to_string()); + } + } + return std::string(path.baseName()); +} + +StorePath EvalState::mountInput( + fetchers::Input & input, const fetchers::Input & originalInput, ref accessor, bool requireLockable) +{ + auto storePath = settings.lazyTrees + ? StorePath::random(input.getName()) + : fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName()); + + allowPath(storePath); // FIXME: should just whitelist the entire virtual store + + std::optional _narHash; + + auto getNarHash = [&]() { + if (!_narHash) { + if (store->isValidPath(storePath)) + _narHash = store->queryPathInfo(storePath)->narHash; + else + _narHash = fetchToStore2(fetchSettings, *store, accessor, FetchMode::DryRun, input.getName()).second; + } + return _narHash; + }; + + storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor); + + if (requireLockable && (!settings.lazyTrees || !settings.lazyLocks || !input.isLocked()) && !input.getNarHash()) + input.attrs.insert_or_assign("narHash", getNarHash()->to_string(HashFormat::SRI, true)); + + if (originalInput.getNarHash() && *getNarHash() != *originalInput.getNarHash()) + throw Error( + (unsigned int) 102, + "NAR hash mismatch in input '%s', expected '%s' but got '%s'", + originalInput.to_string(), + getNarHash()->to_string(HashFormat::SRI, true), + originalInput.getNarHash()->to_string(HashFormat::SRI, true)); + + return storePath; +} + } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f9f834a62f0..f510a66ed91 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -15,6 +15,7 @@ #include "nix/expr/primops.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/sort.hh" +#include "nix/util/mounted-source-accessor.hh" #include #include @@ -76,7 +77,10 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { - ensureValid(o.path); + // We consider virtual store paths valid here. They'll + // be devirtualized if needed elsewhere. + if (!storeFS->getMount(CanonPath(store->printStorePath(o.path)))) + ensureValid(o.path); if (maybePathsOut) maybePathsOut->emplace(o.path); }, @@ -86,6 +90,9 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS if (maybePathsOut) maybePathsOut->emplace(d.drvPath); }, + [&](const NixStringContextElem::Path & p) { + // FIXME: do something? + }, }, c.raw); } @@ -957,7 +964,7 @@ static RegisterPrimOp primop_floor({ a NixInt and if `*number* < -9007199254740992` or `*number* > 9007199254740992`. If the datatype of *number* is neither a NixInt (signed 64-bit integer) nor a NixFloat - (IEEE-754 double-precision floating-point number), an evaluation error will be thrown. + (IEEE-754 double-precision floating-point number), an evaluation error is thrown. )", .fun = prim_floor, }); @@ -1005,7 +1012,7 @@ static RegisterPrimOp primop_tryEval({ `false` if an error was thrown) and `value`, equalling *e* if successful and `false` otherwise. `tryEval` only prevents errors created by `throw` or `assert` from being thrown. - Errors `tryEval` doesn't catch are, for example, those created + Errors that `tryEval` doesn't catch are, for example, those created by `abort` and type errors generated by builtins. Also note that this doesn't evaluate *e* deeply, so `let e = { x = throw ""; }; in (builtins.tryEval e).success` is `true`. Using @@ -1153,7 +1160,7 @@ static RegisterPrimOp primop_warn({ [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace) or [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn) option is set to `true` and the `--debugger` flag is given, the - interactive debugger will be started when `warn` is called (like + interactive debugger is started when `warn` is called (like [`break`](@docroot@/language/builtins.md#builtins-break)). If the @@ -1449,6 +1456,10 @@ static void derivationStrictInternal( /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ + StringMap rewrites; + + std::optional drvS; + for (auto & c : context) { std::visit(overloaded { /* Since this allows the builder to gain access to every @@ -1471,11 +1482,24 @@ static void derivationStrictInternal( drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output); }, [&](const NixStringContextElem::Opaque & o) { - drv.inputSrcs.insert(o.path); + drv.inputSrcs.insert(state.devirtualize(o.path, &rewrites)); + }, + [&](const NixStringContextElem::Path & p) { + if (!drvS) drvS = drv.unparse(*state.store, true); + if (drvS->find(p.storePath.to_string()) != drvS->npos) { + auto devirtualized = state.devirtualize(p.storePath, &rewrites); + warn( + "Using 'builtins.derivation' to create a derivation named '%s' that references the store path '%s' without a proper context. " + "The resulting derivation will not have a correct store reference, so this is unreliable and may stop working in the future.", + drvName, + state.store->printStorePath(devirtualized)); + } }, }, c.raw); } + drv.applyRewrites(rewrites); + /* Do we have all required attributes? */ if (drv.builder == "") state.error("required attribute 'builder' missing") @@ -2376,13 +2400,24 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val { NixStringContext context; auto name = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.toFile"); - auto contents = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile"); + std::string contents(state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile")); StorePathSet refs; + StringMap rewrites; for (auto c : context) { if (auto p = std::get_if(&c.raw)) refs.insert(p->path); + else if (auto p = std::get_if(&c.raw)) { + if (contents.find(p->storePath.to_string()) != contents.npos) { + auto devirtualized = state.devirtualize(p->storePath, &rewrites); + warn( + "Using 'builtins.toFile' to create a file named '%s' that references the store path '%s' without a proper context. " + "The resulting file will not have a correct store reference, so this is unreliable and may stop working in the future.", + name, + state.store->printStorePath(devirtualized)); + } + } else state.error( "files created by %1% may not reference derivations, but %2% references %3%", @@ -2392,6 +2427,8 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val ).atPos(pos).debugThrow(); } + contents = rewriteStrings(contents, rewrites); + auto storePath = settings.readOnlyMode ? state.store->makeFixedOutputPathFromCA(name, TextInfo { .hash = hashString(HashAlgorithm::SHA256, contents), @@ -2541,6 +2578,7 @@ static void addPath( {})); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { + // FIXME: make this lazy? auto dstPath = fetchToStore( state.fetchSettings, *state.store, @@ -2572,7 +2610,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); - addPath(state, pos, path.baseName(), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context); + addPath(state, pos, state.computeBaseName(path, pos), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 56962d6a872..f90a649d971 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -7,9 +7,15 @@ namespace nix { static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - NixStringContext context; + NixStringContext context, filtered; + auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext"); - v.mkString(*s); + + for (auto & c : context) + if (auto * p = std::get_if(&c.raw)) + filtered.insert(*p); + + v.mkString(*s, filtered); } static RegisterPrimOp primop_unsafeDiscardStringContext({ @@ -21,12 +27,19 @@ static RegisterPrimOp primop_unsafeDiscardStringContext({ .fun = prim_unsafeDiscardStringContext, }); +bool hasContext(const NixStringContext & context) +{ + for (auto & c : context) + if (!std::get_if(&c.raw)) + return true; + return false; +} static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { NixStringContext context; state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.hasContext"); - v.mkBool(!context.empty()); + v.mkBool(hasContext(context)); } static RegisterPrimOp primop_hasContext({ @@ -103,7 +116,7 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.addDrvOutputDependencies"); - auto contextSize = context.size(); + auto contextSize = context.size(); if (contextSize != 1) { state.error( "context of string '%s' must have exactly one element, but has %d", @@ -136,6 +149,11 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V above does not make much sense. */ return std::move(c); }, + [&](const NixStringContextElem::Path & p) -> NixStringContextElem::DrvDeep { + state.error( + "`addDrvOutputDependencies` does not work on a string without context" + ).atPos(pos).debugThrow(); + }, }, context.begin()->raw) }), }; @@ -206,6 +224,8 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, [&](NixStringContextElem::Opaque && o) { contextInfos[std::move(o.path)].path = true; }, + [&](NixStringContextElem::Path && p) { + }, }, ((NixStringContextElem &&) i).raw); } diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index ea6145f6f9e..e74c8484d4e 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -124,12 +124,12 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg for (auto & attr : *args[0]->attrs()) { const auto & attrName = state.symbols[attr.name]; auto attrHint = [&]() -> std::string { - return "while evaluating the '" + attrName + "' attribute passed to builtins.fetchClosure"; + return fmt("while evaluating the attribute '%s' passed to builtins.fetchClosure", attrName); }; if (attrName == "fromPath") { NixStringContext context; - fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint()); + fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint()); // FIXME: overflow } else if (attrName == "toPath") { diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 189bd1f73d7..843beb4d195 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -64,7 +64,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a if (rev) attrs.insert_or_assign("rev", rev->gitRev()); auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); - auto [storePath, input2] = input.fetchToStore(state.store); + auto [storePath, accessor, input2] = input.fetchToStore(state.store); auto attrs2 = state.buildBindings(8); state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath)); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 5b6dd65317b..d8efa1e8a7f 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -10,6 +10,8 @@ #include "nix/util/url.hh" #include "nix/expr/value-to-json.hh" #include "nix/fetchers/fetch-to-store.hh" +#include "nix/fetchers/input-cache.hh" +#include "nix/util/mounted-source-accessor.hh" #include @@ -172,15 +174,11 @@ static void fetchTree( } input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); } else { - if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) - state.error( - "passing a string argument to '%s' requires the 'flakes' experimental feature", fetcher - ).atPos(pos).debugThrow(); input = fetchers::Input::fromURL(state.fetchSettings, url); } } - if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) + if (!state.settings.pureEval && !input.isDirect()) input = lookupInRegistries(state.store, input, fetchers::UseRegistries::Limited).first; if (state.settings.pureEval && !input.isLocked()) { @@ -204,11 +202,11 @@ static void fetchTree( throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string()); } - auto [storePath, input2] = input.fetchToStore(state.store); + auto cachedInput = state.inputCache->getAccessor(state.store, input, fetchers::UseRegistries::No); - state.allowPath(storePath); + auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor, true); - emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false); + emitTreeAttrs(state, storePath, cachedInput.lockedInput, v, params.emptyRevFallback, false); } static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -406,7 +404,6 @@ static RegisterPrimOp primop_fetchTree({ - `"mercurial"` *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). - The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled. > **Example** > @@ -443,7 +440,6 @@ static RegisterPrimOp primop_fetchTree({ > ``` )", .fun = prim_fetchTree, - .experimentalFeature = Xp::FetchTree, }); void prim_fetchFinalTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -777,7 +773,7 @@ static RegisterPrimOp primop_fetchGit({ name in the `ref` attribute. However, if the revision you're looking for is in a future - branch for the non-default branch you will need to specify the + branch for the non-default branch you need to specify the the `ref` attribute as well. ```nix diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index 2a0b009ebfb..e966b3f02f4 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -7,10 +7,10 @@ namespace nix { // See: https://github.com/NixOS/nix/issues/9730 void printAmbiguous( - Value &v, - const SymbolTable &symbols, - std::ostream &str, - std::set *seen, + EvalState & state, + Value & v, + std::ostream & str, + std::set * seen, int depth) { checkInterrupt(); @@ -26,9 +26,13 @@ void printAmbiguous( case nBool: printLiteralBool(str, v.boolean()); break; - case nString: - printLiteralString(str, v.string_view()); + case nString: { + NixStringContext context; + copyContext(v, context); + // FIXME: make devirtualization configurable? + printLiteralString(str, state.devirtualize(v.string_view(), context)); break; + } case nPath: str << v.path().to_string(); // !!! escaping? break; @@ -40,9 +44,9 @@ void printAmbiguous( str << "«repeated»"; else { str << "{ "; - for (auto & i : v.attrs()->lexicographicOrder(symbols)) { - str << symbols[i->name] << " = "; - printAmbiguous(*i->value, symbols, str, seen, depth - 1); + for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) { + str << state.symbols[i->name] << " = "; + printAmbiguous(state, *i->value, str, seen, depth - 1); str << "; "; } str << "}"; @@ -58,7 +62,7 @@ void printAmbiguous( str << "[ "; for (auto v2 : v.listView()) { if (v2) - printAmbiguous(*v2, symbols, str, seen, depth - 1); + printAmbiguous(state, *v2, str, seen, depth - 1); else str << "(nullptr)"; str << " "; diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index 1f0c592c157..0aaa6b8b0a6 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -249,7 +249,11 @@ class Printer void printString(Value & v) { - printLiteralString(output, v.string_view(), options.maxStringLength, options.ansiColors); + NixStringContext context; + copyContext(v, context); + std::ostringstream s; + printLiteralString(s, v.string_view(), options.maxStringLength, options.ansiColors); + output << state.devirtualize(s.str(), context); } void printPath(Value & v) diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index a9b51afa074..ba98dd66601 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -7,9 +7,10 @@ #include #include - namespace nix { + using json = nlohmann::json; + // TODO: rename. It doesn't print. json printValueAsJSON(EvalState & state, bool strict, Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore) @@ -38,7 +39,7 @@ json printValueAsJSON(EvalState & state, bool strict, case nPath: if (copyToStore) out = state.store->printStorePath( - state.copyPathToStore(context, v.path())); + state.copyPathToStore(context, v.path(), v.determinePos(pos))); else out = v.path().path.abs(); break; diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index 40d08da59ec..cb3e6b691e8 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -57,6 +57,11 @@ NixStringContextElem NixStringContextElem::parse( .drvPath = StorePath { s.substr(1) }, }; } + case '@': { + return NixStringContextElem::Path { + .storePath = StorePath { s.substr(1) }, + }; + } default: { // Ensure no '!' if (s.find("!") != std::string_view::npos) { @@ -100,6 +105,10 @@ std::string NixStringContextElem::to_string() const res += '='; res += d.drvPath.to_string(); }, + [&](const NixStringContextElem::Path & p) { + res += '@'; + res += p.storePath.to_string(); + }, }, raw); return res; diff --git a/src/libfetchers-tests/access-tokens.cc b/src/libfetchers-tests/access-tokens.cc index 93043ba3efd..e7570c31cce 100644 --- a/src/libfetchers-tests/access-tokens.cc +++ b/src/libfetchers-tests/access-tokens.cc @@ -15,10 +15,7 @@ class AccessKeysTest : public ::testing::Test protected: public: - void SetUp() override - { - experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes); - } + void SetUp() override {} void TearDown() override {} }; diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 9a2531ba526..10c21df7a62 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -123,7 +123,8 @@ struct CacheImpl : Cache std::optional lookupStorePath( Key key, - Store & store) override + Store & store, + bool allowInvalid) override { key.second.insert_or_assign("store", store.storeDir); @@ -136,7 +137,7 @@ struct CacheImpl : Cache ResultWithStorePath res2(*res, StorePath(storePathS)); store.addTempRoot(res2.storePath); - if (!store.isValidPath(res2.storePath)) { + if (!allowInvalid && !store.isValidPath(res2.storePath)) { // FIXME: we could try to substitute 'storePath'. debug("ignoring disappeared cache entry '%s:%s' -> '%s'", key.first, @@ -158,7 +159,7 @@ struct CacheImpl : Cache Key key, Store & store) override { - auto res = lookupStorePath(std::move(key), store); + auto res = lookupStorePath(std::move(key), store, false); return res && !res->expired ? res : std::nullopt; } }; diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index f7ab32322ef..d3e416c7fb0 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -4,19 +4,16 @@ namespace nix { -fetchers::Cache::Key makeFetchToStoreCacheKey( - const std::string & name, +fetchers::Cache::Key makeSourcePathToHashCacheKey( const std::string & fingerprint, ContentAddressMethod method, const std::string & path) { - return fetchers::Cache::Key{"fetchToStore", { - {"name", name}, + return fetchers::Cache::Key{"sourcePathToHash", { {"fingerprint", fingerprint}, {"method", std::string{method.render()}}, {"path", path} }}; - } StorePath fetchToStore( @@ -29,38 +26,82 @@ StorePath fetchToStore( PathFilter * filter, RepairFlag repair) { - // FIXME: add an optimisation for the case where the accessor is - // a `PosixSourceAccessor` pointing to a store path. + return fetchToStore2(settings, store, path, mode, name, method, filter, repair).first; +} +std::pair fetchToStore2( + const fetchers::Settings & settings, + Store & store, + const SourcePath & path, + FetchMode mode, + std::string_view name, + ContentAddressMethod method, + PathFilter * filter, + RepairFlag repair) +{ std::optional cacheKey; - if (!filter && path.accessor->fingerprint) { - cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs()); - if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) { - debug("store path cache hit for '%s'", path); - return res->storePath; + auto [subpath, fingerprint] = + filter + ? std::pair>{path.path, std::nullopt} + : path.accessor->getFingerprint(path.path); + + if (fingerprint) { + cacheKey = makeSourcePathToHashCacheKey(*fingerprint, method, subpath.abs()); + if (auto res = settings.getCache()->lookup(*cacheKey)) { + auto hash = Hash::parseSRI(fetchers::getStrAttr(*res, "hash")); + auto storePath = store.makeFixedOutputPathFromCA(name, + ContentAddressWithReferences::fromParts(method, hash, {})); + if (mode == FetchMode::DryRun || store.isValidPath(storePath)) { + debug("source path '%s' cache hit in '%s' (hash '%s')", path, store.printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); + return {storePath, hash}; + } + debug("source path '%s' not in store", path); } - } else + } else { + static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1"; + if (barf) + throw Error("source path '%s' is uncacheable (filter=%d)", path, (bool) filter); + // FIXME: could still provide in-memory caching keyed on `SourcePath`. debug("source path '%s' is uncacheable", path); + } Activity act(*logger, lvlChatty, actUnknown, fmt(mode == FetchMode::DryRun ? "hashing '%s'" : "copying '%s' to the store", path)); auto filter2 = filter ? *filter : defaultPathFilter; - auto storePath = + auto [storePath, hash] = mode == FetchMode::DryRun - ? store.computeStorePath( - name, path, method, HashAlgorithm::SHA256, {}, filter2).first - : store.addToStore( - name, path, method, HashAlgorithm::SHA256, {}, filter2, repair); - - debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath)); + ? ({ + auto [storePath, hash] = store.computeStorePath( + name, path, method, HashAlgorithm::SHA256, {}, filter2); + debug("hashed '%s' to '%s' (hash '%s')", path, store.printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); + std::make_pair(storePath, hash); + }) + : ({ + // FIXME: ideally addToStore() would return the hash + // right away (like computeStorePath()). + auto storePath = store.addToStore( + name, path, method, HashAlgorithm::SHA256, {}, filter2, repair); + auto info = store.queryPathInfo(storePath); + assert(info->references.empty()); + auto hash = + method == ContentAddressMethod::Raw::NixArchive + ? info->narHash + : ({ + if (!info->ca || info->ca->method != method) + throw Error("path '%s' lacks a CA field", store.printStorePath(storePath)); + info->ca->hash; + }); + debug("copied '%s' to '%s' (hash '%s')", path, store.printStorePath(storePath), hash.to_string(HashFormat::SRI, true)); + std::make_pair(storePath, hash); + }); - if (cacheKey && mode == FetchMode::Copy) - settings.getCache()->upsert(*cacheKey, store, {}, storePath); + if (cacheKey) + settings.getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}}); - return storePath; + return {storePath, hash}; } } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 9cb89660172..c947d860a97 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -5,6 +5,7 @@ #include "nix/util/json-utils.hh" #include "nix/fetchers/store-path-accessor.hh" #include "nix/fetchers/fetch-settings.hh" +#include "nix/util/forwarding-source-accessor.hh" #include @@ -189,34 +190,30 @@ bool Input::contains(const Input & other) const } // FIXME: remove -std::pair Input::fetchToStore(ref store) const +std::tuple, Input> Input::fetchToStore(ref store) const { if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); - auto [storePath, input] = [&]() -> std::pair { - try { - auto [accessor, result] = getAccessorUnchecked(store); - - auto storePath = nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName()); + try { + auto [accessor, result] = getAccessorUnchecked(store); - auto narHash = store->queryPathInfo(storePath)->narHash; - result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); + auto storePath = nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName()); - result.attrs.insert_or_assign("__final", Explicit(true)); + auto narHash = store->queryPathInfo(storePath)->narHash; + result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); - assert(result.isFinal()); + result.attrs.insert_or_assign("__final", Explicit(true)); - checkLocks(*this, result); + assert(result.isFinal()); - return {storePath, result}; - } catch (Error & e) { - e.addTrace({}, "while fetching the input '%s'", to_string()); - throw; - } - }(); + checkLocks(*this, result); - return {std::move(storePath), input}; + return {std::move(storePath), accessor, result}; + } catch (Error & e) { + e.addTrace({}, "while fetching the input '%s'", to_string()); + throw; + } } void Input::checkLocks(Input specified, Input & result) @@ -234,6 +231,9 @@ void Input::checkLocks(Input specified, Input & result) if (auto prevNarHash = specified.getNarHash()) specified.attrs.insert_or_assign("narHash", prevNarHash->to_string(HashFormat::SRI, true)); + if (auto narHash = result.getNarHash()) + result.attrs.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); + for (auto & field : specified.attrs) { auto field2 = result.attrs.find(field.first); if (field2 != result.attrs.end() && field.second != field2->second) @@ -294,6 +294,21 @@ std::pair, Input> Input::getAccessor(ref store) const } } +/** + * Helper class that ensures that paths in substituted source trees + * are rendered as `«input»/path` rather than + * `«input»/nix/store/-source/path`. + */ +struct SubstitutedSourceAccessor : ForwardingSourceAccessor +{ + using ForwardingSourceAccessor::ForwardingSourceAccessor; + + std::string showPath(const CanonPath & path) override + { + return displayPrefix + path.abs() + displaySuffix; + } +}; + std::pair, Input> Input::getAccessorUnchecked(ref store) const { // FIXME: cache the accessor @@ -321,10 +336,12 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath)); - auto accessor = makeStorePathAccessor(store, storePath); + auto accessor = make_ref(makeStorePathAccessor(store, storePath)); accessor->fingerprint = getFingerprint(store); + // FIXME: ideally we would use the `showPath()` of the + // "real" accessor for this fetcher type. accessor->setPathDisplay("«" + to_string() + "»"); return {accessor, *this}; @@ -335,8 +352,10 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto auto [accessor, result] = scheme->getAccessor(store, *this); - assert(!accessor->fingerprint); - accessor->fingerprint = result.getFingerprint(store); + if (!accessor->fingerprint) + accessor->fingerprint = result.getFingerprint(store); + else + result.cachedFingerprint = accessor->fingerprint; return {accessor, std::move(result)}; } diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index 72a3fb4ebad..c339cdbdb48 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -14,15 +14,26 @@ std::string FilteringSourceAccessor::readFile(const CanonPath & path) return next->readFile(prefix / path); } +void FilteringSourceAccessor::readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) +{ + checkAccess(path); + return next->readFile(prefix / path, sink, sizeCallback); +} + bool FilteringSourceAccessor::pathExists(const CanonPath & path) { return isAllowed(path) && next->pathExists(prefix / path); } std::optional FilteringSourceAccessor::maybeLstat(const CanonPath & path) +{ + return isAllowed(path) ? next->maybeLstat(prefix / path) : std::nullopt; +} + +SourceAccessor::Stat FilteringSourceAccessor::lstat(const CanonPath & path) { checkAccess(path); - return next->maybeLstat(prefix / path); + return next->lstat(prefix / path); } SourceAccessor::DirEntries FilteringSourceAccessor::readDirectory(const CanonPath & path) @@ -47,6 +58,13 @@ std::string FilteringSourceAccessor::showPath(const CanonPath & path) return displayPrefix + next->showPath(prefix / path) + displaySuffix; } +std::pair> FilteringSourceAccessor::getFingerprint(const CanonPath & path) +{ + if (fingerprint) + return {path, fingerprint}; + return next->getFingerprint(prefix / path); +} + void FilteringSourceAccessor::checkAccess(const CanonPath & path) { if (!isAllowed(path)) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 9fe271fe8ce..8a10517facf 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1252,9 +1252,8 @@ ref GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool export std::unordered_set{CanonPath::root}, std::move(makeNotAllowedError)).cast(); if (exportIgnore) - return make_ref(self, fileAccessor, std::nullopt); - else - return fileAccessor; + fileAccessor = make_ref(self, fileAccessor, std::nullopt); + return fileAccessor; } ref GitRepoImpl::getFileSystemObjectSink() diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index cf255c00183..7b403794164 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -15,6 +15,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/util/json-utils.hh" #include "nix/util/archive.hh" +#include "nix/util/mounted-source-accessor.hh" #include #include @@ -390,10 +391,10 @@ struct GitInputScheme : InputScheme { if (workdirInfo.isDirty) { if (!settings.allowDirty) - throw Error("Git tree '%s' is dirty", locationToArg()); + throw Error("Git tree '%s' has uncommitted changes", locationToArg()); if (settings.warnDirty) - warn("Git tree '%s' is dirty", locationToArg()); + warn("Git tree '%s' has uncommitted changes", locationToArg()); } } @@ -861,7 +862,7 @@ struct GitInputScheme : InputScheme return makeFingerprint(*rev); else { auto repoInfo = getRepoInfo(input); - if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) { + if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.submodules.empty()) { /* Calculate a fingerprint that takes into account the deleted and modified/added files. */ HashSink hashSink{HashAlgorithm::SHA512}; @@ -874,7 +875,7 @@ struct GitInputScheme : InputScheme writeString("deleted:", hashSink); writeString(file.abs(), hashSink); } - return makeFingerprint(*repoInfo.workdirInfo.headRev) + return makeFingerprint(repoInfo.workdirInfo.headRev.value_or(nullRev)) + ";d=" + hashSink.finish().first.to_string(HashFormat::Base16, false); } return std::nullopt; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 7a902d816d0..0888d387c00 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -335,6 +335,13 @@ struct GitArchiveInputScheme : InputScheme false, "«" + input.to_string() + "»"); + if (!input.settings->trustTarballsFromGitForges) + // FIXME: computing the NAR hash here is wasteful if + // copyInputToStore() is just going to hash/copy it as + // well. + input.attrs.insert_or_assign("narHash", + accessor->hashPath(CanonPath::root).to_string(HashFormat::SRI, true)); + return {accessor, input}; } @@ -349,11 +356,6 @@ struct GitArchiveInputScheme : InputScheme input.getNarHash().has_value()); } - std::optional experimentalFeature() const override - { - return Xp::Flakes; - } - std::optional getFingerprint(ref store, const Input & input) const override { if (auto rev = input.getRev()) diff --git a/src/libfetchers/include/nix/fetchers/cache.hh b/src/libfetchers/include/nix/fetchers/cache.hh index 6ac693183f9..3f3089d3f19 100644 --- a/src/libfetchers/include/nix/fetchers/cache.hh +++ b/src/libfetchers/include/nix/fetchers/cache.hh @@ -76,11 +76,12 @@ struct Cache /** * Look up a store path in the cache. The returned store path will - * be valid, but it may be expired. + * be valid (unless `allowInvalid` is true), but it may be expired. */ virtual std::optional lookupStorePath( Key key, - Store & store) = 0; + Store & store, + bool allowInvalid = false) = 0; /** * Look up a store path in the cache. Return nothing if its TTL diff --git a/src/libfetchers/include/nix/fetchers/fetch-settings.hh b/src/libfetchers/include/nix/fetchers/fetch-settings.hh index 9cfd25e0b83..b055fd0e9e3 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-settings.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-settings.hh @@ -87,10 +87,7 @@ struct Settings : public Config are subsequently modified. Therefore lock files with dirty locks should generally only be used for local testing, and should not be pushed to other users. - )", - {}, - true, - Xp::Flakes}; + )"}; Setting trustTarballsFromGitForges{ this, true, "trust-tarballs-from-git-forges", @@ -112,8 +109,7 @@ struct Settings : public Config Path or URI of the global flake registry. When empty, disables the global flake registry. - )", - {}, true, Xp::Flakes}; + )"}; ref getCache() const; diff --git a/src/libfetchers/include/nix/fetchers/fetch-to-store.hh b/src/libfetchers/include/nix/fetchers/fetch-to-store.hh index a52d567ecfb..753bf8c675c 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-to-store.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-to-store.hh @@ -24,7 +24,17 @@ StorePath fetchToStore( PathFilter * filter = nullptr, RepairFlag repair = NoRepair); -fetchers::Cache::Key makeFetchToStoreCacheKey( - const std::string & name, const std::string & fingerprint, ContentAddressMethod method, const std::string & path); +std::pair fetchToStore2( + const fetchers::Settings & settings, + Store & store, + const SourcePath & path, + FetchMode mode, + std::string_view name = "source", + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, + PathFilter * filter = nullptr, + RepairFlag repair = NoRepair); + +fetchers::Cache::Key +makeSourcePathToHashCacheKey(const std::string & fingerprint, ContentAddressMethod method, const std::string & path); } diff --git a/src/libfetchers/include/nix/fetchers/fetchers.hh b/src/libfetchers/include/nix/fetchers/fetchers.hh index 1f8f6bdacd6..cd096b29a43 100644 --- a/src/libfetchers/include/nix/fetchers/fetchers.hh +++ b/src/libfetchers/include/nix/fetchers/fetchers.hh @@ -121,7 +121,7 @@ public: * Fetch the entire input into the Nix store, returning the * location in the Nix store and the locked input. */ - std::pair fetchToStore(ref store) const; + std::tuple, Input> fetchToStore(ref store) const; /** * Check the locking attributes in `result` against diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index 2b59f03ca22..e0228ad9bb6 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -36,8 +36,12 @@ struct FilteringSourceAccessor : SourceAccessor std::string readFile(const CanonPath & path) override; + void readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) override; + bool pathExists(const CanonPath & path) override; + Stat lstat(const CanonPath & path) override; + std::optional maybeLstat(const CanonPath & path) override; DirEntries readDirectory(const CanonPath & path) override; @@ -46,6 +50,8 @@ struct FilteringSourceAccessor : SourceAccessor std::string showPath(const CanonPath & path) override; + std::pair> getFingerprint(const CanonPath & path) override; + /** * Call `makeNotAllowedError` to throw a `RestrictedPathError` * exception if `isAllowed()` returns `false` for `path`. diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 47cb7587cf7..0ff05af0331 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -106,11 +106,6 @@ struct IndirectInputScheme : InputScheme throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); } - std::optional experimentalFeature() const override - { - return Xp::Flakes; - } - bool isDirect(const Input & input) const override { return false; } }; diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index 14592087999..b6b061e2d9f 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -17,7 +17,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-fetchers"; + pname = "determinate-nix-fetchers"; inherit version; workDir = ./.; diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 9239fd27466..e9f205543ae 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -128,8 +128,6 @@ struct PathInputScheme : InputScheme auto absPath = getAbsPath(input); - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath)); - // FIXME: check whether access to 'path' is allowed. auto storePath = store->maybeParseStorePath(absPath.string()); @@ -138,6 +136,7 @@ struct PathInputScheme : InputScheme time_t mtime = 0; if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath)); // FIXME: try to substitute storePath. auto src = sinkToSource([&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); @@ -145,42 +144,22 @@ struct PathInputScheme : InputScheme storePath = store->addToStoreFromDump(*src, "source"); } - // To avoid copying the path again to the /nix/store, we need to add a cache entry. - ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive; - auto fp = getFingerprint(store, input); - if (fp) { - auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/"); - input.settings->getCache()->upsert(cacheKey, *store, {}, *storePath); - } + auto accessor = makeStorePathAccessor(store, *storePath); + + // To prevent `fetchToStore()` copying the path again to Nix + // store, pre-create an entry in the fetcher cache. + auto info = store->queryPathInfo(*storePath); + accessor->fingerprint = fmt("path:%s", store->queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true)); + input.settings->getCache()->upsert( + makeSourcePathToHashCacheKey(*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"), + {{"hash", info->narHash.to_string(HashFormat::SRI, true)}}); /* Trust the lastModified value supplied by the user, if any. It's not a "secure" attribute so we don't care. */ if (!input.getLastModified()) input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); - return {makeStorePathAccessor(store, *storePath), std::move(input)}; - } - - std::optional getFingerprint(ref store, const Input & input) const override - { - if (isRelative(input)) - return std::nullopt; - - /* If this path is in the Nix store, use the hash of the - store object and the subpath. */ - auto path = getAbsPath(input); - try { - auto [storePath, subPath] = store->toStorePath(path.string()); - auto info = store->queryPathInfo(storePath); - return fmt("path:%s:%s", info->narHash.to_string(HashFormat::Base16, false), subPath); - } catch (Error &) { - return std::nullopt; - } - } - - std::optional experimentalFeature() const override - { - return Xp::Flakes; + return {accessor, std::move(input)}; } }; diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix index 8c6883d9cf9..9ae3ec69515 100644 --- a/src/libflake-c/package.nix +++ b/src/libflake-c/package.nix @@ -17,7 +17,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-flake-c"; + pname = "determinate-nix-flake-c"; inherit version; workDir = ./.; diff --git a/src/libflake-tests/flakeref.cc b/src/libflake-tests/flakeref.cc index 1abaffb96a5..ccef8f37919 100644 --- a/src/libflake-tests/flakeref.cc +++ b/src/libflake-tests/flakeref.cc @@ -8,8 +8,6 @@ namespace nix { /* ----------- tests for flake/flakeref.hh --------------------------------------------------*/ TEST(parseFlakeRef, path) { - experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes); - fetchers::Settings fetchSettings; { diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index 593b0e18d21..b7a48b89e56 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -59,7 +59,7 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', - 'NIX_CONFIG': 'extra-experimental-features = flakes', + 'HOME': meson.current_build_dir() / 'test-home', }, protocol : 'gtest', ) diff --git a/src/libflake-tests/package.nix b/src/libflake-tests/package.nix index 714f3791ad9..8344d98d75c 100644 --- a/src/libflake-tests/package.nix +++ b/src/libflake-tests/package.nix @@ -56,18 +56,13 @@ mkMesonExecutable (finalAttrs: { { meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; } - ( - lib.optionalString stdenv.hostPlatform.isWindows '' - export HOME="$PWD/home-dir" - mkdir -p "$HOME" - '' - + '' - export _NIX_TEST_UNIT_DATA=${resolvePath ./data} - export NIX_CONFIG="extra-experimental-features = flakes" - ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} - touch $out - '' - ); + ('' + export _NIX_TEST_UNIT_DATA=${resolvePath ./data} + export HOME="$TMPDIR/home" + mkdir -p "$HOME" + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} + touch $out + ''); }; }; diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index 7c5ce01b269..703463141f3 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -52,7 +52,6 @@ PrimOp getFlake(const Settings & settings) ``` )", .fun = prim_getFlake, - .experimentalFeature = Xp::Flakes, }; } @@ -94,7 +93,6 @@ nix::PrimOp parseFlakeRef({ ``` )", .fun = prim_parseFlakeRef, - .experimentalFeature = Xp::Flakes, }); static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -154,7 +152,6 @@ nix::PrimOp flakeRefToString({ ``` )", .fun = prim_flakeRefToString, - .experimentalFeature = Xp::Flakes, }); } // namespace nix::flake::primops diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 322abaa4a52..a82b9d9c0f4 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -14,6 +14,7 @@ #include "nix/store/local-fs-store.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/memory-source-accessor.hh" +#include "nix/util/mounted-source-accessor.hh" #include "nix/fetchers/input-cache.hh" #include @@ -21,27 +22,10 @@ namespace nix { using namespace flake; +using namespace fetchers; namespace flake { -static StorePath copyInputToStore( - EvalState & state, - fetchers::Input & input, - const fetchers::Input & originalInput, - ref accessor) -{ - auto storePath = fetchToStore(*input.settings, *state.store, accessor, FetchMode::Copy, input.getName()); - - state.allowPath(storePath); - - auto narHash = state.store->queryPathInfo(storePath)->narHash; - input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); - - assert(!originalInput.getNarHash() || storePath == originalInput.computeStorePath(*state.store)); - - return storePath; -} - static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) { if (value.isThunk() && value.isTrivial()) @@ -67,7 +51,7 @@ static std::pair, fetchers::Attrs> parseFlakeInput static void parseFlakeInputAttr( EvalState & state, - const Attr & attr, + const nix::Attr & attr, fetchers::Attrs & attrs) { // Allow selecting a subset of enum values @@ -336,7 +320,8 @@ static Flake getFlake( EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries, - const InputAttrPath & lockRootAttrPath) + const InputAttrPath & lockRootAttrPath, + bool requireLockable) { // Fetch a lazy tree first. auto cachedInput = state.inputCache->getAccessor(state.store, originalRef.input, useRegistries); @@ -359,16 +344,16 @@ static Flake getFlake( lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir); } - // Copy the tree to the store. - auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, cachedInput.accessor); - // Re-parse flake.nix from the store. - return readFlake(state, originalRef, resolvedRef, lockedRef, state.storePath(storePath), lockRootAttrPath); + return readFlake( + state, originalRef, resolvedRef, lockedRef, + state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor, requireLockable)), + lockRootAttrPath); } -Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries) +Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries, bool requireLockable) { - return getFlake(state, originalRef, useRegistries, {}); + return getFlake(state, originalRef, useRegistries, {}, requireLockable); } static LockFile readLockFile( @@ -388,8 +373,6 @@ LockedFlake lockFlake( const FlakeRef & topRef, const LockFlags & lockFlags) { - experimentalFeatureSettings.require(Xp::Flakes); - auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No; auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No; @@ -398,7 +381,8 @@ LockedFlake lockFlake( state, topRef, useRegistriesTop, - {}); + {}, + lockFlags.requireLockable); if (lockFlags.applyNixConfig) { flake.config.apply(settings); @@ -586,8 +570,9 @@ LockedFlake lockFlake( return getFlake( state, ref, - useRegistries, - inputAttrPath); + useRegistriesInputs, + inputAttrPath, + true); } }; @@ -696,6 +681,29 @@ LockedFlake lockFlake( auto inputIsOverride = explicitCliOverrides.contains(inputAttrPath); auto ref = (input2.ref && inputIsOverride) ? *input2.ref : *input.ref; + /* Warn against the use of indirect flakerefs + (but only at top-level since we don't want + to annoy users about flakes that are not + under their control). */ + auto warnRegistry = [&](const FlakeRef & resolvedRef) + { + if (inputAttrPath.size() == 1 && !input.ref->input.isDirect()) { + std::ostringstream s; + printLiteralString(s, resolvedRef.to_string()); + warn( + "Flake input '%1%' uses the flake registry. " + "Using the registry in flake inputs is deprecated in Determinate Nix. " + "To make your flake future-proof, add the following to '%2%':\n" + "\n" + " inputs.%1%.url = %3%;\n" + "\n" + "For more information, see: https://github.com/DeterminateSystems/nix-src/issues/37", + inputAttrPathS, + flake.path, + s.str()); + } + }; + if (input.isFlake) { auto inputFlake = getInputFlake(*input.ref, inputIsOverride ? fetchers::UseRegistries::All : useRegistriesInputs); @@ -715,18 +723,16 @@ LockedFlake lockFlake( Finally cleanup([&]() { parents.pop_back(); }); /* Recursively process the inputs of this - flake. Also, unless we already have this flake - in the top-level lock file, use this flake's - own lock file. */ + flake, using its own lock file. */ nodePaths.emplace(childNode, inputFlake.path.parent()); computeLocks( inputFlake.inputs, childNode, inputAttrPath, - oldLock - ? std::dynamic_pointer_cast(oldLock) - : readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), - oldLock ? followsPrefix : inputAttrPath, + readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), + inputAttrPath, inputFlake.path, false); + + warnRegistry(inputFlake.resolvedRef); } else { @@ -736,17 +742,17 @@ LockedFlake lockFlake( if (auto resolvedPath = resolveRelativePath()) { return {*resolvedPath, *input.ref}; } else { - auto cachedInput = state.inputCache->getAccessor( - state.store, - input.ref->input, - useRegistriesInputs); + auto cachedInput = state.inputCache->getAccessor(state.store, input.ref->input, useRegistriesTop); + auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), input.ref->subdir); auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); - // FIXME: allow input to be lazy. - auto storePath = copyInputToStore(state, lockedRef.input, input.ref->input, cachedInput.accessor); + warnRegistry(resolvedRef); - return {state.storePath(storePath), lockedRef}; + return { + state.storePath(state.mountInput(lockedRef.input, input.ref->input, cachedInput.accessor, true)), + lockedRef + }; } }(); @@ -859,7 +865,8 @@ LockedFlake lockFlake( flake = getFlake( state, topRef, - useRegistriesTop); + useRegistriesTop, + lockFlags.requireLockable); if (lockFlags.commitLockFile && flake.lockedRef.input.getRev() && @@ -909,8 +916,6 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes) { - experimentalFeatureSettings.require(Xp::Flakes); - auto [lockFileStr, keyMap] = lockedFlake.lockFile.to_string(); auto overrides = state.buildBindings(lockedFlake.nodePaths.size()); diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index ed34aa9c8db..8481aaa199e 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -115,7 +115,11 @@ struct Flake } }; -Flake getFlake(EvalState & state, const FlakeRef & flakeRef, fetchers::UseRegistries useRegistries); +Flake getFlake( + EvalState & state, + const FlakeRef & flakeRef, + fetchers::UseRegistries useRegistries, + bool requireLockable = true); /** * Fingerprint of a locked flake; used as a cache key. @@ -213,6 +217,11 @@ struct LockFlags * for those inputs will be ignored. */ std::set inputUpdates; + + /** + * Whether to require a locked input. + */ + bool requireLockable = true; }; LockedFlake lockFlake( diff --git a/src/libflake/include/nix/flake/settings.hh b/src/libflake/include/nix/flake/settings.hh index b3bffad4ccf..c9f82218c95 100644 --- a/src/libflake/include/nix/flake/settings.hh +++ b/src/libflake/include/nix/flake/settings.hh @@ -20,13 +20,7 @@ struct Settings : public Config void configureEvalSettings(nix::EvalSettings & evalSettings) const; Setting useRegistries{ - this, - true, - "use-registries", - "Whether to use flake registries to resolve flake references.", - {}, - true, - Xp::Flakes}; + this, true, "use-registries", "Whether to use flake registries to resolve flake references.", {}, true}; Setting acceptFlakeConfig{ this, @@ -34,8 +28,7 @@ struct Settings : public Config "accept-flake-config", "Whether to accept Nix configuration settings from a flake without prompting.", {}, - true, - Xp::Flakes}; + true}; Setting commitLockFileSummary{ this, @@ -46,8 +39,7 @@ struct Settings : public Config empty, the summary is generated based on the action performed. )", {"commit-lockfile-summary"}, - true, - Xp::Flakes}; + true}; }; } diff --git a/src/libflake/package.nix b/src/libflake/package.nix index dd442a44ec9..2b0c827a09c 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -18,7 +18,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-flake"; + pname = "determinate-nix-flake"; inherit version; workDir = ./.; diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index f019a917d36..17858d56f2e 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -17,7 +17,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-main-c"; + pname = "determinate-nix-main-c"; inherit version; workDir = ./.; diff --git a/src/libmain/package.nix b/src/libmain/package.nix index 7b0a4dee7da..119e1f1aca5 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -18,7 +18,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-main"; + pname = "determinate-nix-main"; inherit version; workDir = ./.; diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 0982810d1a7..853554099af 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -285,7 +285,7 @@ void parseCmdLine(const std::string & programName, const Strings & args, void printVersion(const std::string & programName) { - std::cout << fmt("%1% (Nix) %2%", programName, nixVersion) << std::endl; + std::cout << fmt("%s (Determinate Nix %s) %s", programName, determinateNixVersion, nixVersion) << std::endl; if (verbosity > lvlInfo) { Strings cfg; #if NIX_USE_BOEHMGC diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index fde17c78e01..0ce37e44c01 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -15,7 +15,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-store-c"; + pname = "determinate-nix-store-c"; inherit version; workDir = ./.; diff --git a/src/libstore-test-support/package.nix b/src/libstore-test-support/package.nix index 391ddeefda2..2561dd791eb 100644 --- a/src/libstore-test-support/package.nix +++ b/src/libstore-test-support/package.nix @@ -18,7 +18,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-store-test-support"; + pname = "determinate-nix-store-test-support"; inherit version; workDir = ./.; diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 8a1ff40f074..8b9893b2335 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -100,6 +100,7 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + 'HOME': meson.current_build_dir() / 'test-home', }, protocol : 'gtest', ) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 3d9f7908b3f..05373cb8847 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -28,10 +28,6 @@ TEST_F(nix_api_store_test, nix_store_get_uri) TEST_F(nix_api_util_context, nix_store_get_storedir_default) { - if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { - // skipping test in sandbox because nix_store_open tries to create /nix/var/nix/profiles - GTEST_SKIP(); - } nix_libstore_init(ctx); Store * store = nix_store_open(ctx, nullptr, nullptr); assert_ctx_ok(); @@ -141,10 +137,6 @@ TEST_F(nix_api_store_test, nix_store_real_path) TEST_F(nix_api_util_context, nix_store_real_path_relocated) { - if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { - // Can't open default store from within sandbox - GTEST_SKIP(); - } auto tmp = nix::createTempDir(); std::string storeRoot = tmp + "/store"; std::string stateDir = tmp + "/state"; @@ -184,13 +176,7 @@ TEST_F(nix_api_util_context, nix_store_real_path_relocated) TEST_F(nix_api_util_context, nix_store_real_path_binary_cache) { - if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { - // TODO: override NIX_CACHE_HOME? - // skipping test in sandbox because narinfo cache can't be written - GTEST_SKIP(); - } - - Store * store = nix_store_open(ctx, "https://cache.nixos.org", nullptr); + Store * store = nix_store_open(ctx, nix::fmt("file://%s/binary-cache", nix::createTempDir()).c_str(), nullptr); assert_ctx_ok(); ASSERT_NE(store, nullptr); diff --git a/src/libstore-tests/package.nix b/src/libstore-tests/package.nix index b39ee7fa73c..1f3701c7fc6 100644 --- a/src/libstore-tests/package.nix +++ b/src/libstore-tests/package.nix @@ -73,17 +73,13 @@ mkMesonExecutable (finalAttrs: { { meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; } - ( - lib.optionalString stdenv.hostPlatform.isWindows '' - export HOME="$PWD/home-dir" - mkdir -p "$HOME" - '' - + '' - export _NIX_TEST_UNIT_DATA=${data + "/src/libstore-tests/data"} - ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} - touch $out - '' - ); + ('' + export _NIX_TEST_UNIT_DATA=${data + "/src/libstore-tests/data"} + export HOME="$TMPDIR/home" + mkdir -p "$HOME" + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} + touch $out + ''); }; }; diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index 09166133786..26442461344 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -1,8 +1,49 @@ #include "nix/store/build-result.hh" +#include + namespace nix { bool BuildResult::operator==(const BuildResult &) const noexcept = default; std::strong_ordering BuildResult::operator<=>(const BuildResult &) const noexcept = default; +void to_json(nlohmann::json & json, const BuildResult & buildResult) +{ + json = nlohmann::json::object(); + json["status"] = BuildResult::statusToString(buildResult.status); + if (buildResult.errorMsg != "") + json["errorMsg"] = buildResult.errorMsg; + if (buildResult.timesBuilt) + json["timesBuilt"] = buildResult.timesBuilt; + if (buildResult.isNonDeterministic) + json["isNonDeterministic"] = buildResult.isNonDeterministic; + if (buildResult.startTime) + json["startTime"] = buildResult.startTime; + if (buildResult.stopTime) + json["stopTime"] = buildResult.stopTime; +} + +void to_json(nlohmann::json & json, const KeyedBuildResult & buildResult) +{ + to_json(json, (const BuildResult &) buildResult); + auto path = nlohmann::json::object(); + std::visit( + overloaded{ + [&](const DerivedPathOpaque & opaque) { path["opaque"] = opaque.path.to_string(); }, + [&](const DerivedPathBuilt & drv) { + path["drvPath"] = drv.drvPath->getBaseStorePath().to_string(); + path["outputs"] = drv.outputs; + auto outputs = nlohmann::json::object(); + for (auto & [name, output] : buildResult.builtOutputs) + outputs[name] = { + {"path", output.outPath.to_string()}, + {"signatures", output.signatures}, + }; + json["builtOutputs"] = std::move(outputs); + }, + }, + buildResult.path.raw()); + json["path"] = std::move(path); +} + } diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index 9a91b3592ce..53b5f7eb3b1 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -632,6 +632,7 @@ Goal::Co DerivationBuildingGoal::tryToBuild() *drvOptions, inputPaths, initialOutputs, + act }); } @@ -753,9 +754,7 @@ void DerivationBuildingGoal::appendLogTailErrorMsg(std::string & msg) msg += line; msg += "\n"; } - auto nixLogCommand = experimentalFeatureSettings.isEnabled(Xp::NixCommand) - ? "nix log" - : "nix-store -l"; + auto nixLogCommand = "nix log"; // The command is on a separate line for easy copying, such as with triple click. // This message will be indented elsewhere, so removing the indentation before the // command will not put it at the start of the line unfortunately. @@ -1244,6 +1243,14 @@ Goal::Done DerivationBuildingGoal::done( fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; } + logger->result( + act ? act->id : getCurActivity(), + resBuildResult, + nlohmann::json( + KeyedBuildResult( + buildResult, + DerivedPath::Built{.drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All{}}))); + return amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3fcc376ed95..9d0ec21ba7b 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -572,6 +572,14 @@ Goal::Done DerivationGoal::done( fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; } + logger->result( + getCurActivity(), + resBuildResult, + nlohmann::json( + KeyedBuildResult( + buildResult, + DerivedPath::Built{.drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All{}}))); + return amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); } diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 9ffc8219d97..428fec25b1d 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -4,8 +4,11 @@ #include "nix/store/nar-info.hh" #include "nix/util/finally.hh" #include "nix/util/signals.hh" + #include +#include + namespace nix { PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) @@ -36,6 +39,15 @@ Goal::Done PathSubstitutionGoal::done( debug(*errorMsg); buildResult.errorMsg = *errorMsg; } + + logger->result( + getCurActivity(), + resBuildResult, + nlohmann::json( + KeyedBuildResult( + buildResult, + DerivedPath::Opaque{storePath}))); + return amDone(result); } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index bf4a9d95906..b946ccbb519 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -730,6 +730,7 @@ static void performOp(TunnelLogger * logger, ref store, options.action = (GCOptions::GCAction) readInt(conn.from); options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); conn.from >> options.ignoreLiveness >> options.maxFreed; + options.censor = !trusted; // obsolete fields readInt(conn.from); readInt(conn.from); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 7e29d00e6c0..50e0fcf2a0f 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -345,7 +345,9 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_MAXREDIRS, 10); curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(req, CURLOPT_USERAGENT, - ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + + ("curl/" LIBCURL_VERSION + " Nix/" + nixVersion + + " DeterminateNix/" + determinateNixVersion + (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : "")).c_str()); #if LIBCURL_VERSION_NUM >= 0x072b00 curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 48467381d3f..d1bbe1571d6 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -208,7 +208,7 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) while ((end = contents.find((char) 0, pos)) != std::string::npos) { Path root(contents, pos, end - pos); debug("got temporary root '%s'", root); - tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid)); + tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{nix-process:%d}", pid)); pos = end + 1; } } @@ -458,13 +458,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) bool gcKeepOutputs = settings.gcKeepOutputs; bool gcKeepDerivations = settings.gcKeepDerivations; - std::unordered_set roots, dead, alive; + Roots roots; + std::unordered_set dead, alive; struct Shared { // The temp roots only store the hash part to make it easier to // ignore suffixes like '.lock', '.chroot' and '.check'. - std::unordered_set tempRoots; + std::unordered_map tempRoots; // Hash part of the store path currently being deleted, if // any. @@ -573,7 +574,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) debug("got new GC root '%s'", path); auto hashPart = std::string(storePath->hashPart()); auto shared(_shared.lock()); - shared->tempRoots.insert(hashPart); + // FIXME: could get the PID from the socket. + shared->tempRoots.insert_or_assign(hashPart, "{nix-process:unknown}"); /* If this path is currently being deleted, then we have to wait until deletion is finished to ensure that @@ -612,19 +614,18 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ printInfo("finding garbage collector roots..."); - Roots rootMap; if (!options.ignoreLiveness) - findRootsNoTemp(rootMap, true); - - for (auto & i : rootMap) roots.insert(i.first); + findRootsNoTemp(roots, options.censor); /* Read the temporary roots created before we acquired the global GC root. Any new roots will be sent to our socket. */ - Roots tempRoots; - findTempRoots(tempRoots, true); - for (auto & root : tempRoots) { - _shared.lock()->tempRoots.insert(std::string(root.first.hashPart())); - roots.insert(root.first); + { + Roots tempRoots; + findTempRoots(tempRoots, options.censor); + for (auto & root : tempRoots) + _shared.lock()->tempRoots.insert_or_assign( + std::string(root.first.hashPart()), + *root.second.begin()); } /* Synchronisation point for testing, see tests/functional/gc-non-blocking.sh. */ @@ -716,21 +717,35 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } catch (InvalidPath &) { } }; + if (options.action == GCOptions::gcDeleteSpecific + && !options.pathsToDelete.count(*path)) + { + throw Error( + "Cannot delete path '%s' because it's referenced by path '%s'.", + printStorePath(start), + printStorePath(*path)); + } + /* If this is a root, bail out. */ - if (roots.count(*path)) { + if (auto i = roots.find(*path); i != roots.end()) { + if (options.action == GCOptions::gcDeleteSpecific) + throw Error( + "Cannot delete path '%s' because it's referenced by the GC root '%s'.", + printStorePath(start), + *i->second.begin()); debug("cannot delete '%s' because it's a root", printStorePath(*path)); return markAlive(); } - if (options.action == GCOptions::gcDeleteSpecific - && !options.pathsToDelete.count(*path)) - return; - { auto hashPart = std::string(path->hashPart()); auto shared(_shared.lock()); - if (shared->tempRoots.count(hashPart)) { - debug("cannot delete '%s' because it's a temporary root", printStorePath(*path)); + if (auto i = shared->tempRoots.find(hashPart); i != shared->tempRoots.end()) { + if (options.action == GCOptions::gcDeleteSpecific) + throw Error( + "Cannot delete path '%s' because it's in use by '%s'.", + printStorePath(start), + i->second); return markAlive(); } shared->pending = hashPart; @@ -789,12 +804,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) for (auto & i : options.pathsToDelete) { deleteReferrersClosure(i); - if (!dead.count(i)) - throw Error( - "Cannot delete path '%1%' since it is still alive. " - "To find out why, use: " - "nix-store --query --roots and nix-store --query --referrers", - printStorePath(i)); + assert(dead.count(i)); } } else if (options.maxFreed > 0) { diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index de512834783..9f51d90d92f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -244,6 +244,8 @@ Path Settings::getDefaultSSLCertFile() std::string nixVersion = PACKAGE_VERSION; +const std::string determinateNixVersion = DETERMINATE_NIX_VERSION; + NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { {SandboxMode::smEnabled, true}, {SandboxMode::smRelaxed, "relaxed"}, @@ -307,6 +309,21 @@ unsigned int MaxBuildJobsSetting::parse(const std::string & str) const } } +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Settings::ExternalBuilder, systems, program, args); + +template<> Settings::ExternalBuilders BaseSetting::parse(const std::string & str) const +{ + try { + return nlohmann::json::parse(str).template get(); + } catch (std::exception & e) { + throw UsageError("parsing setting '%s': %s", name, e.what()); + } +} + +template<> std::string BaseSetting::to_string() const +{ + return nlohmann::json(value).dump(); +} static void preloadNSS() { diff --git a/src/libstore/include/nix/store/build-result.hh b/src/libstore/include/nix/store/build-result.hh index 088b057b65c..23ced29cb4e 100644 --- a/src/libstore/include/nix/store/build-result.hh +++ b/src/libstore/include/nix/store/build-result.hh @@ -8,6 +8,8 @@ #include #include +#include + namespace nix { struct BuildResult @@ -46,28 +48,32 @@ struct BuildResult */ std::string errorMsg; + static std::string_view statusToString(Status status) + { + switch (status) { + case Built: return "Built"; + case Substituted: return "Substituted"; + case AlreadyValid: return "AlreadyValid"; + case PermanentFailure: return "PermanentFailure"; + case InputRejected: return "InputRejected"; + case OutputRejected: return "OutputRejected"; + case TransientFailure: return "TransientFailure"; + case CachedFailure: return "CachedFailure"; + case TimedOut: return "TimedOut"; + case MiscFailure: return "MiscFailure"; + case DependencyFailed: return "DependencyFailed"; + case LogLimitExceeded: return "LogLimitExceeded"; + case NotDeterministic: return "NotDeterministic"; + case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid"; + case NoSubstituters: return "NoSubstituters"; + default: return "Unknown"; + }; + } + std::string toString() const { - auto strStatus = [&]() { - switch (status) { - case Built: return "Built"; - case Substituted: return "Substituted"; - case AlreadyValid: return "AlreadyValid"; - case PermanentFailure: return "PermanentFailure"; - case InputRejected: return "InputRejected"; - case OutputRejected: return "OutputRejected"; - case TransientFailure: return "TransientFailure"; - case CachedFailure: return "CachedFailure"; - case TimedOut: return "TimedOut"; - case MiscFailure: return "MiscFailure"; - case DependencyFailed: return "DependencyFailed"; - case LogLimitExceeded: return "LogLimitExceeded"; - case NotDeterministic: return "NotDeterministic"; - case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid"; - case NoSubstituters: return "NoSubstituters"; - default: return "Unknown"; - }; - }(); - return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg); + return + std::string(statusToString(status)) + + ((errorMsg == "") ? "" : " : " + errorMsg); } /** @@ -130,4 +136,7 @@ struct KeyedBuildResult : BuildResult { } }; +void to_json(nlohmann::json & json, const BuildResult & buildResult); +void to_json(nlohmann::json & json, const KeyedBuildResult & buildResult); + } diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index 8b25ec8d4cb..8d9a83e67ac 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -7,8 +7,11 @@ namespace nix { +// FIXME: should turn this into an std::variant to represent the +// several root types. +using GcRootInfo = std::string; -typedef std::unordered_map> Roots; +typedef std::unordered_map> Roots; struct GCOptions @@ -53,6 +56,12 @@ struct GCOptions * Stop after at least `maxFreed` bytes have been freed. */ uint64_t maxFreed{std::numeric_limits::max()}; + + /** + * Whether to hide potentially sensitive information about GC + * roots (such as PIDs). + */ + bool censor = false; }; diff --git a/src/libstore/include/nix/store/globals.hh b/src/libstore/include/nix/store/globals.hh index 0ac689b55c1..fdc0c0827a5 100644 --- a/src/libstore/include/nix/store/globals.hh +++ b/src/libstore/include/nix/store/globals.hh @@ -380,7 +380,7 @@ public: R"( If set to `true`, Nix instructs [remote build machines](#conf-builders) to use their own [`substituters`](#conf-substituters) if available. - It means that remote build hosts fetches as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all. + It means that remote build hosts fetch as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all. This can drastically reduce build times if the network connection between the local machine and the remote build host is slow. )"}; @@ -448,7 +448,7 @@ public: by the Nix account, its group should be the group specified here, and its mode should be `1775`. - If the build users group is empty, builds areperformed under + If the build users group is empty, builds are performed under the uid of the Nix process (that is, the uid of the caller if `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if `NIX_REMOTE` is `daemon`). Obviously, this should not be used @@ -741,8 +741,8 @@ public: 4. The path to the build's scratch directory. This directory exists only if the build was run with `--keep-failed`. - The stderr and stdout output from the diff hook isn't - displayed to the user. Instead, it print to the nix-daemon's log. + The stderr and stdout output from the diff hook isn't displayed + to the user. Instead, it prints to the nix-daemon's log. When using the Nix daemon, `diff-hook` must be set in the `nix.conf` configuration file, and cannot be passed at the command line. @@ -1211,11 +1211,12 @@ public: Setting upgradeNixStorePathUrl{ this, - "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix", + "", "upgrade-nix-store-path-url", R"( - Used by `nix upgrade-nix`, the URL of the file that contains the - store paths of the latest Nix release. + Deprecated. This option was used to configure how `nix upgrade-nix` operated. + + Using this setting has no effect. It will be removed in a future release of Determinate Nix. )" }; @@ -1230,6 +1231,104 @@ public: Set it to 1 to warn on all paths. )" }; + + struct ExternalBuilder + { + std::vector systems; + Path program; + std::vector args; + }; + + using ExternalBuilders = std::vector; + + Setting externalBuilders{ + this, + {}, + "external-builders", + R"( + Helper programs that execute derivations. + + The program is passed a JSON document that describes the build environment as the final argument. + The JSON document looks like this: + + { + "args": [ + "-e", + "/nix/store/vj1c3wf9c11a0qs6p3ymfvrnsdgsdcbq-source-stdenv.sh", + "/nix/store/shkw4qm9qcw5sc5n1k5jznc83ny02r39-default-builder.sh" + ], + "builder": "/nix/store/s1qkj0ph0ma64a6743mvkwnabrbw1hsc-bash-5.2p37/bin/bash", + "env": { + "HOME": "/homeless-shelter", + "NIX_BUILD_CORES": "14", + "NIX_BUILD_TOP": "/build", + "NIX_LOG_FD": "2", + "NIX_STORE": "/nix/store", + "PATH": "/path-not-set", + "PWD": "/build", + "TEMP": "/build", + "TEMPDIR": "/build", + "TERM": "xterm-256color", + "TMP": "/build", + "TMPDIR": "/build", + "__structuredAttrs": "", + "buildInputs": "", + "builder": "/nix/store/s1qkj0ph0ma64a6743mvkwnabrbw1hsc-bash-5.2p37/bin/bash", + "cmakeFlags": "", + "configureFlags": "", + "depsBuildBuild": "", + "depsBuildBuildPropagated": "", + "depsBuildTarget": "", + "depsBuildTargetPropagated": "", + "depsHostHost": "", + "depsHostHostPropagated": "", + "depsTargetTarget": "", + "depsTargetTargetPropagated": "", + "doCheck": "1", + "doInstallCheck": "1", + "mesonFlags": "", + "name": "hello-2.12.2", + "nativeBuildInputs": "/nix/store/l31j72f1h33hsa4nq4iyhsmsqjyndq9f-version-check-hook", + "out": "/nix/store/2yx2prgxmzbkrnbb4liy6n4zkzb1cqai-hello-2.12.2", + "outputs": "out", + "patches": "", + "pname": "hello", + "postInstallCheck": "stat \"${!outputBin}/bin/hello\"\n", + "propagatedBuildInputs": "", + "propagatedNativeBuildInputs": "", + "src": "/nix/store/dw402azxjrgrzrk6j0p66wkqrab5mwgw-hello-2.12.2.tar.gz", + "stdenv": "/nix/store/i8bw5nqg1225m281zr6lgsz42bw04z7g-stdenv-linux", + "strictDeps": "", + "system": "aarch64-linux", + "version": "2.12.2" + }, + "realStoreDir": "/nix/store", + "storeDir": "/nix/store", + "system": "aarch64-linux", + "tmpDir": "/private/tmp/nix-build-hello-2.12.2.drv-0/build", + "tmpDirInSandbox": "/build", + "topTmpDir": "/private/tmp/nix-build-hello-2.12.2.drv-0" + } + )", + {}, // aliases + true, // document default + // NOTE(cole-h): even though we can make the experimental feature required here, the errors + // are not as good (it just becomes a warning if you try to use this setting without the + // experimental feature) + // + // With this commented out: + // + // error: experimental Nix feature 'external-builders' is disabled; add '--extra-experimental-features external-builders' to enable it + // + // With this uncommented: + // + // warning: Ignoring setting 'external-builders' because experimental feature 'external-builders' is not enabled + // error: Cannot build '/nix/store/vwsp4qd8a62jqa36p26d15hin4xnj949-opentofu-1.10.2.drv'. + // Reason: required system or feature not available + // Required system: 'aarch64-linux' with features {} + // Current system: 'aarch64-darwin' with features {apple-virt, benchmark, big-parallel, nixos-test} + // Xp::ExternalBuilders + }; }; @@ -1257,6 +1356,8 @@ std::vector getUserConfigFiles(); */ extern std::string nixVersion; +extern const std::string determinateNixVersion; + /** * @param loadConfig Whether to load configuration from `nix.conf`, `NIX_CONFIG`, etc. May be disabled for unit tests. * @note When using libexpr, and/or libmain, This is not sufficient. See initNix(). diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 94b8951fdd9..2aff1729077 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -11,6 +11,8 @@ project('nix-store', 'cpp', license : 'LGPL-2.1-or-later', ) +fs = import('fs') + cxx = meson.get_compiler('cpp') subdir('nix-meson-build-support/deps-lists') @@ -21,6 +23,8 @@ configdata_priv = configuration_data() # TODO rename, because it will conflict with downstream projects configdata_priv.set_quoted('PACKAGE_VERSION', meson.project_version()) +configdata_priv.set_quoted('DETERMINATE_NIX_VERSION', fs.read('../../.version-determinate').strip()) + # Used in public header. configdata_pub.set_quoted( 'NIX_LOCAL_SYSTEM', @@ -183,8 +187,6 @@ if get_option('embedded-sandbox-shell') generated_headers += embedded_sandbox_shell_gen endif -fs = import('fs') - prefix = get_option('prefix') # For each of these paths, assume that it is relative to the prefix unless # it is already an absolute path (which is the default for store-dir, localstatedir, and log-dir). diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 775776139ae..20478d9d3c2 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -32,15 +32,17 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-store"; + pname = "determinate-nix-store"; inherit version; workDir = ./.; fileset = fileset.unions [ ../../nix-meson-build-support ./nix-meson-build-support + # FIXME: get rid of these symlinks. ../../.version ./.version + ../../.version-determinate ./meson.build ./meson.options ./include/nix/store/meson.build diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 55b967ed679..04f514d66b0 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -93,7 +93,7 @@ SQLite::~SQLite() void SQLite::isCache() { exec("pragma synchronous = off"); - exec("pragma main.journal_mode = truncate"); + exec("pragma main.journal_mode = wal"); } void SQLite::exec(const std::string & stmt) @@ -250,7 +250,7 @@ void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning) if (now > nextWarning) { nextWarning = now + 10; logWarning({ - .msg = HintFmt(e.what()) + .msg = e.info().msg }); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 9aeab1d1f6b..39de6808da1 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -215,8 +215,12 @@ StorePath Store::addToStore( auto sink = sourceToSink([&](Source & source) { LengthSource lengthSource(source); storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair); - if (settings.warnLargePathThreshold && lengthSource.total >= settings.warnLargePathThreshold) - warn("copied large path '%s' to the store (%s)", path, renderSize(lengthSource.total)); + if (settings.warnLargePathThreshold && lengthSource.total >= settings.warnLargePathThreshold) { + static bool failOnLargePath = getEnv("_NIX_TEST_FAIL_ON_LARGE_PATH").value_or("") == "1"; + if (failOnLargePath) + throw Error("doesn't copy large path '%s' to the store (%d)", path, renderSize(lengthSource.total)); + warn("copied large path '%s' to the store (%d)", path, renderSize(lengthSource.total)); + } }); dumpPath(path, *sink, fsm, filter); sink->finish(); diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index fd62aa664a4..88a5ef13082 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -206,6 +206,12 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder return acquireUserLock(1, false); } + /** + * Throw an exception if we can't do this derivation because of + * missing system features. + */ + virtual void checkSystem(); + /** * Return the paths that should be made available in the sandbox. * This includes: @@ -710,13 +716,8 @@ static bool checkNotWorldWritable(std::filesystem::path path) return true; } -void DerivationBuilderImpl::startBuilder() +void DerivationBuilderImpl::checkSystem() { - /* Make sure that no other processes are executing under the - sandbox uids. This must be done before any chownToBuilder() - calls. */ - prepareUser(); - /* Right platform? */ if (!drvOptions.canBuildLocally(store, drv)) { auto msg = fmt( @@ -732,10 +733,20 @@ void DerivationBuilderImpl::startBuilder() // since aarch64-darwin has Rosetta 2, this user can actually run x86_64-darwin on their hardware - we should tell them to run the command to install Darwin 2 if (drv.platform == "x86_64-darwin" && settings.thisSystem == "aarch64-darwin") - msg += fmt("\nNote: run `%s` to run programs for x86_64-darwin", Magenta("/usr/sbin/softwareupdate --install-rosetta && launchctl stop org.nixos.nix-daemon")); + msg += fmt("\nNote: run `%s` to run programs for x86_64-darwin", Magenta("/usr/sbin/softwareupdate --install-rosetta && launchctl stop org.nixos.nix-daemon")); throw BuildError(msg); } +} + +void DerivationBuilderImpl::startBuilder() +{ + checkSystem(); + + /* Make sure that no other processes are executing under the + sandbox uids. This must be done before any chownToBuilder() + calls. */ + prepareUser(); auto buildDir = getLocalStore(store).config->getBuildDir(); @@ -902,16 +913,26 @@ DerivationBuilderImpl::PathsInChroot DerivationBuilderImpl::getPathsInSandbox() optional = true; i.pop_back(); } + size_t p = i.find('='); - if (p == std::string::npos) - pathsInChroot[i] = {i, optional}; - else - pathsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; + std::string inside, outside; + if (p == std::string::npos) { + inside = i; + outside = i; + } else { + inside = i.substr(0, p); + outside = i.substr(p + 1); + } + + if (!optional && !maybeLstat(outside)) + throw SysError("path '%s' is configured as part of the `sandbox-paths` option, but is inaccessible", outside); + + pathsInChroot[inside] = {outside, optional}; } + if (hasPrefix(store.storeDir, tmpDirInSandbox())) - { throw Error("`sandbox-build-dir` must not contain the storeDir"); - } + pathsInChroot[tmpDirInSandbox()] = tmpDir; /* Add the closure of store paths to the chroot. */ @@ -1043,7 +1064,7 @@ void DerivationBuilderImpl::processSandboxSetupMessages() e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", store.printStorePath(drvPath), statusToString(status), - concatStringsSep("|", msgs)); + concatStringsSep("\n", msgs)); throw; } }(); @@ -1789,6 +1810,12 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() store.printStorePath(drvPath), wanted.to_string(HashFormat::SRI, true), got.to_string(HashFormat::SRI, true))); + act->result(resHashMismatch, + { + {"storePath", store.printStorePath(drvPath)}, + {"wanted", wanted}, + {"got", got}, + }); } if (!newInfo0.references.empty()) { auto numViolations = newInfo.references.size(); @@ -2176,6 +2203,7 @@ StorePath DerivationBuilderImpl::makeFallbackPath(const StorePath & path) // FIXME: do this properly #include "linux-derivation-builder.cc" #include "darwin-derivation-builder.cc" +#include "external-derivation-builder.cc" namespace nix { @@ -2184,6 +2212,9 @@ std::unique_ptr makeDerivationBuilder( std::unique_ptr miscMethods, DerivationBuilderParams params) { + if (auto builder = ExternalDerivationBuilder::newIfSupported(store, miscMethods, params)) + return builder; + bool useSandbox = false; /* Are we doing a sandboxed build? */ diff --git a/src/libstore/unix/build/external-derivation-builder.cc b/src/libstore/unix/build/external-derivation-builder.cc new file mode 100644 index 00000000000..a393d75d9d1 --- /dev/null +++ b/src/libstore/unix/build/external-derivation-builder.cc @@ -0,0 +1,128 @@ +namespace nix { + +struct ExternalDerivationBuilder : DerivationBuilderImpl +{ + Settings::ExternalBuilder externalBuilder; + + ExternalDerivationBuilder( + Store & store, + std::unique_ptr miscMethods, + DerivationBuilderParams params, + Settings::ExternalBuilder externalBuilder) + : DerivationBuilderImpl(store, std::move(miscMethods), std::move(params)) + , externalBuilder(std::move(externalBuilder)) + { + experimentalFeatureSettings.require(Xp::ExternalBuilders); + } + + static std::unique_ptr newIfSupported( + Store & store, std::unique_ptr & miscMethods, DerivationBuilderParams & params) + { + for (auto & handler : settings.externalBuilders.get()) { + for (auto & system : handler.systems) + if (params.drv.platform == system) + return std::make_unique( + store, std::move(miscMethods), std::move(params), handler); + } + return {}; + } + + bool prepareBuild() override + { + return DerivationBuilderImpl::prepareBuild(); + } + + Path tmpDirInSandbox() override + { + /* In a sandbox, for determinism, always use the same temporary + directory. */ + return "/build"; + } + + void setBuildTmpDir() override + { + tmpDir = topTmpDir + "/build"; + createDir(tmpDir, 0700); + } + + void prepareUser() override + { + DerivationBuilderImpl::prepareUser(); + } + + void setUser() override + { + DerivationBuilderImpl::setUser(); + } + + void checkSystem() override + { + // FIXME: should check system features. + } + + void startChild() override + { + if (drvOptions.getRequiredSystemFeatures(drv).count("recursive-nix")) + throw Error("'recursive-nix' is not supported yet by external derivation builders"); + + auto json = nlohmann::json::object(); + + json.emplace("builder", drv.builder); + { + auto l = nlohmann::json::array(); + for (auto & i : drv.args) + l.push_back(rewriteStrings(i, inputRewrites)); + json.emplace("args", std::move(l)); + } + { + auto j = nlohmann::json::object(); + for (auto & [name, value] : env) + j.emplace(name, rewriteStrings(value, inputRewrites)); + json.emplace("env", std::move(j)); + } + json.emplace("topTmpDir", topTmpDir); + json.emplace("tmpDir", tmpDir); + json.emplace("tmpDirInSandbox", tmpDirInSandbox()); + json.emplace("storeDir", store.storeDir); + json.emplace("realStoreDir", getLocalStore(store).config->realStoreDir.get()); + json.emplace("system", drv.platform); + + // TODO(cole-h): writing this to stdin is too much effort right now, if we want to revisit + // that, see this comment by Eelco about how to make it not suck: + // https://github.com/DeterminateSystems/nix-src/pull/141#discussion_r2205493257 + auto jsonFile = std::filesystem::path{topTmpDir} / "build.json"; + writeFile(jsonFile, json.dump()); + + pid = startProcess([&]() { + openSlave(); + try { + commonChildInit(); + + Strings args = {externalBuilder.program}; + + if (!externalBuilder.args.empty()) { + args.insert(args.end(), externalBuilder.args.begin(), externalBuilder.args.end()); + } + + args.insert(args.end(), jsonFile); + + if (chdir(tmpDir.c_str()) == -1) + throw SysError("changing into '%1%'", tmpDir); + + chownToBuilder(topTmpDir); + + setUser(); + + debug("executing external builder: %s", concatStringsSep(" ", args)); + execv(externalBuilder.program.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("executing '%s'", externalBuilder.program); + } catch (...) { + handleChildException(true); + _exit(1); + } + }); + } +}; + +} diff --git a/src/libstore/unix/include/nix/store/build/derivation-builder.hh b/src/libstore/unix/include/nix/store/build/derivation-builder.hh index 5ce38e034eb..2dddfdff8ed 100644 --- a/src/libstore/unix/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/unix/include/nix/store/build/derivation-builder.hh @@ -58,6 +58,11 @@ struct DerivationBuilderParams const BuildMode & buildMode; + /** + * The activity corresponding to the build. + */ + std::unique_ptr & act; + DerivationBuilderParams( const StorePath & drvPath, const BuildMode & buildMode, @@ -66,7 +71,8 @@ struct DerivationBuilderParams const StructuredAttrs * parsedDrv, const DerivationOptions & drvOptions, const StorePathSet & inputPaths, - std::map & initialOutputs) + std::map & initialOutputs, + std::unique_ptr & act) : drvPath{drvPath} , buildResult{buildResult} , drv{drv} @@ -75,6 +81,7 @@ struct DerivationBuilderParams , inputPaths{inputPaths} , initialOutputs{initialOutputs} , buildMode{buildMode} + , act{act} { } DerivationBuilderParams(DerivationBuilderParams &&) = default; diff --git a/src/libstore/unix/user-lock.cc b/src/libstore/unix/user-lock.cc index 6a07cb7cc83..f5d164e5b18 100644 --- a/src/libstore/unix/user-lock.cc +++ b/src/libstore/unix/user-lock.cc @@ -197,7 +197,7 @@ bool useBuildUsers() #ifdef __linux__ static bool b = (settings.buildUsersGroup != "" || settings.autoAllocateUids) && isRootUser(); return b; - #elif defined(__APPLE__) && defined(__FreeBSD__) + #elif defined(__APPLE__) || defined(__FreeBSD__) static bool b = settings.buildUsersGroup != "" && isRootUser(); return b; #else diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index f26f57775d4..a1605bf5bb8 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -14,7 +14,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-util-c"; + pname = "determinate-nix-util-c"; inherit version; workDir = ./.; diff --git a/src/libutil-test-support/package.nix b/src/libutil-test-support/package.nix index f8e92c27113..40ff65d6135 100644 --- a/src/libutil-test-support/package.nix +++ b/src/libutil-test-support/package.nix @@ -17,7 +17,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-util-test-support"; + pname = "determinate-nix-util-test-support"; inherit version; workDir = ./.; diff --git a/src/libutil-tests/config.cc b/src/libutil-tests/config.cc index bc7db251b87..28b680d9c15 100644 --- a/src/libutil-tests/config.cc +++ b/src/libutil-tests/config.cc @@ -191,7 +191,7 @@ namespace nix { "description", {}, true, - Xp::Flakes, + Xp::CaDerivations, }; setting.assign("value"); @@ -203,7 +203,7 @@ namespace nix { "description": "description\n", "documentDefault": true, "value": "value", - "experimentalFeature": "flakes" + "experimentalFeature": "ca-derivations" } })#"_json); } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index d8d004e6fc3..80cd0296905 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -579,7 +579,7 @@ Strings argvToStrings(int argc, char * * argv) std::optional Command::experimentalFeature () { - return { Xp::NixCommand }; + return {}; } MultiCommand::MultiCommand(std::string_view commandName, const Commands & commands_) diff --git a/src/libutil/configuration.cc b/src/libutil/configuration.cc index 314ae34db4b..c3c2325dc74 100644 --- a/src/libutil/configuration.cc +++ b/src/libutil/configuration.cc @@ -355,11 +355,11 @@ template<> std::set BaseSetting res; for (auto & s : tokenizeString(str)) { - if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) { + if (auto thisXpFeature = parseExperimentalFeature(s)) res.insert(thisXpFeature.value()); - if (thisXpFeature.value() == Xp::Flakes) - res.insert(Xp::FetchTree); - } else + else if (stabilizedFeatures.count(s)) + debug("experimental feature '%s' is now stable", s); + else warn("unknown experimental feature '%s'", s); } return res; diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 88f3783f552..075b90ec58e 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -70,19 +70,12 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "https://github.com/NixOS/nix/milestone/42", }, - { - .tag = Xp::Flakes, - .name = "flakes", - .description = R"( - Enable flakes. See the manual entry for [`nix - flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. - )", - .trackingUrl = "https://github.com/NixOS/nix/milestone/27", - }, { .tag = Xp::FetchTree, .name = "fetch-tree", .description = R"( + *Enabled for Determinate Nix Installer users since 2.24* + Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language. `fetchTree` exposes a generic interface for fetching remote file system trees from different types of remote sources. @@ -93,15 +86,6 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "https://github.com/NixOS/nix/milestone/31", }, - { - .tag = Xp::NixCommand, - .name = "nix-command", - .description = R"( - Enable the new `nix` subcommands. See the manual on - [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. - )", - .trackingUrl = "https://github.com/NixOS/nix/milestone/28", - }, { .tag = Xp::GitHashing, .name = "git-hashing", @@ -170,7 +154,7 @@ constexpr std::array xpFeatureDetails "http://foo" ``` - But enabling this experimental feature will cause the Nix parser to + But enabling this experimental feature causes the Nix parser to throw an error when encountering a URL literal: ``` @@ -304,6 +288,14 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "https://github.com/NixOS/nix/milestone/55", }, + { + .tag = Xp::ExternalBuilders, + .name = "external-builders", + .description = R"( + Enables support for external builders / sandbox providers. + )", + .trackingUrl = "", + }, { .tag = Xp::BLAKE3Hashes, .name = "blake3-hashes", @@ -317,12 +309,18 @@ constexpr std::array xpFeatureDetails static_assert( []() constexpr { for (auto [index, feature] : enumerate(xpFeatureDetails)) - if (index != (size_t)feature.tag) + if (index != (size_t) feature.tag) return false; return true; }(), "array order does not match enum tag order"); +/** + * A set of previously experimental features that are now considered + * stable. We don't warn if users have these in `experimental-features`. + */ +std::set stabilizedFeatures{"flakes", "nix-command"}; + const std::optional parseExperimentalFeature(const std::string_view & name) { using ReverseXpMap = std::map; diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 319eb795e6b..6d279f3c803 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -16,6 +16,8 @@ #include #include +#include + #include namespace nix { @@ -487,4 +489,13 @@ std::string_view printHashAlgo(HashAlgorithm ha) } } +void to_json(nlohmann::json & json, const Hash & hash) +{ + json = nlohmann::json::object( + { + {"algo", printHashAlgo(hash.algo)}, + {"base16", hash.to_string(HashFormat::Base16, false)}, + }); +} + } diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libutil/include/nix/util/configuration.hh index 24b42f02c84..fb0325fdcf8 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libutil/include/nix/util/configuration.hh @@ -389,7 +389,7 @@ struct ExperimentalFeatureSettings : Config { Example: ``` - experimental-features = nix-command flakes + experimental-features = ca-derivations ``` The following experimental features are available: diff --git a/src/libutil/include/nix/util/experimental-features.hh b/src/libutil/include/nix/util/experimental-features.hh index 8923517bace..5a01d960ca4 100644 --- a/src/libutil/include/nix/util/experimental-features.hh +++ b/src/libutil/include/nix/util/experimental-features.hh @@ -19,9 +19,7 @@ enum struct ExperimentalFeature { CaDerivations, ImpureDerivations, - Flakes, FetchTree, - NixCommand, GitHashing, RecursiveNix, NoUrlLiterals, @@ -37,9 +35,12 @@ enum struct ExperimentalFeature MountedSSHStore, VerifiedFetches, PipeOperators, + ExternalBuilders, BLAKE3Hashes, }; +extern std::set stabilizedFeatures; + /** * Just because writing `ExperimentalFeature::CaDerivations` is way too long */ diff --git a/src/libutil/include/nix/util/forwarding-source-accessor.hh b/src/libutil/include/nix/util/forwarding-source-accessor.hh new file mode 100644 index 00000000000..bdba2addcb0 --- /dev/null +++ b/src/libutil/include/nix/util/forwarding-source-accessor.hh @@ -0,0 +1,57 @@ +#pragma once + +#include "source-accessor.hh" + +namespace nix { + +/** + * A source accessor that just forwards every operation to another + * accessor. This is not useful in itself but can be used as a + * superclass for accessors that do change some operations. + */ +struct ForwardingSourceAccessor : SourceAccessor +{ + ref next; + + ForwardingSourceAccessor(ref next) + : next(next) + { + } + + std::string readFile(const CanonPath & path) override + { + return next->readFile(path); + } + + void readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) override + { + next->readFile(path, sink, sizeCallback); + } + + std::optional maybeLstat(const CanonPath & path) override + { + return next->maybeLstat(path); + } + + DirEntries readDirectory(const CanonPath & path) override + { + return next->readDirectory(path); + } + + std::string readLink(const CanonPath & path) override + { + return next->readLink(path); + } + + std::string showPath(const CanonPath & path) override + { + return next->showPath(path); + } + + std::optional getPhysicalPath(const CanonPath & path) override + { + return next->getPhysicalPath(path); + } +}; + +} diff --git a/src/libutil/include/nix/util/hash.hh b/src/libutil/include/nix/util/hash.hh index 71553745662..1c7b8ed9c55 100644 --- a/src/libutil/include/nix/util/hash.hh +++ b/src/libutil/include/nix/util/hash.hh @@ -6,6 +6,8 @@ #include "nix/util/serialise.hh" #include "nix/util/file-system.hh" +#include + namespace nix { @@ -210,6 +212,10 @@ std::optional parseHashAlgoOpt(std::string_view s); */ std::string_view printHashAlgo(HashAlgorithm ha); +/** + * Write a JSON serialisation of the format `{"algo":"","base16":""}`. + */ +void to_json(nlohmann::json & json, const Hash & hash); union Ctx; diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index dabfac48390..2b71c417155 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -39,6 +39,8 @@ typedef enum { resSetExpected = 106, resPostBuildLogLine = 107, resFetchStatus = 108, + resHashMismatch = 109, + resBuildResult = 110, } ResultType; typedef uint64_t ActivityId; @@ -55,7 +57,7 @@ struct LoggerSettings : Config Setting jsonLogPath{ this, "", "json-log-path", R"( - A file or unix socket to which JSON records of Nix's log output are + A file or Unix domain socket to which JSON records of Nix's log output are written, in the same format as `--log-format internal-json` (without the `@nix ` prefixes on each line). Concurrent writes to the same file by multiple Nix processes are not supported and @@ -129,6 +131,8 @@ public: virtual void result(ActivityId act, ResultType type, const Fields & fields) { }; + virtual void result(ActivityId act, ResultType type, const nlohmann::json & json) { }; + virtual void writeToStdout(std::string_view s); template @@ -181,6 +185,11 @@ struct Activity void setExpected(ActivityType type2, uint64_t expected) const { result(resSetExpected, type2, expected); } + void result(ResultType type, const nlohmann::json & json) const + { + logger.result(id, type, json); + } + template void result(ResultType type, const Args & ... args) const { diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index 22438c1d075..e3be662a35b 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -34,6 +34,7 @@ headers = files( 'file-system.hh', 'finally.hh', 'fmt.hh', + 'forwarding-source-accessor.hh', 'fs-sink.hh', 'git.hh', 'hash.hh', @@ -43,6 +44,7 @@ headers = files( 'logging.hh', 'lru-cache.hh', 'memory-source-accessor.hh', + 'mounted-source-accessor.hh', 'muxable-pipe.hh', 'os-string.hh', 'pool.hh', diff --git a/src/libutil/include/nix/util/mounted-source-accessor.hh b/src/libutil/include/nix/util/mounted-source-accessor.hh new file mode 100644 index 00000000000..2e8d45dd69b --- /dev/null +++ b/src/libutil/include/nix/util/mounted-source-accessor.hh @@ -0,0 +1,20 @@ +#pragma once + +#include "source-accessor.hh" + +namespace nix { + +struct MountedSourceAccessor : SourceAccessor +{ + virtual void mount(CanonPath mountPoint, ref accessor) = 0; + + /** + * Return the accessor mounted on `mountPoint`, or `nullptr` if + * there is no such mount point. + */ + virtual std::shared_ptr getMount(CanonPath mountPoint) = 0; +}; + +ref makeMountedSourceAccessor(std::map> mounts); + +} diff --git a/src/libutil/include/nix/util/pos-idx.hh b/src/libutil/include/nix/util/pos-idx.hh index 0bf59301aff..423f8b03206 100644 --- a/src/libutil/include/nix/util/pos-idx.hh +++ b/src/libutil/include/nix/util/pos-idx.hh @@ -15,12 +15,12 @@ class PosIdx private: uint32_t id; +public: explicit PosIdx(uint32_t id) : id(id) { } -public: PosIdx() : id(0) { @@ -45,6 +45,11 @@ public: { return std::hash{}(id); } + + uint32_t get() const + { + return id; + } }; inline PosIdx noPos = {}; diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index 92a9adc46e9..c0e8528db4a 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -118,7 +118,7 @@ struct SourceAccessor : std::enable_shared_from_this std::string typeString(); }; - Stat lstat(const CanonPath & path); + virtual Stat lstat(const CanonPath & path); virtual std::optional maybeLstat(const CanonPath & path) = 0; @@ -182,6 +182,27 @@ struct SourceAccessor : std::enable_shared_from_this */ std::optional fingerprint; + /** + * Return the fingerprint for `path`. This is usually the + * fingerprint of the current accessor, but for composite + * accessors (like `MountedSourceAccessor`), we want to return the + * fingerprint of the "inner" accessor if the current one lacks a + * fingerprint. + * + * So this method is intended to return the most-outer accessor + * that has a fingerprint for `path`. It also returns the path that `path` + * corresponds to in that accessor. + * + * For example: in a `MountedSourceAccessor` that has + * `/nix/store/foo` mounted, + * `getFingerprint("/nix/store/foo/bar")` will return the path + * `/bar` and the fingerprint of the `/nix/store/foo` accessor. + */ + virtual std::pair> getFingerprint(const CanonPath & path) + { + return {path, fingerprint}; + } + /** * Return the maximum last-modified time of the files in this * tree, if available. @@ -214,8 +235,6 @@ ref getFSSourceAccessor(); */ ref makeFSSourceAccessor(std::filesystem::path root); -ref makeMountedSourceAccessor(std::map> mounts); - /** * Construct an accessor that presents a "union" view of a vector of * underlying accessors. Earlier accessors take precedence over later. diff --git a/src/libutil/include/nix/util/unix-domain-socket.hh b/src/libutil/include/nix/util/unix-domain-socket.hh index 3aaaddf823d..2dce9f9f2e3 100644 --- a/src/libutil/include/nix/util/unix-domain-socket.hh +++ b/src/libutil/include/nix/util/unix-domain-socket.hh @@ -87,4 +87,9 @@ void connect(Socket fd, const std::filesystem::path & path); */ AutoCloseFD connect(const std::filesystem::path & path); +/** + * Connect to a Unix domain socket. + */ +AutoCloseFD connect(const std::filesystem::path & path); + } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 4dadf15501f..5a14b63be29 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -303,6 +303,16 @@ struct JSONLogger : Logger { addFields(json, fields); write(json); } + + void result(ActivityId act, ResultType type, const nlohmann::json & j) override + { + nlohmann::json json; + json["action"] = "result"; + json["id"] = act; + json["type"] = type; + json["payload"] = j; + write(json); + } }; std::unique_ptr makeJSONLogger(Descriptor fd, bool includeNixPrefix) diff --git a/src/libutil/mounted-source-accessor.cc b/src/libutil/mounted-source-accessor.cc index b7de2afbf03..ed62fd2a37d 100644 --- a/src/libutil/mounted-source-accessor.cc +++ b/src/libutil/mounted-source-accessor.cc @@ -1,12 +1,12 @@ -#include "nix/util/source-accessor.hh" +#include "nix/util/mounted-source-accessor.hh" namespace nix { -struct MountedSourceAccessor : SourceAccessor +struct MountedSourceAccessorImpl : MountedSourceAccessor { std::map> mounts; - MountedSourceAccessor(std::map> _mounts) + MountedSourceAccessorImpl(std::map> _mounts) : mounts(std::move(_mounts)) { displayPrefix.clear(); @@ -23,6 +23,12 @@ struct MountedSourceAccessor : SourceAccessor return accessor->readFile(subpath); } + Stat lstat(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->lstat(subpath); + } + std::optional maybeLstat(const CanonPath & path) override { auto [accessor, subpath] = resolve(path); @@ -69,11 +75,34 @@ struct MountedSourceAccessor : SourceAccessor auto [accessor, subpath] = resolve(path); return accessor->getPhysicalPath(subpath); } + + void mount(CanonPath mountPoint, ref accessor) override + { + // FIXME: thread-safety + mounts.insert_or_assign(std::move(mountPoint), accessor); + } + + std::shared_ptr getMount(CanonPath mountPoint) override + { + auto i = mounts.find(mountPoint); + if (i != mounts.end()) + return i->second; + else + return nullptr; + } + + std::pair> getFingerprint(const CanonPath & path) override + { + if (fingerprint) + return {path, fingerprint}; + auto [accessor, subpath] = resolve(path); + return accessor->getFingerprint(subpath); + } }; -ref makeMountedSourceAccessor(std::map> mounts) +ref makeMountedSourceAccessor(std::map> mounts) { - return make_ref(std::move(mounts)); + return make_ref(std::move(mounts)); } } diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 46f56e07e6d..ba580b1b30c 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -22,7 +22,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-util"; + pname = "determinate-nix-util"; inherit version; workDir = ./.; diff --git a/src/libutil/tee-logger.cc b/src/libutil/tee-logger.cc index 55334a821fb..c539ae5d1a2 100644 --- a/src/libutil/tee-logger.cc +++ b/src/libutil/tee-logger.cc @@ -65,6 +65,12 @@ struct TeeLogger : Logger logger->result(act, type, fields); } + void result(ActivityId act, ResultType type, const nlohmann::json & json) override + { + for (auto & logger : loggers) + logger->result(act, type, json); + } + void writeToStdout(std::string_view s) override { for (auto & logger : loggers) { diff --git a/src/libutil/union-source-accessor.cc b/src/libutil/union-source-accessor.cc index 9950f604960..69cf04c186b 100644 --- a/src/libutil/union-source-accessor.cc +++ b/src/libutil/union-source-accessor.cc @@ -72,6 +72,18 @@ struct UnionSourceAccessor : SourceAccessor } return std::nullopt; } + + std::pair> getFingerprint(const CanonPath & path) override + { + if (fingerprint) + return {path, fingerprint}; + for (auto & accessor : accessors) { + auto [subpath, fingerprint] = accessor->getFingerprint(path); + if (fingerprint) + return {subpath, fingerprint}; + } + return {path, std::nullopt}; + } }; ref makeUnionSourceAccessor(std::vector> && accessors) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index c9cc80fef6c..23dafe8c9f4 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -190,8 +190,10 @@ void ignoreExceptionInDestructor(Verbosity lvl) try { try { throw; + } catch (Error & e) { + printMsg(lvl, ANSI_RED "error (ignored):" ANSI_NORMAL " %s", e.info().msg); } catch (std::exception & e) { - printMsg(lvl, "error (ignored): %1%", e.what()); + printMsg(lvl, ANSI_RED "error (ignored):" ANSI_NORMAL " %s", e.what()); } } catch (...) { } } @@ -202,8 +204,10 @@ void ignoreExceptionExceptInterrupt(Verbosity lvl) throw; } catch (const Interrupted & e) { throw; + } catch (Error & e) { + printMsg(lvl, ANSI_RED "error (ignored):" ANSI_NORMAL " %s", e.info().msg); } catch (std::exception & e) { - printMsg(lvl, "error (ignored): %1%", e.what()); + printMsg(lvl, ANSI_RED "error (ignored):" ANSI_NORMAL " %s", e.what()); } } diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index c4a05865823..e6d2a89ad5f 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -166,6 +166,11 @@ static void update(const StringSet & channelNames) static int main_nix_channel(int argc, char ** argv) { + warn( + "nix-channel is deprecated in favor of flakes in Determinate Nix. \ +See https://zero-to-nix.com for a guide to Nix flakes. \ +For details and to offer feedback on the deprecation process, see: https://github.com/DeterminateSystems/nix-src/issues/34."); + { // Figure out the name of the `.nix-channels' file to use auto home = getHome(); diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index e149b6aeb7f..c49f2885d22 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -110,7 +110,7 @@ bool createUserEnv(EvalState & state, PackageInfos & elems, environment. */ auto manifestFile = ({ std::ostringstream str; - printAmbiguous(manifest, state.symbols, str, nullptr, std::numeric_limits::max()); + printAmbiguous(state, manifest, str, nullptr, std::numeric_limits::max()); StringSource source { toView(str) }; state.store->addToStoreFromDump( source, "env-manifest.nix", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index f7b218efce4..f327454ec91 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -17,6 +17,7 @@ #include #include +#include using namespace nix; @@ -52,18 +53,25 @@ void processExpr(EvalState & state, const Strings & attrPaths, else state.autoCallFunction(autoArgs, v, vRes); if (output == okRaw) - std::cout << *state.coerceToString(noPos, vRes, context, "while generating the nix-instantiate output"); + std::cout << + state.devirtualize( + *state.coerceToString(noPos, vRes, context, "while generating the nix-instantiate output"), + context); // We intentionally don't output a newline here. The default PS1 for Bash in NixOS starts with a newline // and other interactive shells like Zsh are smart enough to print a missing newline before the prompt. - else if (output == okXML) - printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); + else if (output == okXML) { + std::ostringstream s; + printValueAsXML(state, strict, location, vRes, s, context, noPos); + std::cout << state.devirtualize(s.str(), context); + } else if (output == okJSON) { - printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context); - std::cout << std::endl; - } else { + auto j = printValueAsJSON(state, strict, vRes, v.determinePos(noPos), context); + std::cout << state.devirtualize(j.dump(), context) << std::endl; + } + else { if (strict) state.forceValueDeep(vRes); std::set seen; - printAmbiguous(vRes, state.symbols, std::cout, &seen, std::numeric_limits::max()); + printAmbiguous(state, vRes, std::cout, &seen, std::numeric_limits::max()); std::cout << std::endl; } } else { diff --git a/src/nix/app.cc b/src/nix/app.cc index c9a9f9caf7d..d3c14c06202 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -92,6 +92,9 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) .path = o.path, }; }, + [&](const NixStringContextElem::Path & p) -> DerivedPath { + throw Error("'program' attribute of an 'app' output cannot have no context"); + }, }, c.raw)); } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 37bce6ca078..b0818e50bc7 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -1,5 +1,6 @@ #include "nix/util/config-global.hh" #include "nix/expr/eval.hh" +#include "nix/fetchers/fetch-settings.hh" #include "nix/cmd/installable-flake.hh" #include "nix/cmd/command-installable-value.hh" #include "nix/main/common-args.hh" @@ -586,6 +587,11 @@ struct CmdDevelop : Common, MixEnvironment ; } + void preRun(ref store) override + { + fetchSettings.warnDirty = false; + } + void run(ref store, ref installable) override { auto [buildEnvironment, gcroot] = getBuildEnvironment(store, installable); diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index ff9f9db4098..fa3d51ae7b3 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -49,10 +49,10 @@ GroupedPaths getClosureInfo(ref store, const StorePath & toplevel) std::string showVersions(const StringSet & versions) { - if (versions.empty()) return "∅"; + if (versions.empty()) return "(absent)"; StringSet versions2; for (auto & version : versions) - versions2.insert(version.empty() ? "ε" : version); + versions2.insert(version.empty() ? "(no version)" : version); return concatStringsSep(", ", versions2); } @@ -97,8 +97,13 @@ void printClosureDiff( if (showDelta || !removed.empty() || !added.empty()) { std::vector items; - if (!removed.empty() || !added.empty()) + if (!removed.empty() && !added.empty()) { items.push_back(fmt("%s → %s", showVersions(removed), showVersions(added))); + } else if (!removed.empty()) { + items.push_back(fmt("%s removed", showVersions(removed))); + } else if (!added.empty()) { + items.push_back(fmt("%s added", showVersions(added))); + } if (showDelta) items.push_back(fmt("%s%+.1f KiB" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, sizeDelta / 1024.0)); logger->cout("%s%s: %s", indent, name, concatStringsSep(", ", items)); diff --git a/src/nix/diff-closures.md b/src/nix/diff-closures.md index 0294c0d8def..6b07af28f95 100644 --- a/src/nix/diff-closures.md +++ b/src/nix/diff-closures.md @@ -11,8 +11,8 @@ R""( baloo-widgets: 20.08.1 → 20.08.2 bluez-qt: +12.6 KiB dolphin: 20.08.1 → 20.08.2, +13.9 KiB - kdeconnect: 20.08.2 → ∅, -6597.8 KiB - kdeconnect-kde: ∅ → 20.08.2, +6599.7 KiB + kdeconnect: 20.08.2 removed, -6597.8 KiB + kdeconnect-kde: 20.08.2 added, +6599.7 KiB … ``` @@ -34,9 +34,9 @@ dolphin: 20.08.1 → 20.08.2, +13.9 KiB No size change is shown if it's below the threshold. If the package does not exist in either the *before* or *after* closures, it is -represented using `∅` (empty set) on the appropriate side of the -arrow. If a package has an empty version string, the version is -rendered as `ε` (epsilon). +represented using `added` or `removed`. +If a package has an empty version string, the version is +rendered as `(no version)`. There may be multiple versions of a package in each closure. In that case, only the changed versions are shown. Thus, diff --git a/src/nix/env.cc b/src/nix/env.cc index 277bd0fdd45..a0b0e976b73 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -6,6 +6,7 @@ #include "run.hh" #include "nix/util/strings.hh" #include "nix/util/executable-path.hh" +#include "nix/util/mounted-source-accessor.hh" using namespace nix; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index be064e5527a..e5b0aa968d0 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -114,11 +114,20 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption else if (raw) { logger->stop(); - writeFull(getStandardOutput(), *state->coerceToString(noPos, *v, context, "while generating the eval command output")); + writeFull( + getStandardOutput(), + state->devirtualize( + *state->coerceToString(noPos, *v, context, "while generating the eval command output"), + context)); } else if (json) { - printJSON(printValueAsJSON(*state, true, *v, pos, context, false)); + // FIXME: use printJSON + auto j = printValueAsJSON(*state, true, *v, pos, context, false); + logger->cout("%s", + state->devirtualize( + outputPretty ? j.dump(2) : j.dump(), + context)); } else { diff --git a/src/nix/flake-prefetch-inputs.cc b/src/nix/flake-prefetch-inputs.cc new file mode 100644 index 00000000000..9ee4b546e70 --- /dev/null +++ b/src/nix/flake-prefetch-inputs.cc @@ -0,0 +1,73 @@ +#include "flake-command.hh" +#include "nix/fetchers/fetch-to-store.hh" +#include "nix/util/thread-pool.hh" +#include "nix/store/filetransfer.hh" +#include "nix/util/exit.hh" + +#include + +using namespace nix; +using namespace nix::flake; + +struct CmdFlakePrefetchInputs : FlakeCommand +{ + std::string description() override + { + return "fetch the inputs of a flake"; + } + + std::string doc() override + { + return +#include "flake-prefetch-inputs.md" + ; + } + + void run(nix::ref store) override + { + auto flake = lockFlake(); + + ThreadPool pool{fileTransferSettings.httpConnections}; + + struct State + { + std::set done; + }; + + Sync state_; + + std::atomic nrFailed{0}; + + std::function visit; + visit = [&](const Node & node) { + if (!state_.lock()->done.insert(&node).second) + return; + + if (auto lockedNode = dynamic_cast(&node)) { + try { + Activity act(*logger, lvlInfo, actUnknown, fmt("fetching '%s'", lockedNode->lockedRef)); + auto accessor = lockedNode->lockedRef.input.getAccessor(store).first; + if (!evalSettings.lazyTrees) + fetchToStore( + fetchSettings, *store, accessor, FetchMode::Copy, lockedNode->lockedRef.input.getName()); + } catch (Error & e) { + printError("%s", e.what()); + nrFailed++; + } + } + + for (auto & [inputName, input] : node.inputs) { + if (auto inputNode = std::get_if<0>(&input)) + pool.enqueue(std::bind(visit, **inputNode)); + } + }; + + pool.enqueue(std::bind(visit, *flake.lockFile.root)); + + pool.process(); + + throw Exit(nrFailed ? 1 : 0); + } +}; + +static auto rCmdFlakePrefetchInputs = registerCommand2({"flake", "prefetch-inputs"}); diff --git a/src/nix/flake-prefetch-inputs.md b/src/nix/flake-prefetch-inputs.md new file mode 100644 index 00000000000..a69f7d36791 --- /dev/null +++ b/src/nix/flake-prefetch-inputs.md @@ -0,0 +1,17 @@ +R""( + +# Examples + +* Fetch the inputs of the `hydra` flake: + + ```console + # nix flake prefetch-inputs github:NixOS/hydra + ``` + +# Description + +Fetch the inputs of a flake. This ensures that they are already available for any subsequent evaluation of the flake. + +This operation is recursive: it will fetch not just the direct inputs of the top-level flake, but also transitive inputs. + +)"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 1d20add02ce..eb6326e1805 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -125,6 +125,7 @@ struct CmdFlakeUpdate : FlakeCommand lockFlags.recreateLockFile = updateAll; lockFlags.writeLockFile = true; lockFlags.applyNixConfig = true; + lockFlags.requireLockable = false; lockFlake(); } @@ -157,6 +158,7 @@ struct CmdFlakeLock : FlakeCommand lockFlags.writeLockFile = true; lockFlags.failOnUnlocked = true; lockFlags.applyNixConfig = true; + lockFlags.requireLockable = false; lockFlake(); } @@ -203,11 +205,17 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON void run(nix::ref store) override { + lockFlags.requireLockable = false; auto lockedFlake = lockFlake(); auto & flake = lockedFlake.flake; - // Currently, all flakes are in the Nix store via the rootFS accessor. - auto storePath = store->printStorePath(store->toStorePath(flake.path.path.abs()).first); + /* Hack to show the store path if available. */ + std::optional storePath; + if (store->isInStore(flake.path.path.abs())) { + auto path = store->toStorePath(flake.path.path.abs()).first; + if (store->isValidPath(path)) + storePath = path; + } if (json) { nlohmann::json j; @@ -229,7 +237,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; - j["path"] = storePath; + if (storePath) + j["path"] = store->printStorePath(*storePath); j["locks"] = lockedFlake.lockFile.toJSON().first; if (auto fingerprint = lockedFlake.getFingerprint(store, fetchSettings)) j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); @@ -246,9 +255,10 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout( ANSI_BOLD "Description:" ANSI_NORMAL " %s", *flake.description); - logger->cout( - ANSI_BOLD "Path:" ANSI_NORMAL " %s", - storePath); + if (storePath) + logger->cout( + ANSI_BOLD "Path:" ANSI_NORMAL " %s", + store->printStorePath(*storePath)); if (auto rev = flake.lockedRef.input.getRev()) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", @@ -628,7 +638,7 @@ struct CmdFlakeCheck : FlakeCommand if (name == "checks") { state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs()) { - std::string_view attr_name = state->symbols[attr.name]; + const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); if (checkSystemType(attr_name, attr.pos)) { state->forceAttrs(*attr.value, attr.pos, ""); @@ -823,8 +833,31 @@ struct CmdFlakeCheck : FlakeCommand if (build && !drvPaths.empty()) { Activity act(*logger, lvlInfo, actUnknown, fmt("running %d flake checks", drvPaths.size())); - store->buildPaths(drvPaths); + + auto missing = store->queryMissing(drvPaths); + + /* This command doesn't need to actually substitute + derivation outputs if they're missing but + substitutable. So filter out derivations that are + substitutable or already built. */ + std::vector toBuild; + for (auto & path : drvPaths) { + std::visit(overloaded { + [&](const DerivedPath::Built & bfd) { + auto drvPathP = std::get_if(&*bfd.drvPath); + if (!drvPathP || missing.willBuild.contains(drvPathP->path)) + toBuild.push_back(path); + }, + [&](const DerivedPath::Opaque & bo) { + if (!missing.willSubstitute.contains(bo.path)) + toBuild.push_back(path); + }, + }, path.raw()); + } + + store->buildPaths(toBuild); } + if (hasErrors) throw Error("some errors were encountered during the evaluation"); @@ -1079,7 +1112,10 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun StorePathSet sources; - auto storePath = store->toStorePath(flake.flake.path.path.abs()).first; + auto storePath = + dryRun + ? flake.flake.lockedRef.input.computeStorePath(*store) + : std::get(flake.flake.lockedRef.input.fetchToStore(store)); sources.insert(storePath); @@ -1095,7 +1131,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) - : (*inputNode)->lockedRef.input.fetchToStore(store).first; + : std::get((*inputNode)->lockedRef.input.fetchToStore(store)); sources.insert(*storePath); } if (json) { @@ -1534,12 +1570,6 @@ struct CmdFlake : NixMultiCommand #include "flake.md" ; } - - void run() override - { - experimentalFeatureSettings.require(Xp::Flakes); - NixMultiCommand::run(); - } }; static auto rCmdFlake = registerCommand("flake"); diff --git a/src/nix/main.cc b/src/nix/main.cc index 6144f746ffe..1c1ba95c779 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -115,7 +115,6 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs .description = "Print full build logs on standard error.", .category = loggingCategory, .handler = {[&]() { logger->setPrintBuildLogs(true); }}, - .experimentalFeature = Xp::NixCommand, }); addFlag({ @@ -131,7 +130,6 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs .description = "Disable substituters and consider all previously downloaded files up-to-date.", .category = miscCategory, .handler = {[&]() { useNet = false; }}, - .experimentalFeature = Xp::NixCommand, }); addFlag({ @@ -139,7 +137,6 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs .description = "Consider all previously downloaded files out-of-date.", .category = miscCategory, .handler = {[&]() { refresh = true; }}, - .experimentalFeature = Xp::NixCommand, }); aliases = { @@ -398,7 +395,6 @@ void mainWrapped(int argc, char * * argv) if (argc == 2 && std::string(argv[1]) == "__dump-language") { experimentalFeatureSettings.experimentalFeatures = { - Xp::Flakes, Xp::FetchClosure, Xp::DynamicDerivations, Xp::FetchTree, diff --git a/src/nix/meson.build b/src/nix/meson.build index 0ba8bdd4640..91c8261e4e7 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -78,6 +78,7 @@ nix_sources = [config_priv_h] + files( 'env.cc', 'eval.cc', 'flake.cc', + 'flake-prefetch-inputs.cc', 'formatter.cc', 'hash.cc', 'log.cc', diff --git a/src/nix/nix.md b/src/nix/nix.md index b88bd9a949a..add98907d95 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -48,11 +48,6 @@ manual](https://nixos.org/manual/nix/stable/). # Installables -> **Warning** \ -> Installables are part of the unstable -> [`nix-command` experimental feature](@docroot@/development/experimental-features.md#xp-feature-nix-command), -> and subject to change without notice. - Many `nix` subcommands operate on one or more *installables*. These are command line arguments that represent something that can be realised in the Nix store. @@ -72,13 +67,6 @@ That is, Nix will operate on the default flake output attribute of the flake in ### Flake output attribute -> **Warning** \ -> Flake output attribute installables depend on both the -> [`flakes`](@docroot@/development/experimental-features.md#xp-feature-flakes) -> and -> [`nix-command`](@docroot@/development/experimental-features.md#xp-feature-nix-command) -> experimental features, and subject to change without notice. - Example: `nixpkgs#hello` These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a diff --git a/src/nix/profile-history.md b/src/nix/profile-history.md index f0bfe503791..0c9a340ddf0 100644 --- a/src/nix/profile-history.md +++ b/src/nix/profile-history.md @@ -7,7 +7,7 @@ R""( ```console # nix profile history Version 508 (2020-04-10): - flake:nixpkgs#legacyPackages.x86_64-linux.awscli: ∅ -> 1.17.13 + flake:nixpkgs#legacyPackages.x86_64-linux.awscli: 1.17.13 added Version 509 (2020-05-16) <- 508: flake:nixpkgs#legacyPackages.x86_64-linux.awscli: 1.17.13 -> 1.18.211 @@ -20,7 +20,7 @@ between subsequent versions of a profile. It only shows top-level packages, not dependencies; for that, use [`nix profile diff-closures`](./nix3-profile-diff-closures.md). -The addition of a package to a profile is denoted by the string `∅ ->` -*version*, whereas the removal is denoted by *version* `-> ∅`. +The addition of a package to a profile is denoted by the string +*version* `added`, whereas the removal is denoted by *version* ` removed`. )"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 2c593729f49..5aa7013c532 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -289,12 +289,12 @@ struct ProfileManifest while (i != prev.elements.end() || j != cur.elements.end()) { if (j != cur.elements.end() && (i == prev.elements.end() || i->first > j->first)) { - logger->cout("%s%s: ∅ -> %s", indent, j->second.identifier(), j->second.versions()); + logger->cout("%s%s: %s added", indent, j->second.identifier(), j->second.versions()); changes = true; ++j; } else if (i != prev.elements.end() && (j == cur.elements.end() || i->first < j->first)) { - logger->cout("%s%s: %s -> ∅", indent, i->second.identifier(), i->second.versions()); + logger->cout("%s%s: %s removed", indent, i->second.identifier(), i->second.versions()); changes = true; ++i; } diff --git a/src/nix/repl.md b/src/nix/repl.md index 32c08e24b24..e608dabf6f9 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -36,7 +36,7 @@ R""( Loading Installable ''... Added 1 variables. - # nix repl --extra-experimental-features 'flakes' nixpkgs + # nix repl nixpkgs Loading Installable 'flake:nixpkgs#'... Added 5 variables. diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index fb868baa1f2..89ed7b91d56 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -3,6 +3,7 @@ #include "nix/main/shared.hh" #include "nix/store/store-open.hh" #include "nix/util/thread-pool.hh" +#include "nix/store/filetransfer.hh" #include @@ -28,6 +29,13 @@ struct CmdCopySigs : StorePathsCommand return "copy store path signatures from substituters"; } + std::string doc() override + { + return + #include "store-copy-sigs.md" + ; + } + void run(ref store, StorePaths && storePaths) override { if (substituterUris.empty()) @@ -38,7 +46,7 @@ struct CmdCopySigs : StorePathsCommand for (auto & s : substituterUris) substituters.push_back(openStore(s)); - ThreadPool pool; + ThreadPool pool{fileTransferSettings.httpConnections}; std::atomic added{0}; diff --git a/src/nix/store-copy-sigs.md b/src/nix/store-copy-sigs.md new file mode 100644 index 00000000000..67875622156 --- /dev/null +++ b/src/nix/store-copy-sigs.md @@ -0,0 +1,30 @@ +R""( + +# Examples + +* To copy signatures from a binary cache to the local store: + + ```console + # nix store copy-sigs --substituter https://cache.nixos.org \ + --recursive /nix/store/y1x7ng5bmc9s8lqrf98brcpk1a7lbcl5-hello-2.12.1 + ``` + +* To copy signatures from one binary cache to another: + + ```console + # nix store copy-sigs --substituter https://cache.nixos.org \ + --store file:///tmp/binary-cache \ + --recursive -v \ + /nix/store/y1x7ng5bmc9s8lqrf98brcpk1a7lbcl5-hello-2.12.1 + imported 2 signatures + ``` + +# Description + +`nix store copy-sigs` copies store path signatures from one store to another. + +It is not advised to copy signatures to binary cache stores. Binary cache signatures are stored in `.narinfo` files. Since these are cached aggressively, clients may not see the new signatures quickly. It is therefore better to set any required signatures when the paths are first uploaded to the binary cache. + +Store paths are processed in parallel. The amount of parallelism is controlled by the [`http-connections`](@docroot@/command-ref/conf-file.md#conf-http-connections) settings. + +)"" diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 64824110460..3f05622e15b 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -14,26 +14,6 @@ using namespace nix; struct CmdUpgradeNix : MixDryRun, StoreCommand { - std::filesystem::path profileDir; - - CmdUpgradeNix() - { - addFlag({ - .longName = "profile", - .shortName = 'p', - .description = "The path to the Nix profile to upgrade.", - .labels = {"profile-dir"}, - .handler = {&profileDir}, - }); - - addFlag({ - .longName = "nix-store-paths-url", - .description = "The URL of the file that contains the store paths of the latest Nix release.", - .labels = {"url"}, - .handler = {&(std::string&) settings.upgradeNixStorePathUrl}, - }); - } - /** * This command is stable before the others */ @@ -44,7 +24,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand std::string description() override { - return "upgrade Nix to the latest stable version"; + return "deprecated in favor of determinate-nixd upgrade"; } std::string doc() override @@ -58,101 +38,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand void run(ref store) override { - evalSettings.pureEval = true; - - if (profileDir == "") - profileDir = getProfileDir(store); - - printInfo("upgrading Nix in profile %s", profileDir); - - auto storePath = getLatestNix(store); - - auto version = DrvName(storePath.name()).version; - - if (dryRun) { - logger->stop(); - warn("would upgrade to version %s", version); - return; - } - - { - Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", store->printStorePath(storePath))); - store->ensurePath(storePath); - } - - { - Activity act(*logger, lvlInfo, actUnknown, fmt("verifying that '%s' works...", store->printStorePath(storePath))); - auto program = store->printStorePath(storePath) + "/bin/nix-env"; - auto s = runProgram(program, false, {"--version"}); - if (s.find("Nix") == std::string::npos) - throw Error("could not verify that '%s' works", program); - } - - logger->stop(); - - { - Activity act(*logger, lvlInfo, actUnknown, - fmt("installing '%s' into profile %s...", store->printStorePath(storePath), profileDir)); - - // FIXME: don't call an external process. - runProgram(getNixBin("nix-env").string(), false, - {"--profile", profileDir.string(), "-i", store->printStorePath(storePath), "--no-sandbox"}); - } - - printInfo(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version); - } - - /* Return the profile in which Nix is installed. */ - std::filesystem::path getProfileDir(ref store) - { - auto whereOpt = ExecutablePath::load().findName(OS_STR("nix-env")); - if (!whereOpt) - throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); - const auto & where = whereOpt->parent_path(); - - printInfo("found Nix in %s", where); - - if (hasPrefix(where.string(), "/run/current-system")) - throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); - - auto profileDir = where.parent_path(); - - // Resolve profile to /nix/var/nix/profiles/ link. - while (canonPath(profileDir.string()).find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir)) - profileDir = readLink(profileDir.string()); - - printInfo("found profile %s", profileDir); - - Path userEnv = canonPath(profileDir.string(), true); - - if (std::filesystem::exists(profileDir / "manifest.json")) - throw Error("directory %s is managed by 'nix profile' and currently cannot be upgraded by 'nix upgrade-nix'", profileDir); - - if (!std::filesystem::exists(profileDir / "manifest.nix")) - throw Error("directory %s does not appear to be part of a Nix profile", profileDir); - - if (!store->isValidPath(store->parseStorePath(userEnv))) - throw Error("directory '%s' is not in the Nix store", userEnv); - - return profileDir; - } - - /* Return the store path of the latest stable Nix. */ - StorePath getLatestNix(ref store) - { - Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); - - // FIXME: use nixos.org? - auto req = FileTransferRequest((std::string&) settings.upgradeNixStorePathUrl); - auto res = getFileTransfer()->download(req); - - auto state = std::make_unique(LookupPath{}, store, fetchSettings, evalSettings); - auto v = state->allocValue(); - state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); - Bindings & bindings(*state->allocBindings(0)); - auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v).first; - - return store->parseStorePath(state->forceString(*v2, noPos, "while evaluating the path tho latest nix version")); + throw Error("The upgrade-nix command isn't available in Determinate Nix; use %s instead", "sudo determinate-nixd upgrade"); } }; diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md index 3a3bf61b9b0..bb515717582 100644 --- a/src/nix/upgrade-nix.md +++ b/src/nix/upgrade-nix.md @@ -1,33 +1,11 @@ R""( -# Examples - -* Upgrade Nix to the stable version declared in Nixpkgs: - - ```console - # nix upgrade-nix - ``` - -* Upgrade Nix in a specific profile: - - ```console - # nix upgrade-nix --profile ~alice/.local/state/nix/profiles/profile - ``` - # Description -This command upgrades Nix to the stable version. - -By default, the latest stable version is defined by Nixpkgs, in -[nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix) -and updated manually. It may not always be the latest tagged release. - -By default, it locates the directory containing the `nix` binary in the `$PATH` -environment variable. If that directory is a Nix profile, it will -upgrade the `nix` package in that profile to the latest stable binary -release. +This command isn't available in Determinate Nix but is present in order to guide +users to the new upgrade path. -You cannot use this command to upgrade Nix in the system profile of a -NixOS system (that is, if `nix` is found in `/run/current-system`). +Use `sudo determinate-nixd upgrade` to upgrade Determinate Nix on systems that manage it imperatively. +In practice, this is any system that isn't running NixOS. )"" diff --git a/src/perl/package.nix b/src/perl/package.nix index 5841570cd09..f7e81b66a23 100644 --- a/src/perl/package.nix +++ b/src/perl/package.nix @@ -18,7 +18,7 @@ in perl.pkgs.toPerlModule ( mkMesonDerivation (finalAttrs: { - pname = "nix-perl"; + pname = "determinate-nix-perl"; inherit version; workDir = ./.; diff --git a/tests/functional/ca/derivation-json.sh b/tests/functional/ca/derivation-json.sh index 0b8bcac0cc8..2103707a2e8 100644 --- a/tests/functional/ca/derivation-json.sh +++ b/tests/functional/ca/derivation-json.sh @@ -19,7 +19,7 @@ drvPath3=$(nix derivation add --dry-run < "$TEST_HOME"/foo.json) [[ ! -e "$drvPath3" ]] # But the JSON is rejected without the experimental feature -expectStderr 1 nix derivation add < "$TEST_HOME"/foo.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'ca-derivations' is disabled" +expectStderr 1 nix derivation add < "$TEST_HOME"/foo.json --experimental-features '' | grepQuiet "experimental Nix feature 'ca-derivations' is disabled" # Without --dry-run it is actually written drvPath4=$(nix derivation add < "$TEST_HOME"/foo.json) diff --git a/tests/functional/ca/selfref-gc.sh b/tests/functional/ca/selfref-gc.sh index 24877889459..a730bdab694 100755 --- a/tests/functional/ca/selfref-gc.sh +++ b/tests/functional/ca/selfref-gc.sh @@ -4,7 +4,7 @@ source common.sh requireDaemonNewerThan "2.4pre20210626" -enableFeatures "ca-derivations nix-command flakes" +enableFeatures "ca-derivations" export NIX_TESTS_CA_BY_DEFAULT=1 cd .. diff --git a/tests/functional/common/functions.sh b/tests/functional/common/functions.sh index 1b2ec8fe0e8..fd59385762d 100644 --- a/tests/functional/common/functions.sh +++ b/tests/functional/common/functions.sh @@ -73,6 +73,7 @@ startDaemon() { fi # Start the daemon, wait for the socket to appear. rm -f "$NIX_DAEMON_SOCKET_PATH" + # TODO: remove the nix-command feature when we're no longer testing against old daemons. PATH=$DAEMON_PATH nix --extra-experimental-features 'nix-command' daemon & _NIX_TEST_DAEMON_PID=$! export _NIX_TEST_DAEMON_PID @@ -132,11 +133,11 @@ restartDaemon() { } isDaemonNewer () { - [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0 - local requiredVersion="$1" - local daemonVersion - daemonVersion=$("$NIX_DAEMON_PACKAGE/bin/nix" daemon --version | cut -d' ' -f3) - [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]] + [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0 + local requiredVersion="$1" + local daemonVersion + daemonVersion=$("$NIX_DAEMON_PACKAGE/bin/nix" daemon --version | sed 's/.*) //') + [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]] } skipTest () { diff --git a/tests/functional/common/init.sh b/tests/functional/common/init.sh index 66b44c76f69..7f28a09d753 100755 --- a/tests/functional/common/init.sh +++ b/tests/functional/common/init.sh @@ -12,7 +12,7 @@ if isTestOnNixOS; then ! test -e "$test_nix_conf" cat > "$test_nix_conf" < "$NIX_CONF_DIR"/nix.conf < "$NIX_CONF_DIR"/nix.conf.extra <"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +xpFeature=auto-allocate-uids +gatedSetting=auto-allocate-uids + +# Experimental feature is disabled before, ignore and warn. +NIX_CONFIG=" + experimental-features = + $gatedSetting = true +" expect 1 nix config show $gatedSetting 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr [[ $(cat "$TEST_ROOT/stdout") = '' ]] -grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" "$TEST_ROOT/stderr" -grepQuiet "error: could not find setting 'accept-flake-config'" "$TEST_ROOT/stderr" +grepQuiet "error: could not find setting '$gatedSetting'" "$TEST_ROOT/stderr" -# 'flakes' experimental-feature is disabled after, ignore and warn -NIX_CONFIG=' - accept-flake-config = true - experimental-features = nix-command -' expect 1 nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +# Experimental feature is disabled after, ignore and warn. +NIX_CONFIG=" + $gatedSetting = true + experimental-features = +" expect 1 nix config show $gatedSetting 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr [[ $(cat "$TEST_ROOT/stdout") = '' ]] -grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" "$TEST_ROOT/stderr" -grepQuiet "error: could not find setting 'accept-flake-config'" "$TEST_ROOT/stderr" +grepQuiet "error: could not find setting '$gatedSetting'" "$TEST_ROOT/stderr" -# 'flakes' experimental-feature is enabled before, process -NIX_CONFIG=' - experimental-features = nix-command flakes - accept-flake-config = true -' nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +# Experimental feature is enabled before, process. +NIX_CONFIG=" + experimental-features = $xpFeature + $gatedSetting = true +" nix config show $gatedSetting 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr grepQuiet "true" "$TEST_ROOT/stdout" -grepQuietInverse "Ignoring setting 'accept-flake-config'" "$TEST_ROOT/stderr" -# 'flakes' experimental-feature is enabled after, process -NIX_CONFIG=' - accept-flake-config = true - experimental-features = nix-command flakes -' nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +# Experimental feature is enabled after, process. +NIX_CONFIG=" + $gatedSetting = true + experimental-features = $xpFeature +" nix config show $gatedSetting 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr grepQuiet "true" "$TEST_ROOT/stdout" -grepQuietInverse "Ignoring setting 'accept-flake-config'" "$TEST_ROOT/stderr" +grepQuietInverse "Ignoring setting '$gatedSetting'" "$TEST_ROOT/stderr" function exit_code_both_ways { - expect 1 nix --experimental-features 'nix-command' "$@" 1>/dev/null - nix --experimental-features 'nix-command flakes' "$@" 1>/dev/null + expect 1 nix --experimental-features '' "$@" 1>/dev/null + nix --experimental-features "$xpFeature" "$@" 1>/dev/null # Also, the order should not matter - expect 1 nix "$@" --experimental-features 'nix-command' 1>/dev/null - nix "$@" --experimental-features 'nix-command flakes' 1>/dev/null + expect 1 nix "$@" --experimental-features '' 1>/dev/null + nix "$@" --experimental-features "$xpFeature" 1>/dev/null } -exit_code_both_ways show-config --flake-registry 'https://no' +exit_code_both_ways config show --auto-allocate-uids # Double check these are stable nix --experimental-features '' --help 1>/dev/null nix --experimental-features '' doctor --help 1>/dev/null nix --experimental-features '' repl --help 1>/dev/null nix --experimental-features '' upgrade-nix --help 1>/dev/null - -# These 3 arguments are currently given to all commands, which is wrong (as not -# all care). To deal with fixing later, we simply make them require the -# nix-command experimental features --- it so happens that the commands we wish -# stabilizing to do not need them anyways. -for arg in '--print-build-logs' '--offline' '--refresh'; do - nix --experimental-features 'nix-command' "$arg" --help 1>/dev/null - expect 1 nix --experimental-features '' "$arg" --help 1>/dev/null -done diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index 422cab96cc2..77bc030605f 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -2,6 +2,8 @@ source ../common.sh +export _NIX_TEST_BARF_ON_UNCACHEABLE=1 + # shellcheck disable=SC2034 # this variable is used by tests that source this file registry=$TEST_ROOT/registry.json diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index fe5acf26dec..a7d86698de8 100755 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -62,8 +62,8 @@ flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule # Check that dirtying a submodule makes the entire thing dirty. [[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) != null ]] echo '"foo"' > "$rootRepo"/submodule/sub.nix -[[ $(nix eval --json "$flakeref#sub" ) = '"foo"' ]] -[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] +[[ $(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval --json "$flakeref#sub" ) = '"foo"' ]] +[[ $(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] # Test that `nix flake metadata` parses `submodule` correctly. cat > "$rootRepo"/flake.nix < "$flake1Dir/foo" git -C "$flake1Dir" add $flake1Dir/foo [[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] +[[ $(_NIX_TEST_FAIL_ON_LARGE_PATH=1 nix flake metadata flake1 --json --refresh --warn-large-path-threshold 1 --lazy-trees | jq -r .dirtyRevision) == "$hash1-dirty" ]] [[ "$(nix flake metadata flake1 --json | jq -r .fingerprint)" != null ]] echo -n '# foo' >> "$flake1Dir/flake.nix" @@ -109,6 +112,11 @@ nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir#default" nix build -o "$TEST_ROOT/result" "$flake1Dir?ref=HEAD#default" nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" +# Check that the fetcher cache works. +if [[ $(nix config show lazy-trees) = false ]]; then + nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" -vvvvv 2>&1 | grepQuiet "source path.*cache hit" +fi + # Check that relative paths are allowed for git flakes. # This may change in the future once git submodule support is refined. # See: https://discourse.nixos.org/t/57783 and #9708. @@ -160,7 +168,12 @@ expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --commit-lock-file [[ -e "$flake2Dir/flake.lock" ]] [[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] -[[ $(jq --indent 0 --compact-output . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'.*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +[[ $(jq --indent 0 --compact-output . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'[0-9]*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +if [[ $(nix config show lazy-trees) = true ]]; then + # Test that `lazy-locks` causes NAR hashes to be omitted from the lock file. + nix flake update --flake "$flake2Dir" --commit-lock-file --lazy-locks + [[ $(jq --indent 0 --compact-output . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'[0-9]*',"ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +fi # Rerunning the build should not change the lockfile. nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" @@ -361,6 +374,7 @@ nix build -o $TEST_ROOT/result git+file://$flakeGitBare mkdir -p $flake5Dir writeDependentFlake $flake5Dir nix flake lock path://$flake5Dir +[[ "$(nix flake metadata path://$flake5Dir --json | jq -r .fingerprint)" != null ]] # Test tarball flakes. tar cfz $TEST_ROOT/flake.tar.gz -C $TEST_ROOT flake5 @@ -432,3 +446,41 @@ nix flake metadata "$flake2Dir" --reference-lock-file $TEST_ROOT/flake2-overridd # reference-lock-file can only be used if allow-dirty is set. expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock + +# After changing an input (flake2 from newFlake2Rev to prevFlake2Rev), we should have the transitive inputs locked by revision $prevFlake2Rev of flake2. +prevFlake1Rev=$(nix flake metadata --json "$flake1Dir" | jq -r .revision) +prevFlake2Rev=$(nix flake metadata --json "$flake2Dir" | jq -r .revision) + +echo "# bla" >> "$flake1Dir/flake.nix" +git -C "$flake1Dir" commit flake.nix -m 'bla' + +nix flake update --flake "$flake2Dir" +git -C "$flake2Dir" commit flake.lock -m 'bla' + +newFlake1Rev=$(nix flake metadata --json "$flake1Dir" | jq -r .revision) +newFlake2Rev=$(nix flake metadata --json "$flake2Dir" | jq -r .revision) + +cat > "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" <&1 | grep '/flakeB.*is forbidden in pure evaluation mode' -expect 1 nix flake lock --impure $flakeFollowsA 2>&1 | grep '/flakeB.*does not exist' +expect 1 nix flake lock --impure $flakeFollowsA 2>&1 | grep '/flakeB.*does not exist' # FIXME # Test relative non-flake inputs. cat > $flakeFollowsA/flake.nix < "$repo/flake.nix" < "$repo/foo" + +expectStderr 1 nix eval "$repo#z" | grepQuiet "error: Path 'foo' in the repository \"$repo\" is not tracked by Git." +expectStderr 1 nix eval "$repo#a" | grepQuiet "error: Path 'foo' in the repository \"$repo\" is not tracked by Git." + +git -C "$repo" add "$repo/foo" + +[[ $(nix eval --raw "$repo#z") = 123 ]] + +expectStderr 1 nix eval "$repo#b" | grepQuiet "error: Path 'dir' does not exist in Git repository \"$repo\"." + +mkdir -p "$repo/dir" +echo 456 > "$repo/dir/default.nix" + +expectStderr 1 nix eval "$repo#b" | grepQuiet "error: Path 'dir' in the repository \"$repo\" is not tracked by Git." + +git -C "$repo" add "$repo/dir/default.nix" + +[[ $(nix eval "$repo#b") = 456 ]] diff --git a/tests/functional/flakes/unlocked-override.sh b/tests/functional/flakes/unlocked-override.sh index ed05440de03..ed4d131b7ad 100755 --- a/tests/functional/flakes/unlocked-override.sh +++ b/tests/functional/flakes/unlocked-override.sh @@ -36,6 +36,7 @@ expectStderr 1 nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/f grepQuiet "Not writing lock file.*because it has an unlocked input" nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" --allow-dirty-locks +_NIX_TEST_FAIL_ON_LARGE_PATH=1 nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" --allow-dirty-locks --warn-large-path-threshold 1 --lazy-trees # Using a lock file with a dirty lock does not require --allow-dirty-locks, but should print a warning. expectStderr 0 nix eval "$flake2Dir#x" | diff --git a/tests/functional/gc-runtime.nix b/tests/functional/gc-runtime.nix index ee5980bdff9..df7f8ad1647 100644 --- a/tests/functional/gc-runtime.nix +++ b/tests/functional/gc-runtime.nix @@ -9,6 +9,7 @@ mkDerivation { cat > $out/program < \$TEST_ROOT/fifo sleep 10000 EOF diff --git a/tests/functional/gc-runtime.sh b/tests/functional/gc-runtime.sh index 0cccaaf16ab..34e99415d5c 100755 --- a/tests/functional/gc-runtime.sh +++ b/tests/functional/gc-runtime.sh @@ -21,11 +21,16 @@ nix-env -p "$profiles/test" -f ./gc-runtime.nix -i gc-runtime outPath=$(nix-env -p "$profiles/test" -q --no-name --out-path gc-runtime) echo "$outPath" +fifo="$TEST_ROOT/fifo" +mkfifo "$fifo" + echo "backgrounding program..." -"$profiles"/test/program & -sleep 2 # hack - wait for the program to get started +"$profiles"/test/program "$fifo" & child=$! echo PID=$child +cat "$fifo" + +expectStderr 1 nix-store --delete "$outPath" | grepQuiet "Cannot delete path.*because it's referenced by the GC root '/proc/" nix-env -p "$profiles/test" -e gc-runtime nix-env -p "$profiles/test" --delete-generations old diff --git a/tests/functional/gc.sh b/tests/functional/gc.sh index c58f47021f8..92ac7fac41d 100755 --- a/tests/functional/gc.sh +++ b/tests/functional/gc.sh @@ -23,10 +23,10 @@ if nix-store --gc --print-dead | grep -E "$outPath"$; then false; fi nix-store --gc --print-dead inUse=$(readLink "$outPath/reference-to-input-2") -if nix-store --delete "$inUse"; then false; fi +expectStderr 1 nix-store --delete "$inUse" | grepQuiet "Cannot delete path.*because it's referenced by path '" test -e "$inUse" -if nix-store --delete "$outPath"; then false; fi +expectStderr 1 nix-store --delete "$outPath" | grepQuiet "Cannot delete path.*because it's referenced by the GC root " test -e "$outPath" for i in "$NIX_STORE_DIR"/*; do diff --git a/tests/functional/impure-derivations.sh b/tests/functional/impure-derivations.sh index 5dea220fec7..69884c2932e 100755 --- a/tests/functional/impure-derivations.sh +++ b/tests/functional/impure-derivations.sh @@ -21,7 +21,7 @@ drvPath2=$(nix derivation add < $TEST_HOME/impure-drv.json) [[ "$drvPath" = "$drvPath2" ]] # But only with the experimental feature! -expectStderr 1 nix derivation add < $TEST_HOME/impure-drv.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'impure-derivations' is disabled" +expectStderr 1 nix derivation add < $TEST_HOME/impure-drv.json --experimental-features '' | grepQuiet "experimental Nix feature 'impure-derivations' is disabled" nix build --dry-run --json --file ./impure-derivations.nix impure.all json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all) diff --git a/tests/functional/lang/eval-fail-hashfile-missing.err.exp b/tests/functional/lang/eval-fail-hashfile-missing.err.exp index 0d3747a6d57..901dea2b544 100644 --- a/tests/functional/lang/eval-fail-hashfile-missing.err.exp +++ b/tests/functional/lang/eval-fail-hashfile-missing.err.exp @@ -10,4 +10,4 @@ error: … while calling the 'hashFile' builtin - error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory + error: path '/pwd/lang/this-file-is-definitely-not-there-7392097' does not exist diff --git a/tests/functional/lang/eval-fail-missing-arg-import.err.exp b/tests/functional/lang/eval-fail-missing-arg-import.err.exp new file mode 100644 index 00000000000..45774f0032d --- /dev/null +++ b/tests/functional/lang/eval-fail-missing-arg-import.err.exp @@ -0,0 +1,12 @@ +error: + … from call site + at /pwd/lang/eval-fail-missing-arg-import.nix:1:1: + 1| import ./non-eval-trivial-lambda-formals.nix { } + | ^ + 2| + + error: function 'anonymous lambda' called without required argument 'a' + at /pwd/lang/non-eval-trivial-lambda-formals.nix:1:1: + 1| { a }: a + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-missing-arg-import.nix b/tests/functional/lang/eval-fail-missing-arg-import.nix new file mode 100644 index 00000000000..7cb33f2b516 --- /dev/null +++ b/tests/functional/lang/eval-fail-missing-arg-import.nix @@ -0,0 +1 @@ +import ./non-eval-trivial-lambda-formals.nix { } diff --git a/tests/functional/lang/eval-fail-undeclared-arg-import.err.exp b/tests/functional/lang/eval-fail-undeclared-arg-import.err.exp new file mode 100644 index 00000000000..ca797d3eca2 --- /dev/null +++ b/tests/functional/lang/eval-fail-undeclared-arg-import.err.exp @@ -0,0 +1,13 @@ +error: + … from call site + at /pwd/lang/eval-fail-undeclared-arg-import.nix:1:1: + 1| import ./non-eval-trivial-lambda-formals.nix { + | ^ + 2| a = "a"; + + error: function 'anonymous lambda' called with unexpected argument 'b' + at /pwd/lang/non-eval-trivial-lambda-formals.nix:1:1: + 1| { a }: a + | ^ + 2| + Did you mean a? diff --git a/tests/functional/lang/eval-fail-undeclared-arg-import.nix b/tests/functional/lang/eval-fail-undeclared-arg-import.nix new file mode 100644 index 00000000000..e8454c725a7 --- /dev/null +++ b/tests/functional/lang/eval-fail-undeclared-arg-import.nix @@ -0,0 +1,4 @@ +import ./non-eval-trivial-lambda-formals.nix { + a = "a"; + b = "b"; +} diff --git a/tests/functional/lang/non-eval-trivial-lambda-formals.nix b/tests/functional/lang/non-eval-trivial-lambda-formals.nix new file mode 100644 index 00000000000..46a7ea4f494 --- /dev/null +++ b/tests/functional/lang/non-eval-trivial-lambda-formals.nix @@ -0,0 +1 @@ +{ a }: a diff --git a/tests/functional/linux-sandbox.sh b/tests/functional/linux-sandbox.sh index abb635f1195..e02ff5326a2 100755 --- a/tests/functional/linux-sandbox.sh +++ b/tests/functional/linux-sandbox.sh @@ -96,3 +96,8 @@ nix-sandbox-build symlink-derivation.nix -A test_sandbox_paths \ --option extra-sandbox-paths "/dir=$TEST_ROOT" \ --option extra-sandbox-paths "/symlinkDir=$symlinkDir" \ --option extra-sandbox-paths "/symlink=$symlinkcert" + +# Nonexistent sandbox paths should error early in the build process +expectStderr 1 nix-sandbox-build --option extra-sandbox-paths '/does-not-exist' \ + -E 'with import '"${config_nix}"'; mkDerivation { name = "trivial"; buildCommand = "echo > $out"; }' | + grepQuiet "path '/does-not-exist' is configured as part of the \`sandbox-paths\` option, but is inaccessible" diff --git a/tests/functional/local-overlay-store/delete-refs-inner.sh b/tests/functional/local-overlay-store/delete-refs-inner.sh index 385eeadc923..01b6162c529 100644 --- a/tests/functional/local-overlay-store/delete-refs-inner.sh +++ b/tests/functional/local-overlay-store/delete-refs-inner.sh @@ -22,14 +22,14 @@ input2=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg input3=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2 -A passthru.input3 -j0) # Can't delete because referenced -expectStderr 1 nix-store --delete $input1 | grepQuiet "Cannot delete path" -expectStderr 1 nix-store --delete $input2 | grepQuiet "Cannot delete path" -expectStderr 1 nix-store --delete $input3 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --delete $input1 | grepQuiet "Cannot delete path.*because it's referenced by path" +expectStderr 1 nix-store --delete $input2 | grepQuiet "Cannot delete path.*because it's referenced by path" +expectStderr 1 nix-store --delete $input3 | grepQuiet "Cannot delete path.*because it's referenced by path" # These same paths are referenced in the lower layer (by the seed 1 # build done in `initLowerStore`). -expectStderr 1 nix-store --store "$storeA" --delete $input2 | grepQuiet "Cannot delete path" -expectStderr 1 nix-store --store "$storeA" --delete $input3 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --store "$storeA" --delete $input2 | grepQuiet "Cannot delete path.*because it's referenced by path" +expectStderr 1 nix-store --store "$storeA" --delete $input3 | grepQuiet "Cannot delete path.*because it's referenced by path" # Can delete nix-store --delete $hermetic diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index b1cfef6b0b2..a96abbbdff5 100755 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -58,8 +58,8 @@ nix profile list | grep -A4 'Name:.*flake1' | grep 'Locked flake URL:.*narHash' [ -e $TEST_HOME/.nix-profile/share/man ] (! [ -e $TEST_HOME/.nix-profile/include ]) nix profile history -nix profile history | grep "packages.$system.default: ∅ -> 1.0" -nix profile diff-closures | grep 'env-manifest.nix: ε → ∅' +nix profile history | grep "packages.$system.default: 1.0, 1.0-man added" +nix profile diff-closures | grep 'env-manifest.nix: (no version) removed' # Test XDG Base Directories support export NIX_CONFIG="use-xdg-base-directories = true" @@ -128,7 +128,7 @@ nix profile rollback [ -e $TEST_HOME/.nix-profile/bin/foo ] nix profile remove foo 2>&1 | grep 'removed 1 packages' (! [ -e $TEST_HOME/.nix-profile/bin/foo ]) -nix profile history | grep 'foo: 1.0 -> ∅' +nix profile history | grep 'foo: 1.0 removed' nix profile diff-closures | grep 'Version 3 -> 4' # Test installing a non-flake package. diff --git a/tests/functional/package.nix b/tests/functional/package.nix index 43f2f25a200..799026ebe62 100644 --- a/tests/functional/package.nix +++ b/tests/functional/package.nix @@ -26,6 +26,9 @@ # For running the functional tests against a different pre-built Nix. test-daemon ? null, + + # Whether to run tests with lazy trees enabled. + lazyTrees ? false, }: let @@ -95,6 +98,8 @@ mkMesonDerivation ( mkdir $out ''; + _NIX_TEST_EXTRA_CONFIG = lib.optionalString lazyTrees "lazy-trees = true"; + meta = { platforms = lib.platforms.unix; }; diff --git a/tests/functional/recursive.nix b/tests/functional/recursive.nix index be9e55da37e..aa2aa26c549 100644 --- a/tests/functional/recursive.nix +++ b/tests/functional/recursive.nix @@ -17,7 +17,7 @@ mkDerivation rec { buildCommand = '' mkdir $out - opts="--experimental-features nix-command ${ + opts="${ if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else "" }" diff --git a/tests/functional/recursive.sh b/tests/functional/recursive.sh index 640fb92d2c5..fb0aa69752e 100755 --- a/tests/functional/recursive.sh +++ b/tests/functional/recursive.sh @@ -13,7 +13,7 @@ rm -f $TEST_ROOT/result export unreachable=$(nix store add-path ./recursive.sh) -NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --file ./recursive.nix +NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'recursive-nix' build -o $TEST_ROOT/result -L --impure --file ./recursive.nix [[ $(cat $TEST_ROOT/result/inner1) =~ blaat ]] diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index bfe18c9e586..d75b80bb0b0 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -155,7 +155,7 @@ EOF testReplResponse ' foo + baz ' "3" \ - ./flake ./flake\#bar --experimental-features 'flakes' + ./flake ./flake\#bar testReplResponse $' :a { a = 1; b = 2; longerName = 3; "with spaces" = 4; } @@ -190,7 +190,7 @@ testReplResponseNoRegex $' # - Check that the result has changed mkfifo repl_fifo touch repl_output -nix repl ./flake --experimental-features 'flakes' < repl_fifo >> repl_output 2>&1 & +nix repl ./flake < repl_fifo >> repl_output 2>&1 & repl_pid=$! exec 3>repl_fifo # Open fifo for writing echo "changingThing" >&3 @@ -314,7 +314,7 @@ import $testDir/lang/parse-fail-eof-pos.nix badDiff=0 badExitCode=0 -nixVersion="$(nix eval --impure --raw --expr 'builtins.nixVersion' --extra-experimental-features nix-command)" +nixVersion="$(nix eval --impure --raw --expr 'builtins.nixVersion')" # TODO: write a repl interacter for testing. Papering over the differences between readline / editline and between platforms is a pain. diff --git a/tests/functional/store-info.sh b/tests/functional/store-info.sh index b1e0772b5ae..dfab2ddf4d3 100755 --- a/tests/functional/store-info.sh +++ b/tests/functional/store-info.sh @@ -10,7 +10,7 @@ echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" echo "$LEGACY_STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then - DAEMON_VERSION=$("$NIX_DAEMON_PACKAGE"/bin/nix daemon --version | cut -d' ' -f3) + DAEMON_VERSION=$("$NIX_DAEMON_PACKAGE"/bin/nix daemon --version | sed 's/.*) //') echo "$STORE_INFO" | grep "Version: $DAEMON_VERSION" [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] fi diff --git a/tests/installer/default.nix b/tests/installer/default.nix index d48537dd0d0..dc831cc97b1 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -232,7 +232,7 @@ let source /etc/bashrc || true nix-env --version - nix --extra-experimental-features nix-command store info + nix store info out=\$(nix-build --no-substitute -E 'derivation { name = "foo"; system = "x86_64-linux"; builder = "/bin/sh"; args = ["-c" "echo foobar > \$out"]; }') [[ \$(cat \$out) = foobar ]] diff --git a/tests/nixos/authorization.nix b/tests/nixos/authorization.nix index 6540e9fa337..944e5925925 100644 --- a/tests/nixos/authorization.nix +++ b/tests/nixos/authorization.nix @@ -13,8 +13,6 @@ users.users.alice.isNormalUser = true; users.users.bob.isNormalUser = true; users.users.mallory.isNormalUser = true; - - nix.settings.experimental-features = "nix-command"; }; testScript = diff --git a/tests/nixos/cgroups/default.nix b/tests/nixos/cgroups/default.nix index a6b4bca8c76..4161aba2ca2 100644 --- a/tests/nixos/cgroups/default.nix +++ b/tests/nixos/cgroups/default.nix @@ -9,7 +9,7 @@ { virtualisation.additionalPaths = [ pkgs.stdenvNoCC ]; nix.extraOptions = '' - extra-experimental-features = nix-command auto-allocate-uids cgroups + extra-experimental-features = auto-allocate-uids cgroups extra-system-features = uid-range ''; nix.settings.use-cgroups = true; diff --git a/tests/nixos/chroot-store.nix b/tests/nixos/chroot-store.nix index 0a4fff99222..ecac371e152 100644 --- a/tests/nixos/chroot-store.nix +++ b/tests/nixos/chroot-store.nix @@ -25,7 +25,6 @@ in virtualisation.writableStore = true; virtualisation.additionalPaths = [ pkgA ]; environment.systemPackages = [ pkgB ]; - nix.extraOptions = "experimental-features = nix-command"; }; }; diff --git a/tests/nixos/containers/containers.nix b/tests/nixos/containers/containers.nix index b590dc8498f..8d07c80b6a3 100644 --- a/tests/nixos/containers/containers.nix +++ b/tests/nixos/containers/containers.nix @@ -23,7 +23,7 @@ virtualisation.memorySize = 4096; nix.settings.substituters = lib.mkForce [ ]; nix.extraOptions = '' - extra-experimental-features = nix-command auto-allocate-uids cgroups + extra-experimental-features = auto-allocate-uids cgroups extra-system-features = uid-range ''; nix.nixPath = [ "nixpkgs=${nixpkgs}" ]; diff --git a/tests/nixos/fetch-git/testsupport/setup.nix b/tests/nixos/fetch-git/testsupport/setup.nix index c13386c7223..3c9f4bddea1 100644 --- a/tests/nixos/fetch-git/testsupport/setup.nix +++ b/tests/nixos/fetch-git/testsupport/setup.nix @@ -81,10 +81,6 @@ in environment.variables = { _NIX_FORCE_HTTP = "1"; }; - nix.settings.experimental-features = [ - "nix-command" - "flakes" - ]; }; setupScript = ''''; testScript = '' diff --git a/tests/nixos/fetchurl.nix b/tests/nixos/fetchurl.nix index e8663debbcd..d75cc2017de 100644 --- a/tests/nixos/fetchurl.nix +++ b/tests/nixos/fetchurl.nix @@ -64,8 +64,6 @@ in ]; virtualisation.writableStore = true; - - nix.settings.experimental-features = "nix-command"; }; }; diff --git a/tests/nixos/fsync.nix b/tests/nixos/fsync.nix index e215e5b3c25..50105f1ccd9 100644 --- a/tests/nixos/fsync.nix +++ b/tests/nixos/fsync.nix @@ -23,7 +23,6 @@ in { virtualisation.emptyDiskImages = [ 1024 ]; environment.systemPackages = [ pkg1 ]; - nix.settings.experimental-features = [ "nix-command" ]; nix.settings.fsync-store-paths = true; nix.settings.require-sigs = false; boot.supportedFilesystems = [ diff --git a/tests/nixos/git-submodules.nix b/tests/nixos/git-submodules.nix index c6f53ada2dc..9105eb79bd7 100644 --- a/tests/nixos/git-submodules.nix +++ b/tests/nixos/git-submodules.nix @@ -24,7 +24,6 @@ { programs.ssh.extraConfig = "ConnectTimeout 30"; environment.systemPackages = [ pkgs.git ]; - nix.extraOptions = "experimental-features = nix-command flakes"; }; }; diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index 06142c2efda..67e2d41e906 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -81,7 +81,7 @@ let mkdir -p $out/archive dir=NixOS-nixpkgs-${nixpkgs.shortRev} - cp -prd ${nixpkgs} $dir + cp -rd --preserve=ownership,timestamps ${nixpkgs} $dir # Set the correct timestamp in the tarball. find $dir -print0 | xargs -0 touch -h -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${ builtins.substring 12 2 nixpkgs.lastModifiedDate @@ -163,7 +163,6 @@ in ]; virtualisation.memorySize = 4096; nix.settings.substituters = lib.mkForce [ ]; - nix.extraOptions = "experimental-features = nix-command flakes"; networking.hosts.${(builtins.head nodes.github.networking.interfaces.eth1.ipv4.addresses).address} = [ "channels.nixos.org" @@ -205,7 +204,7 @@ in cat_log() # ... otherwise it should use the API - out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0") + out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0 --no-trust-tarballs-from-git-forges") print(out) info = json.loads(out) assert info["revision"] == "${private-flake-rev}", f"revision mismatch: {info['revision']} != ${private-flake-rev}" @@ -224,6 +223,10 @@ in hash = client.succeed(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr '(fetchTree {info['url']}).narHash'") assert hash == info['locked']['narHash'] + # Fetching with an incorrect NAR hash should fail. + out = client.fail(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr '(fetchTree \"github:fancy-enterprise/private-flake/{info['revision']}?narHash=sha256-HsrRFZYg69qaVe/wDyWBYLeS6ca7ACEJg2Z%2BGpEFw4A%3D\").narHash' 2>&1") + assert "NAR hash mismatch in input" in out, "NAR hash check did not fail with the expected error" + # Fetching without a narHash should succeed if trust-github is set and fail otherwise. client.succeed(f"nix eval --raw --expr 'builtins.fetchTree github:github:fancy-enterprise/private-flake/{info['revision']}'") out = client.fail(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr 'builtins.fetchTree github:github:fancy-enterprise/private-flake/{info['revision']}' 2>&1") diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix index 64de622de76..a7f0a6a326f 100644 --- a/tests/nixos/nix-copy.nix +++ b/tests/nixos/nix-copy.nix @@ -39,7 +39,6 @@ in pkgD.drvPath ]; nix.settings.substituters = lib.mkForce [ ]; - nix.settings.experimental-features = [ "nix-command" ]; services.getty.autologinUser = "root"; programs.ssh.extraConfig = '' Host * diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index a22e4c2c28f..1f79e8cf969 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -34,7 +34,6 @@ in virtualisation.additionalPaths = [ pkgA ]; environment.systemPackages = [ pkgs.minio-client ]; nix.extraOptions = '' - experimental-features = nix-command substituters = ''; services.minio = { @@ -53,7 +52,6 @@ in { virtualisation.writableStore = true; nix.extraOptions = '' - experimental-features = nix-command substituters = ''; }; diff --git a/tests/nixos/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix index 61670ccf346..5b40866d1fa 100644 --- a/tests/nixos/sourcehut-flakes.nix +++ b/tests/nixos/sourcehut-flakes.nix @@ -48,7 +48,7 @@ let nixpkgs-repo = pkgs.runCommand "nixpkgs-flake" { } '' dir=NixOS-nixpkgs-${nixpkgs.shortRev} - cp -prd ${nixpkgs} $dir + cp -rd --preserve=ownership,timestamps ${nixpkgs} $dir # Set the correct timestamp in the tarball. find $dir -print0 | xargs -0 touch -h -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${ @@ -119,7 +119,6 @@ in virtualisation.memorySize = 4096; nix.settings.substituters = lib.mkForce [ ]; nix.extraOptions = '' - experimental-features = nix-command flakes flake-registry = https://git.sr.ht/~NixOS/flake-registry/blob/master/flake-registry.json ''; environment.systemPackages = [ pkgs.jq ]; diff --git a/tests/nixos/tarball-flakes.nix b/tests/nixos/tarball-flakes.nix index 7b3638b64b8..a68d64bfdd8 100644 --- a/tests/nixos/tarball-flakes.nix +++ b/tests/nixos/tarball-flakes.nix @@ -13,7 +13,7 @@ let set -x dir=nixpkgs-${nixpkgs.shortRev} - cp -prd ${nixpkgs} $dir + cp -rd --preserve=ownership,timestamps ${nixpkgs} $dir # Set the correct timestamp in the tarball. find $dir -print0 | xargs -0 touch -h -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${ builtins.substring 12 2 nixpkgs.lastModifiedDate @@ -61,7 +61,6 @@ in ]; virtualisation.memorySize = 4096; nix.settings.substituters = lib.mkForce [ ]; - nix.extraOptions = "experimental-features = nix-command flakes"; }; }; diff --git a/tests/repl-completion.nix b/tests/repl-completion.nix index 07406e969cd..9ae37796bf5 100644 --- a/tests/repl-completion.nix +++ b/tests/repl-completion.nix @@ -15,7 +15,7 @@ runCommand "repl-completion" ]; expectScript = '' # Regression https://github.com/NixOS/nix/pull/10778 - spawn nix repl --offline --extra-experimental-features nix-command + spawn nix repl --offline expect "nix-repl>" send "foo = import ./does-not-exist.nix\n" expect "nix-repl>"