diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 02ab247c76..dddac3ce96 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,9 +1,5 @@ # The whole SDK team owns the whole repo for now. * @dfinity/dx -# The infra team can still approve Nix PRs. -**/*.nix @dfinity-lab/infra -nix/** @dfinity-lab/infra - -# Allows sources.json to be updated without full approval (only CI). -nix/sources.json @ghost +# Allows dfx-asset-sources.json to be updated without full approval (only CI). +src/dfx/assets/dfx-asset-sources.json @ghost diff --git a/.github/repo_policies/BOT_APPROVED_FILES b/.github/repo_policies/BOT_APPROVED_FILES index de51b66e92..2b332a9f94 100644 --- a/.github/repo_policies/BOT_APPROVED_FILES +++ b/.github/repo_policies/BOT_APPROVED_FILES @@ -2,6 +2,5 @@ # This is to increase security and prevent accidentally updating files that shouldn't be changed by a bot Cargo.lock CHANGELOG.md -nix/sources.json src/dfx/Cargo.toml -src/dfx/assets/dfx-asset-sources.toml +src/dfx/assets/dfx-asset-sources.json diff --git a/.github/workflows/bitcoin-canister-update.yml b/.github/workflows/bitcoin-canister-update.yml index 9672bb89b3..78596224d3 100644 --- a/.github/workflows/bitcoin-canister-update.yml +++ b/.github/workflows/bitcoin-canister-update.yml @@ -23,6 +23,9 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} + - name: Install script dependencies + run: sudo apt-get install -y jq moreutils + - name: Fetch Bitcoin Canister latest release tag env: GH_TOKEN: "${{ secrets.NIV_UPDATER_TOKEN }}" @@ -33,8 +36,7 @@ jobs: - name: Check if the latest release tag has been updated run: | - URL_ENCODED_CURRENT_TAG=$(jq -r '.["ic-btc-canister"].version' nix/sources.json) - CURRENT_TAG=$(python -c "import sys, urllib.parse as ul; print(ul.unquote_plus(sys.argv[1]))" "$URL_ENCODED_CURRENT_TAG") + CURRENT_TAG=$(jq -r '.commmon."ic-btc-canister".version' src/dfx/assets/dfx-asset-sources.json) echo "Current tag is $CURRENT_TAG" if [[ "$CURRENT_TAG" == "$LATEST_TAG" ]]; then echo "No update is required." @@ -43,24 +45,10 @@ jobs: echo "An update is required." fi - - name: install Nix - uses: cachix/install-nix-action@v21 - with: - nix_path: nixpkgs=channel:nixos-unstable - - - name: install niv (dependency manager for Nix projects) - run: nix-env -i niv -f '' - - - name: install packages from nix/sources.json - run: niv update - - - name: update sources - run: | - URL_ENCODED_LATEST_TAG=$(echo -n "$LATEST_TAG" | python -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.stdin.read().strip(), safe=""))') - niv update ic-btc-canister -a version=$URL_ENCODED_LATEST_TAG - ./scripts/write-dfx-asset-sources.sh + - name: Update sources to use the latest bitcoin canister version + run: ./scripts/update-btc-canister.sh "$LATEST_TAG" - - name: Update dfx to use the latest Bitcoin Canister version + - name: Create PR env: GH_TOKEN: "${{ secrets.NIV_UPDATER_TOKEN }}" run: | diff --git a/.github/workflows/consistent-sources.yml b/.github/workflows/consistent-sources.yml deleted file mode 100644 index 9610578bc5..0000000000 --- a/.github/workflows/consistent-sources.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Check that nix/sources.json and scripts/dfx-asset-sources.sh are consistent -on: - pull_request: - push: - branches: - - master - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - # When getting Rust dependencies, retry on network error: - CARGO_NET_RETRY: 10 - # Use the local .curlrc - CURL_HOME: . - # Disable DFX telemetry - DFX_TELEMETRY: 'off' - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: cachix/install-nix-action@v18 - with: - nix_path: nixpkgs=channel:nixos-21.05-small - - name: Install dependencies - run: sudo apt-get install --yes jq - - name: Check - run: | - ./scripts/write-dfx-asset-sources.sh - if [ -z "$(git status --porcelain)" ] - then - exit 0 - else - git diff - echo "scripts/dfx-asset-sources.sh is out of date. Please run:" - echo " scripts/write-dfx-asset-sources.sh" - exit 1 - fi diff --git a/.github/workflows/update-motoko.yml b/.github/workflows/update-motoko.yml index 53e9261105..2897378f3f 100644 --- a/.github/workflows/update-motoko.yml +++ b/.github/workflows/update-motoko.yml @@ -38,17 +38,6 @@ jobs: echo "MOTOKO_VERSION=${{ github.event.inputs.motokoVersion }}" >> $GITHUB_ENV fi grep -s "MOTOKO_VERSION" $GITHUB_ENV - - - name: install Nix - uses: cachix/install-nix-action@v21 - with: - nix_path: nixpkgs=channel:nixos-unstable - - - name: install niv (dependency manager for Nix projects) - run: nix-env -i niv -f '' - - - name: install packages from nix/sources.json - run: niv update - name: update Motoko run: | diff --git a/.github/workflows/update-replica-version.yml b/.github/workflows/update-replica-version.yml index fac1154bae..b2c8ac7904 100644 --- a/.github/workflows/update-replica-version.yml +++ b/.github/workflows/update-replica-version.yml @@ -45,17 +45,6 @@ jobs: echo "REPLICA_VERSION=${{ github.event.inputs.customReplicaVersion }}" >> $GITHUB_ENV fi grep -s "REPLICA_VERSION" $GITHUB_ENV - - - name: install Nix - uses: cachix/install-nix-action@v21 - with: - nix_path: nixpkgs=channel:nixos-unstable - - - name: install niv (dependency manager for Nix projects) - run: nix-env -i niv -f '' - - - name: install packages from nix/sources.json - run: niv update - name: update replica run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 954214ef52..ee9f065199 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,7 +103,6 @@ What that means is your PR title should start with one of the following prefix: #### Locally To update the replica to a given $SHA from the dfinity repo, execute the following: ``` bash -# Requires niv to run. To install niv, run nix-env -iA nixpkgs.niv ./scripts/update-replica.sh $SHA ``` @@ -114,7 +113,6 @@ To update Motoko to a given $VERSION from the motoko and motoko-base repos, run You can also execute the following locally: ``` bash -# Requires niv to run. To install niv, run nix-env -iA nixpkgs.niv ./scripts/update-motoko.sh $VERSION ``` diff --git a/Cargo.lock b/Cargo.lock index 7b2520bb9b..08e411553f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1622,7 +1622,6 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "toml", "url", "uuid", "walkdir", @@ -4951,7 +4950,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ - "toml_edit 0.22.23", + "toml_edit", ] [[package]] @@ -5777,15 +5776,6 @@ dependencies = [ "syn 2.0.98", ] -[[package]] -name = "serde_spanned" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" -dependencies = [ - "serde", -] - [[package]] name = "serde_tokenstream" version = "0.1.7" @@ -6504,39 +6494,11 @@ dependencies = [ "tokio", ] -[[package]] -name = "toml" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit 0.19.15", -] - [[package]] name = "toml_datetime" version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.19.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" -dependencies = [ - "indexmap 2.7.1", - "serde", - "serde_spanned", - "toml_datetime", - "winnow 0.5.40", -] [[package]] name = "toml_edit" @@ -6546,7 +6508,7 @@ checksum = "02a8b472d1a3d7c18e2d61a489aee3453fd9031c33e4f55bd533f4a7adca1bee" dependencies = [ "indexmap 2.7.1", "toml_datetime", - "winnow 0.7.0", + "winnow", ] [[package]] @@ -7250,15 +7212,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "winnow" -version = "0.5.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" -dependencies = [ - "memchr", -] - [[package]] name = "winnow" version = "0.7.0" diff --git a/e2e/tests-dfx/info.bash b/e2e/tests-dfx/info.bash index a0dafa8220..77c4809ec0 100644 --- a/e2e/tests-dfx/info.bash +++ b/e2e/tests-dfx/info.bash @@ -53,8 +53,8 @@ teardown() { } @test "displays the replica revision included in dfx" { - nix_sources_path="${BATS_TEST_DIRNAME}/../../nix/sources.json" - expected_rev="$(jq -r '."pocket-ic-x86_64-linux".rev' "$nix_sources_path")" + sources_path="${BATS_TEST_DIRNAME}/../../src/dfx/assets/dfx-asset-sources.json" + expected_rev="$(jq -r '."x86_64-linux"."pocket-ic".rev' "$sources_path")" assert_command dfx info replica-rev assert_eq "$expected_rev" diff --git a/nix/README.adoc b/nix/README.adoc deleted file mode 100644 index e04dae8d6d..0000000000 --- a/nix/README.adoc +++ /dev/null @@ -1,7 +0,0 @@ -= Nix - -== Niv - -We use the https://github.com/nmattia/niv[niv] tool to manage references to -external source code. All references to external source code should be specified -in `sources.json`. `niv` can then be used to add, update or drop dependencies. diff --git a/nix/sources.json b/nix/sources.json deleted file mode 100644 index 3409cdace7..0000000000 --- a/nix/sources.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "ic-btc-canister": { - "sha256": "0giwl43l1rlwc116zrnr4ss8cb23c0jxk2jxshvmy189pqaadbfh", - "type": "file", - "url": "https://github.com/dfinity/bitcoin-canister/releases/download/release%2F2024-08-30/ic-btc-canister.wasm.gz", - "url_template": "https://github.com/dfinity/bitcoin-canister/releases/download//ic-btc-canister.wasm.gz", - "version": "release%2F2024-08-30" - }, - "motoko-base": { - "builtin": false, - "description": "The Motoko base library", - "owner": "dfinity", - "sha256": "0w03fh54jwkzic9a92f6kfmv8r0w9f1jk1r0g4r91p79h6ylwfba", - "type": "tarball", - "url": "https://github.com/dfinity/motoko/releases/download/0.14.9/motoko-base-library.tar.gz", - "url_template": "https://github.com/dfinity/motoko/releases/download//motoko-base-library.tar.gz", - "version": "0.14.9" - }, - "motoko-x86_64-darwin": { - "builtin": false, - "sha256": "12i7s8mim65537h8xl8z1qh4012jr7bsnsglqspdficncfyc4vk1", - "type": "file", - "url": "https://github.com/dfinity/motoko/releases/download/0.14.9/motoko-Darwin-x86_64-0.14.9.tar.gz", - "url_template": "https://github.com/dfinity/motoko/releases/download//motoko-Darwin-x86_64-.tar.gz", - "version": "0.14.9" - }, - "motoko-x86_64-linux": { - "builtin": false, - "sha256": "0pxllp9v6mfpzfnvpnazzv1rjz8q3h6cid0hf57cvgm1c1v9iql9", - "type": "file", - "url": "https://github.com/dfinity/motoko/releases/download/0.14.9/motoko-Linux-x86_64-0.14.9.tar.gz", - "url_template": "https://github.com/dfinity/motoko/releases/download//motoko-Linux-x86_64-.tar.gz", - "version": "0.14.9" - }, - "pocket-ic-x86_64-darwin": { - "rev": "579b8ba3a31341f354f4ddb3d60ac44548a91bc2", - "sha256": "19jhnq2i6ss27id263f33v9zqy9yy6i0nkgd8knqxvml0w1xdd5h", - "type": "file", - "url": "https://download.dfinity.systems/ic/579b8ba3a31341f354f4ddb3d60ac44548a91bc2/binaries/x86_64-darwin/pocket-ic.gz", - "url_template": "https://download.dfinity.systems/ic//binaries/x86_64-darwin/pocket-ic.gz" - }, - "pocket-ic-x86_64-linux": { - "rev": "579b8ba3a31341f354f4ddb3d60ac44548a91bc2", - "sha256": "031wpv5wh5rwvbh1php3490yxpilrg3r7xs7gfrd10sdaazm5s5k", - "type": "file", - "url": "https://download.dfinity.systems/ic/579b8ba3a31341f354f4ddb3d60ac44548a91bc2/binaries/x86_64-linux/pocket-ic.gz", - "url_template": "https://download.dfinity.systems/ic//binaries/x86_64-linux/pocket-ic.gz" - } -} diff --git a/scripts/update-btc-canister.sh b/scripts/update-btc-canister.sh new file mode 100755 index 0000000000..79c4c98d5c --- /dev/null +++ b/scripts/update-btc-canister.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +# $1 not set || not running in repo root +if [ -z ${1+x} ] || [ ! "$0" -ef ./scripts/update-btc-canister.sh ]; then + echo "Usage: run ./scripts/update-btc-canister.sh in repo root" + exit 1 +fi +if ! command -v curl jq sponge &>/dev/null; then + echo "This script requires curl, jq, and moreutils to be installed" + exit 1 +fi + +urlencode() { + printf '%s' "$1" | jq -sRr @uri +} + +sources="src/dfx/assets/dfx-asset-sources.json" +version=$1 +echo "Updating sources to version $version" +btc_canister_url=$(printf 'https://github.com/dfinity/bitcoin-canister/releases/download/%s/ic-btc-canister.wasm.gz' "$(urlencode "$version")") +btc_canister_sha=$(curl --proto '=https' --tlsv1.2 -sSfL "$btc_canister_url" | sha256sum | head -c 64) +jq '.common."ic-btc-canister" = {url: $url, sha256: $sha256, version: $version}' --arg version "$version" \ + --arg url "$btc_canister_url" --arg sha256 "$btc_canister_sha" "$sources" | sponge "$sources" + +echo "Done. Don't forget to update CHANGELOG.md" diff --git a/scripts/update-motoko.sh b/scripts/update-motoko.sh index 21d8e55fac..8add0a6503 100755 --- a/scripts/update-motoko.sh +++ b/scripts/update-motoko.sh @@ -1,20 +1,37 @@ #!/usr/bin/env bash -set -e +set -euo pipefail # $1 not set || not running in repo root -if [ -z ${1+x} ] || [ ! -f ./scripts/write-dfx-asset-sources.sh ]; then +if [ -z ${1+x} ] || [ ! "$0" -ef ./scripts/update-motoko.sh ]; then echo "Usage: run ./scripts/update-motoko.sh in repo root" exit 1 fi +if ! command -v curl jq sponge &>/dev/null; then + echo "This script requires curl, jq, and moreutils to be installed" + exit 1 +fi + +urlencode() { + printf '%s' "$1" | jq -sRr @uri +} + +sources="src/dfx/assets/dfx-asset-sources.json" +version=$1 +echo "Updating sources to version ${version}" -VERSION=$1 -echo "Updating sources to version ${VERSION}" -niv update motoko-base -a version="$VERSION" -niv update motoko-x86_64-darwin -a version="$VERSION" -niv update motoko-x86_64-linux -a version="$VERSION" +motoko_base_url=$(printf 'https://github.com/dfinity/motoko/releases/download/%s/motoko-base-library.tar.gz' "$(urlencode "$version")") +motoko_base_sha=$(curl --proto '=https' --tlsv1.2 -sSfL "$motoko_base_url" | sha256sum | head -c 64) +jq '.common."motoko-base" = {url: $url, sha256: $sha256, version: $version}' --arg version "$version" \ + --arg url "$motoko_base_url" --arg sha256 "$motoko_base_sha" "$sources" | sponge "$sources" -echo "Writing asset sources" -./scripts/write-dfx-asset-sources.sh +declare -A variants=([x86_64-darwin]=Darwin-x86_64 [x86_64-linux]=Linux-x86_64) +for platform in "${!variants[@]}"; do + motoko_url=$(printf 'https://github.com/dfinity/motoko/releases/download/%s/motoko-%s-%s.tar.gz' \ + "$(urlencode "$version")" "$(urlencode "${variants[$platform]}")" "$(urlencode "$version")") + motoko_sha=$(curl --proto '=https' --tlsv1.2 -sSfL "$motoko_url" | sha256sum | head -c 64) + jq '.[$platform].motoko = {url: $url, sha256: $sha256, version: $version}' --arg platform "$platform" --arg version "$version" \ + --arg url "$motoko_url" --arg sha256 "$motoko_sha" "$sources" | sponge "$sources" +done echo "Done. Don't forget to update CHANGELOG.md" diff --git a/scripts/update-replica.sh b/scripts/update-replica.sh index c393648c14..39816ca3b7 100755 --- a/scripts/update-replica.sh +++ b/scripts/update-replica.sh @@ -1,23 +1,30 @@ #!/usr/bin/env bash -set -e +set -euo pipefail # $1 not set || not running in repo root -if [ -z ${1+x} ] || [ ! -f ./scripts/write-dfx-asset-sources.sh ]; then +if [ -z ${1+x} ] || [ ! "$0" -ef ./scripts/update-replica.sh ]; then echo "Usage: run ./scripts/update-replica.sh in repo root" exit 1 fi +if ! command -v jq sponge curl cargo &>/dev/null; then + echo "This script requires Rust, jq, moreutils, and curl to be installed" + exit 1 +fi +sources="src/dfx/assets/dfx-asset-sources.json" -SHA=$1 -echo "Updating sources to rev ${SHA}" -niv update pocket-ic-x86_64-darwin -a rev="$SHA" -niv update pocket-ic-x86_64-linux -a rev="$SHA" +rev=$1 +echo "Updating sources to rev ${rev}" +jq '."replica-rev" = $rev' --arg rev "$rev" "$sources" | sponge "$sources" +for platform in x86_64-darwin x86_64-linux; do + pocketic_url=$(printf 'https://download.dfinity.systems/ic/%s/binaries/%s/pocket-ic.gz' "$rev" "$platform") + pocketic_sha=$(curl --proto '=https' --tlsv1.2 -sSfL "$pocketic_url" | sha256sum | head -c 64) + jq '.[$platform]."pocket-ic" = {url: $url, sha256: $sha256, rev: $rev}' --arg platform "$platform" --arg rev "$rev" \ + --arg url "$pocketic_url" --arg sha256 "$pocketic_sha" "$sources" | sponge "$sources" +done -# pocket-ic client needs to be upgraded to the same SHA as the pocket-ic server -perl -i.bak -pe "s/(pocket-ic = {[^}]*rev = \")[a-f0-9]+(\")/\${1}$SHA\${2}/" src/dfx/Cargo.toml +# pocket-ic client needs to be upgraded to the same rev as the pocket-ic server +perl -i.bak -pe "s/(pocket-ic = {[^}]*rev = \")[a-f0-9]+(\")/\${1}$rev\${2}/" src/dfx/Cargo.toml cargo update -p pocket-ic # refresh the lock file -echo "Writing asset sources" -./scripts/write-dfx-asset-sources.sh - echo "Done. Don't forget to update CHANGELOG.md" diff --git a/scripts/write-dfx-asset-sources.sh b/scripts/write-dfx-asset-sources.sh deleted file mode 100755 index 055d92dda7..0000000000 --- a/scripts/write-dfx-asset-sources.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash - -# `niv update` puts base32 "nix hashes" (https://nixos.wiki/wiki/Nix_Hash) in nix/sources.json. -# These hashes can't be compared trivially with equivalent hashes output by shasum -a 256. -# This script converts the needed base32 hashes into base16 hashes that can be trivially compared. -# -# This script also makes it so there is no build-time dependency on jq (in addition to nix). -# -# Run this script when updating agent-rs, replica, ic-starter, or pocket-ic in nix/sources.json. - -set -e - -which jq >/dev/null || ( echo "Please install jq in order to run this script." ; exit 1 ) -which nix >/dev/null || ( echo "Please install nix in order to run this script." ; exit 1 ) -which curl >/dev/null || ( echo "Please install curl in order to run this script." ; exit 1 ) - -SDK_ROOT_DIR="$( cd -- "$(dirname -- "$( dirname -- "${BASH_SOURCE[0]}" )" )" &> /dev/null && pwd )" - -DFX_ASSET_SOURCES="$SDK_ROOT_DIR/src/dfx/assets/dfx-asset-sources.toml" -NIX_SOURCES_JSON="$SDK_ROOT_DIR/nix/sources.json" - -read_sha256_from_nix_sources() { - KEY="$1" - - SHA256_BASE32=$(jq -r .'"'"$KEY"'".sha256' "$NIX_SOURCES_JSON") - - nix-hash --to-base16 --type sha256 "$SHA256_BASE32" -} - -read_url_from_nix_sources() { - KEY="$1" - - jq -r .'"'"$KEY"'".url' "$NIX_SOURCES_JSON" -} - -read_rev_from_nix_sources() { - KEY="$1" - - jq -r .'"'"$KEY"'".rev' "$NIX_SOURCES_JSON" -} - -write_entry() { - NAME="$1" - PLATFORM="$2" - KEY="${3:-"${NAME}-${PLATFORM}"}" - URL=$(read_url_from_nix_sources "$KEY") - SHA256="${4:-"$(read_sha256_from_nix_sources "$KEY")"}" - cat >>"$DFX_ASSET_SOURCES" <<<" -[${PLATFORM}.${NAME}] -url = '${URL}' -sha256 = '${SHA256}'" -} - -calculate_sha256() { - KEY="$1" - URL=$(read_url_from_nix_sources "$KEY") - TEMP_FILE="$(mktemp)" - TEMP_DIR="$(mktemp -d)" - curl --silent --location --fail --output "$TEMP_FILE" "$URL" - - tar -xf "$TEMP_FILE" -C "$TEMP_DIR" - EXPECTED_BASE32_SHA256=$(jq -r .'"'"$KEY"'".sha256' "$NIX_SOURCES_JSON") - ACTUAL_BASE32_SHA256="$(nix-hash --base32 --type sha256 "$(realpath "$TEMP_DIR")")" - - SHA256="$(shasum -a 256 "$TEMP_FILE" | awk '{print $1}' )" - - chmod -R 0744 "$TEMP_DIR" - rm "$TEMP_FILE" - rm -rf "$TEMP_DIR" - - if [ "$EXPECTED_BASE32_SHA256" != "$ACTUAL_BASE32_SHA256" ]; then - echo "SHA256 mismatch for $URL: expected $EXPECTED_BASE32_SHA256, got $ACTUAL_BASE32_SHA256" - exit 1 - fi - echo "$SHA256" -} - -write_replica_rev() { - REV=$(read_rev_from_nix_sources "pocket-ic-x86_64-darwin") - - echo "replica-rev = '$REV'" >>"$DFX_ASSET_SOURCES" -} - -echo "# generated by write-dfx-asset-sources.sh" >"$DFX_ASSET_SOURCES" - -write_replica_rev -motoko_base_sha="$(calculate_sha256 "motoko-base")" -for platform in "darwin" "linux"; -do - for name in "motoko" "pocket-ic"; - do - write_entry "$name" "x86_64-${platform}" - done - write_entry "motoko-base" "x86_64-${platform}" "motoko-base" "$motoko_base_sha" - write_entry "ic-btc-canister" "x86_64-${platform}" "ic-btc-canister" -done diff --git a/src/dfx/Cargo.toml b/src/dfx/Cargo.toml index 21542fdaec..2611a8529a 100644 --- a/src/dfx/Cargo.toml +++ b/src/dfx/Cargo.toml @@ -27,7 +27,7 @@ serde = { version = "1.0", features = ["derive"] } sha2 = "0.10.6" tar = "0.4.26" tokio = { version = "1.43.1", features = ["full"] } -toml = "0.7.3" +serde_json = "1.0" walkdir = "2.3.2" [dependencies] diff --git a/src/dfx/assets/build.rs b/src/dfx/assets/build.rs index 242ac2a071..7513cd93c1 100644 --- a/src/dfx/assets/build.rs +++ b/src/dfx/assets/build.rs @@ -13,7 +13,7 @@ use walkdir::WalkDir; mod prepare_assets; const INPUTS: &[&str] = &[ - "nix/sources.json", + "src/dfx/assets/dfx-asset-sources.json", "src/dfx/assets/prepare_assets.rs", "src/dfx/assets/build.rs", "src/distributed/assetstorage.did", @@ -64,6 +64,7 @@ struct Sources { x86_64_linux: HashMap, #[serde(rename = "x86_64-darwin")] x86_64_darwin: HashMap, + common: HashMap, #[serde(rename = "replica-rev")] replica_rev: String, } @@ -94,7 +95,7 @@ fn find_assets(sources: Sources) -> PathBuf { } } - let source_set = match ( + let mut source_set = match ( &*env::var("CARGO_CFG_TARGET_ARCH").unwrap(), &*env::var("CARGO_CFG_TARGET_OS").unwrap(), ) { @@ -102,6 +103,7 @@ fn find_assets(sources: Sources) -> PathBuf { ("x86_64", "linux" | "windows") => sources.x86_64_linux, (arch, os) => panic!("Unsupported OS type {arch}-{os}"), }; + source_set.extend(sources.common); prepare_assets::prepare(&dfx_assets_path, source_set); fs::write(last_hash_of_inputs_path, hash_of_inputs) @@ -345,11 +347,11 @@ fn define_replica_rev(replica_rev: &str) { } fn main() { - let sources: Sources = toml::from_str( - &fs::read_to_string("assets/dfx-asset-sources.toml") - .expect("unable to read dfx-asset-sources.toml"), + let sources: Sources = serde_json::from_str( + &fs::read_to_string("assets/dfx-asset-sources.json") + .expect("unable to read dfx-asset-sources.json"), ) - .expect("unable to parse dfx-asset-sources.toml"); + .expect("unable to parse dfx-asset-sources.json"); define_replica_rev(&sources.replica_rev); add_assets(sources); define_dfx_version(); diff --git a/src/dfx/assets/dfx-asset-sources.json b/src/dfx/assets/dfx-asset-sources.json new file mode 100644 index 0000000000..7ac21ced6b --- /dev/null +++ b/src/dfx/assets/dfx-asset-sources.json @@ -0,0 +1,39 @@ +{ + "replica-rev": "579b8ba3a31341f354f4ddb3d60ac44548a91bc2", + "x86_64-darwin": { + "motoko": { + "url": "https://github.com/dfinity/motoko/releases/download/0.14.8/motoko-Darwin-x86_64-0.14.8.tar.gz", + "sha256": "7b8c6bb263c0c34a2719f848bc56dabf17ae7fb41e1555e3413502bc92eb1ffd", + "version": "0.14.8" + }, + "pocket-ic": { + "url": "https://download.dfinity.systems/ic/579b8ba3a31341f354f4ddb3d60ac44548a91bc2/binaries/x86_64-darwin/pocket-ic.gz", + "sha256": "b0b4d60307b4ee8eed44ed4d0ba2f13e79fcd31ec30d235a3c426b1305b650a6", + "rev": "579b8ba3a31341f354f4ddb3d60ac44548a91bc2" + } + }, + "x86_64-linux": { + "motoko": { + "url": "https://github.com/dfinity/motoko/releases/download/0.14.8/motoko-Linux-x86_64-0.14.8.tar.gz", + "sha256": "8236a7deade717d3626b0b45b367778cfac8ec3353bb0996ee5933857b53c2e9", + "version": "0.14.8" + }, + "pocket-ic": { + "url": "https://download.dfinity.systems/ic/579b8ba3a31341f354f4ddb3d60ac44548a91bc2/binaries/x86_64-linux/pocket-ic.gz", + "sha256": "b3e852bf524d83d0b27b47f793c7cb34deee4122e3c21be0da3c17c8cbbe3c0c", + "rev": "579b8ba3a31341f354f4ddb3d60ac44548a91bc2" + } + }, + "common": { + "motoko-base": { + "url": "https://github.com/dfinity/motoko/releases/download/0.14.8/motoko-base-library.tar.gz", + "sha256": "634d414a597355c4154a3d052453aa39c3f1736cc8ad789c58ac54b43f5f47f6", + "version": "0.14.8" + }, + "ic-btc-canister": { + "url": "https://github.com/dfinity/bitcoin-canister/releases/download/release%2F2024-08-30/ic-btc-canister.wasm.gz", + "sha256": "d0ada614be09055f37d45d8ad92560432c86b426d9e66f42609ce64007a13c3e", + "version": "release/2024-08-30" + } + } +} diff --git a/src/dfx/assets/dfx-asset-sources.toml b/src/dfx/assets/dfx-asset-sources.toml deleted file mode 100644 index 9d9c5a390a..0000000000 --- a/src/dfx/assets/dfx-asset-sources.toml +++ /dev/null @@ -1,34 +0,0 @@ -# generated by write-dfx-asset-sources.sh -replica-rev = '579b8ba3a31341f354f4ddb3d60ac44548a91bc2' - -[x86_64-darwin.motoko] -url = 'https://github.com/dfinity/motoko/releases/download/0.14.9/motoko-Darwin-x86_64-0.14.9.tar.gz' -sha256 = '616ec2bc639645d7aec6f469abd7c9520440200e1fd18ee019a5981a2bd2278a' - -[x86_64-darwin.pocket-ic] -url = 'https://download.dfinity.systems/ic/579b8ba3a31341f354f4ddb3d60ac44548a91bc2/binaries/x86_64-darwin/pocket-ic.gz' -sha256 = 'b0b4d60307b4ee8eed44ed4d0ba2f13e79fcd31ec30d235a3c426b1305b650a6' - -[x86_64-darwin.motoko-base] -url = 'https://github.com/dfinity/motoko/releases/download/0.14.9/motoko-base-library.tar.gz' -sha256 = 'df6fcc13142bd384f9401504660a880d80e1d2d35086092f45694147dc962730' - -[x86_64-darwin.ic-btc-canister] -url = 'https://github.com/dfinity/bitcoin-canister/releases/download/release%2F2024-08-30/ic-btc-canister.wasm.gz' -sha256 = 'd0ada614be09055f37d45d8ad92560432c86b426d9e66f42609ce64007a13c3e' - -[x86_64-linux.motoko] -url = 'https://github.com/dfinity/motoko/releases/download/0.14.9/motoko-Linux-x86_64-0.14.9.tar.gz' -sha256 = '89e2987660a1becd4e7110b4c80c1c187d99c3fe5fd9bbadfbd755b3d3a5b45f' - -[x86_64-linux.pocket-ic] -url = 'https://download.dfinity.systems/ic/579b8ba3a31341f354f4ddb3d60ac44548a91bc2/binaries/x86_64-linux/pocket-ic.gz' -sha256 = 'b3e852bf524d83d0b27b47f793c7cb34deee4122e3c21be0da3c17c8cbbe3c0c' - -[x86_64-linux.motoko-base] -url = 'https://github.com/dfinity/motoko/releases/download/0.14.9/motoko-base-library.tar.gz' -sha256 = 'df6fcc13142bd384f9401504660a880d80e1d2d35086092f45694147dc962730' - -[x86_64-linux.ic-btc-canister] -url = 'https://github.com/dfinity/bitcoin-canister/releases/download/release%2F2024-08-30/ic-btc-canister.wasm.gz' -sha256 = 'd0ada614be09055f37d45d8ad92560432c86b426d9e66f42609ce64007a13c3e'