diff --git a/.cirrus.yml b/.cirrus.yml index c5b68e7dc52..b4902039d15 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,8 +1,10 @@ env: # Global defaults SECP256K1_TEST_ITERS: 16 # ELEMENTS: avoid test timeouts on arm + CIRRUS_CLONE_DEPTH: 1 PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y" MAKEJOBS: "-j3" # ELEMENTS: reduced from j4 TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache + CI_FAILFAST_TEST_LEAVE_DANGLING: "1" # Cirrus CI does not care about dangling process and setting this variable avoids killing the CI script itself on error CCACHE_SIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" CCACHE_NOHASHDIR: "1" # Debug info might contain a stale path if the build dir changes, but this is fine @@ -24,14 +26,12 @@ filter_template: &FILTER_TEMPLATE base_template: &BASE_TEMPLATE << : *FILTER_TEMPLATE merge_base_script: - # Unconditionally install git (used in fingerprint_script) and set the - # default git author name (used in verify-commits.py) + # Unconditionally install git (used in fingerprint_script). - bash -c "$PACKAGE_MANAGER_INSTALL git" - - git config --global user.email "ci@ci.ci" - - git config --global user.name "ci" - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi - - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH - - git merge FETCH_HEAD # Merge base to detect silent merge conflicts + - git fetch --depth=1 $CIRRUS_REPO_CLONE_URL "pull/${CIRRUS_PR}/merge" + - git checkout FETCH_HEAD # Use merged changes to detect silent merge conflicts + # Also, the merge commit is used to lint COMMIT_RANGE="HEAD~..HEAD" main_template: &MAIN_TEMPLATE timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out @@ -40,7 +40,7 @@ main_template: &MAIN_TEMPLATE ci_script: - ./ci/test_run_all.sh -global_task_template: &GLOBAL_TASK_TEMPLATE +container_depends_template: &CONTAINER_DEPENDS_TEMPLATE << : *BASE_TEMPLATE container: # https://cirrus-ci.org/faq/#are-there-any-limits @@ -48,17 +48,13 @@ global_task_template: &GLOBAL_TASK_TEMPLATE cpu: 2 greedy: true memory: 8G # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers + dockerfile: ci/test_imagefile # https://cirrus-ci.org/guide/docker-builder-vm/#dockerfile-as-a-ci-environment depends_built_cache: folder: "depends/built" - fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-list -1 HEAD ./depends) - << : *MAIN_TEMPLATE + fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-parse HEAD:depends) -macos_native_task_template: &MACOS_NATIVE_TASK_TEMPLATE - << : *BASE_TEMPLATE - check_clang_script: - - clang --version - brew_install_script: - - brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt +global_task_template: &GLOBAL_TASK_TEMPLATE + << : *CONTAINER_DEPENDS_TEMPLATE << : *MAIN_TEMPLATE compute_credits_template: &CREDITS_TEMPLATE @@ -67,53 +63,68 @@ compute_credits_template: &CREDITS_TEMPLATE use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'ElementsProject/elements' && $CIRRUS_PR != "" task: - name: 'lint [bionic]' + name: 'lint [bookworm]' << : *BASE_TEMPLATE container: - image: ubuntu:bionic # For python 3.6, oldest supported version according to doc/dependencies.md + image: debian:bookworm cpu: 1 memory: 1G # For faster CI feedback, immediately schedule the linters << : *CREDITS_TEMPLATE + python_cache: + folder: "/tmp/python" + fingerprint_script: cat .python-version /etc/os-release + unshallow_script: + - git fetch --unshallow --no-tags lint_script: - ./ci/lint_run_all.sh env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV task: - name: "Win64 native [msvc]" + name: 'tidy [24.04]' + << : *GLOBAL_TASK_TEMPLATE + container: + cpu: 2 + memory: 5G + docker_arguments: + CI_IMAGE_NAME_TAG: ubuntu:24.04 + FILE_ENV: "./ci/test/00_setup_env_native_tidy.sh" + # For faster CI feedback, immediately schedule the linters + << : *CREDITS_TEMPLATE + env: + << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + +task: + name: "Win64 native [vs2022]" << : *FILTER_TEMPLATE windows_container: - cpu: 4 - memory: 8G - image: cirrusci/windowsservercore:visualstudio2019 + cpu: 6 + memory: 12G + image: cirrusci/windowsservercore:visualstudio2022 timeout_in: 120m env: - PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin;%PATH%' + PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\MSBuild\Current\Bin;%PATH%' PYTHONUTF8: 1 - CI_VCPKG_TAG: '2021.05.12' + CI_VCPKG_TAG: '2023.01.09' VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads' VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' - QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.3/single/qt-everywhere-opensource-src-5.15.3.zip' - QT_LOCAL_PATH: 'C:\qt-everywhere-opensource-src-5.15.3.zip' - QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.3' + CCACHE_DIR: 'C:\Users\ContainerAdministrator\AppData\Local\ccache' + WRAPPED_CL: 'C:\Users\ContainerAdministrator\AppData\Local\Temp\cirrus-ci-build\ci\test\wrapped-cl.bat' + QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip' + QT_LOCAL_PATH: 'C:\qt-everywhere-opensource-src-5.15.5.zip' + QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.5' QTBASEDIR: 'C:\Qt_static' - x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"' + x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat"' + QT_CONFIGURE_COMMAND: '..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml' IgnoreWarnIntDirInTempDetected: 'true' merge_script: - - git config --global user.email "ci@ci.ci" - - git config --global user.name "ci" - # Windows filesystem loses the executable bit, and all of the executable - # files are considered "modified" now. It will break the following `git merge` - # command. The next two commands make git ignore this issue. - - git config core.filemode false - - git reset --hard - - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL $env:CIRRUS_BASE_BRANCH; git merge FETCH_HEAD; } + - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL pull/$env:CIRRUS_PR/merge; git reset --hard FETCH_HEAD; } msvc_qt_built_cache: folder: "%QTBASEDIR%" reupload_on_changes: false fingerprint_script: - - echo %QT_DOWNLOAD_URL% + - echo %QT_DOWNLOAD_URL% %QT_CONFIGURE_COMMAND% - msbuild -version populate_script: - curl -L -o C:\jom.zip http://download.qt.io/official_releases/jom/jom.zip @@ -125,7 +136,7 @@ task: - cd %QT_SOURCE_DIR% - mkdir build - cd build - - ..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml -prefix %QTBASEDIR% + - '%QT_CONFIGURE_COMMAND% -prefix %QTBASEDIR%' - jom - jom install vcpkg_tools_cache: @@ -139,12 +150,17 @@ task: reupload_on_changes: true fingerprint_script: - echo %CI_VCPKG_TAG% + - type build_msvc\vcpkg.json - msbuild -version populate_script: - mkdir %VCPKG_DEFAULT_BINARY_CACHE% - install_python_script: + ccache_cache: + folder: '%CCACHE_DIR%' + install_tools_script: + - choco install --yes --no-progress ccache --version=4.7.4 - choco install --yes --no-progress python3 --version=3.9.6 - pip install zmq + - ccache --version - python -VV install_vcpkg_script: - cd .. @@ -156,23 +172,26 @@ task: - .\vcpkg integrate install - .\vcpkg version build_script: + - '%x64_NATIVE_TOOLS%' - cd %CIRRUS_WORKING_DIR% + - ccache --zero-stats --max-size=%CCACHE_SIZE% - python build_msvc\msvc-autogen.py - msbuild build_msvc\bitcoin.sln -property:Configuration=Release -maxCpuCount -verbosity:minimal -noLogo - unit_tests_script: + - ccache --show-stats + check_script: - src\test_elements.exe -l test_suite - - src\bench_elements.exe > NUL + - src\bench_elements.exe --sanity-check - python test\util\test_runner.py - python test\util\rpcauth-test.py functional_tests_script: # Increase the dynamic port range to the maximum allowed value to mitigate "OSError: [WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted". - # See: https://docs.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance + # See: https://learn.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance - netsh int ipv4 set dynamicport tcp start=1025 num=64511 - netsh int ipv6 set dynamicport tcp start=1025 num=64511 # Exclude feature_dbcrash for now due to timeout # Exclude also wallet_avoidreuse due to timeout # Ignore failures for now, need to investigate but we really don't use native win64 builds - - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=4 --timeout-factor=8 --extended --exclude feature_dbcrash,wallet_avoidreuse || true + - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=99999999 --jobs=4 --timeout-factor=8 --extended --exclude feature_dbcrash,wallet_avoidreuse || true task: name: 'ARM [unit tests, no functional tests] [bullseye]' @@ -190,23 +209,28 @@ task: name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [jammy]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:jammy + docker_arguments: + CI_IMAGE_NAME_TAG: ubuntu:jammy + FILE_ENV: "./ci/test/00_setup_env_win64.sh" + << : *CREDITS_TEMPLATE env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_win64.sh" task: name: '32-bit + dash [gui] [Rocky 8]' << : *GLOBAL_TASK_TEMPLATE container: - image: quay.io/rockylinux/rockylinux:8 + docker_arguments: + CI_IMAGE_NAME_TAG: quay.io/rockylinux/rockylinux:8 + FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh" + # For faster CI feedback, immediately schedule one task that runs all tests + << : *CREDITS_TEMPLATE env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV PACKAGE_MANAGER_INSTALL: "yum install -y" - FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh" task: - name: '[previous releases, uses qt5 dev package and some depends packages, DEBUG] [unsigned char] [bionic]' + name: '[previous releases, uses qt5 dev package and some depends packages, DEBUG] [unsigned char] [buster]' previous_releases_cache: folder: "releases" << : *GLOBAL_TASK_TEMPLATE @@ -219,44 +243,54 @@ task: name: '[TSan, depends, gui] [2404]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:24.04 cpu: 6 # Increase CPU and Memory to avoid timeout memory: 24G + docker_arguments: + CI_IMAGE_NAME_TAG: ubuntu:24.04 + FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" MAKEJOBS: "-j2" # Avoid excessive memory use due to MSan task: - name: '[MSan, depends] [focal]' + name: '[MSan, depends] [24.04]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:focal + docker_arguments: + CI_IMAGE_NAME_TAG: ubuntu:24.04 + FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" + MAKEJOBS: "-j4" # Avoid excessive memory use due to MSan task: - name: '[ASan + LSan + UBSan + integer, no depends] [jammy]' + name: '[ASan + LSan + UBSan + integer, no depends, USDT] [jammy]' << : *GLOBAL_TASK_TEMPLATE - container: - image: ubuntu:jammy - memory: 16G # ELEMENTS: need more memory + # We can't use a 'container' for the USDT interface tests as the CirrusCI + # containers don't have privileges to hook into bitcoind. CirrusCI uses + # Google Compute Engine instances: https://cirrus-ci.org/guide/custom-vms/ + # Images can be found here: https://cloud.google.com/compute/docs/images/os-details + compute_engine_instance: + image_project: ubuntu-os-cloud + image: family/ubuntu-2204-lts # when upgrading, check if we can drop "ADD_UNTRUSTED_BPFCC_PPA" cpu: 4 # ELEMENTS: cirrus wants more CPUs if you want more memory + memory: 16G # ELEMENTS: need more memory env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + HOME: /root/ # Only needed for compute_engine_instance FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" task: - name: '[fuzzer,address,undefined,integer, no depends] [jammy]' + name: '[fuzzer,address,undefined,integer, no depends] [24.04]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:jammy cpu: 8 # Increase CPU and memory to avoid timeout memory: 16G + docker_arguments: + CI_IMAGE_NAME_TAG: ubuntu:24.04 + FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" task: name: '[multiprocess, i686, DEBUG] [focal]' @@ -264,42 +298,52 @@ task: only_if: false << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:focal cpu: 4 memory: 16G # The default memory is sometimes just a bit too small, so double everything + docker_arguments: + CI_IMAGE_NAME_TAG: ubuntu:focal + FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" task: - name: '[no wallet, libbitcoinkernel] [bionic]' + name: '[no wallet, libbitcoinkernel] [focal]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:bionic + docker_arguments: + CI_IMAGE_NAME_TAG: ubuntu:focal + FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh" + << : *CREDITS_TEMPLATE env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh" task: name: 'macOS 10.15 [gui, no tests] [focal]' - << : *BASE_TEMPLATE + << : *CONTAINER_DEPENDS_TEMPLATE + container: + docker_arguments: + CI_IMAGE_NAME_TAG: ubuntu:focal + FILE_ENV: "./ci/test/00_setup_env_mac.sh" + << : *CREDITS_TEMPLATE macos_sdk_cache: folder: "depends/SDKs/$MACOS_SDK" fingerprint_key: "$MACOS_SDK" << : *MAIN_TEMPLATE - container: - image: ubuntu:focal env: MACOS_SDK: "Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers" << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_mac.sh" task: name: 'macOS 13 native arm64 [gui, sqlite only] [no depends]' macos_instance: # Use latest image, but hardcode version to avoid silent upgrades (and breaks) image: ghcr.io/cirruslabs/macos-ventura-xcode:14.1 # https://cirrus-ci.org/guide/macOS - << : *MACOS_NATIVE_TASK_TEMPLATE + << : *BASE_TEMPLATE + check_clang_script: + - clang --version + brew_install_script: + - brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt + << : *MAIN_TEMPLATE env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV CI_USE_APT_INSTALL: "no" @@ -307,18 +351,21 @@ task: FILE_ENV: "./ci/test/00_setup_env_mac_native_arm64.sh" task: - name: 'ARM64 Android APK [focal]' + name: 'ARM64 Android APK [jammy]' # Disable for Elements; Android build is broken and unsupported. only_if: false + << : *CONTAINER_DEPENDS_TEMPLATE + container: + docker_arguments: + CI_IMAGE_NAME_TAG: ubuntu:jammy + FILE_ENV: "./ci/test/00_setup_env_android.sh" + << : *CREDITS_TEMPLATE android_sdk_cache: folder: "depends/SDKs/android" - fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.1.7779620" + fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.2.8568313" depends_sources_cache: folder: "depends/sources" - fingerprint_script: git rev-list -1 HEAD ./depends + fingerprint_script: git rev-parse HEAD:depends/packages << : *MAIN_TEMPLATE - container: - image: ubuntu:focal env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_android.sh" diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index eedeeb4e54c..00000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 00000000000..83922b54cbf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,93 @@ +name: Bug report +description: Submit a new bug report. +labels: [bug] +body: + - type: markdown + attributes: + value: | + ## This issue tracker is only for technical issues related to Bitcoin Core. + + * General bitcoin questions and/or support requests should use Bitcoin StackExchange at https://bitcoin.stackexchange.com. + * For reporting security issues, please read instructions at https://bitcoincore.org/en/contact/. + * If the node is "stuck" during sync or giving "block checksum mismatch" errors, please ensure your hardware is stable by running `memtest` and observe CPU temperature with a load-test tool such as `linpack` before creating an issue. + + ---- + - type: checkboxes + attributes: + label: Is there an existing issue for this? + description: Please search to see if an issue already exists for the bug you encountered. + options: + - label: I have searched the existing issues + required: true + - type: textarea + id: current-behaviour + attributes: + label: Current behaviour + description: Tell us what went wrong + validations: + required: true + - type: textarea + id: expected-behaviour + attributes: + label: Expected behaviour + description: Tell us what you expected to happen + validations: + required: true + - type: textarea + id: reproduction-steps + attributes: + label: Steps to reproduce + description: | + Tell us how to reproduce your bug. Please attach related screenshots if necessary. + * Run-time or compile-time configuration options + * Actions taken + validations: + required: true + - type: textarea + id: logs + attributes: + label: Relevant log output + description: | + Please copy and paste any relevant log output or attach a debug log file. + + You can find the debug.log in your [data dir.](https://github.com/bitcoin/bitcoin/blob/master/doc/files.md#data-directory-location) + + Please be aware that the debug log might contain personally identifying information. + validations: + required: false + - type: dropdown + attributes: + label: How did you obtain Bitcoin Core + multiple: false + options: + - Compiled from source + - Pre-built binaries + - Package manager + - Other + validations: + required: true + - type: input + id: core-version + attributes: + label: What version of Bitcoin Core are you using? + description: Run `bitcoind --version` or in Bitcoin-QT use `Help > About Bitcoin Core` + placeholder: e.g. v24.0.1 or master@e1bf547 + validations: + required: true + - type: input + id: os + attributes: + label: Operating system and version + placeholder: e.g. "MacOS Ventura 13.2" or "Ubuntu 22.04 LTS" + validations: + required: true + - type: textarea + id: machine-specs + attributes: + label: Machine specifications + description: | + What are the specifications of the host machine? + e.g. OS/CPU and disk type, network connectivity + validations: + required: false + diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index fb91208954e..00000000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve (use this for suspected bugs only, if not sure, open a regular issue below) -title: '' -labels: Bug -assignees: '' - ---- - - - - - -**Expected behavior** - - - -**Actual behavior** - - - -**To reproduce** - - - -**System information** - - - - - - - - - diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000..40370284a6d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: true +contact_links: + - name: Bitcoin Core Security Policy + url: https://github.com/bitcoin/bitcoin/blob/master/SECURITY.md + about: View security policy + - name: Bitcoin Core Developers + url: https://bitcoincore.org + about: Bitcoin Core homepage diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 2d5685185ea..00000000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: '' -labels: Feature -assignees: '' - ---- - -**Is your feature request related to a problem? Please describe.** - - -**Describe the solution you'd like** - - -**Describe alternatives you've considered** - - -**Additional context** - diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 00000000000..4622fd98191 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,36 @@ +name: Feature Request +description: Suggest an idea for this project. +labels: [Feature] +body: + - type: textarea + id: feature + attributes: + label: Please describe the feature you'd like to see added. + description: Attach screenshots or logs if applicable. + validations: + required: true + - type: textarea + id: related-problem + attributes: + label: Is your feature related to a problem, if so please describe it. + description: Attach screenshots or logs if applicable. + validations: + required: false + - type: textarea + id: solution + attributes: + label: Describe the solution you'd like + validations: + required: false + - type: textarea + id: alternatives + attributes: + label: Describe any alternatives you've considered + validations: + required: false + - type: textarea + id: additional-context + attributes: + label: Please leave any additional context + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/good_first_issue.md b/.github/ISSUE_TEMPLATE/good_first_issue.md deleted file mode 100644 index d32e22d3607..00000000000 --- a/.github/ISSUE_TEMPLATE/good_first_issue.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -name: Good first issue -about: '(Regular devs only): Suggest a new good first issue' -title: '' -labels: '' -assignees: '' - ---- - - - - - - - -#### Useful skills: - - - -#### Want to work on this issue? - -For guidance on contributing, please read [CONTRIBUTING.md](https://github.com/bitcoin/bitcoin/blob/master/CONTRIBUTING.md) before opening your pull request. diff --git a/.github/ISSUE_TEMPLATE/good_first_issue.yml b/.github/ISSUE_TEMPLATE/good_first_issue.yml new file mode 100644 index 00000000000..c40dad9687a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/good_first_issue.yml @@ -0,0 +1,42 @@ +name: Good First Issue +description: (Regular devs only) Suggest a new good first issue +labels: [good first issue] +body: + - type: markdown + attributes: + value: | + Please add the label "good first issue" manually before or after opening + + A good first issue is an uncontroversial issue, that has a relatively unique and obvious solution + + Motivate the issue and explain the solution briefly + - type: textarea + id: motivation + attributes: + label: Motivation + description: Motivate the issue + validations: + required: true + - type: textarea + id: solution + attributes: + label: Possible solution + description: Describe a possible solution + validations: + required: false + - type: textarea + id: useful-skills + attributes: + label: Useful Skills + description: For example, “`std::thread`”, “Qt5 GUI and async GUI design” or “basic understanding of Bitcoin mining and the Bitcoin Core RPC interface”. + validations: + required: false + - type: textarea + attributes: + label: Guidance for new contributors + description: Please leave this to automatically add the footer for new contributors + value: | + Want to work on this issue? + + For guidance on contributing, please read [CONTRIBUTING.md](https://github.com/bitcoin/bitcoin/blob/master/CONTRIBUTING.md) before opening your pull request. + diff --git a/.github/ISSUE_TEMPLATE/gui_issue.md b/.github/ISSUE_TEMPLATE/gui_issue.md deleted file mode 100644 index 37acc81e21e..00000000000 --- a/.github/ISSUE_TEMPLATE/gui_issue.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -name: An issue or feature request related to the GUI -about: Any report, issue or feature request related to the GUI should be reported at https://github.com/bitcoin-core/gui/issues/ -title: Any report, issue or feature request related to the GUI should be reported at https://github.com/bitcoin-core/gui/issues/ -labels: GUI -assignees: '' - ---- - -Any report, issue or feature request related to the GUI should be reported at -https://github.com/bitcoin-core/gui/issues/ diff --git a/.github/ISSUE_TEMPLATE/gui_issue.yml b/.github/ISSUE_TEMPLATE/gui_issue.yml new file mode 100644 index 00000000000..4fe578e9b5b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/gui_issue.yml @@ -0,0 +1,18 @@ +name: Issue or feature request related to the GUI +description: Any report, issue or feature request related to the GUI +labels: [GUI] +body: +- type: checkboxes + id: acknowledgement + attributes: + label: Issues, reports or feature requests related to the GUI should be opened directly on the GUI repo + description: https://github.com/bitcoin-core/gui/issues/ + options: + - label: I still think this issue should be opened here + required: true +- type: textarea + id: gui-request + attributes: + label: Report + validations: + required: true diff --git a/.gitignore b/.gitignore index fcb8fddf63b..b8256008a55 100644 --- a/.gitignore +++ b/.gitignore @@ -44,8 +44,6 @@ src/obj share/setup.nsi share/qt/Info.plist -src/univalue/gen - src/qt/*.moc src/qt/moc_*.cpp src/qt/forms/ui_*.h @@ -77,7 +75,6 @@ src/qt/bitcoin-qt.includes *.log *.trs *.dmg -*.iso *.json.h *.raw.h @@ -151,3 +148,5 @@ osx_volname dist/ /guix-build-* + +/ci/scratch/ diff --git a/.python-version b/.python-version index 8b7b0b52e55..36f601f10e9 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.6.12 +3.7.16 diff --git a/.tx/config b/.tx/config index c4fe7cc324d..ab1fb933d83 100644 --- a/.tx/config +++ b/.tx/config @@ -1,7 +1,7 @@ [main] host = https://www.transifex.com -[bitcoin.qt-translation-023x] +[o:bitcoin:p:bitcoin:r:qt-translation-025x] file_filter = src/qt/locale/bitcoin_.xlf source_file = src/qt/locale/bitcoin_en.xlf source_lang = en diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e81b477714d..be3d48cddbb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,10 +10,9 @@ First, in terms of structure, there is no particular concept of "Bitcoin Core developers" in the sense of privileged people. Open source often naturally revolves around a meritocracy where contributors earn trust from the developer community over time. Nevertheless, some hierarchy is necessary for practical -purposes. As such, there are repository "maintainers" who are responsible for -merging pull requests, as well as a "lead maintainer" who is responsible for the -[release cycle](/doc/release-process.md) as well as overall merging, moderation -and appointment of maintainers. +purposes. As such, there are repository maintainers who are responsible for +merging pull requests, the [release cycle](/doc/release-process.md), and +moderation. Getting Started --------------- @@ -153,7 +152,8 @@ the pull request affects. Valid areas as: - `test`, `qa` or `ci` for changes to the unit tests, QA tests or CI code - `util` or `lib` for changes to the utils or libraries - `wallet` for changes to the wallet code - - `build` for changes to the GNU Autotools or reproducible builds + - `build` for changes to the GNU Autotools or MSVC builds + - `guix` for changes to the GUIX reproducible builds Examples: @@ -216,9 +216,9 @@ Please update the resulting commit message, if needed. It should read as a coherent message. In most cases, this means not just listing the interim commits. -If you have problems with squashing or other git workflows, you can enable -"Allow edits from maintainers" in the right-hand sidebar of the GitHub web -interface and ask for help in the pull request. +If your change contains a merge commit, the above workflow may not work and you +will need to remove the merge commit first. See the next section for details on +how to rebase. Please refrain from creating several pull requests for the same change. Use the pull request that is already open (or was created earlier) to amend @@ -231,7 +231,9 @@ pull request to pull request. ### Rebasing Changes When a pull request conflicts with the target branch, you may be asked to rebase it on top of the current target branch. -The `git rebase` command will take care of rebuilding your commits on top of the new base. + + git fetch https://github.com/bitcoin/bitcoin # Fetch the latest upstream commit + git rebase FETCH_HEAD # Rebuild commits on top of the new base This project aims to have a clean git history, where code changes are only made in non-merge commits. This simplifies auditability because merge commits can be assumed to not contain arbitrary code changes. Merge commits should be signed, @@ -292,7 +294,7 @@ projects such as libsecp256k1), and is not to be confused with overall Bitcoin Network Protocol consensus changes. Whether a pull request is merged into Bitcoin Core rests with the project merge -maintainers and ultimately the project lead. +maintainers. Maintainers will take into consideration if a patch is in line with the general principles of the project; meets the minimum standards for inclusion; and will diff --git a/COPYING b/COPYING index b157c5fe490..2f7add71aca 100644 --- a/COPYING +++ b/COPYING @@ -1,7 +1,7 @@ The MIT License (MIT) -Copyright (c) 2009-2022 The Bitcoin Core developers -Copyright (c) 2009-2022 Bitcoin Developers +Copyright (c) 2009-2023 The Bitcoin Core developers +Copyright (c) 2009-2023 Bitcoin Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile.am b/Makefile.am index 59fb949699e..58fbc6da94f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -12,7 +12,7 @@ if ENABLE_MAN SUBDIRS += doc/man endif .PHONY: deploy FORCE -.INTERMEDIATE: $(OSX_TEMP_ISO) $(COVERAGE_INFO) +.INTERMEDIATE: $(COVERAGE_INFO) export PYTHONPATH @@ -23,6 +23,7 @@ endif BITCOIND_BIN=$(top_builddir)/src/$(BITCOIN_DAEMON_NAME)$(EXEEXT) BITCOIN_QT_BIN=$(top_builddir)/src/qt/$(BITCOIN_GUI_NAME)$(EXEEXT) +BITCOIN_TEST_BIN=$(top_builddir)/src/test/$(BITCOIN_TEST_NAME)$(EXEEXT) BITCOIN_CLI_BIN=$(top_builddir)/src/$(BITCOIN_CLI_NAME)$(EXEEXT) BITCOIN_TX_BIN=$(top_builddir)/src/$(BITCOIN_TX_NAME)$(EXEEXT) BITCOIN_UTIL_BIN=$(top_builddir)/src/$(BITCOIN_UTIL_NAME)$(EXEEXT) @@ -37,8 +38,6 @@ space := $(empty) $(empty) OSX_APP=Elements-Qt.app OSX_VOLNAME = $(subst $(space),-,$(PACKAGE_NAME)) OSX_DMG = $(OSX_VOLNAME).dmg -OSX_TEMP_ISO = $(OSX_DMG:.dmg=).temp.iso -OSX_BACKGROUND_IMAGE=$(top_srcdir)/contrib/macdeploy/background.tiff OSX_DEPLOY_SCRIPT=$(top_srcdir)/contrib/macdeploy/macdeployqtplus OSX_INSTALLER_ICONS=$(top_srcdir)/src/qt/res/icons/bitcoin.icns OSX_PLIST=$(top_builddir)/share/qt/Info.plist #not installed @@ -48,7 +47,8 @@ DIST_CONTRIB = \ $(top_srcdir)/test/sanitizer_suppressions/tsan \ $(top_srcdir)/test/sanitizer_suppressions/ubsan \ $(top_srcdir)/contrib/linearize/linearize-data.py \ - $(top_srcdir)/contrib/linearize/linearize-hashes.py + $(top_srcdir)/contrib/linearize/linearize-hashes.py \ + $(top_srcdir)/contrib/signet/miner DIST_SHARE = \ $(top_srcdir)/share/genbuild.sh \ @@ -64,7 +64,6 @@ WINDOWS_PACKAGING = $(top_srcdir)/share/pixmaps/bitcoin.ico \ $(top_srcdir)/doc/README_windows.txt OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_INSTALLER_ICONS) \ - $(top_srcdir)/contrib/macdeploy/detached-sig-apply.sh \ $(top_srcdir)/contrib/macdeploy/detached-sig-create.sh COVERAGE_INFO = $(COV_TOOL_WRAPPER) baseline.info \ @@ -80,6 +79,7 @@ $(BITCOIN_WIN_INSTALLER): all-recursive $(MKDIR_P) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIND_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_QT_BIN) $(top_builddir)/release + STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_TEST_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_CLI_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_TX_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_WALLET_BIN) $(top_builddir)/release @@ -130,31 +130,16 @@ $(OSX_DMG): $(OSX_APP_BUILT) $(OSX_PACKAGING) deploydir: $(OSX_DMG) else !BUILD_DARWIN APP_DIST_DIR=$(top_builddir)/dist -APP_DIST_EXTRAS=$(APP_DIST_DIR)/.background/background.tiff $(APP_DIST_DIR)/.DS_Store $(APP_DIST_DIR)/Applications -$(APP_DIST_DIR)/Applications: - @rm -f $@ - @cd $(@D); $(LN_S) /Applications $(@F) - -$(APP_DIST_EXTRAS): $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Elements-Qt - -$(OSX_TEMP_ISO): $(APP_DIST_EXTRAS) +$(OSX_DMG): deploydir $(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ $(APP_DIST_DIR) -- $(if $(SOURCE_DATE_EPOCH),-volume_date all_file_dates =$(SOURCE_DATE_EPOCH)) -$(OSX_DMG): $(OSX_TEMP_ISO) - $(DMG) dmg "$<" "$@" - -$(APP_DIST_DIR)/.background/background.tiff: - $(MKDIR_P) $(@D) - cp $(OSX_BACKGROUND_IMAGE) $@ - $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Elements-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING) - INSTALLNAMETOOL=$(INSTALLNAMETOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) + INSTALL_NAME_TOOL=$(INSTALL_NAME_TOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR) -deploydir: $(APP_DIST_EXTRAS) +deploydir: $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Elements-Qt endif !BUILD_DARWIN -appbundle: $(OSX_APP_BUILT) deploy: $(OSX_DMG) endif @@ -191,7 +176,6 @@ LCOV_FILTER_PATTERN = \ -p "src/leveldb/" \ -p "src/crc32c/" \ -p "src/bench/" \ - -p "src/univalue" \ -p "src/crypto/ctaes" \ -p "src/minisketch" \ -p "src/secp256k1" \ diff --git a/REVIEWERS b/REVIEWERS deleted file mode 100644 index a4b05f86c08..00000000000 --- a/REVIEWERS +++ /dev/null @@ -1,140 +0,0 @@ -# ============================================================================== -# Bitcoin Core REVIEWERS -# ============================================================================== - -# Configuration of automated review requests for the bitcoin/bitcoin repo -# via DrahtBot. - -# Order is not important; if a modified file or directory matches a fnmatch, -# the reviewer will be mentioned in a PR comment requesting a review. - -# Regular contributors are free to add their names to specific directories or -# files provided that they are willing to provide a review. - -# Absence from this list should not be interpreted as a discouragement to -# review a pull request. Peer review is always welcome and is a critical -# component of the progress of the codebase. Information on peer review -# guidelines can be found in the CONTRIBUTING.md doc. - - -# Maintainers -# @achow101 -# @fanquake -# @hebasto -# @laanwj -# @marcofalke -# @sipa - -# Docs -/doc/*[a-zA-Z-].md @harding -/doc/Doxyfile.in @fanquake -/doc/REST-interface.md @jonasschnelli -/doc/benchmarking.md @ariard -/doc/bitcoin-conf.md @hebasto -/doc/build-freebsd.md @fanquake -/doc/build-netbsd.md @fanquake -/doc/build-openbsd.md @laanwj -/doc/build-osx.md @fanquake -/doc/build-unix.md @laanwj -/doc/build-windows.md @sipsorcery -/doc/dependencies.md @fanquake -/doc/developer-notes.md @laanwj -/doc/files.md @hebasto -/doc/reduce-memory.md @fanquake -/doc/reduce-traffic.md @jonasschnelli -/doc/release-process.md @laanwj -/doc/translation_strings_policy.md @laanwj - -# Build aux -/build-aux/m4/bitcoin_qt.m4 @hebasto - -# MSVC build system -/build_msvc/ @sipsorcery - -# Settings -/src/util/settings.* @ryanofsky - -# Fuzzing - -# Tests -/src/test/net_peer_eviction_tests.cpp @jonatack -/test/functional/mempool_updatefromblock.py @hebasto -/test/functional/feature_asmap.py @jonatack -/test/functional/interface_bitcoin_cli.py @jonatack - -# Backwards compatibility tests -*_compatibility.py @sjors -/test/functional/wallet_upgradewallet.py @sjors @achow101 -/test/get_previous_releases.py @sjors - -# Translations -/src/util/translation.h @hebasto - -# Dev Tools -/contrib/devtools/security-check.py @fanquake -/contrib/devtools/test-security-check.py @fanquake -/contrib/devtools/symbol-check.py @fanquake - -# Guix -/contrib/guix/ @dongcarl - -# Compatibility -/src/compat/glibc_* @fanquake - -# GUI -/src/qt/forms/ @hebasto - -# Wallet -/src/wallet/ @achow101 - -# CLI -/src/bitcoin-cli.cpp @jonatack - -# Coinstats -/src/node/coinstats.* @fjahr - -# Index -/src/index/ @fjahr - -# Descriptors -*descriptor* @achow101 @sipa - -# External signer -*external_signer* @sjors -/doc/external-signer.md @sjors -*signer.py @sjors - -# Interfaces -/src/interfaces/ @ryanofsky - -# DB -/src/txdb.* @jamesob -/src/dbwrapper.* @jamesob - -# Linter -/test/lint/lint-shell.sh @hebasto - -# Bech32 -/src/bech32.* @sipa -/src/bench/bech32.* @sipa - -# PSBT -/src/psbt* @achow101 -/src/node/psbt* @achow101 -/doc/psbt.md @achow101 - -# P2P -/src/net_processing.* @sipa -/src/protocol.* @sipa - -# Consensus -/src/coins.* @sipa @jamesob -/src/script/script.* @sipa -/src/script/interpreter.* @sipa -/src/validation.* @sipa -/src/consensus/ @sipa - -# Tracing -/doc/tracing.md @jb55 @0xB10C -/src/util/trace.h @jb55 @0xB10C -/contrib/tracing/ @jb55 @0xB10C diff --git a/build-aux/m4/ax_boost_base.m4 b/build-aux/m4/ax_boost_base.m4 index 7aac53c8155..f6620882a21 100644 --- a/build-aux/m4/ax_boost_base.m4 +++ b/build-aux/m4/ax_boost_base.m4 @@ -8,21 +8,23 @@ # # DESCRIPTION # -# Test for the Boost C++ libraries of a particular version (or newer) +# Test for the Boost C++ headers of a particular version (or newer) # # If no path to the installed boost library is given the macro searchs -# under /usr, /usr/local, /opt, /opt/local and /opt/homebrew and evaluates the -# $BOOST_ROOT environment variable. Further documentation is available at -# . +# under /usr, /usr/local, /opt, /opt/local and /opt/homebrew and evaluates +# the $BOOST_ROOT environment variable. Further documentation is available +# at . # # This macro calls: # -# AC_SUBST(BOOST_CPPFLAGS) / AC_SUBST(BOOST_LDFLAGS) +# AC_SUBST(BOOST_CPPFLAGS) # # And sets: # # HAVE_BOOST # +# Note that this macro has been modified compared to upstream. +# # LICENSE # # Copyright (c) 2008 Thomas Porschberg @@ -33,7 +35,7 @@ # and this notice are preserved. This file is offered as-is, without any # warranty. -#serial 48 +#serial 51 # example boost program (need to pass version) m4_define([_AX_BOOST_BASE_PROGRAM], @@ -59,26 +61,10 @@ AC_ARG_WITH([boost], ], [want_boost="yes"]) - -AC_ARG_WITH([boost-libdir], - [AS_HELP_STRING([--with-boost-libdir=LIB_DIR], - [Force given directory for boost libraries. - Note that this will override library path detection, - so use this parameter only if default library detection fails - and you know exactly where your boost libraries are located.])], - [ - AS_IF([test -d "$withval"], - [_AX_BOOST_BASE_boost_lib_path="$withval"], - [AC_MSG_ERROR([--with-boost-libdir expected directory name])]) - ], - [_AX_BOOST_BASE_boost_lib_path=""]) - -BOOST_LDFLAGS="" BOOST_CPPFLAGS="" AS_IF([test "x$want_boost" = "xyes"], [_AX_BOOST_BASE_RUNDETECT([$1],[$2],[$3])]) AC_SUBST(BOOST_CPPFLAGS) -AC_SUBST(BOOST_LDFLAGS) ]) @@ -114,7 +100,7 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ AS_CASE([${host_cpu}], [x86_64],[libsubdirs="lib64 libx32 lib lib64"], [mips*64*],[libsubdirs="lib64 lib32 lib lib64"], - [ppc64|powerpc64|s390x|sparc64|aarch64|ppc64le|powerpc64le|riscv64],[libsubdirs="lib64 lib lib64"], + [ppc64|powerpc64|s390x|sparc64|aarch64|ppc64le|powerpc64le|riscv64|e2k],[libsubdirs="lib64 lib lib64"], [libsubdirs="lib"] ) @@ -128,7 +114,7 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ ) dnl first we check the system location for boost libraries - dnl this location ist chosen if boost libraries are installed with the --layout=system option + dnl this location is chosen if boost libraries are installed with the --layout=system option dnl or if you install boost with RPM AS_IF([test "x$_AX_BOOST_BASE_boost_path" != "x"],[ AC_MSG_CHECKING([for boostlib >= $1 ($WANT_BOOST_VERSION) includes in "$_AX_BOOST_BASE_boost_path/include"]) @@ -139,7 +125,6 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ AC_MSG_CHECKING([for boostlib >= $1 ($WANT_BOOST_VERSION) lib path in "$_AX_BOOST_BASE_boost_path/$_AX_BOOST_BASE_boost_path_tmp"]) AS_IF([test -d "$_AX_BOOST_BASE_boost_path/$_AX_BOOST_BASE_boost_path_tmp" && test -r "$_AX_BOOST_BASE_boost_path/$_AX_BOOST_BASE_boost_path_tmp" ],[ AC_MSG_RESULT([yes]) - BOOST_LDFLAGS="-L$_AX_BOOST_BASE_boost_path/$_AX_BOOST_BASE_boost_path_tmp"; break; ], [AC_MSG_RESULT([no])]) @@ -151,32 +136,22 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ else search_libsubdirs="$multiarch_libsubdir $libsubdirs" fi - for _AX_BOOST_BASE_boost_path_tmp in /usr /usr/local /opt /opt/local /opt/homebrew/; do + for _AX_BOOST_BASE_boost_path_tmp in /usr /usr/local /opt /opt/local /opt/homebrew ; do if test -d "$_AX_BOOST_BASE_boost_path_tmp/include/boost" && test -r "$_AX_BOOST_BASE_boost_path_tmp/include/boost" ; then for libsubdir in $search_libsubdirs ; do if ls "$_AX_BOOST_BASE_boost_path_tmp/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi done - BOOST_LDFLAGS="-L$_AX_BOOST_BASE_boost_path_tmp/$libsubdir" BOOST_CPPFLAGS="-I$_AX_BOOST_BASE_boost_path_tmp/include" break; fi done ]) - dnl overwrite ld flags if we have required special directory with - dnl --with-boost-libdir parameter - AS_IF([test "x$_AX_BOOST_BASE_boost_lib_path" != "x"], - [BOOST_LDFLAGS="-L$_AX_BOOST_BASE_boost_lib_path"]) - - AC_MSG_CHECKING([for boostlib >= $1 ($WANT_BOOST_VERSION)]) + AC_MSG_CHECKING([for Boost headers >= $1 ($WANT_BOOST_VERSION)]) CPPFLAGS_SAVED="$CPPFLAGS" CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" export CPPFLAGS - LDFLAGS_SAVED="$LDFLAGS" - LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" - export LDFLAGS - AC_REQUIRE([AC_PROG_CXX]) AC_LANG_PUSH(C++) AC_COMPILE_IFELSE([_AX_BOOST_BASE_PROGRAM($WANT_BOOST_VERSION)],[ @@ -193,11 +168,8 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ dnl built and installed without the --layout=system option or for a staged(not installed) version if test "x$succeeded" != "xyes" ; then CPPFLAGS="$CPPFLAGS_SAVED" - LDFLAGS="$LDFLAGS_SAVED" BOOST_CPPFLAGS= - if test -z "$_AX_BOOST_BASE_boost_lib_path" ; then - BOOST_LDFLAGS= - fi + _version=0 if test -n "$_AX_BOOST_BASE_boost_path" ; then if test -d "$_AX_BOOST_BASE_boost_path" && test -r "$_AX_BOOST_BASE_boost_path"; then @@ -216,14 +188,6 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ BOOST_CPPFLAGS="-I$_AX_BOOST_BASE_boost_path" fi fi - dnl if we found something and BOOST_LDFLAGS was unset before - dnl (because "$_AX_BOOST_BASE_boost_lib_path" = ""), set it here. - if test -n "$BOOST_CPPFLAGS" && test -z "$BOOST_LDFLAGS"; then - for libsubdir in $libsubdirs ; do - if ls "$_AX_BOOST_BASE_boost_path/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi - done - BOOST_LDFLAGS="-L$_AX_BOOST_BASE_boost_path/$libsubdir" - fi fi else if test "x$cross_compiling" != "xyes" ; then @@ -242,12 +206,6 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ VERSION_UNDERSCORE=`echo $_version | sed 's/\./_/'` BOOST_CPPFLAGS="-I$best_path/include/boost-$VERSION_UNDERSCORE" - if test -z "$_AX_BOOST_BASE_boost_lib_path" ; then - for libsubdir in $libsubdirs ; do - if ls "$best_path/$libsubdir/libboost_"* >/dev/null 2>&1 ; then break; fi - done - BOOST_LDFLAGS="-L$best_path/$libsubdir" - fi fi if test -n "$BOOST_ROOT" ; then @@ -259,10 +217,9 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ stage_version=`echo $version_dir | sed 's/boost_//' | sed 's/_/./g'` stage_version_shorten=`expr $stage_version : '\([[0-9]]*\.[[0-9]]*\)'` V_CHECK=`expr $stage_version_shorten \>\= $_version` - if test "x$V_CHECK" = "x1" && test -z "$_AX_BOOST_BASE_boost_lib_path" ; then + if test "x$V_CHECK" = "x1" ; then AC_MSG_NOTICE(We will use a staged boost library from $BOOST_ROOT) BOOST_CPPFLAGS="-I$BOOST_ROOT" - BOOST_LDFLAGS="-L$BOOST_ROOT/stage/$libsubdir" fi fi fi @@ -270,8 +227,6 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" export CPPFLAGS - LDFLAGS="$LDFLAGS $BOOST_LDFLAGS" - export LDFLAGS AC_LANG_PUSH(C++) AC_COMPILE_IFELSE([_AX_BOOST_BASE_PROGRAM($WANT_BOOST_VERSION)],[ @@ -298,6 +253,4 @@ AC_DEFUN([_AX_BOOST_BASE_RUNDETECT],[ fi CPPFLAGS="$CPPFLAGS_SAVED" - LDFLAGS="$LDFLAGS_SAVED" - ]) diff --git a/build-aux/m4/ax_cxx_compile_stdcxx.m4 b/build-aux/m4/ax_cxx_compile_stdcxx.m4 index 43087b2e688..51a35054d08 100644 --- a/build-aux/m4/ax_cxx_compile_stdcxx.m4 +++ b/build-aux/m4/ax_cxx_compile_stdcxx.m4 @@ -10,13 +10,13 @@ # # Check for baseline language coverage in the compiler for the specified # version of the C++ standard. If necessary, add switches to CXX and -# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard) -# or '14' (for the C++14 standard). +# CXXCPP to enable support. VERSION may be '11', '14', '17', or '20' for +# the respective C++ standard version. # # The second argument, if specified, indicates whether you insist on an # extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. # -std=c++11). If neither is specified, you get whatever works, with -# preference for an extended mode. +# preference for no added switch, and then for an extended mode. # # The third argument, if specified 'mandatory' or if left unspecified, # indicates that baseline support for the specified C++ standard is @@ -35,13 +35,15 @@ # Copyright (c) 2015 Moritz Klammler # Copyright (c) 2016, 2018 Krzesimir Nowak # Copyright (c) 2019 Enji Cooper +# Copyright (c) 2020 Jason Merrill +# Copyright (c) 2021 Jörn Heusipp # # Copying and distribution of this file, with or without modification, are # permitted in any medium without royalty provided the copyright notice # and this notice are preserved. This file is offered as-is, without any # warranty. -#serial 11 +#serial 14 dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro dnl (serial version number 13). @@ -50,6 +52,7 @@ AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], [$1], [14], [ax_cxx_compile_alternatives="14 1y"], [$1], [17], [ax_cxx_compile_alternatives="17 1z"], + [$1], [20], [ax_cxx_compile_alternatives="20"], [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl m4_if([$2], [], [], [$2], [ext], [], @@ -62,6 +65,16 @@ AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl AC_LANG_PUSH([C++])dnl ac_success=no + m4_if([$2], [], [dnl + AC_CACHE_CHECK(whether $CXX supports C++$1 features by default, + ax_cv_cxx_compile_cxx$1, + [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [ax_cv_cxx_compile_cxx$1=yes], + [ax_cv_cxx_compile_cxx$1=no])]) + if test x$ax_cv_cxx_compile_cxx$1 = xyes; then + ac_success=yes + fi]) + m4_if([$2], [noext], [], [dnl if test x$ac_success = xno; then for alternative in ${ax_cxx_compile_alternatives}; do @@ -140,7 +153,6 @@ m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 ) - dnl Test body for checking C++14 support m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], @@ -148,12 +160,24 @@ m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 ) +dnl Test body for checking C++17 support + m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 ) +dnl Test body for checking C++20 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_20], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_20 +) + + dnl Tests for new features in C++11 m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ @@ -949,3 +973,33 @@ namespace cxx17 #endif // __cplusplus < 201703L ]]) + + +dnl Tests for new features in C++20 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_20], [[ + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 202002L + +#error "This is not a C++20 compiler" + +#else + +#include + +namespace cxx20 +{ + +// As C++20 supports feature test macros in the standard, there is no +// immediate need to actually test for feature availability on the +// Autoconf side. + +} // namespace cxx20 + +#endif // __cplusplus < 202002L + +]]) diff --git a/build-aux/m4/bitcoin_qt.m4 b/build-aux/m4/bitcoin_qt.m4 index 1454e33f6e2..a716cd9a277 100644 --- a/build-aux/m4/bitcoin_qt.m4 +++ b/build-aux/m4/bitcoin_qt.m4 @@ -116,8 +116,8 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ BITCOIN_QT_CHECK([ TEMP_CPPFLAGS=$CPPFLAGS TEMP_CXXFLAGS=$CXXFLAGS - CPPFLAGS="$QT_INCLUDES $CPPFLAGS" - CXXFLAGS="$PIC_FLAGS $CXXFLAGS" + CPPFLAGS="$QT_INCLUDES $CORE_CPPFLAGS $CPPFLAGS" + CXXFLAGS="$PIC_FLAGS $CORE_CXXFLAGS $CXXFLAGS" _BITCOIN_QT_IS_STATIC if test "$bitcoin_cv_static_qt" = "yes"; then _BITCOIN_QT_CHECK_STATIC_LIBS @@ -178,8 +178,8 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ AC_MSG_CHECKING([whether -fPIE can be used with this Qt config]) TEMP_CPPFLAGS=$CPPFLAGS TEMP_CXXFLAGS=$CXXFLAGS - CPPFLAGS="$QT_INCLUDES $CPPFLAGS" - CXXFLAGS="$PIE_FLAGS $CXXFLAGS" + CPPFLAGS="$QT_INCLUDES $CORE_CPPFLAGS $CPPFLAGS" + CXXFLAGS="$PIE_FLAGS $CORE_CXXFLAGS $CXXFLAGS" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #ifndef QT_VERSION @@ -201,7 +201,7 @@ AC_DEFUN([BITCOIN_QT_CONFIGURE],[ BITCOIN_QT_CHECK([ AC_MSG_CHECKING([whether -fPIC is needed with this Qt config]) TEMP_CPPFLAGS=$CPPFLAGS - CPPFLAGS="$QT_INCLUDES $CPPFLAGS" + CPPFLAGS="$QT_INCLUDES $CORE_CPPFLAGS $CPPFLAGS" AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include #ifndef QT_VERSION diff --git a/build-aux/m4/l_atomic.m4 b/build-aux/m4/l_atomic.m4 index 40639dfe618..602b57fe432 100644 --- a/build-aux/m4/l_atomic.m4 +++ b/build-aux/m4/l_atomic.m4 @@ -18,7 +18,7 @@ m4_define([_CHECK_ATOMIC_testbody], [[ int main() { std::atomic lock{true}; - std::atomic_exchange(&lock, false); + lock.exchange(false); std::atomic t{0s}; t.store(2s); @@ -34,6 +34,8 @@ m4_define([_CHECK_ATOMIC_testbody], [[ AC_DEFUN([CHECK_ATOMIC], [ AC_LANG_PUSH(C++) + TEMP_CXXFLAGS="$CXXFLAGS" + CXXFLAGS="$CXXFLAGS $PTHREAD_CFLAGS" AC_MSG_CHECKING([whether std::atomic can be used without link library]) @@ -51,5 +53,6 @@ AC_DEFUN([CHECK_ATOMIC], [ ]) ]) + CXXFLAGS="$TEMP_CXXFLAGS" AC_LANG_POP ]) diff --git a/build_msvc/.gitignore b/build_msvc/.gitignore index b0e557bc0ca..b2eb9313a06 100644 --- a/build_msvc/.gitignore +++ b/build_msvc/.gitignore @@ -22,6 +22,7 @@ bench_bitcoin/bench_bitcoin.vcxproj libtest_util/libtest_util.vcxproj /bitcoin_config.h +/common.init.vcxproj */Win32 libbitcoin_qt/QtGeneratedFiles/* diff --git a/build_msvc/README.md b/build_msvc/README.md index 7feee6b7661..ba6171fee7a 100644 --- a/build_msvc/README.md +++ b/build_msvc/README.md @@ -3,9 +3,13 @@ Building Bitcoin Core with Visual Studio Introduction --------------------- -Solution and project files to build Bitcoin Core with `msbuild` or Visual Studio can be found in the `build_msvc` directory. The build has been tested with Visual Studio 2019 (building with earlier versions of Visual Studio should not be expected to work). +Visual Studio 2022 is minimum required to build Bitcoin Core. -To build Bitcoin Core from the command-line, it is sufficient to only install the Visual Studio Build Tools component. +Solution and project files to build with `msbuild` or Visual Studio can be found in the `build_msvc` directory. + +To build Bitcoin Core from the command-line, it is sufficient to only install the [Visual Studio Build Tools](https://visualstudio.microsoft.com/downloads/) component. + +The "Desktop development with C++" workload must be installed as well. Building with Visual Studio is an alternative to the Linux based [cross-compiler build](../doc/build-windows.md). @@ -13,7 +17,7 @@ Building with Visual Studio is an alternative to the Linux based [cross-compiler Prerequisites --------------------- To build [dependencies](../doc/dependencies.md) (except for [Qt](#qt)), -the default approach is to use the [vcpkg](https://docs.microsoft.com/en-us/cpp/vcpkg) package manager from Microsoft: +the default approach is to use the [vcpkg](https://vcpkg.io) package manager from Microsoft: 1. [Install](https://vcpkg.io/en/getting-started.html) vcpkg. @@ -28,9 +32,9 @@ Qt --------------------- To build Bitcoin Core with the GUI, a static build of Qt is required. -1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-opensource-src-5.15.3.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.3/single/qt-everywhere-opensource-src-5.15.3.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. +1. Download a single ZIP archive of Qt source code from https://download.qt.io/official_releases/qt/ (e.g., [`qt-everywhere-opensource-src-5.15.5.zip`](https://download.qt.io/official_releases/qt/5.15/5.15.5/single/qt-everywhere-opensource-src-5.15.5.zip)), and expand it into a dedicated folder. The following instructions assume that this folder is `C:\dev\qt-source`. -2. Open "x64 Native Tools Command Prompt for VS 2019", and input the following commands: +2. Open "x64 Native Tools Command Prompt for VS 2022", and input the following commands: ```cmd cd C:\dev\qt-source mkdir build @@ -47,25 +51,25 @@ To build Bitcoin Core without Qt, unload or disable the `bitcoin-qt`, `libbitcoi Building --------------------- -1. Use Python to generate `*.vcxproj` from Makefile: +1. Use Python to generate `*.vcxproj` for the Visual Studio 2022 toolchain from Makefile: -``` -PS >py -3 msvc-autogen.py +```cmd +python build_msvc\msvc-autogen.py ``` 2. An optional step is to adjust the settings in the `build_msvc` directory and the `common.init.vcxproj` file. This project file contains settings that are common to all projects such as the runtime library version and target Windows SDK version. The Qt directories can also be set. To specify a non-default path to a static Qt package directory, use the `QTBASEDIR` environment variable. -3. To build from the command-line with the Visual Studio 2019 toolchain use: +3. To build from the command-line with the Visual Studio toolchain use: ```cmd -msbuild -property:Configuration=Release -maxCpuCount -verbosity:minimal bitcoin.sln +msbuild build_msvc\bitcoin.sln -property:Configuration=Release -maxCpuCount -verbosity:minimal ``` -Alternatively, open the `build_msvc/bitcoin.sln` file in Visual Studio 2019. +Alternatively, open the `build_msvc/bitcoin.sln` file in Visual Studio. Security --------------------- -[Base address randomization](https://docs.microsoft.com/en-us/cpp/build/reference/dynamicbase-use-address-space-layout-randomization?view=msvc-160) is used to make Bitcoin Core more secure. When building Bitcoin using the `build_msvc` process base address randomization can be disabled by editing `common.init.vcproj` to change `RandomizedBaseAddress` from `true` to `false` and then rebuilding the project. +[Base address randomization](https://learn.microsoft.com/en-us/cpp/build/reference/dynamicbase-use-address-space-layout-randomization) is used to make Bitcoin Core more secure. When building Bitcoin using the `build_msvc` process base address randomization can be disabled by editing `common.init.vcproj` to change `RandomizedBaseAddress` from `true` to `false` and then rebuilding the project. To check if `bitcoind` has `RandomizedBaseAddress` enabled or disabled run diff --git a/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj b/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj index 9e05afe2166..f88891dd2a0 100644 --- a/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj +++ b/build_msvc/bitcoin-cli/bitcoin-cli.vcxproj @@ -16,6 +16,9 @@ {0667528c-d734-4009-adf9-c0d6c4a5a5a6} + + {7c87e378-df58-482e-aa2f-1bc129bc19ce} + {6190199c-6cf4-4dad-bfbd-93fa72a760c1} diff --git a/build_msvc/bitcoin_config.h.in b/build_msvc/bitcoin_config.h.in index 91b9c694f2e..0512b368ec3 100644 --- a/build_msvc/bitcoin_config.h.in +++ b/build_msvc/bitcoin_config.h.in @@ -41,18 +41,12 @@ /* Define to 1 to enable ZMQ functions */ #define ENABLE_ZMQ 1 -/* define if the Boost library is available */ -#define HAVE_BOOST /**/ - /* define if external signer support is enabled (requires Boost::Process) */ #define ENABLE_EXTERNAL_SIGNER /**/ /* Define this symbol if the consensus lib has been built */ #define HAVE_CONSENSUS_LIB 1 -/* define if the compiler supports basic C++17 syntax */ -#define HAVE_CXX17 1 - /* Define to 1 if you have the declaration of `be16toh', and to 0 if you don't. */ #define HAVE_DECL_BE16TOH 0 @@ -121,53 +115,9 @@ */ #define HAVE_DECL_SETSID 0 -/* Define to 1 if you have the declaration of `strerror_r', and to 0 if you - don't. */ -#define HAVE_DECL_STRERROR_R 0 - -/* Define to 1 if you have the declaration of `strnlen', and to 0 if you - don't. */ -#define HAVE_DECL_STRNLEN 1 - /* Define if the dllexport attribute is supported. */ #define HAVE_DLLEXPORT_ATTRIBUTE 1 -/* Define to 1 if you have the header file. */ -#define HAVE_INTTYPES_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_MEMORY_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_MINIUPNPC_MINIUPNPC_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_MINIUPNPC_UPNPCOMMANDS_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_MINIUPNPC_UPNPERRORS_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDINT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDIO_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDLIB_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRINGS_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRING_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_STAT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_TYPES_H 1 - /* Define to the address where bug reports for this package should be sent. */ #define PACKAGE_BUGREPORT "https://github.com/ElementsProject/elements/issues" diff --git a/build_msvc/common.init.vcxproj b/build_msvc/common.init.vcxproj.in similarity index 92% rename from build_msvc/common.init.vcxproj rename to build_msvc/common.init.vcxproj.in index c5e76b17593..11709330dfd 100644 --- a/build_msvc/common.init.vcxproj +++ b/build_msvc/common.init.vcxproj.in @@ -39,7 +39,7 @@ false false - v142 + @TOOLSET@ Unicode No $(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\ @@ -49,7 +49,7 @@ true true - v142 + @TOOLSET@ Unicode $(SolutionDir)$(Platform)\$(Configuration)\$(ProjectName)\ $(Platform)\$(Configuration)\$(ProjectName)\ @@ -87,11 +87,10 @@ Level3 NotUsing - stdcpp17 - /utf-8 /Zc:__cplusplus %(AdditionalOptions) - 4018;4244;4267;4334;4715;4805;4834 + /utf-8 /Zc:__cplusplus /std:c++20 %(AdditionalOptions) + 4018;4244;4267;4715;4805 true - _SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;SECP256K1_STATIC;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions) + SECP256K1_STATIC;_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;ZMQ_STATIC;NOMINMAX;WIN32;HAVE_CONFIG_H;_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_CONSOLE;_WIN32_WINNT=0x0601;_WIN32_IE=0x0501;WIN32_LEAN_AND_MEAN;%(PreprocessorDefinitions) ..\..\src;..\..\src\minisketch\include;..\..\src\univalue\include;..\..\src\secp256k1\include;..\..\src\simplicity\include;..\..\src\leveldb\include;..\..\src\leveldb\helpers\memenv;%(AdditionalIncludeDirectories) diff --git a/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in b/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in index 9ad4ef07c40..e71c71f5418 100644 --- a/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in +++ b/build_msvc/libbitcoin_common/libbitcoin_common.vcxproj.in @@ -12,6 +12,7 @@ + @SOURCE_FILES@ diff --git a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj index a64ae881f25..9f9dc9d5fa0 100644 --- a/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj +++ b/build_msvc/libbitcoin_qt/libbitcoin_qt.vcxproj @@ -53,6 +53,7 @@ + diff --git a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in index 6ec40461c2a..adf4fa03540 100644 --- a/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in +++ b/build_msvc/libbitcoin_util/libbitcoin_util.vcxproj.in @@ -8,7 +8,6 @@ StaticLibrary - @SOURCE_FILES@ diff --git a/build_msvc/libleveldb/libleveldb.vcxproj b/build_msvc/libleveldb/libleveldb.vcxproj index 009be30decb..2914eb2cfbb 100644 --- a/build_msvc/libleveldb/libleveldb.vcxproj +++ b/build_msvc/libleveldb/libleveldb.vcxproj @@ -50,7 +50,7 @@ - HAVE_CRC32C=0;HAVE_SNAPPY=0;__STDC_LIMIT_MACROS;LEVELDB_IS_BIG_ENDIAN=0;_UNICODE;UNICODE;_CRT_NONSTDC_NO_DEPRECATE;LEVELDB_PLATFORM_WINDOWS;LEVELDB_ATOMIC_PRESENT;%(PreprocessorDefinitions) + HAVE_CRC32C=0;HAVE_SNAPPY=0;LEVELDB_IS_BIG_ENDIAN=0;_UNICODE;UNICODE;_CRT_NONSTDC_NO_DEPRECATE;LEVELDB_PLATFORM_WINDOWS;%(PreprocessorDefinitions) 4244;4267 ..\..\src\leveldb;..\..\src\leveldb\include;%(AdditionalIncludeDirectories) diff --git a/build_msvc/libsecp256k1_config.h b/build_msvc/libsecp256k1_config.h index 57f2f144ff6..2b1a980e273 100644 --- a/build_msvc/libsecp256k1_config.h +++ b/build_msvc/libsecp256k1_config.h @@ -8,23 +8,6 @@ #define BITCOIN_LIBSECP256K1_CONFIG_H #undef USE_ASM_X86_64 -#undef USE_ENDOMORPHISM -#undef USE_FIELD_10X26 -#undef USE_FIELD_5X52 -#undef USE_FIELD_INV_BUILTIN -#undef USE_FIELD_INV_NUM -#undef USE_NUM_GMP -#undef USE_NUM_NONE -#undef USE_SCALAR_4X64 -#undef USE_SCALAR_8X32 -#undef USE_SCALAR_INV_BUILTIN -#undef USE_SCALAR_INV_NUM - -#define USE_NUM_NONE 1 -#define USE_FIELD_INV_BUILTIN 1 -#define USE_SCALAR_INV_BUILTIN 1 -#define USE_FIELD_10X26 1 -#define USE_SCALAR_8X32 1 #define ECMULT_GEN_PREC_BITS 4 #define ECMULT_WINDOW_SIZE 15 diff --git a/build_msvc/msvc-autogen.py b/build_msvc/msvc-autogen.py index 2a70cd93324..e02e3abdfa1 100755 --- a/build_msvc/msvc-autogen.py +++ b/build_msvc/msvc-autogen.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2021 The Bitcoin Core developers +# Copyright (c) 2016-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,7 @@ from shutil import copyfile SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'src')) -DEFAULT_PLATFORM_TOOLSET = R'v142' +DEFAULT_PLATFORM_TOOLSET = R'v143' libs = [ 'libbitcoin_cli', @@ -50,13 +50,6 @@ def parse_makefile(makefile): lib_sources[current_lib] = [] break -def set_common_properties(toolset): - with open(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), 'r', encoding='utf-8') as rfile: - s = rfile.read() - s = re.sub('.*?', ''+toolset+'', s) - with open(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), 'w', encoding='utf-8',newline='\n') as wfile: - wfile.write(s) - def parse_config_into_btc_config(): def find_between( s, first, last ): try: @@ -92,13 +85,18 @@ def find_between( s, first, last ): with open(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), "w", encoding="utf8") as btc_config: btc_config.writelines(template) +def set_properties(vcxproj_filename, placeholder, content): + with open(vcxproj_filename + '.in', 'r', encoding='utf-8') as vcxproj_in_file: + with open(vcxproj_filename, 'w', encoding='utf-8') as vcxproj_file: + vcxproj_file.write(vcxproj_in_file.read().replace(placeholder, content)) + def main(): parser = argparse.ArgumentParser(description='Bitcoin-core msbuild configuration initialiser.') - parser.add_argument('-toolset', nargs='?',help='Optionally sets the msbuild platform toolset, e.g. v142 for Visual Studio 2019.' + parser.add_argument('-toolset', nargs='?', default=DEFAULT_PLATFORM_TOOLSET, + help='Optionally sets the msbuild platform toolset, e.g. v143 for Visual Studio 2022.' ' default is %s.'%DEFAULT_PLATFORM_TOOLSET) args = parser.parse_args() - if args.toolset: - set_common_properties(args.toolset) + set_properties(os.path.join(SOURCE_DIR, '../build_msvc/common.init.vcxproj'), '@TOOLSET@', args.toolset) for makefile_name in os.listdir(SOURCE_DIR): if 'Makefile' in makefile_name: @@ -110,10 +108,7 @@ def main(): content += ' \n' content += ' $(IntDir)' + object_filename + '\n' content += ' \n' - with open(vcxproj_filename + '.in', 'r', encoding='utf-8') as vcxproj_in_file: - with open(vcxproj_filename, 'w', encoding='utf-8') as vcxproj_file: - vcxproj_file.write(vcxproj_in_file.read().replace( - '@SOURCE_FILES@\n', content)) + set_properties(vcxproj_filename, '@SOURCE_FILES@\n', content) parse_config_into_btc_config() copyfile(os.path.join(SOURCE_DIR,'../build_msvc/bitcoin_config.h'), os.path.join(SOURCE_DIR, 'config/bitcoin-config.h')) copyfile(os.path.join(SOURCE_DIR,'../build_msvc/libsecp256k1_config.h'), os.path.join(SOURCE_DIR, 'secp256k1/src/libsecp256k1-config.h')) diff --git a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj index 91c1b0a18a3..f6ab415a592 100644 --- a/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj +++ b/build_msvc/test_bitcoin-qt/test_bitcoin-qt.vcxproj @@ -56,6 +56,9 @@ {792d487f-f14c-49fc-a9de-3fc150f31c3f} + + {1e065f03-3566-47d0-8fa9-daa72b084e7d} + {18430fef-6b61-4c53-b396-718e02850f1b} diff --git a/build_msvc/vcpkg.json b/build_msvc/vcpkg.json index d8753ec21f4..3557269be0c 100644 --- a/build_msvc/vcpkg.json +++ b/build_msvc/vcpkg.json @@ -8,11 +8,17 @@ "boost-signals2", "boost-test", "sqlite3", - "double-conversion", { "name": "libevent", "features": ["thread"] }, "zeromq" + ], + "builtin-baseline": "f14984af3738e69f197bf0e647a8dca12de92996", + "overrides": [ + { + "name": "libevent", + "version": "2.1.12#7" + } ] } diff --git a/ci/README.md b/ci/README.md index 3c5f04c39e3..de798607df9 100644 --- a/ci/README.md +++ b/ci/README.md @@ -8,8 +8,7 @@ Be aware that the tests will be built and run in-place, so please run at your ow If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first. The ci needs to perform various sysadmin tasks such as installing packages or writing to the user's home directory. -While most of the actions are done inside a docker container, this is not possible for all. Thus, cache directories, -such as the depends cache, previous release binaries, or ccache, are mounted as read-write into the docker container. While it should be fine to run +While it should be fine to run the ci system locally on you development box, the ci scripts can generally be assumed to have received less review and testing compared to other parts of the codebase. If you want to keep the work tree clean, you might want to run the ci system in a virtual machine with a Linux operating system of your choice. diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index 1518c033f4f..f7147582dc1 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -1,22 +1,45 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C ${CI_RETRY_EXE} apt-get update -${CI_RETRY_EXE} apt-get install -y clang-format-9 python3-pip curl git gawk jq -update-alternatives --install /usr/bin/clang-format clang-format "$(which clang-format-9 )" 100 -update-alternatives --install /usr/bin/clang-format-diff clang-format-diff "$(which clang-format-diff-9)" 100 +# Lint dependencies: +# - curl/xz-utils (to install shellcheck) +# - git (used in many lint scripts) +# - gpg (used by verify-commits) +${CI_RETRY_EXE} apt-get install -y curl xz-utils git gpg -${CI_RETRY_EXE} pip3 install codespell==2.0.0 -${CI_RETRY_EXE} pip3 install flake8==3.8.3 -${CI_RETRY_EXE} pip3 install mypy==0.910 -${CI_RETRY_EXE} pip3 install pyzmq==22.3.0 -${CI_RETRY_EXE} pip3 install vulture==2.3 +if [ -z "${SKIP_PYTHON_INSTALL}" ]; then + PYTHON_PATH=/tmp/python + if [ ! -d "${PYTHON_PATH}/bin" ]; then + ( + git clone https://github.com/pyenv/pyenv.git + cd pyenv/plugins/python-build || exit 1 + ./install.sh + ) + # For dependencies see https://github.com/pyenv/pyenv/wiki#suggested-build-environment + ${CI_RETRY_EXE} apt-get install -y build-essential libssl-dev zlib1g-dev \ + libbz2-dev libreadline-dev libsqlite3-dev curl llvm \ + libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \ + clang + env CC=clang python-build "$(cat "${BASE_ROOT_DIR}/.python-version")" "${PYTHON_PATH}" + fi + export PATH="${PYTHON_PATH}/bin:${PATH}" + command -v python3 + python3 --version +fi + +${CI_RETRY_EXE} pip3 install codespell==2.2.1 +${CI_RETRY_EXE} pip3 install flake8==5.0.4 +${CI_RETRY_EXE} pip3 install mypy==0.971 +${CI_RETRY_EXE} pip3 install pyzmq==24.0.1 +${CI_RETRY_EXE} pip3 install vulture==2.6 SHELLCHECK_VERSION=v0.8.0 -curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ -export PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}" +curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | \ + tar --xz -xf - --directory /tmp/ +mv "/tmp/shellcheck-${SHELLCHECK_VERSION}/shellcheck" /usr/bin/ diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index 52d434afef5..8786f97618b 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -1,15 +1,23 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C -GIT_HEAD=$(git rev-parse HEAD) -if [ -n "$CIRRUS_PR" ]; then - COMMIT_RANGE="${CIRRUS_BASE_SHA}..$GIT_HEAD" +if [ -n "$LOCAL_BRANCH" ]; then + # To faithfully recreate CI linting locally, specify all commits on the current + # branch. + COMMIT_RANGE="$(git merge-base HEAD master)..HEAD" +elif [ -n "$CIRRUS_PR" ]; then + COMMIT_RANGE="HEAD~..HEAD" + echo + git log --no-merges --oneline "$COMMIT_RANGE" + echo test/lint/commit-script-check.sh "$COMMIT_RANGE" +else + COMMIT_RANGE="SKIP_EMPTY_NOT_A_PR" fi export COMMIT_RANGE @@ -19,11 +27,10 @@ test/lint/git-subtree-check.sh src/crypto/ctaes test/lint/git-subtree-check.sh src/secp256k1 test/lint/git-subtree-check.sh src/simplicity test/lint/git-subtree-check.sh src/minisketch -test/lint/git-subtree-check.sh src/univalue test/lint/git-subtree-check.sh src/leveldb test/lint/git-subtree-check.sh src/crc32c test/lint/check-doc.py -test/lint/lint-all.sh +test/lint/all-lint.py if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ "$CIRRUS_PR" = "" ] ; then # Sanity check only the last few commits to get notified of missing sigs, @@ -33,11 +40,8 @@ if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ "$CIRRUS_PR" = "" ] ; t git log HEAD~10 -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit git log HEAD~10 -1 --format='%H' > ./contrib/verify-commits/trusted-git-root mapfile -t KEYS < contrib/verify-commits/trusted-keys + git config user.email "ci@ci.ci" + git config user.name "ci" ${CI_RETRY_EXE} gpg --keyserver hkps://keys.openpgp.org --recv-keys "${KEYS[@]}" && ./contrib/verify-commits/verify-commits.py; fi - -if [ -n "$COMMIT_RANGE" ]; then - echo - git --no-pager log --no-merges --oneline "$COMMIT_RANGE" -fi diff --git a/ci/lint/Dockerfile b/ci/lint/Dockerfile new file mode 100644 index 00000000000..03c20c72860 --- /dev/null +++ b/ci/lint/Dockerfile @@ -0,0 +1,29 @@ +# See test/lint/README.md for usage. +# +# This container basically has to live in this directory in order to pull in the CI +# install scripts. If it lived in the root directory, it would have to pull in the +# entire repo as docker context during build; if it lived elsewhere, it wouldn't be +# able to make back-references to pull in the install scripts. So here it lives. + +FROM python:3.7-buster + +ENV DEBIAN_FRONTEND=noninteractive +ENV LC_ALL=C.UTF-8 + +# This is used by the 04_install.sh script; we can't read the Python version from +# .python-version for the same reasons as above, and it's more efficient to pull a +# preexisting Python image than it is to build from source. +ENV SKIP_PYTHON_INSTALL=1 + +# Must be built from ./ci/lint/ for these paths to work. +COPY ./docker-entrypoint.sh /entrypoint.sh +COPY ./04_install.sh /install.sh + +RUN /install.sh && \ + echo 'alias lint="./ci/lint/06_script.sh"' >> ~/.bashrc && \ + chmod 755 /entrypoint.sh && \ + rm -rf /var/lib/apt/lists/* + + +WORKDIR /bitcoin +ENTRYPOINT ["/entrypoint.sh"] diff --git a/ci/lint/docker-entrypoint.sh b/ci/lint/docker-entrypoint.sh new file mode 100755 index 00000000000..3fdbbb0761c --- /dev/null +++ b/ci/lint/docker-entrypoint.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +export LC_ALL=C + +# Fixes permission issues when there is a container UID/GID mismatch with the owner +# of the mounted bitcoin src dir. +git config --global --add safe.directory /bitcoin + +if [ -z "$1" ]; then + LOCAL_BRANCH=1 bash -ic "./ci/lint/06_script.sh" +else + exec "$@" +fi diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index e806683128b..4a54f47b03f 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,11 +8,10 @@ export LC_ALL=C.UTF-8 # The root dir. # The ci system copies this folder. -# This is where the depends build is done. BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) export BASE_ROOT_DIR # The depends dir. -# This folder exists on the ci host and ci guest. Changes are propagated back and forth. +# This folder exists only on the ci guest, and on the ci host as a volume. export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...) # This folder only exists on the ci host. @@ -37,6 +36,7 @@ export USE_BUSY_BOX=${USE_BUSY_BOX:-false} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} +export RUN_TIDY=${RUN_TIDY:-false} export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} # By how much to scale the test_runner timeouts (option --timeout-factor). # This is needed because some ci machines have slow CPU or disk, so sanitizers @@ -44,10 +44,9 @@ export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} export TEST_RUNNER_TIMEOUT_FACTOR=${TEST_RUNNER_TIMEOUT_FACTOR:-40} export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-} export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} -export EXPECTED_TESTS_DURATION_IN_SECONDS=${EXPECTED_TESTS_DURATION_IN_SECONDS:-1000} export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed} -export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:20.04} +export CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG:-ubuntu:20.04} # Randomize test order. # See https://www.boost.org/doc/libs/1_71_0/libs/test/doc/html/boost_test/utf_reference/rt_param_reference/random.html export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1} @@ -57,15 +56,18 @@ export CCACHE_SIZE=${CCACHE_SIZE:-100M} export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} # The cache dir. -# This folder exists on the ci host and ci guest. Changes are propagated back and forth. +# This folder exists only on the ci guest, and on the ci host as a volume. export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache} # Folder where the build result is put (bin and lib). export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} # Folder where the build is done (dist and out-of-tree build). export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build} +# The folder for previous release binaries. +# This folder exists only on the ci guest, and on the ci host as a volume. export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST} +export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu" export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} -export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison} +export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH diff --git a/ci/test/00_setup_env_android.sh b/ci/test/00_setup_env_android.sh index 522a5497fac..1834bd0bc4a 100755 --- a/ci/test/00_setup_env_android.sh +++ b/ci/test/00_setup_env_android.sh @@ -1,23 +1,23 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export HOST=aarch64-linux-android -export PACKAGES="clang llvm unzip openjdk-8-jdk gradle" +export PACKAGES="unzip openjdk-8-jdk gradle" export CONTAINER_NAME=ci_android -export DOCKER_NAME_TAG="ubuntu:focal" +export CI_IMAGE_NAME_TAG="ubuntu:jammy" export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export ANDROID_API_LEVEL=28 export ANDROID_BUILD_TOOLS_VERSION=28.0.3 -export ANDROID_NDK_VERSION=23.1.7779620 -export ANDROID_TOOLS_URL=https://dl.google.com/android/repository/commandlinetools-linux-6609375_latest.zip +export ANDROID_NDK_VERSION=23.2.8568313 +export ANDROID_TOOLS_URL=https://dl.google.com/android/repository/commandlinetools-linux-8512546_latest.zip export ANDROID_HOME="${DEPENDS_DIR}/SDKs/android" export ANDROID_NDK_HOME="${ANDROID_HOME}/ndk/${ANDROID_NDK_VERSION}" export DEP_OPTS="ANDROID_SDK=${ANDROID_HOME} ANDROID_NDK=${ANDROID_NDK_HOME} ANDROID_API_LEVEL=${ANDROID_API_LEVEL} ANDROID_TOOLCHAIN_BIN=${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin/" diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 932be4b43dc..ac0c0be96a7 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -18,7 +18,7 @@ if [ -n "$QEMU_USER_CMD" ]; then fi export CONTAINER_NAME=ci_arm_linux # Use debian to avoid 404 apt errors when cross compiling -export DOCKER_NAME_TAG="debian:bullseye" +export CI_IMAGE_NAME_TAG="debian:bullseye" export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index 28583c6e63b..a03eb9b6213 100755 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,9 +8,12 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_centos -export DOCKER_NAME_TAG=quay.io/rockylinux/rockylinux:8 +export CI_IMAGE_NAME_TAG=quay.io/rockylinux/rockylinux:8 export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-zmq which patch lbzip2 xz procps-ng dash rsync coreutils bison" export GOAL="install" +export NO_WERROR=1 # GCC 8 export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports" export CONFIG_SHELL="/bin/dash" export TEST_RUNNER_ENV="LC_ALL=en_US.UTF-8" + +export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-zmq which patch lbzip2 xz procps-ng dash rsync coreutils bison" diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index ec3b14fe832..5bac325fe13 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,11 +8,10 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_multiprocess -export DOCKER_NAME_TAG=ubuntu:20.04 -export PACKAGES="cmake python3 python3-pip llvm clang g++-multilib" +export CI_IMAGE_NAME_TAG=ubuntu:20.04 +export PACKAGES="cmake python3 llvm clang g++-multilib" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" export BITCOIN_CONFIG="--enable-debug CC='clang -m32' CXX='clang++ -m32' LDFLAGS='--rtlib=compiler-rt -lgcc_s'" export TEST_RUNNER_ENV="BITCOIND=elements-node" export TEST_RUNNER_EXTRA="--nosandbox" -export PIP_PACKAGES="lief" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index c4f22c8f9ea..fe42871c316 100755 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross -export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos +export CI_IMAGE_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos export HOST=x86_64-apple-darwin export PACKAGES="cmake libz-dev libtinfo5 python3-setuptools xorriso" export XCODE_VERSION=12.2 diff --git a/ci/test/00_setup_env_mac_native_arm64.sh b/ci/test/00_setup_env_mac_native_arm64.sh index ade2d2787c1..068f04c2a59 100755 --- a/ci/test/00_setup_env_mac_native_arm64.sh +++ b/ci/test/00_setup_env_mac_native_arm64.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index b03a7edb546..bb3f6997f36 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -1,14 +1,26 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 +# Only install BCC tracing packages in Cirrus CI. +if [[ "${CIRRUS_CI}" == "true" ]]; then + # We install an up-to-date 'bpfcc-tools' package from an untrusted PPA. + # This can be dropped with the next Ubuntu or Debian release that includes up-to-date packages. + # See the if-then in ci/test/04_install.sh too. + export ADD_UNTRUSTED_BPFCC_PPA=true + export BPFCC_PACKAGE="bpfcc-tools" +else + export ADD_UNTRUSTED_BPFCC_PPA=false + export BPFCC_PACKAGE="" +fi + export CONTAINER_NAME=ci_native_asan -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev" -export DOCKER_NAME_TAG=ubuntu:22.04 +export PACKAGES="systemtap-sdt-dev clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" +export CI_IMAGE_NAME_TAG=ubuntu:22.04 export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 13edd3ba133..9e692b3dad8 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -1,19 +1,19 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:22.04" +export CI_IMAGE_NAME_TAG="ubuntu:24.04" export CONTAINER_NAME=ci_native_fuzz -export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev" +export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--exclude=coins_view" # work around https://github.com/bitcoin/bitcoin/issues/22233 export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC='clang -ftrivial-auto-var-init=pattern' CXX='clang++ -ftrivial-auto-var-init=pattern'" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC='clang-16 -ftrivial-auto-var-init=pattern' CXX='clang++-16 -ftrivial-auto-var-init=pattern'" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 619acc46773..35a0de8034e 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -1,22 +1,23 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:20.04" +export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version). LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" -export CONTAINER_NAME="ci_native_msan" -export PACKAGES="clang-9 llvm-9 cmake" -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" +export CONTAINER_NAME="ci_native_fuzz_msan" +export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake" +# BDB generates false-positives and will be removed in future +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_UNIT_TESTS="false" export RUN_FUNCTIONAL_TESTS="false" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index a7715a6ded0..5fee10e37e5 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -1,19 +1,20 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:22.04" +export CI_IMAGE_NAME_TAG="debian:bookworm" export CONTAINER_NAME=ci_native_fuzz_valgrind -export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev valgrind" +export PACKAGES="clang llvm libclang-rt-dev python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev valgrind" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--valgrind" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++" +# Temporarily pin dwarf 4, until using Valgrind 3.20 or later +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang CXX=clang++ CFLAGS='-gdwarf-4' CXXFLAGS='-gdwarf-4'" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index b8418106c6a..acb4d2767b8 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -1,23 +1,23 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:20.04" +export CI_IMAGE_NAME_TAG="ubuntu:24.04" LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" -export BDB_PREFIX="${BASE_ROOT_DIR}/db4" export CONTAINER_NAME="ci_native_msan" -export PACKAGES="clang-9 llvm-9 cmake" -export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" +export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake" +# BDB generates false-positives and will be removed in future +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'" +export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_FUNCTIONAL_TESTS="false" export CCACHE_SIZE=250M diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index 89d62562988..3fdb49765c7 100755 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -1,14 +1,16 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel -export DOCKER_NAME_TAG=ubuntu:18.04 # Use bionic to have one config run the tests in python3.6, see doc/dependencies.md -export PACKAGES="python3-zmq clang-7 llvm-7 libc++abi-7-dev libc++-7-dev" # Use clang-7 to test C++17 compatibility, see doc/dependencies.md -export DEP_OPTS="NO_WALLET=1 CC=clang-7 CXX='clang++-7 -stdlib=libc++'" +export CI_IMAGE_NAME_TAG=ubuntu:focal +# Use minimum supported python3.7 (or python3.8, as best-effort) and clang-8, see doc/dependencies.md +export PACKAGES="python3-zmq clang-8 llvm-8 libc++abi-8-dev libc++-8-dev" +export DEP_OPTS="NO_WALLET=1 CC=clang-8 CXX='clang++-8 -stdlib=libc++'" export GOAL="install" -export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-7 CXX='clang++-7 -stdlib=libc++' --enable-experimental-util-chainstate" +export NO_WERROR=1 +export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-8 CXX='clang++-8 -stdlib=libc++' --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index f8b366e77f4..5cc0addd33c 100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -1,19 +1,21 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_qt5 -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic gcc-8 can compile our C++17 and run our functional tests in python3, see doc/dependencies.md +export CI_IMAGE_NAME_TAG=debian:buster +# Use minimum supported python3.7 and gcc-8, see doc/dependencies.md export PACKAGES="gcc-8 g++-8 python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1 CC=gcc-8 CXX=g++-8" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" -export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1 v0.20.1 v0.21.0 v22.0" +export NO_WERROR=1 +export DOWNLOAD_PREVIOUS_RELEASES="true" export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-reduce-exports \ ---enable-debug --disable-fuzz-binary CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8" +--enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" CC=gcc-8 CXX=g++-8" diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh new file mode 100755 index 00000000000..ed7a6b07c4d --- /dev/null +++ b/ci/test/00_setup_env_native_tidy.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +export CI_IMAGE_NAME_TAG="ubuntu:24.04" +export CONTAINER_NAME=ci_native_tidy +export PACKAGES="clang-16 libclang-16-dev llvm-16-dev libomp-16-dev clang-tidy-16 bear cmake libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" +export NO_DEPENDS=1 +export RUN_UNIT_TESTS=false +export RUN_FUNCTIONAL_TESTS=false +export RUN_FUZZ_TESTS=false +export RUN_TIDY=true +export GOAL="install" +export BITCOIN_CONFIG="CC=clang-16 CXX=clang++-16 --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0 -I/usr/lib/llvm-16/lib/clang/16/include'" +export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 0b3505cb0b6..88d4b2573d6 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,4 +11,4 @@ export DOCKER_NAME_TAG="docker.io/ubuntu:24.04" export PACKAGES="clang-18 llvm-18 libclang-rt-18-dev libc++abi-18-dev libc++-18-dev python3-zmq" export DEP_OPTS="CC=clang-18 CXX='clang++-18 -stdlib=libc++'" export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' CXXFLAGS='-g' --with-sanitizers=thread" +export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION' CXXFLAGS='-g' --with-sanitizers=thread" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 646070a84e7..97b85755ef9 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -1,16 +1,17 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:22.04" +export CI_IMAGE_NAME_TAG="debian:bookworm" export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" +export PACKAGES="valgrind clang llvm libclang-rt-dev python3-zmq libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 export TEST_RUNNER_EXTRA="--nosandbox --exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++" # TODO enable GUI +# Temporarily pin dwarf 4, until using Valgrind 3.20 or later +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no CC=clang CXX=clang++ CFLAGS='-gdwarf-4' CXXFLAGS='-gdwarf-4'" # TODO enable GUI diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 136edb6662b..af18703ce11 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,7 +18,7 @@ if [ -n "$QEMU_USER_CMD" ]; then fi # Use debian to avoid 404 apt errors export CONTAINER_NAME=ci_s390x -export DOCKER_NAME_TAG="debian:bookworm" +export CI_IMAGE_NAME_TAG="debian:bookworm" export TEST_RUNNER_ENV="LC_ALL=C" export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export RUN_FUNCTIONAL_TESTS=true diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 3f43bf227be..3adfbf6e479 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -1,16 +1,16 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 -export DOCKER_NAME_TAG=ubuntu:22.04 # Check that Jammy can cross-compile to win64 +export CI_IMAGE_NAME_TAG=ubuntu:22.04 # Check that Jammy can cross-compile to win64 export HOST=x86_64-w64-mingw32 export DPKG_ADD_ARCH="i386" -export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 wine32 file" +export PACKAGES="python3 nsis g++-mingw-w64-x86-64-posix wine-binfmt wine64 wine32 file" export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="--enable-reduce-exports --disable-external-signer --disable-gui-tests" +export BITCOIN_CONFIG="--enable-reduce-exports --enable-external-signer --disable-gui-tests" diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh new file mode 100755 index 00000000000..e2608377f6b --- /dev/null +++ b/ci/test/01_base_install.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +CFG_DONE="ci.base-install-done" # Use a global git setting to remember whether this script ran to avoid running it twice + +if [ "$(git config --global ${CFG_DONE})" == "true" ]; then + echo "Skip base install" + exit 0 +fi + +if [ -n "$DPKG_ADD_ARCH" ]; then + dpkg --add-architecture "$DPKG_ADD_ARCH" +fi + +if [[ $CI_IMAGE_NAME_TAG == *centos* ]] || [[ $CI_IMAGE_NAME_TAG == *rocky* ]]; then + ${CI_RETRY_EXE} bash -c "dnf -y install epel-release" + ${CI_RETRY_EXE} bash -c "dnf -y --allowerasing install $CI_BASE_PACKAGES $PACKAGES" +elif [ "$CI_USE_APT_INSTALL" != "no" ]; then + if [[ "${ADD_UNTRUSTED_BPFCC_PPA}" == "true" ]]; then + # Ubuntu 22.04 LTS and Debian 11 both have an outdated bpfcc-tools packages. + # The iovisor PPA is outdated as well. The next Ubuntu and Debian releases will contain updated + # packages. Meanwhile, use an untrusted PPA to install an up-to-date version of the bpfcc-tools + # package. + # TODO: drop this once we can use newer images in GCE + add-apt-repository ppa:hadret/bpfcc + fi + if [[ -n "${APPEND_APT_SOURCES_LIST}" ]]; then + echo "${APPEND_APT_SOURCES_LIST}" >> /etc/apt/sources.list + fi + ${CI_RETRY_EXE} apt-get update + ${CI_RETRY_EXE} bash -c "apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $CI_BASE_PACKAGES" +fi + +if [ -n "$PIP_PACKAGES" ]; then + if [ "$CI_OS_NAME" == "macos" ]; then + sudo -H pip3 install --upgrade --break-system-packages pip + # shellcheck disable=SC2086 + IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES + else + # shellcheck disable=SC2086 + ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES + fi +fi + +if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then + update-alternatives --install /usr/bin/clang++ clang++ "$(which clang++-16)" 100 + update-alternatives --install /usr/bin/clang clang "$(which clang-16)" 100 + git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.1 "${BASE_SCRATCH_DIR}"/msan/llvm-project + cmake -B "${BASE_SCRATCH_DIR}"/msan/build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF -DLIBCXX_ENABLE_DEBUG_MODE=ON -DLIBCXX_ENABLE_ASSERTIONS=ON -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes + make -C "${BASE_SCRATCH_DIR}"/msan/build/ "$MAKEJOBS" +fi + +if [[ "${RUN_TIDY}" == "true" ]]; then + git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_16 "${DIR_IWYU}"/include-what-you-use + cmake -B "${DIR_IWYU}"/build/ -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-16 -S "${DIR_IWYU}"/include-what-you-use + make -C "${DIR_IWYU}"/build/ install "$MAKEJOBS" +fi + +mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" + +OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers" + +if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then + OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz" + OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}" + if [ ! -f "$OSX_SDK_PATH" ]; then + curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" + fi + tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" +fi + +if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then + ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip + if [ ! -f "$ANDROID_TOOLS_PATH" ]; then + curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" + fi + mkdir -p "$ANDROID_HOME" + unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME" + yes | "${ANDROID_HOME}"/cmdline-tools/bin/sdkmanager --sdk_root="${ANDROID_HOME}" --install "build-tools;${ANDROID_BUILD_TOOLS_VERSION}" "platform-tools" "platforms;android-${ANDROID_API_LEVEL}" "ndk;${ANDROID_NDK_VERSION}" +fi + +git config --global ${CFG_DONE} "true" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 1e8ec188408..0c1e0ac3edc 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,12 +10,6 @@ if [[ $QEMU_USER_CMD == qemu-s390* ]]; then export LC_ALL=C fi -if [ "$CI_OS_NAME" == "macos" ]; then - sudo -H pip3 install --upgrade pip - # shellcheck disable=SC2086 - IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES -fi - # Create folders that are mounted into the docker mkdir -p "${CCACHE_DIR}" mkdir -p "${PREVIOUS_RELEASES_DIR}" @@ -26,14 +20,23 @@ export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/t export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|PREVIOUS_RELEASES_DIR)' | tee /tmp/env if [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) - DOCKER_ADMIN="--cap-add SYS_PTRACE" + CI_CONTAINER_CAP="--cap-add SYS_PTRACE" fi export P_CI_DIR="$PWD" +export BINS_SCRATCH_DIR="${BASE_SCRATCH_DIR}/bins/" if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then - echo "Creating $DOCKER_NAME_TAG container to run in" - ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" + echo "Creating $CI_IMAGE_NAME_TAG container to run in" + DOCKER_BUILDKIT=1 ${CI_RETRY_EXE} docker build \ + --file "${BASE_ROOT_DIR}/ci/test_imagefile" \ + --build-arg "CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG}" \ + --build-arg "FILE_ENV=${FILE_ENV}" \ + --tag="${CONTAINER_NAME}" \ + "${BASE_ROOT_DIR}" + docker volume create "${CONTAINER_NAME}_ccache" || true + docker volume create "${CONTAINER_NAME}_depends" || true + docker volume create "${CONTAINER_NAME}_previous_releases" || true if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then echo "Restart docker before run to stop and clear all containers started with --rm" @@ -41,40 +44,34 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then fi # shellcheck disable=SC2086 - DOCKER_ID=$(docker run $DOCKER_ADMIN --rm --interactive --detach --tty \ + CI_CONTAINER_ID=$(docker run $CI_CONTAINER_CAP --rm --interactive --detach --tty \ --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ - --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \ - --mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \ - --mount type=bind,src=$PREVIOUS_RELEASES_DIR,dst=$PREVIOUS_RELEASES_DIR \ + --mount "type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" \ + --mount "type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR" \ + --mount "type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" \ -w $BASE_ROOT_DIR \ --env-file /tmp/env \ --name $CONTAINER_NAME \ - $DOCKER_NAME_TAG) - export DOCKER_CI_CMD_PREFIX="docker exec $DOCKER_ID" + $CONTAINER_NAME) + export CI_CONTAINER_ID + export CI_EXEC_CMD_PREFIX="docker exec ${CI_CONTAINER_ID}" else echo "Running on host system without docker wrapper" fi CI_EXEC () { - $DOCKER_CI_CMD_PREFIX bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd \"$P_CI_DIR\" && $*" + $CI_EXEC_CMD_PREFIX bash -c "export PATH=${BINS_SCRATCH_DIR}:\$PATH && cd \"$P_CI_DIR\" && $*" } export -f CI_EXEC -if [ -n "$DPKG_ADD_ARCH" ]; then - CI_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH" -fi +CI_EXEC rsync --archive --stats --human-readable /ci_base_install/ "${BASE_ROOT_DIR}" || echo "/ci_base_install/ missing" +CI_EXEC "${BASE_ROOT_DIR}/ci/test/01_base_install.sh" +CI_EXEC rsync --archive --stats --human-readable /ro_base/ "${BASE_ROOT_DIR}" || echo "Nothing to copy from ro_base" +# Fixes permission issues when there is a container UID/GID mismatch with the owner +# of the git source code directory. +CI_EXEC git config --global --add safe.directory \"*\" -if [[ $DOCKER_NAME_TAG == *centos* ]] || [[ $DOCKER_NAME_TAG == *rocky* ]]; then - ${CI_RETRY_EXE} CI_EXEC dnf -y install epel-release - ${CI_RETRY_EXE} CI_EXEC dnf -y --allowerasing install "$DOCKER_PACKAGES" "$PACKAGES" -elif [ "$CI_USE_APT_INSTALL" != "no" ]; then - ${CI_RETRY_EXE} CI_EXEC apt-get update - ${CI_RETRY_EXE} CI_EXEC apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$DOCKER_PACKAGES" - if [ -n "$PIP_PACKAGES" ]; then - # shellcheck disable=SC2086 - ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES - fi -fi +CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}" if [ "$CI_OS_NAME" == "macos" ]; then top -l 1 -s 0 | awk ' /PhysMem/ {print}' @@ -102,28 +99,12 @@ fi CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" -if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - CI_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-9) 100" - CI_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-9) 100" - CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/" - CI_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project" - CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Memory -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" - CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx" -fi - -if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then - echo "Create $BASE_ROOT_DIR" - CI_EXEC rsync -a /ro_base/ "$BASE_ROOT_DIR" -fi - if [ "$USE_BUSY_BOX" = "true" ]; then echo "Setup to use BusyBox utils" - CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/bins/" # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) - # find excluded for now because it does not recognize the -delete option in ./depends (fixed in later BusyBox version) # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed) # shellcheck disable=SC1010 - CI_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \| grep -v "^find$"\)\; do ln -s \$\(command -v busybox\) "${BASE_SCRATCH_DIR}/bins/\$util"\; done + CI_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \)\; do ln -s \$\(command -v busybox\) "${BINS_SCRATCH_DIR}/\$util"\; done # Print BusyBox version CI_EXEC patch --help fi diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index b1cb1e300f3..584488c0967 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -13,38 +13,8 @@ else CI_EXEC echo \> \$HOME/.bitcoin fi -CI_EXEC mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" - -OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers" - -if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then - OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz" - OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}" - if [ ! -f "$OSX_SDK_PATH" ]; then - CI_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" - fi - CI_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH" -fi - -if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then - ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip - if [ ! -f "$ANDROID_TOOLS_PATH" ]; then - CI_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH" - fi - CI_EXEC mkdir -p "${ANDROID_HOME}/cmdline-tools" - CI_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "${ANDROID_HOME}/cmdline-tools" - CI_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/tools/bin/sdkmanager --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\"" -fi - -if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - # Use BDB compiled using install_db4.sh script to work around linking issue when using BDB - # from depends. See https://github.com/bitcoin/bitcoin/pull/18288#discussion_r433189350 for - # details. - CI_EXEC "contrib/install_db4.sh \$(pwd) --enable-umrw CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" -fi - if [ -z "$NO_DEPENDS" ]; then - if [[ $DOCKER_NAME_TAG == *centos* ]] || [[ $DOCKER_NAME_TAG == *rocky* ]]; then + if [[ $CI_IMAGE_NAME_TAG == *centos* ]] || [[ $CI_IMAGE_NAME_TAG == *rocky* ]]; then # CentOS has problems building the depends if the config shell is not explicitly set # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to # an error as the first command is executed) @@ -52,8 +22,8 @@ if [ -z "$NO_DEPENDS" ]; then else SHELL_OPTS="CONFIG_SHELL=" fi - CI_EXEC "$SHELL_OPTS" make "$MAKEJOBS" -C depends HOST="$HOST" "$DEP_OPTS" + CI_EXEC "$SHELL_OPTS" make "$MAKEJOBS" -C depends HOST="$HOST" "$DEP_OPTS" LOG=1 fi -if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then - CI_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" +if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then + CI_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" fi diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index 4d63042a1e2..7ccb5af4433 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -1,24 +1,32 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 +BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking" +if [ -z "$NO_DEPENDS" ]; then + BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} CONFIG_SITE=$DEPENDS_DIR/$HOST/share/config.site" +fi +if [ -z "$NO_WERROR" ]; then + BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" +fi + +CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" +PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats" + if [ -n "$ANDROID_TOOLS_URL" ]; then CI_EXEC make distclean || true CI_EXEC ./autogen.sh - CI_EXEC ./configure "$BITCOIN_CONFIG" --prefix="${DEPENDS_DIR}/aarch64-linux-android" || ( (CI_EXEC cat config.log) && false) + CI_EXEC ./configure "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) CI_EXEC "make $MAKEJOBS && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk" + CI_EXEC "${PRINT_CCACHE_STATISTICS}" exit 0 fi -BITCOIN_CONFIG_ALL="--enable-external-signer --enable-suppress-external-warnings --disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" -if [ -z "$NO_WERROR" ]; then - BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" -fi -CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" +BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --prefix=$BASE_OUTDIR" if [ -n "$CONFIG_SHELL" ]; then CI_EXEC "$CONFIG_SHELL" -c "./autogen.sh" @@ -48,8 +56,13 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then CI_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' fi -CI_EXEC make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false ) +if [[ "${RUN_TIDY}" == "true" ]]; then + MAYBE_BEAR="bear --config src/.bear-tidy-config" + MAYBE_TOKEN="--" +fi + +CI_EXEC "${MAYBE_BEAR}" "${MAYBE_TOKEN}" make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false ) -CI_EXEC "ccache --version | head -n 1 && ccache --show-stats" +CI_EXEC "${PRINT_CCACHE_STATISTICS}" CI_EXEC du -sh "${DEPENDS_DIR}"/*/ CI_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index d459704f659..db09885050f 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,12 +9,14 @@ export LC_ALL=C.UTF-8 if [[ $HOST = *-mingw32 ]]; then # Generate all binaries, so that they can be wrapped CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 + CI_EXEC make "$MAKEJOBS" -C src minisketch/test.exe VERBOSE=1 CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" fi if [ -n "$QEMU_USER_CMD" ]; then # Generate all binaries, so that they can be wrapped CI_EXEC make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 + CI_EXEC make "$MAKEJOBS" -C src minisketch/test VERBOSE=1 CI_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" fi @@ -27,11 +29,65 @@ if [ "$RUN_UNIT_TESTS" = "true" ]; then fi if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_BUILD_DIR}/elements-*/src/test/test_bitcoin*" --catch_system_errors=no -l test_suite + CI_EXEC "${TEST_RUNNER_ENV}" DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${BASE_OUTDIR}/bin/test_bitcoin" --catch_system_errors=no -l test_suite fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then - CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast + CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=99999999 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast +fi + +if [ "${RUN_TIDY}" = "true" ]; then + set -eo pipefail + export P_CI_DIR="${BASE_BUILD_DIR}/elements-$HOST/src/" + ( CI_EXEC run-clang-tidy-16 -quiet "${MAKEJOBS}" ) | grep -C5 "error" + export P_CI_DIR="${BASE_BUILD_DIR}/elements-$HOST/" + CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/iwyu_tool.py"\ + " src/common/init.cpp"\ + " src/common/url.cpp"\ + " src/compat"\ + " src/dbwrapper.cpp"\ + " src/init"\ + " src/kernel"\ + " src/node/chainstate.cpp"\ + " src/node/chainstatemanager_args.cpp"\ + " src/node/mempool_args.cpp"\ + " src/node/minisketchwrapper.cpp"\ + " src/node/utxo_snapshot.cpp"\ + " src/node/validation_cache_args.cpp"\ + " src/policy/feerate.cpp"\ + " src/policy/packages.cpp"\ + " src/policy/settings.cpp"\ + " src/primitives/transaction.cpp"\ + " src/random.cpp"\ + " src/rpc/fees.cpp"\ + " src/rpc/signmessage.cpp"\ + " src/test/fuzz/string.cpp"\ + " src/test/fuzz/txorphan.cpp"\ + " src/test/fuzz/util/"\ + " src/test/util/coins.cpp"\ + " src/uint256.cpp"\ + " src/util/bip32.cpp"\ + " src/util/bytevectorhash.cpp"\ + " src/util/check.cpp"\ + " src/util/error.cpp"\ + " src/util/exception.cpp"\ + " src/util/getuniquepath.cpp"\ + " src/util/hasher.cpp"\ + " src/util/message.cpp"\ + " src/util/moneystr.cpp"\ + " src/util/serfloat.cpp"\ + " src/util/spanparsing.cpp"\ + " src/util/strencodings.cpp"\ + " src/util/string.cpp"\ + " src/util/syserror.cpp"\ + " src/util/threadinterrupt.cpp"\ + " src/zmq"\ + " -p . ${MAKEJOBS}"\ + " -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/elements-$HOST/contrib/devtools/iwyu/bitcoin.core.imp"\ + " |& tee /tmp/iwyu_ci.out" + export P_CI_DIR="${BASE_ROOT_DIR}/src" + CI_EXEC "python3 ${DIR_IWYU}/include-what-you-use/fix_includes.py --nosafe_headers < /tmp/iwyu_ci.out" + CI_EXEC "git --no-pager diff" fi if [ "$RUN_SECURITY_TESTS" = "true" ]; then @@ -41,3 +97,8 @@ fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/fuzz/test_runner.py "${FUZZ_TESTS_CONFIG}" "$MAKEJOBS" -l DEBUG "${DIR_FUZZ_IN}" fi + +if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then + echo "Stop and remove CI container by ID" + docker container kill "${CI_CONTAINER_ID}" +fi diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh index fcd56f533e2..e028ede3788 100755 --- a/ci/test/wrap-qemu.sh +++ b/ci/test/wrap-qemu.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{no_nul,test_json,unitester,object}}; do +for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{test_json,unitester,object}}; do # shellcheck disable=SC2044 for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do echo "Wrap $b ..." diff --git a/ci/test/wrap-wine.sh b/ci/test/wrap-wine.sh index 525db9eded5..90e53887bc4 100755 --- a/ci/test/wrap-wine.sh +++ b/ci/test/wrap-wine.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{no_nul,test_json,unitester,object}}.exe; do +for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{test_json,unitester,object}}.exe; do # shellcheck disable=SC2044 for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename "$b_name")"); do if (file "$b" | grep "Windows"); then diff --git a/ci/test/wrapped-cl.bat b/ci/test/wrapped-cl.bat new file mode 100644 index 00000000000..fc2a604c580 --- /dev/null +++ b/ci/test/wrapped-cl.bat @@ -0,0 +1 @@ +ccache cl %* diff --git a/ci/test_imagefile b/ci/test_imagefile new file mode 100644 index 00000000000..4854708d1a9 --- /dev/null +++ b/ci/test_imagefile @@ -0,0 +1,10 @@ +ARG CI_IMAGE_NAME_TAG +FROM ${CI_IMAGE_NAME_TAG} + +ARG FILE_ENV +ENV FILE_ENV=${FILE_ENV} + +COPY ./ci/retry/retry /usr/bin/retry +COPY ./ci/test/00_setup_env.sh ./${FILE_ENV} ./ci/test/01_base_install.sh /ci_base_install/ci/test/ + +RUN ["bash", "-c", "cd /ci_base_install/ && set -o errexit && source ./ci/test/00_setup_env.sh && ./ci/test/01_base_install.sh"] diff --git a/configure.ac b/configure.ac index c35002dfc0f..d68f3171652 100644 --- a/configure.ac +++ b/configure.ac @@ -1,10 +1,10 @@ AC_PREREQ([2.69]) -define(_CLIENT_VERSION_MAJOR, 23) +define(_CLIENT_VERSION_MAJOR, 25) define(_CLIENT_VERSION_MINOR, 99) define(_CLIENT_VERSION_BUILD, 0) define(_CLIENT_VERSION_RC, 0) define(_CLIENT_VERSION_IS_RELEASE, false) -define(_COPYRIGHT_YEAR, 2022) +define(_COPYRIGHT_YEAR, 2023) define(_COPYRIGHT_HOLDERS,[The %s developers]) define(_COPYRIGHT_HOLDERS_SUBSTITUTION,[[Elements Project]]) AC_INIT([Elements Core],m4_join([.], _CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MINOR, _CLIENT_VERSION_BUILD)m4_if(_CLIENT_VERSION_RC, [0], [], [rc]_CLIENT_VERSION_RC),[https://github.com/ElementsProject/elements/issues],[elements],[https://elementsproject.org/]) @@ -19,8 +19,19 @@ if test "$PKG_CONFIG" = ""; then AC_MSG_ERROR([pkg-config not found]) fi +# When compiling with depends, the `PKG_CONFIG_PATH` and `PKG_CONFIG_LIBDIR` variables, +# being set in a `config.site` file, are not exported to let the `--config-cache` option +# work properly. +if test -n "$PKG_CONFIG_PATH"; then + PKG_CONFIG="env PKG_CONFIG_PATH=$PKG_CONFIG_PATH $PKG_CONFIG" +fi +if test -n "$PKG_CONFIG_LIBDIR"; then + PKG_CONFIG="env PKG_CONFIG_LIBDIR=$PKG_CONFIG_LIBDIR $PKG_CONFIG" +fi + BITCOIN_DAEMON_NAME=elementsd BITCOIN_GUI_NAME=elements-qt +BITCOIN_TEST_NAME=test_bitcoin BITCOIN_CLI_NAME=elements-cli BITCOIN_TX_NAME=elements-tx BITCOIN_UTIL_NAME=elements-util @@ -31,6 +42,7 @@ BITCOIN_MP_NODE_NAME=elements-node BITCOIN_MP_GUI_NAME=elements-gui dnl Unless the user specified ARFLAGS, force it to be cr +dnl This is also the default as-of libtool 2.4.7 AC_ARG_VAR([ARFLAGS], [Flags for the archiver, defaults to if not set]) if test "${ARFLAGS+set}" != "set"; then ARFLAGS="cr" @@ -42,14 +54,9 @@ AH_TOP([#ifndef BITCOIN_CONFIG_H]) AH_TOP([#define BITCOIN_CONFIG_H]) AH_BOTTOM([#endif //BITCOIN_CONFIG_H]) -dnl faketime breaks configure and is only needed for make. Disable it here. -unset FAKETIME - dnl Automake init set-up and checks AM_INIT_AUTOMAKE([1.13 no-define subdir-objects foreign]) -dnl faketime messes with timestamps and causes configure to be re-run. -dnl --disable-maintainer-mode can be used to bypass this. AM_MAINTAINER_MODE([enable]) dnl make the compilation flags quiet unless V=1 is used @@ -69,6 +76,12 @@ dnl we have those under control, re-enable that functionality. case $host in *mingw*) lt_cv_deplibs_check_method="pass_all" + + dnl Remove unwanted -DDLL_EXPORT from these variables. + dnl We do not use this macro, but system headers may export unwanted symbols + dnl if it's set. + lt_cv_prog_compiler_pic="-DPIC" + lt_cv_prog_compiler_pic_CXX="-DPIC" ;; esac @@ -78,11 +91,18 @@ AC_ARG_WITH([seccomp], [seccomp_found=$withval], [seccomp_found=auto]) +AC_ARG_ENABLE([c++20], + [AS_HELP_STRING([--enable-c++20], + [enable compilation in c++20 mode (disabled by default)])], + [use_cxx20=$enableval], + [use_cxx20=no]) + dnl Require C++17 compiler (no GNU extensions) +if test "$use_cxx20" = "no"; then AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory]) - -dnl Check if -latomic is required for -CHECK_ATOMIC +else +AX_CXX_COMPILE_STDCXX([20], [noext], [mandatory]) +fi dnl check if additional link flags are required for std::filesystem CHECK_FILESYSTEM @@ -96,22 +116,18 @@ fi AC_PROG_OBJCXX ]) -dnl Since libtool 1.5.2 (released 2004-01-25), on Linux libtool no longer -dnl sets RPATH for any directories in the dynamic linker search path. -dnl See more: https://wiki.debian.org/RpathIssue -LT_PREREQ([1.5.2]) +dnl OpenBSD ships with 2.4.2 +LT_PREREQ([2.4.2]) dnl Libtool init checks. LT_INIT([pic-only win32-dll]) dnl Check/return PATH for base programs. AC_PATH_TOOL([AR], [ar]) -AC_PATH_TOOL([RANLIB], [ranlib]) -AC_PATH_TOOL([STRIP], [strip]) AC_PATH_TOOL([GCOV], [gcov]) AC_PATH_TOOL([LLVM_COV], [llvm-cov]) AC_PATH_PROG([LCOV], [lcov]) -dnl Python 3.6 is specified in .python-version and should be used if available, see doc/dependencies.md -AC_PATH_PROGS([PYTHON], [python3.6 python3.7 python3.8 python3.9 python3.10 python3.11 python3 python]) +dnl Python 3.7 is specified in .python-version and should be used if available, see doc/dependencies.md +AC_PATH_PROGS([PYTHON], [python3.7 python3.8 python3.9 python3.10 python3.11 python3.12 python3 python]) AC_PATH_PROG([GENHTML], [genhtml]) AC_PATH_PROG([GIT], [git]) AC_PATH_PROG([CCACHE], [ccache]) @@ -153,24 +169,12 @@ AC_ARG_WITH([miniupnpc], [use_upnp=$withval], [use_upnp=auto]) -AC_ARG_ENABLE([upnp-default], - [AS_HELP_STRING([--enable-upnp-default], - [if UPNP is enabled, turn it on at startup (default is no)])], - [use_upnp_default=$enableval], - [use_upnp_default=no]) - AC_ARG_WITH([natpmp], [AS_HELP_STRING([--with-natpmp], [enable NAT-PMP (default is yes if libnatpmp is found)])], [use_natpmp=$withval], [use_natpmp=auto]) -AC_ARG_ENABLE([natpmp-default], - [AS_HELP_STRING([--enable-natpmp-default], - [if NAT-PMP is enabled, turn it on at startup (default is no)])], - [use_natpmp_default=$enableval], - [use_natpmp_default=no]) - AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[do not compile tests (default is to compile)]), [use_tests=$enableval], @@ -332,7 +336,7 @@ AC_ARG_ENABLE([werror], [enable_werror=no]) AC_ARG_ENABLE([external-signer], - [AS_HELP_STRING([--enable-external-signer],[compile external signer support (default is yes, requires Boost::Process)])], + [AS_HELP_STRING([--enable-external-signer],[compile external signer support (default is auto, requires Boost::Process)])], [use_external_signer=$enableval], [use_external_signer=auto]) @@ -367,7 +371,9 @@ case $host in esac if test "$enable_debug" = "yes"; then - dnl Clear default -g -O2 flags + dnl If debugging is enabled, and the user hasn't overridden CXXFLAGS, clear + dnl them, to prevent autoconfs "-g -O2" being added. Otherwise we'd end up + dnl with "-O0 -g3 -g -O2". if test "$CXXFLAGS_overridden" = "no"; then CXXFLAGS="" fi @@ -384,6 +390,8 @@ if test "$enable_debug" = "yes"; then AX_CHECK_PREPROC_FLAG([-DDEBUG], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG"], [], [$CXXFLAG_WERROR]) AX_CHECK_PREPROC_FLAG([-DDEBUG_LOCKORDER], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG_LOCKORDER"], [], [$CXXFLAG_WERROR]) + AX_CHECK_PREPROC_FLAG([-DDEBUG_LOCKCONTENTION], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DDEBUG_LOCKCONTENTION"], [], [$CXXFLAG_WERROR]) + AX_CHECK_PREPROC_FLAG([-DRPC_DOC_CHECK], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DRPC_DOC_CHECK"], [], [$CXXFLAG_WERROR]) AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-ftrapv], [DEBUG_CXXFLAGS="$DEBUG_CXXFLAGS -ftrapv"], [], [$CXXFLAG_WERROR]) fi @@ -443,7 +451,7 @@ if test "$CXXFLAGS_overridden" = "no"; then AX_CHECK_COMPILE_FLAG([-Wvla], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wvla"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wshadow-field], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wshadow-field"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wthread-safety], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wthread-safety"], [], [$CXXFLAG_WERROR]) - AX_CHECK_COMPILE_FLAG([-Wloop-analysis], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wrange-loop-analysis"], [], [$CXXFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-Wloop-analysis], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wloop-analysis"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wredundant-decls], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wredundant-decls"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wunused-member-function], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wunused-member-function"], [], [$CXXFLAG_WERROR]) AX_CHECK_COMPILE_FLAG([-Wdate-time], [WARN_CXXFLAGS="$WARN_CXXFLAGS -Wdate-time"], [], [$CXXFLAG_WERROR]) @@ -474,8 +482,10 @@ if test "$CXXFLAGS_overridden" = "no"; then fi dnl Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review. -AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers], [CXXFLAGS="$CXXFLAGS -fno-extended-identifiers"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-fno-extended-identifiers], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fno-extended-identifiers"], [], [$CXXFLAG_WERROR]) +enable_arm_crc=no +enable_arm_shani=no enable_sse42=no enable_sse41=no enable_avx2=no @@ -512,7 +522,7 @@ if test "$enable_clmul" = "yes"; then fi TEMP_CXXFLAGS="$CXXFLAGS" -CXXFLAGS="$CXXFLAGS $SSE42_CXXFLAGS" +CXXFLAGS="$SSE42_CXXFLAGS $CXXFLAGS" AC_MSG_CHECKING([for SSE4.2 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include @@ -534,7 +544,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" -CXXFLAGS="$CXXFLAGS $SSE41_CXXFLAGS" +CXXFLAGS="$SSE41_CXXFLAGS $CXXFLAGS" AC_MSG_CHECKING([for SSE4.1 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include @@ -549,7 +559,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" -CXXFLAGS="$CXXFLAGS $AVX2_CXXFLAGS" +CXXFLAGS="$AVX2_CXXFLAGS $CXXFLAGS" AC_MSG_CHECKING([for AVX2 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include @@ -564,7 +574,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" -CXXFLAGS="$CXXFLAGS $X86_SHANI_CXXFLAGS" +CXXFLAGS="$X86_SHANI_CXXFLAGS $CXXFLAGS" AC_MSG_CHECKING([for x86 SHA-NI intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include @@ -581,11 +591,11 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ CXXFLAGS="$TEMP_CXXFLAGS" # ARM -AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto], [ARM_CRC_CXXFLAGS="-march=armv8-a+crc+crypto"], [], [$CXXFLAG_WERROR]) -AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc+crypto], [ARM_SHANI_CXXFLAGS="-march=armv8-a+crc+crypto"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-march=armv8-a+crc], [ARM_CRC_CXXFLAGS="-march=armv8-a+crc"], [], [$CXXFLAG_WERROR]) +AX_CHECK_COMPILE_FLAG([-march=armv8-a+crypto], [ARM_SHANI_CXXFLAGS="-march=armv8-a+crypto"], [], [$CXXFLAG_WERROR]) TEMP_CXXFLAGS="$CXXFLAGS" -CXXFLAGS="$CXXFLAGS $ARM_CRC_CXXFLAGS" +CXXFLAGS="$ARM_CRC_CXXFLAGS $CXXFLAGS" AC_MSG_CHECKING([for ARMv8 CRC32 intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include @@ -604,7 +614,7 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ CXXFLAGS="$TEMP_CXXFLAGS" TEMP_CXXFLAGS="$CXXFLAGS" -CXXFLAGS="$CXXFLAGS $ARM_SHANI_CXXFLAGS" +CXXFLAGS="$ARM_SHANI_CXXFLAGS $CXXFLAGS" AC_MSG_CHECKING([for ARMv8 SHA-NI intrinsics]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ #include @@ -623,7 +633,7 @@ CXXFLAGS="$TEMP_CXXFLAGS" fi -CPPFLAGS="$CPPFLAGS -DHAVE_BUILD_INFO" +CORE_CPPFLAGS="$CORE_CPPFLAGS -DHAVE_BUILD_INFO" AC_ARG_WITH([utils], [AS_HELP_STRING([--with-utils], @@ -667,6 +677,12 @@ AC_ARG_WITH([libs], [build_bitcoin_libs=$withval], [build_bitcoin_libs=yes]) +AC_ARG_WITH([experimental-kernel-lib], + [AS_HELP_STRING([--with-experimental-kernel-lib], + [build experimental bitcoinkernel library (default is to build if we're building libraries and the experimental build-chainstate executable)])], + [build_experimental_kernel_lib=$withval], + [build_experimental_kernel_lib=auto]) + AC_ARG_WITH([daemon], [AS_HELP_STRING([--with-daemon], [build bitcoind daemon (default=yes)])], @@ -706,7 +722,10 @@ case $host in fi CPPFLAGS="$CPPFLAGS -DSECP256K1_STATIC" - CPPFLAGS="$CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN" + CORE_CPPFLAGS="$CORE_CPPFLAGS -D_MT -DWIN32 -D_WINDOWS -D_WIN32_WINNT=0x0601 -D_WIN32_IE=0x0501 -DWIN32_LEAN_AND_MEAN" + dnl Prevent the definition of min/max macros. + dnl We always want to use the standard library. + CORE_CPPFLAGS="$CORE_CPPFLAGS -DNOMINMAX" dnl libtool insists upon adding -nostdlib and a list of objects/libs to link against. dnl That breaks our ability to build dll's with static libgcc/libstdc++/libssp. Override @@ -717,7 +736,7 @@ case $host in postdeps_CXX= dnl We require Windows 7 (NT 6.1) or later - AX_CHECK_LINK_FLAG([-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1], [LDFLAGS="$LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"], [], [$LDFLAG_WERROR]) ;; *darwin*) TARGET_OS=darwin @@ -731,6 +750,16 @@ case $host in dnl It's safe to add these paths even if the functionality is disabled by dnl the user (--without-wallet or --without-gui for example). + dnl Homebrew may create symlinks in /usr/local/include for some packages. + dnl Because MacOS's clang internally adds "-I /usr/local/include" to its search + dnl paths, this will negate efforts to use -isystem for those packages, as they + dnl will be found first in /usr/local. Use the internal "-internal-isystem" + dnl option to system-ify all /usr/local/include paths without adding it to the list + dnl of search paths in case it's not already there. + if test "$suppress_external_warnings" != "no"; then + AX_CHECK_PREPROC_FLAG([-Xclang -internal-isystem/usr/local/include], [CORE_CPPFLAGS="$CORE_CPPFLAGS -Xclang -internal-isystem/usr/local/include"], [], [$CXXFLAG_WERROR]) + fi + if test "$use_bdb" != "no" && $BREW list --versions berkeley-db@4 >/dev/null && test "$BDB_CFLAGS" = "" && test "$BDB_LIBS" = ""; then bdb_prefix=$($BREW --prefix berkeley-db@4 2>/dev/null) dnl This must precede the call to BITCOIN_FIND_BDB48 below. @@ -753,20 +782,20 @@ case $host in if test "$use_upnp" != "no" && $BREW list --versions miniupnpc >/dev/null; then miniupnpc_prefix=$($BREW --prefix miniupnpc 2>/dev/null) if test "$suppress_external_warnings" != "no"; then - CPPFLAGS="$CPPFLAGS -isystem $miniupnpc_prefix/include" + MINIUPNPC_CPPFLAGS="-isystem $miniupnpc_prefix/include" else - CPPFLAGS="$CPPFLAGS -I$miniupnpc_prefix/include" + MINIUPNPC_CPPFLAGS="-I$miniupnpc_prefix/include" fi - LDFLAGS="$LDFLAGS -L$miniupnpc_prefix/lib" + MINIUPNPC_LIBS="-L$miniupnpc_prefix/lib" fi if test "$use_natpmp" != "no" && $BREW list --versions libnatpmp >/dev/null; then libnatpmp_prefix=$($BREW --prefix libnatpmp 2>/dev/null) if test "$suppress_external_warnings" != "no"; then - CPPFLAGS="$CPPFLAGS -isystem $libnatpmp_prefix/include" + NATPMP_CPPFLAGS="-isystem $libnatpmp_prefix/include" else - CPPFLAGS="$CPPFLAGS -I$libnatpmp_prefix/include" + NATPMP_CPPFLAGS="-I$libnatpmp_prefix/include" fi - LDFLAGS="$LDFLAGS -L$libnatpmp_prefix/lib" + NATPMP_LIBS="-L$libnatpmp_prefix/lib" fi ;; esac @@ -778,10 +807,9 @@ case $host in ;; *) AC_PATH_TOOL([DSYMUTIL], [dsymutil], [dsymutil]) - AC_PATH_TOOL([INSTALLNAMETOOL], [install_name_tool], [install_name_tool]) + AC_PATH_TOOL([INSTALL_NAME_TOOL], [install_name_tool], [install_name_tool]) AC_PATH_TOOL([OTOOL], [otool], [otool]) AC_PATH_PROGS([XORRISOFS], [xorrisofs], [xorrisofs]) - AC_PATH_PROGS([DMG], [dmg], [dmg]) dnl libtool will try to strip the static lib, which is a problem for dnl cross-builds because strip attempts to call a hard-coded ld, @@ -792,8 +820,8 @@ case $host in esac fi - AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names], [LDFLAGS="$LDFLAGS -Wl,-headerpad_max_install_names"], [], [$LDFLAG_WERROR]) - CPPFLAGS="$CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0" + AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-headerpad_max_install_names"], [], [$LDFLAG_WERROR]) + CORE_CPPFLAGS="$CORE_CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0" OBJCXXFLAGS="$CXXFLAGS" ;; *android*) @@ -856,11 +884,17 @@ if test "$use_lcov" = "yes"; then AC_SUBST(COV_TOOL_WRAPPER, "cov_tool_wrapper.sh") LCOV="$LCOV --gcov-tool $(pwd)/$COV_TOOL_WRAPPER" - AX_CHECK_LINK_FLAG([--coverage], [LDFLAGS="$LDFLAGS --coverage"], + AX_CHECK_LINK_FLAG([--coverage], [CORE_LDFLAGS="$CORE_LDFLAGS --coverage"], [AC_MSG_ERROR([lcov testing requested but --coverage linker flag does not work])]) - AX_CHECK_COMPILE_FLAG([--coverage],[CXXFLAGS="$CXXFLAGS --coverage";CFLAGS="$CFLAGS --coverage"], + AX_CHECK_COMPILE_FLAG([--coverage],[CORE_CXXFLAGS="$CORE_CXXFLAGS --coverage";CFLAGS="$CFLAGS --coverage"], [AC_MSG_ERROR([lcov testing requested but --coverage flag does not work])]) - CXXFLAGS="$CXXFLAGS -Og" + dnl If coverage is enabled, and the user hasn't overridden CXXFLAGS, clear + dnl them, to prevent autoconfs "-g -O2" being added. Otherwise we'd end up + dnl with "--coverage -Og -O0 -g -O2". + if test "$CXXFLAGS_overridden" = "no"; then + CXXFLAGS="" + fi + CORE_CXXFLAGS="$CORE_CXXFLAGS -Og -O0" fi if test "$use_lcov_branch" != "no"; then @@ -873,6 +907,9 @@ AC_C_BIGENDIAN dnl Check for pthread compile/link requirements AX_PTHREAD +dnl Check if -latomic is required for +CHECK_ATOMIC + dnl The following macro will add the necessary defines to bitcoin-config.h, but dnl they also need to be passed down to any subprojects. Pull the results out of dnl the cache and add them to CPPFLAGS. @@ -883,13 +920,13 @@ AC_FUNC_STRERROR_R if test "$ac_cv_sys_file_offset_bits" != "" && test "$ac_cv_sys_file_offset_bits" != "no" && test "$ac_cv_sys_file_offset_bits" != "unknown"; then - CPPFLAGS="$CPPFLAGS -D_FILE_OFFSET_BITS=$ac_cv_sys_file_offset_bits" + CORE_CPPFLAGS="$CORE_CPPFLAGS -D_FILE_OFFSET_BITS=$ac_cv_sys_file_offset_bits" fi if test "$ac_cv_sys_large_files" != "" && test "$ac_cv_sys_large_files" != "no" && test "$ac_cv_sys_large_files" != "unknown"; then - CPPFLAGS="$CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" + CORE_CPPFLAGS="$CORE_CPPFLAGS -D_LARGE_FILES=$ac_cv_sys_large_files" fi AC_SEARCH_LIBS([clock_gettime],[rt]) @@ -916,7 +953,9 @@ if test "$TARGET_OS" != "windows"; then AX_CHECK_COMPILE_FLAG([-fPIC], [PIC_FLAGS="-fPIC"]) fi -dnl All versions of gcc that we commonly use for building are subject to bug +dnl Versions of gcc prior to 12.1 (commit +dnl https://github.com/gcc-mirror/gcc/commit/551aa75778a4c5165d9533cd447c8fc822f583e1) +dnl are subject to a bug, see the gccbug_90348 test case and dnl https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90348. To work around that, set dnl -fstack-reuse=none for all gcc builds. (Only gcc understands this flag) AX_CHECK_COMPILE_FLAG([-fstack-reuse=none], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-reuse=none"]) @@ -925,10 +964,7 @@ if test "$use_hardening" != "no"; then AX_CHECK_COMPILE_FLAG([-Wstack-protector], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -Wstack-protector"]) AX_CHECK_COMPILE_FLAG([-fstack-protector-all], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fstack-protector-all"]) - dnl -fcf-protection used with Clang 7 causes ld to emit warnings: - dnl ld: error: ... - dnl Use CHECK_LINK_FLAG & --fatal-warnings to ensure we won't use the flag in this case. - AX_CHECK_LINK_FLAG([-fcf-protection=full], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"], [], [$LDFLAG_WERROR]) + AX_CHECK_COMPILE_FLAG([-fcf-protection=full], [HARDENED_CXXFLAGS="$HARDENED_CXXFLAGS -fcf-protection=full"]) case $host in *mingw*) @@ -945,11 +981,11 @@ if test "$use_hardening" != "no"; then dnl However, FORTIFY_SOURCE requires that there is some level of optimization, otherwise it does nothing and just creates a compiler warning. dnl Since FORTIFY_SOURCE is a no-op without optimizations, do not enable it when enable_debug is yes. if test "$enable_debug" != "yes"; then - AX_CHECK_PREPROC_FLAG([-D_FORTIFY_SOURCE=2],[ + AX_CHECK_PREPROC_FLAG([-D_FORTIFY_SOURCE=3],[ AX_CHECK_PREPROC_FLAG([-U_FORTIFY_SOURCE],[ HARDENED_CPPFLAGS="$HARDENED_CPPFLAGS -U_FORTIFY_SOURCE" ]) - HARDENED_CPPFLAGS="$HARDENED_CPPFLAGS -D_FORTIFY_SOURCE=2" + HARDENED_CPPFLAGS="$HARDENED_CPPFLAGS -D_FORTIFY_SOURCE=3" ]) fi @@ -973,18 +1009,17 @@ dnl These flags are specific to ld64, and may cause issues with other linkers. dnl For example: GNU ld will interpret -dead_strip as -de and then try and use dnl "ad_strip" as the symbol for the entry point. if test "$TARGET_OS" = "darwin"; then - AX_CHECK_LINK_FLAG([-Wl,-dead_strip], [LDFLAGS="$LDFLAGS -Wl,-dead_strip"], [], [$LDFLAG_WERROR]) - AX_CHECK_LINK_FLAG([-Wl,-dead_strip_dylibs], [LDFLAGS="$LDFLAGS -Wl,-dead_strip_dylibs"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-dead_strip], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-dead_strip"], [], [$LDFLAG_WERROR]) + AX_CHECK_LINK_FLAG([-Wl,-dead_strip_dylibs], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-dead_strip_dylibs"], [], [$LDFLAG_WERROR]) AX_CHECK_LINK_FLAG([-Wl,-bind_at_load], [HARDENED_LDFLAGS="$HARDENED_LDFLAGS -Wl,-bind_at_load"], [], [$LDFLAG_WERROR]) fi -AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h stdio.h stdlib.h unistd.h strings.h sys/types.h sys/stat.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) +AC_CHECK_HEADERS([endian.h sys/endian.h byteswap.h sys/select.h sys/prctl.h sys/sysctl.h vm/vm_param.h sys/vmmeter.h sys/resources.h]) AC_CHECK_DECLS([getifaddrs, freeifaddrs],[CHECK_SOCKET],, [#include #include ] ) -AC_CHECK_DECLS([strnlen]) dnl These are used for daemonization in bitcoind AC_CHECK_DECLS([fork]) @@ -992,6 +1027,8 @@ AC_CHECK_DECLS([setsid]) AC_CHECK_DECLS([pipe2]) +AC_CHECK_FUNCS([timingsafe_bcmp]) + AC_CHECK_DECLS([le16toh, le32toh, le64toh, htole16, htole32, htole64, be16toh, be32toh, be64toh, htobe16, htobe32, htobe64],,, [#if HAVE_ENDIAN_H #include @@ -1253,8 +1290,8 @@ AC_LINK_IFELSE( AC_MSG_CHECKING([for ::_wsystem]) AC_LINK_IFELSE( [ AC_LANG_PROGRAM( - [[ ]], - [[ int nErr = ::_wsystem(""); ]] + [[ #include ]], + [[ int nErr = ::_wsystem(NULL); ]] )], [ AC_MSG_RESULT([yes]); have_any_system=yes], [ AC_MSG_RESULT([no]) ] @@ -1274,7 +1311,7 @@ dnl Do not change "-I/usr/include" to "-isystem /usr/include" because that dnl is not necessary (/usr/include is already a system directory) and because dnl it would break GCC's #include_next. AC_DEFUN([SUPPRESS_WARNINGS], - [$(echo $1 |${SED} -E -e 's/(^| )-I/\1-isystem /g' -e 's;-isystem /usr/include([/ ]|$);-I/usr/include\1;g')]) + [[$(echo $1 |${SED} -E -e 's/(^| )-I/\1-isystem /g' -e 's;-isystem /usr/include/*( |$);-I/usr/include\1;g')]]) dnl enable-fuzz should disable all other targets if test "$enable_fuzz" = "yes"; then @@ -1291,6 +1328,7 @@ if test "$enable_fuzz" = "yes"; then bitcoin_enable_qt_test=no bitcoin_enable_qt_dbus=no use_bench=no + use_tests=no use_external_signer=no use_upnp=no use_natpmp=no @@ -1298,26 +1336,11 @@ if test "$enable_fuzz" = "yes"; then enable_fuzz_binary=yes AX_CHECK_PREPROC_FLAG([-DABORT_ON_FAILED_ASSUME], [DEBUG_CPPFLAGS="$DEBUG_CPPFLAGS -DABORT_ON_FAILED_ASSUME"], [], [$CXXFLAG_WERROR]) - - AC_MSG_CHECKING([whether main function is needed for fuzz binary]) - AX_CHECK_LINK_FLAG( - [-fsanitize=$use_sanitizers], - [AC_MSG_RESULT([no])], - [AC_MSG_RESULT([yes]); CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION"], - [], - [AC_LANG_PROGRAM([[ - #include - #include - extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { return 0; } - /* comment to remove the main function ... - ]],[[ - */ int not_main() { - ]])]) else BITCOIN_QT_INIT dnl sets $bitcoin_enable_qt, $bitcoin_enable_qt_test, $bitcoin_enable_qt_dbus - BITCOIN_QT_CONFIGURE([5.9.5]) + BITCOIN_QT_CONFIGURE([5.11.3]) dnl Keep a copy of the original $QT_INCLUDES and use it when invoking qt's moc QT_INCLUDES_UNSUPPRESSED=$QT_INCLUDES @@ -1326,8 +1349,25 @@ else QT_DBUS_INCLUDES=SUPPRESS_WARNINGS($QT_DBUS_INCLUDES) QT_TEST_INCLUDES=SUPPRESS_WARNINGS($QT_TEST_INCLUDES) fi +fi + +if test "$enable_fuzz_binary" = "yes"; then + AC_MSG_CHECKING([whether main function is needed for fuzz binary]) + AX_CHECK_LINK_FLAG( + [], + [AC_MSG_RESULT([no])], + [AC_MSG_RESULT([yes]); CORE_CPPFLAGS="$CORE_CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION"], + [$SANITIZER_LDFLAGS], + [AC_LANG_PROGRAM([[ + #include + #include + extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { return 0; } + /* comment to remove the main function ... + ]],[[ + */ int not_main() { + ]])]) - CPPFLAGS="$CPPFLAGS -DPROVIDE_FUZZ_MAIN_FUNCTION" + CHECK_RUNTIME_LIB fi if test "$enable_wallet" != "no"; then @@ -1379,8 +1419,9 @@ if test "$use_usdt" != "no"; then [AC_MSG_RESULT([no]); use_usdt=no;] ) fi +AM_CONDITIONAL([ENABLE_USDT_TRACEPOINTS], [test "$use_usdt" = "yes"]) -if test "$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests" = "nonononononono"; then +if test "$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests" = "nononono"; then use_upnp=no use_natpmp=no use_zmq=no @@ -1388,41 +1429,48 @@ fi dnl Check for libminiupnpc (optional) if test "$use_upnp" != "no"; then + TEMP_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $MINIUPNPC_CPPFLAGS" AC_CHECK_HEADERS( [miniupnpc/miniupnpc.h miniupnpc/upnpcommands.h miniupnpc/upnperrors.h], - [AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS=-lminiupnpc], [have_miniupnpc=no])], + [AC_CHECK_LIB([miniupnpc], [upnpDiscover], [MINIUPNPC_LIBS="$MINIUPNPC_LIBS -lminiupnpc"], [have_miniupnpc=no], [$MINIUPNPC_LIBS])], [have_miniupnpc=no] ) -dnl The minimum supported miniUPnPc API version is set to 10. This keeps compatibility -dnl with Ubuntu 16.04 LTS and Debian 8 libminiupnpc-dev packages. -if test "$have_miniupnpc" != "no"; then - AC_MSG_CHECKING([whether miniUPnPc API version is supported]) - AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ - @%:@include - ]], [[ - #if MINIUPNPC_API_VERSION >= 10 - // Everything is okay - #else - # error miniUPnPc API version is too old - #endif - ]])],[ - AC_MSG_RESULT([yes]) - ],[ - AC_MSG_RESULT([no]) - AC_MSG_WARN([miniUPnPc API version < 10 is unsupported, disabling UPnP support.]) - have_miniupnpc=no - ]) -fi + + dnl The minimum supported miniUPnPc API version is set to 17. This excludes + dnl versions with known vulnerabilities. + if test "$have_miniupnpc" != "no"; then + AC_MSG_CHECKING([whether miniUPnPc API version is supported]) + AC_PREPROC_IFELSE([AC_LANG_PROGRAM([[ + @%:@include + ]], [[ + #if MINIUPNPC_API_VERSION >= 17 + // Everything is okay + #else + # error miniUPnPc API version is too old + #endif + ]])],[ + AC_MSG_RESULT([yes]) + ],[ + AC_MSG_RESULT([no]) + AC_MSG_WARN([miniUPnPc API version < 17 is unsupported, disabling UPnP support.]) + have_miniupnpc=no + ]) + fi + CPPFLAGS="$TEMP_CPPFLAGS" fi dnl Check for libnatpmp (optional). if test "$use_natpmp" != "no"; then + TEMP_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $NATPMP_CPPFLAGS" AC_CHECK_HEADERS([natpmp.h], - [AC_CHECK_LIB([natpmp], [initnatpmp], [NATPMP_LIBS=-lnatpmp], [have_natpmp=no])], + [AC_CHECK_LIB([natpmp], [initnatpmp], [NATPMP_LIBS="$NATPMP_LIBS -lnatpmp"], [have_natpmp=no], [$NATPMP_LIBS])], [have_natpmp=no]) + CPPFLAGS="$TEMP_CPPFLAGS" fi -if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench" = "nonononononono"; then +if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench$enable_fuzz_binary" = "nonononononononono"; then use_boost=no else use_boost=yes @@ -1436,51 +1484,65 @@ if test "$use_boost" = "yes"; then AC_MSG_ERROR([only libbitcoinconsensus can be built without Boost]) fi + dnl we don't use multi_index serialization + BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_MULTI_INDEX_DISABLE_SERIALIZATION" + + dnl Prevent use of std::unary_function, which was removed in C++17, + dnl and will generate warnings with newer compilers for Boost + dnl older than 1.80. + dnl See: https://github.com/boostorg/config/pull/430. + AX_CHECK_PREPROC_FLAG([-DBOOST_NO_CXX98_FUNCTION_BASE], [BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_NO_CXX98_FUNCTION_BASE"], [], [$CXXFLAG_WERROR], + [AC_LANG_PROGRAM([[#include ]])]) + + if test "$enable_debug" = "yes" || test "$enable_fuzz" = "yes"; then + BOOST_CPPFLAGS="$BOOST_CPPFLAGS -DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE" + fi + if test "$suppress_external_warnings" != "no"; then BOOST_CPPFLAGS=SUPPRESS_WARNINGS($BOOST_CPPFLAGS) fi fi if test "$use_external_signer" != "no"; then - case $host in - *mingw*) - dnl Boost Process uses Boost Filesystem when targeting Windows. Also, - dnl since Boost 1.71.0, Process does not work with mingw-w64 without - dnl workarounds. See 67669ab425b52a2b6be3d2f3b3b7e3939b676a2c. - if test "$use_external_signer" = "yes"; then - AC_MSG_ERROR([External signing is not supported on Windows]) - fi - use_external_signer="no"; - ;; - *) - AC_MSG_CHECKING([whether Boost.Process can be used]) - TEMP_CXXFLAGS="$CXXFLAGS" - dnl Boost 1.78 requires the following workaround. - dnl See: https://github.com/boostorg/process/issues/235 - CXXFLAGS="$CXXFLAGS -Wno-error=narrowing" - TEMP_CPPFLAGS="$CPPFLAGS" - CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" - TEMP_LDFLAGS="$LDFLAGS" - dnl Boost 1.73 and older require the following workaround. - LDFLAGS="$LDFLAGS $PTHREAD_CFLAGS" - AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include ]])], - [have_boost_process="yes"], - [have_boost_process="no"]) - LDFLAGS="$TEMP_LDFLAGS" - CPPFLAGS="$TEMP_CPPFLAGS" - CXXFLAGS="$TEMP_CXXFLAGS" - AC_MSG_RESULT([$have_boost_process]) - if test "$have_boost_process" = "yes"; then - use_external_signer="yes" - AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [1], [Define if external signer support is enabled]) - else - if test "$use_external_signer" = "yes"; then - AC_MSG_ERROR([External signing is not supported for this Boost version]) - fi - use_external_signer="no"; - fi - ;; - esac + AC_MSG_CHECKING([whether Boost.Process can be used]) + TEMP_CXXFLAGS="$CXXFLAGS" + dnl Boost 1.78 requires the following workaround. + dnl See: https://github.com/boostorg/process/issues/235 + CXXFLAGS="$CXXFLAGS -Wno-error=narrowing" + TEMP_CPPFLAGS="$CPPFLAGS" + CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS" + TEMP_LDFLAGS="$LDFLAGS" + dnl Boost 1.73 and older require the following workaround. + LDFLAGS="$LDFLAGS $PTHREAD_CFLAGS" + AC_LINK_IFELSE([AC_LANG_PROGRAM([[ + #define BOOST_PROCESS_USE_STD_FS + #include + ]],[[ + namespace bp = boost::process; + bp::opstream stdin_stream; + bp::ipstream stdout_stream; + bp::child c("dummy", bp::std_out > stdout_stream, bp::std_err > stdout_stream, bp::std_in < stdin_stream); + stdin_stream << std::string{"test"} << std::endl; + if (c.running()) c.terminate(); + c.wait(); + c.exit_code(); + ]])], + [have_boost_process="yes"], + [have_boost_process="no"]) + LDFLAGS="$TEMP_LDFLAGS" + CPPFLAGS="$TEMP_CPPFLAGS" + CXXFLAGS="$TEMP_CXXFLAGS" + AC_MSG_RESULT([$have_boost_process]) + if test "$have_boost_process" = "yes"; then + use_external_signer="yes" + AC_DEFINE([ENABLE_EXTERNAL_SIGNER], [1], [Define if external signer support is enabled]) + AC_DEFINE([BOOST_PROCESS_USE_STD_FS], [1], [Defined to avoid Boost::Process trying to use Boost Filesystem]) + else + if test "$use_external_signer" = "yes"; then + AC_MSG_ERROR([External signing is not supported for this Boost version]) + fi + use_external_signer="no"; + fi fi AM_CONDITIONAL([ENABLE_EXTERNAL_SIGNER], [test "$use_external_signer" = "yes"]) @@ -1516,7 +1578,7 @@ AM_CONDITIONAL([ENABLE_SYSCALL_SANDBOX], [test "$use_syscall_sandbox" != "no"]) dnl Check for reduced exports if test "$use_reduce_exports" = "yes"; then - AX_CHECK_COMPILE_FLAG([-fvisibility=hidden], [CXXFLAGS="$CXXFLAGS -fvisibility=hidden"], + AX_CHECK_COMPILE_FLAG([-fvisibility=hidden], [CORE_CXXFLAGS="$CORE_CXXFLAGS -fvisibility=hidden"], [AC_MSG_ERROR([Cannot set hidden symbol visibility. Use --disable-reduce-exports.])], [$CXXFLAG_WERROR]) AX_CHECK_LINK_FLAG([-Wl,--exclude-libs,ALL], [RELDFLAGS="-Wl,--exclude-libs,ALL"], [], [$LDFLAG_WERROR]) fi @@ -1530,10 +1592,11 @@ fi dnl libevent check -if test "$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$use_tests$use_bench" != "nonononono"; then - PKG_CHECK_MODULES([EVENT], [libevent >= 2.0.21], [use_libevent=yes], [AC_MSG_ERROR([libevent version 2.0.21 or greater not found.])]) +use_libevent=no +if test "$build_bitcoin_cli$build_bitcoind$bitcoin_enable_qt$enable_fuzz_binary$use_tests$use_bench" != "nononononono"; then + PKG_CHECK_MODULES([EVENT], [libevent >= 2.1.8], [use_libevent=yes], [AC_MSG_ERROR([libevent version 2.1.8 or greater not found.])]) if test "$TARGET_OS" != "windows"; then - PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.0.21],, [AC_MSG_ERROR([libevent_pthreads version 2.0.21 or greater not found.])]) + PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads >= 2.1.8], [], [AC_MSG_ERROR([libevent_pthreads version 2.1.8 or greater not found.])]) fi if test "$suppress_external_warnings" != "no"; then @@ -1649,15 +1712,24 @@ AM_CONDITIONAL([BUILD_BITCOIN_UTIL], [test $build_bitcoin_util = "yes"]) AC_MSG_RESULT($build_bitcoin_util) AC_MSG_CHECKING([whether to build experimental bitcoin-chainstate]) +if test "$build_bitcoin_chainstate" = "yes"; then + if test "$build_experimental_kernel_lib" = "no"; then + AC_MSG_ERROR([experimental bitcoin-chainstate cannot be built without the experimental bitcoinkernel library. Use --with-experimental-kernel-lib]); + fi +fi AM_CONDITIONAL([BUILD_BITCOIN_CHAINSTATE], [test $build_bitcoin_chainstate = "yes"]) AC_MSG_RESULT($build_bitcoin_chainstate) AC_MSG_CHECKING([whether to build libraries]) AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test $build_bitcoin_libs = "yes"]) + if test "$build_bitcoin_libs" = "yes"; then AC_DEFINE([HAVE_CONSENSUS_LIB], [1], [Define this symbol if the consensus lib has been built]) AC_CONFIG_FILES([libbitcoinconsensus.pc:libbitcoinconsensus.pc.in]) fi + +AM_CONDITIONAL([BUILD_BITCOIN_KERNEL_LIB], [test "$build_experimental_kernel_lib" != "no" && ( test "$build_experimental_kernel_lib" = "yes" || test "$build_bitcoin_chainstate" = "yes" )]) + AC_MSG_RESULT($build_bitcoin_libs) AC_LANG_POP @@ -1704,17 +1776,10 @@ if test "$have_miniupnpc" = "no"; then else if test "$use_upnp" != "no"; then AC_MSG_RESULT([yes]) - AC_MSG_CHECKING([whether to build with UPnP enabled by default]) use_upnp=yes - upnp_setting=0 - if test "$use_upnp_default" != "no"; then - use_upnp_default=yes - upnp_setting=1 - fi - AC_MSG_RESULT([$use_upnp_default]) - AC_DEFINE_UNQUOTED([USE_UPNP],[$upnp_setting],[UPnP support not compiled if undefined, otherwise value (0 or 1) determines default state]) + AC_DEFINE([USE_UPNP], [1], [Define to 1 if UPnP support should be compiled in.]) if test "$TARGET_OS" = "windows"; then - MINIUPNPC_CPPFLAGS="-DSTATICLIB -DMINIUPNP_STATICLIB" + MINIUPNPC_CPPFLAGS="$MINIUPNPC_CPPFLAGS -DMINIUPNP_STATICLIB" fi else AC_MSG_RESULT([no]) @@ -1732,17 +1797,10 @@ if test "$have_natpmp" = "no"; then else if test "$use_natpmp" != "no"; then AC_MSG_RESULT([yes]) - AC_MSG_CHECKING([whether to build with NAT-PMP enabled by default]) use_natpmp=yes - natpmp_setting=0 - if test "$use_natpmp_default" != "no"; then - use_natpmp_default=yes - natpmp_setting=1 - fi - AC_MSG_RESULT($use_natpmp_default) - AC_DEFINE_UNQUOTED([USE_NATPMP], [$natpmp_setting], [NAT-PMP support not compiled if undefined, otherwise value (0 or 1) determines default state]) + AC_DEFINE([USE_NATPMP], [1], [Define to 1 if UPnP support should be compiled in.]) if test "$TARGET_OS" = "windows"; then - NATPMP_CPPFLAGS="-DSTATICLIB -DNATPMP_STATICLIB" + NATPMP_CPPFLAGS="$NATPMP_CPPFLAGS -DSTATICLIB -DNATPMP_STATICLIB" fi else AC_MSG_RESULT([no]) @@ -1809,12 +1867,8 @@ else AC_MSG_RESULT([no]) fi -if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$use_bench$use_tests" = "nononononononono"; then - AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-gui --enable-bench or --enable-tests]) -fi - -if test "$enable_fuzz_binary" = "yes"; then - CHECK_RUNTIME_LIB +if test "$build_bitcoin_wallet$build_bitcoin_cli$build_bitcoin_tx$build_bitcoin_util$build_bitcoin_libs$build_bitcoind$bitcoin_enable_qt$enable_fuzz_binary$use_bench$use_tests" = "nononononononononono"; then + AC_MSG_ERROR([No targets! Please specify at least one of: --with-utils --with-libs --with-daemon --with-gui --enable-fuzz(-binary) --enable-bench or --enable-tests]) fi AM_CONDITIONAL([TARGET_DARWIN], [test "$TARGET_OS" = "darwin"]) @@ -1869,6 +1923,7 @@ AC_SUBST(COPYRIGHT_HOLDERS_SUBSTITUTION, "_COPYRIGHT_HOLDERS_SUBSTITUTION") AC_SUBST(COPYRIGHT_HOLDERS_FINAL, "_COPYRIGHT_HOLDERS_FINAL") AC_SUBST(BITCOIN_DAEMON_NAME) AC_SUBST(BITCOIN_GUI_NAME) +AC_SUBST(BITCOIN_TEST_NAME) AC_SUBST(BITCOIN_CLI_NAME) AC_SUBST(BITCOIN_TX_NAME) AC_SUBST(BITCOIN_UTIL_NAME) @@ -1878,6 +1933,9 @@ AC_SUBST(BITCOIN_MP_NODE_NAME) AC_SUBST(BITCOIN_MP_GUI_NAME) AC_SUBST(RELDFLAGS) +AC_SUBST(CORE_LDFLAGS) +AC_SUBST(CORE_CPPFLAGS) +AC_SUBST(CORE_CXXFLAGS) AC_SUBST(DEBUG_CPPFLAGS) AC_SUBST(WARN_CXXFLAGS) AC_SUBST(NOWARN_CXXFLAGS) @@ -1924,17 +1982,19 @@ AC_SUBST(HAVE_EVHTTP_CONNECTION_GET_PEER_CONST_CHAR) AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/config.ini]) AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh]) AM_COND_IF([HAVE_DOXYGEN], [AC_CONFIG_FILES([doc/Doxyfile])]) -AC_CONFIG_LINKS([contrib/devtools/security-check.py:contrib/devtools/security-check.py]) -AC_CONFIG_LINKS([contrib/devtools/symbol-check.py:contrib/devtools/symbol-check.py]) -AC_CONFIG_LINKS([contrib/devtools/test-security-check.py:contrib/devtools/test-security-check.py]) -AC_CONFIG_LINKS([contrib/devtools/test-symbol-check.py:contrib/devtools/test-symbol-check.py]) +AC_CONFIG_LINKS([contrib/devtools/iwyu/bitcoin.core.imp:contrib/devtools/iwyu/bitcoin.core.imp]) AC_CONFIG_LINKS([contrib/filter-lcov.py:contrib/filter-lcov.py]) AC_CONFIG_LINKS([contrib/macdeploy/background.tiff:contrib/macdeploy/background.tiff]) +AC_CONFIG_LINKS([src/.bear-tidy-config:src/.bear-tidy-config]) +AC_CONFIG_LINKS([src/.clang-tidy:src/.clang-tidy]) AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py]) AC_CONFIG_LINKS([test/bitcoin_functional/functional/test_runner.py:test/bitcoin_functional/functional/test_runner.py]) AC_CONFIG_LINKS([test/fuzz/test_runner.py:test/fuzz/test_runner.py]) AC_CONFIG_LINKS([test/util/test_runner.py:test/util/test_runner.py]) AC_CONFIG_LINKS([test/util/rpcauth-test.py:test/util/rpcauth-test.py]) +AC_CONFIG_LINKS([src/qt/Makefile:src/qt/Makefile]) +AC_CONFIG_LINKS([src/qt/test/Makefile:src/qt/test/Makefile]) +AC_CONFIG_LINKS([src/test/Makefile:src/test/Makefile]) dnl boost's m4 checks do something really nasty: they export these vars. As a dnl result, they leak into secp256k1's configure and crazy things happen. @@ -2011,9 +2071,10 @@ echo " build os = $build_os" echo echo " CC = $CC" echo " CFLAGS = $PTHREAD_CFLAGS $CFLAGS" -echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS" +echo " CPPFLAGS = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CORE_CPPFLAGS $CPPFLAGS" echo " CXX = $CXX" -echo " CXXFLAGS = $LTO_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS" -echo " LDFLAGS = $LTO_LDFLAGS $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS" +echo " CXXFLAGS = $LTO_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CORE_CXXFLAGS $CXXFLAGS" +echo " LDFLAGS = $LTO_LDFLAGS $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $CORE_LDFLAGS $LDFLAGS" +echo " AR = $AR" echo " ARFLAGS = $ARFLAGS" echo diff --git a/contrib/README.md b/contrib/README.md index ae1372e95de..3c6e9780611 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -26,9 +26,6 @@ The [Debian](/contrib/debian) subfolder contains the copyright file. All other packaging related files can be found in the [bitcoin-core/packaging](https://github.com/bitcoin-core/packaging) repository. -### [Builder keys](/contrib/builder-keys) -PGP keys used for signing Bitcoin Core [release](/doc/release-process.md) results. - ### [MacDeploy](/contrib/macdeploy) ### Scripts and notes for Mac builds. @@ -40,3 +37,9 @@ Utilities to generate test vectors for the data-driven Bitcoin tests. ### [Verify Binaries](/contrib/verifybinaries) ### This script attempts to download and verify the signature file SHA256SUMS.asc from bitcoin.org. + +Command Line Tools +--------------------- + +### [Completions](/contrib/completions) ### +Shell completions for bash and fish. diff --git a/contrib/assets_tutorial/assets_tutorial.py b/contrib/assets_tutorial/assets_tutorial.py index 8262d8a7b2f..a6feee2eadd 100755 --- a/contrib/assets_tutorial/assets_tutorial.py +++ b/contrib/assets_tutorial/assets_tutorial.py @@ -63,7 +63,7 @@ e1.getinfo() print ("ERROR: was able to start an elementsd without a working bitcoind") sys.exit(1) -except: +except Exception: pass # 1b. Start bitcoind, then elementsd. Initially, the bitcoind may be warming up and @@ -716,4 +716,3 @@ txid = e1.sendrawtransaction(signedtx["hex"]) print ("Finished!") - diff --git a/contrib/builder-keys/README.md b/contrib/builder-keys/README.md deleted file mode 100644 index 56bd87d0afc..00000000000 --- a/contrib/builder-keys/README.md +++ /dev/null @@ -1,27 +0,0 @@ -## PGP keys of builders and Developers - -The file `keys.txt` contains fingerprints of the public keys of builders and -active developers. - -The associated keys are mainly used to sign git commits or the build results -of Guix builds. - -The most recent version of each pgp key can be found on most pgp key servers. - -Fetch the latest version from the key server to see if any key was revoked in -the meantime. -To fetch the latest version of all pgp keys in your gpg homedir, - -```sh -gpg --refresh-keys -``` - -To fetch keys of builders and active developers, feed the list of fingerprints -of the primary keys into gpg: - -```sh -while read fingerprint keyholder_name; do gpg --keyserver hkps://keys.openpgp.org --recv-keys ${fingerprint}; done < ./keys.txt -``` - -Add your key to the list if you provided Guix attestations for two major or -minor releases of Bitcoin Core. diff --git a/contrib/builder-keys/keys.txt b/contrib/builder-keys/keys.txt deleted file mode 100644 index e8032f66eeb..00000000000 --- a/contrib/builder-keys/keys.txt +++ /dev/null @@ -1,54 +0,0 @@ -9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C Aaron Clauson (sipsorcery) -617C90010B3BD370B0AC7D424BB42E31C79111B8 Akira Takizawa (akx20000) -E944AE667CF960B1004BC32FCA662BE18B877A60 Andreas Schildbach (aschildbach) -152812300785C96444D3334D17565732E08E5E41 Andrew Chow (achow101) -590B7292695AFFA5B672CBB2E13FC145CD3F4304 Antoine Poinsot (darosior) -0AD83877C1F0CD1EE9BD660AD7CC770B81FD22A8 Ben Carman (benthecarman) -912FD3228387123DC97E0E57D5566241A0295FA9 BtcDrak (btcdrak) -04017A2A6D9A0CCDC81D8EC296AB007F1A7ED999 Carl Dong (dongcarl) -C519EBCF3B926298946783EFF6430754120EC2F4 Christian Decker (cdecker) -18AE2F798E0D239755DA4FD24B79F986CBDF8736 Chun Kuan Le (ken2812221) -101598DC823C1B5F9A6624ABA5E0907A0380E6C3 CoinForensics (CoinForensics) -F20F56EF6A067F70E8A5C99FFF95FAA971697405 centaur (centaur) -C060A6635913D98A3587D7DB1C2491FFEB0EF770 Cory Fields (cfields) -BF6273FAEF7CC0BA1F562E50989F6B3048A116B5 Dev Random (devrandom) -6D3170C1DC2C6FD0AEEBCA6743811D1A26623924 Douglas Roark (droark) -1C6621605EC50319C463D56C7F81D87985D61612 Emanuele Cisbani (cisba) -9A1689B60D1B3CCE9262307A2F40A9BF167FBA47 Erik Mossberg (erkmos) -D35176BE9264832E4ACA8986BF0792FBE95DC863 fivepiece (fivepiece) -6F993B250557E7B016ADE5713BDCDA2D87A881D9 Fuzzbawls (Fuzzbawls) -01CDF4627A3B88AAE4A571C87588242FBE38D3A8 Gavin Andresen (gavinandresen) -D1DBF2C4B96F2DEBF4C16654410108112E7EA81F Hennadii Stepanov (hebasto) -A2FD494D0021AA9B4FA58F759102B7AE654A4A5A Ilyas Ridhuan (IlyasRidhuan) -2688F5A9A4BE0F295E921E8A25F27A38A47AD566 James O'Beirne (jamesob) -D3F22A3A4C366C2DCB66D3722DA9C5A7FA81EA35 Jarol Rodriguez (jarolrod) -7480909378D544EA6B6DCEB7535B12980BB8A4D3 Jeffri H Frontz (jhfrontz) -D3CC177286005BB8FF673294C5242A1AB3936517 jl2012 (jl2012) -82921A4B88FD454B7EB8CE3C796C4109063D4EAF Jon Atack (jonatack) -32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC Jonas Schnelli (jonasschnelli) -4B4E840451149DD7FB0D633477DFAB5C3108B9A8 Jorge Timon (jtimon) -C42AFF7C61B3E44A1454CD3557AF762DB3353322 Karl-Johan Alm (kallewoof) -70A1D47DD44F59DF8B22244333E472FE870C7E5D Kristaps Kaupe (kristapsk) -30DE693AE0DE9E37B3E7EB6BBFF0F67810C1EED1 Lisa Neigut (niftynei) -E463A93F5F3117EEDE6C7316BD02942421F4889F Luke Dashjr (luke-jr) -B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B Marco Falke (marco) -07DF3E57A548CCFB7530709189BBB8663E2E65CE Matt Corallo (BlueMatt) -CA03882CB1FC067B5D3ACFE4D300116E1C875A3D MeshCollider (meshcollider) -E777299FC265DD04793070EB944D35F9AC3DB76A Michael Ford (fanquake) -AD5764F4ADCE1B99BDFD179E12335A271D4D62EC Michael Tidwell (miketwenty1) -9692B91BBF0E8D34DFD33B1882C5C009628ECF0C Michagogo (michagogo) -C57E4B42223FDE851D4F69DD28DF2724F241D8EE midnightmagic (midnightmagic) -F4FC70F07310028424EFC20A8E4256593F177720 Oliver Gugger (guggero, Oliver Gugger) -D62A803E27E7F43486035ADBBCD04D8E9CCCAC2A Paul Rabahy (prab) -37EC7D7B0A217CDB4B4E007E7FAB114267E4FA04 Peter Todd (petertodd) -D762373D24904A3E42F33B08B9A408E71DAAC974 Pieter Wuille [Location: Leuven, Belgium] (sipa) -133EAC179436F14A5CF1B794860FEB804E669320 Pieter Wuille (sipa) -6A8F9C266528E25AEB1D7731C2371D91CB716EA7 Sebastian Falbesoner (theStack) -A8FC55F3B04BA3146F3492E79303B33A305224CB Sebastian Kung (TheCharlatan) -ED9BDF7AD6A55E232E84524257FF9BDBCC301009 Sjors Provoost (sjors) -867345026B6763E8B07EE73AB6737117397F5C4F Stephan Oeste (Emzy) -9EDAFF80E080659604F4A76B2EBB056FD847F8A7 Stephan Oeste (Emzy) -6DEEF79B050C4072509B743F8C275BC595448867 Tomas Kanocz (KanoczTomas) -AEC1884398647C47413C1C3FB1179EB7347DC10D Warren Togami (wtogami) -79D00BAC68B56D422F945A8F8E3A8F3247DBCBBF Willy Ko (willyko) -71A3B16735405025D447E8F274810B012346C9A6 Wladimir J. van der Laan (laanwj) diff --git a/contrib/bitcoin-cli.bash-completion b/contrib/completions/bash/bitcoin-cli.bash-completion similarity index 98% rename from contrib/bitcoin-cli.bash-completion rename to contrib/completions/bash/bitcoin-cli.bash-completion index 33ccae39450..5f817f72182 100644 --- a/contrib/bitcoin-cli.bash-completion +++ b/contrib/completions/bash/bitcoin-cli.bash-completion @@ -1,5 +1,5 @@ # bash programmable completion for bitcoin-cli(1) -# Copyright (c) 2012-2019 The Bitcoin Core developers +# Copyright (c) 2012-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/bitcoin-tx.bash-completion b/contrib/completions/bash/bitcoin-tx.bash-completion similarity index 96% rename from contrib/bitcoin-tx.bash-completion rename to contrib/completions/bash/bitcoin-tx.bash-completion index 0764931dc93..ecc7d6608c6 100644 --- a/contrib/bitcoin-tx.bash-completion +++ b/contrib/completions/bash/bitcoin-tx.bash-completion @@ -1,5 +1,5 @@ # bash programmable completion for bitcoin-tx(1) -# Copyright (c) 2016 The Bitcoin Core developers +# Copyright (c) 2016-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/bitcoind.bash-completion b/contrib/completions/bash/bitcoind.bash-completion similarity index 96% rename from contrib/bitcoind.bash-completion rename to contrib/completions/bash/bitcoind.bash-completion index ef67c006102..cf81bca6214 100644 --- a/contrib/bitcoind.bash-completion +++ b/contrib/completions/bash/bitcoind.bash-completion @@ -1,5 +1,5 @@ # bash programmable completion for bitcoind(1) and bitcoin-qt(1) -# Copyright (c) 2012-2019 The Bitcoin Core developers +# Copyright (c) 2012-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/completions/fish/bitcoin-cli.fish b/contrib/completions/fish/bitcoin-cli.fish new file mode 100644 index 00000000000..2f034c475c1 --- /dev/null +++ b/contrib/completions/fish/bitcoin-cli.fish @@ -0,0 +1,99 @@ +# Disable files from being included in completions by default +complete --command bitcoin-cli --no-files + +function __fish_bitcoin_cli_get_commands_helper + set --local cmd (commandline -oc) + + # Don't return commands if '-help or -?' in commandline + if string match --quiet --regex -- '^-help$|^-\?$' $cmd + return + end + + # Strip help cmd from token to avoid duplication errors + set --local cmd (string match --invert --regex -- '^help$' $cmd) + # Strip -stdin* options to avoid waiting for input while we fetch completions + # TODO: this appears to be broken when run as tab completion (requires ctrl+c to exit) + set --local cmd (string match --invert --regex -- '^-stdin.*$' $cmd) + + # Match, format and return commands + for command in ($cmd help 2>&1 | string match --invert -r '^\=\=.*' | string match --invert -r '^\\s*$') + echo $command + end +end + +function __fish_bitcoin_cli_get_commands + argparse 'nohelp' 'commandsonly' -- $argv + set --local commands + + # Exclude description, exclude help + if set -q _flag_nohelp; and set -q _flag_commandsonly + set --append commands (__fish_bitcoin_cli_get_commands_helper | string replace -r ' .*$' '' | string match --invert -r 'help') + # Include description, exclude help + else if set -q _flag_nohelp + set --append commands (__fish_bitcoin_cli_get_commands_helper | string replace ' ' \t | string match --invert -r 'help') + # Exclude description, include help + else if set -q _flag_commandsonly + set --append commands (__fish_bitcoin_cli_get_commands_helper | string replace -r ' .*$' '') + # Include description, include help + else + set --append commands (__fish_bitcoin_cli_get_commands_helper | string replace ' ' \t) + end + + if string match -q -r '^.*error.*$' $commands[1] + # RPC offline or RPC wallet not loaded + return + else + for command in $commands + echo $command + end + end +end + + +function __fish_bitcoin_cli_get_options + argparse 'nofiles' -- $argv + set --local cmd (commandline -oc) + # Don't return options if '-help or -?' in commandline + if string match --quiet --regex -- '^-help$|-\?$' $cmd + return + end + set --local options + + if set -q _flag_nofiles + set --append options ($cmd -help 2>&1 | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=' | string match --invert -r '^.*=$') + else + set --append options ($cmd -help 2>&1 | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=' | string match -r '^.*=$') + end + + for option in $options + echo $option + end +end + +# Add options with file completion +# Don't offer after a command is given +complete \ + --command bitcoin-cli \ + --no-files \ + --condition "not __fish_seen_subcommand_from (__fish_bitcoin_cli_get_commands --commandsonly)" \ + --arguments "(__fish_bitcoin_cli_get_options)" +# Enable file completions only if the commandline now contains a `*.=` style option +complete --command bitcoin-cli \ + --condition 'string match --regex -- ".*=" (commandline -pt)' \ + --force-files + +# Add options without file completion +# Don't offer after a command is given +complete \ + --command bitcoin-cli \ + --no-files \ + --condition "not __fish_seen_subcommand_from (__fish_bitcoin_cli_get_commands --commandsonly)" \ + --arguments "(__fish_bitcoin_cli_get_options --nofiles)" + +# Add commands +# Permit command completions after `bitcoin-cli help` but not after other commands +complete \ + --command bitcoin-cli \ + --no-files \ + --condition "not __fish_seen_subcommand_from (__fish_bitcoin_cli_get_commands --commandsonly --nohelp)" \ + --arguments "(__fish_bitcoin_cli_get_commands)" diff --git a/contrib/completions/fish/bitcoin-qt.fish b/contrib/completions/fish/bitcoin-qt.fish new file mode 100644 index 00000000000..15a355ae88f --- /dev/null +++ b/contrib/completions/fish/bitcoin-qt.fish @@ -0,0 +1,35 @@ +# Disable files from being included in completions by default +complete --command bitcoin-qt --no-files + +# Extract options +function __fish_bitcoinqt_get_options + argparse 'nofiles' -- $argv + set --local cmd (commandline -opc)[1] + set --local options + + if set -q _flag_nofiles + set --append options ($cmd -help-debug | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=' | string match --invert -r '^.*=$') + else + set --append options ($cmd -help-debug | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=' | string match -r '^.*=$') + end + + for option in $options + echo $option + end +end + + +# Add options with file completion +complete \ + --command bitcoin-qt \ + --arguments "(__fish_bitcoinqt_get_options)" +# Enable file completions only if the commandline now contains a `*.=` style option +complete -c bitcoin-qt \ + --condition 'string match --regex -- ".*=" (commandline -pt)' \ + --force-files + +# Add options without file completion +complete \ + --command bitcoin-qt \ + --arguments "(__fish_bitcoinqt_get_options --nofiles)" + diff --git a/contrib/completions/fish/bitcoin-tx.fish b/contrib/completions/fish/bitcoin-tx.fish new file mode 100644 index 00000000000..0ff262b948e --- /dev/null +++ b/contrib/completions/fish/bitcoin-tx.fish @@ -0,0 +1,65 @@ +# Disable files from being included in completions by default +complete --command bitcoin-tx --no-files + +# Modified version of __fish_seen_subcommand_from +# Uses regex to detect cmd= syntax +function __fish_bitcoin_seen_cmd + set -l cmd (commandline -oc) + set -e cmd[1] + for i in $cmd + for j in $argv + if string match --quiet --regex -- "^$j.*" $i + return 0 + end + end + end + return 1 +end + +# Extract options +function __fish_bitcoin_tx_get_options + set --local cmd (commandline -oc)[1] + if string match --quiet --regex -- '^-help$|-\?$' $cmd + return + end + + for option in ($cmd -help 2>&1 | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=') + echo $option + end +end + +# Extract commands +function __fish_bitcoin_tx_get_commands + argparse 'commandsonly' -- $argv + set --local cmd (commandline -oc)[1] + set --local commands + + if set -q _flag_commandsonly + set --append commands ($cmd -help | sed -e '1,/Commands:/d' -e 's/=/=\t/' -e 's/(=/=/' -e '/^ [a-z]/ p' -e d | string replace -r '\ \ ' '' | string replace -r '=.*' '') + else + set --append commands ($cmd -help | sed -e '1,/Commands:/d' -e 's/=/=\t/' -e 's/(=/=/' -e '/^ [a-z]/ p' -e d | string replace -r '\ \ ' '') + end + + for command in $commands + echo $command + end +end + +# Add options +complete \ + --command bitcoin-tx \ + --condition "not __fish_bitcoin_seen_cmd (__fish_bitcoin_tx_get_commands --commandsonly)" \ + --arguments "(__fish_bitcoin_tx_get_options)" \ + --no-files + +# Add commands +complete \ + --command bitcoin-tx \ + --arguments "(__fish_bitcoin_tx_get_commands)" \ + --no-files + +# Add file completions for load and set commands +complete \ + --command bitcoin-tx \ + --condition 'string match --regex -- "(load|set)=" (commandline -pt)' \ + --force-files diff --git a/contrib/completions/fish/bitcoin-util.fish b/contrib/completions/fish/bitcoin-util.fish new file mode 100644 index 00000000000..0650bf2cb6d --- /dev/null +++ b/contrib/completions/fish/bitcoin-util.fish @@ -0,0 +1,38 @@ +# Disable files from being included in completions by default +complete --command bitcoin-util --no-files + +# Extract options +function __fish_bitcoin_util_get_options + set --local cmd (commandline -opc)[1] + set --local options + + set --append options ($cmd -help 2>&1 | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=') + + for option in $options + echo $option + end +end + +# Extract commands +function __fish_bitcoin_util_get_commands + set --local cmd (commandline -opc)[1] + set --local commands + + set --append commands ($cmd -help | sed -e '1,/Commands:/d' -e 's/=/=\t/' -e 's/(=/=/' -e '/^ [a-z]/ p' -e d | string replace -r '\ \ ' '') + for command in $commands + echo $command + end +end + +# Add options +complete \ + --command bitcoin-util \ + --condition "not __fish_seen_subcommand_from (__fish_bitcoin_util_get_commands)" \ + --arguments "(__fish_bitcoin_util_get_options)" + +# Add commands +complete \ + --command bitcoin-util \ + --condition "not __fish_seen_subcommand_from (__fish_bitcoin_util_get_commands)" \ + --arguments "(__fish_bitcoin_util_get_commands)" + diff --git a/contrib/completions/fish/bitcoin-wallet.fish b/contrib/completions/fish/bitcoin-wallet.fish new file mode 100644 index 00000000000..82d8277c9b4 --- /dev/null +++ b/contrib/completions/fish/bitcoin-wallet.fish @@ -0,0 +1,35 @@ +# Disable files from being included in completions by default +complete --command bitcoin-wallet --no-files + +# Extract options +function __fish_bitcoin_wallet_get_options + set --local cmd (commandline -opc)[1] + for option in ($cmd -help 2>&1 | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=') + echo $option + end +end + +# Extract commands +function __fish_bitcoin_wallet_get_commands + set --local cmd (commandline -opc)[1] + for command in ($cmd -help | sed -e '1,/Commands:/d' -e 's/=/=\t/' -e 's/(=/=/' -e '/^ [a-z]/ p' -e d | string replace -r '\ \ ' '') + echo $command + end +end + +# Add options +complete \ + --command bitcoin-wallet \ + --condition "not __fish_seen_subcommand_from (__fish_bitcoin_wallet_get_commands)" \ + --arguments "(__fish_bitcoin_wallet_get_options)" + +# Add commands +complete \ + --command bitcoin-wallet \ + --condition "not __fish_seen_subcommand_from (__fish_bitcoin_wallet_get_commands)" \ + --arguments "(__fish_bitcoin_wallet_get_commands)" + +# Add file completions for load and set commands +complete --command bitcoin-wallet \ + --condition "string match -r -- '(dumpfile|datadir)*=' (commandline -pt)" \ + --force-files diff --git a/contrib/completions/fish/bitcoind.fish b/contrib/completions/fish/bitcoind.fish new file mode 100644 index 00000000000..fa245ae17f4 --- /dev/null +++ b/contrib/completions/fish/bitcoind.fish @@ -0,0 +1,35 @@ +# Disable files from being included in completions by default +complete --command bitcoind --no-files + +# Extract options +function __fish_bitcoind_get_options + argparse 'nofiles' -- $argv + set --local cmd (commandline -opc)[1] + set --local options + + if set -q _flag_nofiles + set --append options ($cmd -help-debug | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=' | string match --invert -r '^.*=$') + else + set --append options ($cmd -help-debug | string match -r '^ -.*' | string replace -r ' -' '-' | string replace -r '=.*' '=' | string match -r '^.*=$') + end + + for option in $options + echo $option + end +end + + +# Add options with file completion +complete \ + --command bitcoind \ + --arguments "(__fish_bitcoind_get_options)" +# Enable file completions only if the commandline now contains a `*.=` style option +complete --command bitcoind \ + --condition 'string match --regex -- ".*=" (commandline -pt)' \ + --force-files + +# Add options without file completion +complete \ + --command bitcoind \ + --arguments "(__fish_bitcoind_get_options --nofiles)" + diff --git a/contrib/debian/copyright b/contrib/debian/copyright index 95a281ce054..ca430170a1d 100644 --- a/contrib/debian/copyright +++ b/contrib/debian/copyright @@ -5,24 +5,16 @@ Upstream-Contact: Satoshi Nakamoto Source: https://github.com/bitcoin/bitcoin Files: * -Copyright: 2009-2022, Bitcoin Core Developers +Copyright: 2009-2023, Bitcoin Core Developers License: Expat -Comment: The Bitcoin Core Developers encompasses the current developers listed on bitcoin.org, - as well as the numerous contributors to the project. +Comment: The Bitcoin Core Developers encompasses all contributors to the + project, listed in the release notes or the git log. Files: debian/* Copyright: 2010-2011, Jonas Smedegaard 2011, Matt Corallo License: GPL-2+ -Files: src/secp256k1/build-aux/m4/ax_jni_include_dir.m4 -Copyright: 2008 Don Anderson -License: GNU-All-permissive-License - -Files: src/secp256k1/build-aux/m4/ax_prog_cc_for_build.m4 -Copyright: 2008 Paolo Bonzini -License: GNU-All-permissive-License - Files: src/qt/res/icons/add.png src/qt/res/icons/address-book.png src/qt/res/icons/chevron.png @@ -112,12 +104,6 @@ License: Expat TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -License: GNU-All-permissive-License - Copying and distribution of this file, with or without modification, are - permitted in any medium without royalty provided the copyright notice - and this notice are preserved. This file is offered as-is, without any - warranty. - License: GPL-2+ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 79b0134adc2..54b1a855881 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -90,6 +90,21 @@ example: BUILDDIR=$PWD/build contrib/devtools/gen-manpages.py ``` +gen-bitcoin-conf.sh +=================== + +Generates a bitcoin.conf file in `share/examples/` by parsing the output from `bitcoind --help`. This script is run during the +release process to include a bitcoin.conf with the release binaries and can also be run by users to generate a file locally. +When generating a file as part of the release process, make sure to commit the changes after running the script. + +With in-tree builds this tool can be run from any directory within the +repository. To use this tool with out-of-tree builds set `BUILDDIR`. For +example: + +```bash +BUILDDIR=$PWD/build contrib/devtools/gen-bitcoin-conf.sh +``` + security-check.py and test-security-check.py ============================================ diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py index 98eee67f430..420bf7ff330 100755 --- a/contrib/devtools/clang-format-diff.py +++ b/contrib/devtools/clang-format-diff.py @@ -146,7 +146,7 @@ def main(): stdout=subprocess.PIPE, stderr=None, stdin=subprocess.PIPE, - universal_newlines=True) + text=True) stdout, stderr = p.communicate() if p.returncode != 0: sys.exit(p.returncode) diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 38f3df77c95..3dddffe3242 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2016-2021 The Bitcoin Core developers +# Copyright (c) 2016-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -35,7 +35,6 @@ "src/leveldb/", "src/minisketch", "src/secp256k1/", - "src/univalue/", "src/crc32c/", ] @@ -320,15 +319,13 @@ def get_most_recent_git_change_year(filename): ################################################################################ def read_file_lines(filename): - f = open(filename, 'r', encoding="utf8") - file_lines = f.readlines() - f.close() + with open(filename, 'r', encoding="utf8") as f: + file_lines = f.readlines() return file_lines def write_file_lines(filename, file_lines): - f = open(filename, 'w', encoding="utf8") - f.write(''.join(file_lines)) - f.close() + with open(filename, 'w', encoding="utf8") as f: + f.write(''.join(file_lines)) ################################################################################ # update header years execution diff --git a/contrib/devtools/gen-bitcoin-conf.sh b/contrib/devtools/gen-bitcoin-conf.sh new file mode 100755 index 00000000000..2ebbd420223 --- /dev/null +++ b/contrib/devtools/gen-bitcoin-conf.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C +TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)} +BUILDDIR=${BUILDDIR:-$TOPDIR} +BINDIR=${BINDIR:-$BUILDDIR/src} +BITCOIND=${BITCOIND:-$BINDIR/bitcoind} +SHARE_EXAMPLES_DIR=${SHARE_EXAMPLES_DIR:-$TOPDIR/share/examples} +EXAMPLE_CONF_FILE=${EXAMPLE_CONF_FILE:-$SHARE_EXAMPLES_DIR/bitcoin.conf} + +[ ! -x "$BITCOIND" ] && echo "$BITCOIND not found or not executable." && exit 1 + +DIRTY="" +VERSION_OUTPUT=$($BITCOIND --version) +if [[ $VERSION_OUTPUT == *"dirty"* ]]; then + DIRTY="${DIRTY}${BITCOIND}\n" +fi + +if [ -n "$DIRTY" ] +then + echo -e "WARNING: $BITCOIND was built from a dirty tree.\n" + echo -e "To safely generate a bitcoin.conf file, please commit your changes to $BITCOIND, rebuild, then run this script again.\n" +fi + +echo 'Generating example bitcoin.conf file in share/examples/' + +# create the directory, if it doesn't exist +mkdir -p "${SHARE_EXAMPLES_DIR}" + +# create the header text +cat > "${EXAMPLE_CONF_FILE}" << 'EOF' +## +## bitcoin.conf configuration file. +## Generated by contrib/devtools/gen-bitcoin-conf.sh. +## +## Lines beginning with # are comments. +## All possible configuration options are provided. To use, copy this file +## to your data directory (default or specified by -datadir), uncomment +## options you would like to change, and save the file. +## + + +### Options +EOF + +# parse the output from bitcoind --help +# adding newlines is a bit funky to ensure portability for BSD +# see here for more details: https://stackoverflow.com/a/24575385 +${BITCOIND} --help \ + | sed '1,/Print this help message and exit/d' \ + | sed -E 's/^[[:space:]]{2}\-/#/' \ + | sed -E 's/^[[:space:]]{7}/# /' \ + | sed -E '/[=[:space:]]/!s/#.*$/&=1/' \ + | awk '/^#[a-z]/{x=$0;next}{if (NF==0) print x"\n",x="";else print}' \ + | sed 's,\(^[[:upper:]].*\)\:$,\ +### \1,' \ + | sed 's/[[:space:]]*$//' >> "${EXAMPLE_CONF_FILE}" + +# create the footer text +cat >> "${EXAMPLE_CONF_FILE}" << 'EOF' + +# [Sections] +# Most options will apply to all networks. To confine an option to a specific +# network, add it under the relevant section below. +# +# Note: If not specified under a network section, the options addnode, connect, +# port, bind, rpcport, rpcbind, and wallet will only apply to mainnet. + +# Options for mainnet +[main] + +# Options for testnet +[test] + +# Options for signet +[signet] + +# Options for regtest +[regtest] +EOF diff --git a/contrib/devtools/gen-manpages.py b/contrib/devtools/gen-manpages.py index 03cd3ddab40..ecf58c2e36b 100755 --- a/contrib/devtools/gen-manpages.py +++ b/contrib/devtools/gen-manpages.py @@ -23,7 +23,7 @@ # If not otherwise specified, get top directory from git. topdir = os.getenv('TOPDIR') if not topdir: - r = subprocess.run([git, 'rev-parse', '--show-toplevel'], stdout=subprocess.PIPE, check=True, universal_newlines=True) + r = subprocess.run([git, 'rev-parse', '--show-toplevel'], stdout=subprocess.PIPE, check=True, text=True) topdir = r.stdout.rstrip() # Get input and output directories. @@ -36,7 +36,7 @@ for relpath in BINARIES: abspath = os.path.join(builddir, relpath) try: - r = subprocess.run([abspath, '--version'], stdout=subprocess.PIPE, universal_newlines=True) + r = subprocess.run([abspath, "--version"], stdout=subprocess.PIPE, check=True, text=True) except IOError: print(f'{abspath} not found or not an executable', file=sys.stderr) sys.exit(1) diff --git a/contrib/devtools/iwyu/bitcoin.core.imp b/contrib/devtools/iwyu/bitcoin.core.imp new file mode 100644 index 00000000000..919ffab102d --- /dev/null +++ b/contrib/devtools/iwyu/bitcoin.core.imp @@ -0,0 +1,7 @@ +# Fixups / upstreamed changes +[ + { include: [ "", private, "", public ] }, + { include: [ "", private, "", public ] }, + { include: [ "", private, "", public ] }, + { include: [ "", private, "", public ] }, +] diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index e6a29b73b92..6cd022ef171 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2021 The Bitcoin Core developers +# Copyright (c) 2015-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' @@ -12,10 +12,6 @@ import lief #type:ignore -# temporary constant, to be replaced with lief.ELF.ARCH.RISCV -# https://github.com/lief-project/LIEF/pull/562 -LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243) - def check_ELF_RELRO(binary) -> bool: ''' Check for read-only relocations. @@ -38,7 +34,7 @@ def check_ELF_RELRO(binary) -> bool: flags = binary.get(lief.ELF.DYNAMIC_TAGS.FLAGS) if flags.value & lief.ELF.DYNAMIC_FLAGS.BIND_NOW: have_bindnow = True - except: + except Exception: have_bindnow = False return have_gnu_relro and have_bindnow @@ -101,7 +97,6 @@ def check_ELF_separate_code(binary): for segment in binary.segments: if segment.type == lief.ELF.SEGMENT_TYPES.LOAD: for section in segment.sections: - assert(section.name not in flags_per_section) flags_per_section[section.name] = segment.flags # Spot-check ELF LOAD program header flags per section # If these sections exist, check them against the expected R/W/E flags @@ -151,6 +146,12 @@ def check_PE_control_flow(binary) -> bool: return True return False +def check_PE_Canary(binary) -> bool: + ''' + Check for use of stack canary + ''' + return binary.has_symbol('__stack_chk_fail') + def check_MACHO_NOUNDEFS(binary) -> bool: ''' Check for no undefined references. @@ -208,6 +209,7 @@ def check_MACHO_control_flow(binary) -> bool: ('NX', check_NX), ('RELOC_SECTION', check_PE_RELOC_SECTION), ('CONTROL_FLOW', check_PE_control_flow), + ('Canary', check_PE_Canary), ] BASE_MACHO = [ @@ -222,7 +224,7 @@ def check_MACHO_control_flow(binary) -> bool: lief.ARCHITECTURES.ARM: BASE_ELF, lief.ARCHITECTURES.ARM64: BASE_ELF, lief.ARCHITECTURES.PPC: BASE_ELF, - LIEF_ELF_ARCH_RISCV: BASE_ELF, + lief.ARCHITECTURES.RISCV: BASE_ELF, }, lief.EXE_FORMATS.PE: { lief.ARCHITECTURES.X86: BASE_PE, @@ -250,12 +252,9 @@ def check_MACHO_control_flow(binary) -> bool: continue if arch == lief.ARCHITECTURES.NONE: - if binary.header.machine_type == LIEF_ELF_ARCH_RISCV: - arch = LIEF_ELF_ARCH_RISCV - else: - print(f'{filename}: unknown architecture') - retval = 1 - continue + print(f'{filename}: unknown architecture') + retval = 1 + continue failed: List[str] = [] for (name, func) in CHECKS[etype][arch]: diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 461132ae63a..f26236dd59e 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -15,23 +15,19 @@ import lief #type:ignore -# temporary constant, to be replaced with lief.ELF.ARCH.RISCV -# https://github.com/lief-project/LIEF/pull/562 -LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243) - -# Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases +# Debian 10 (Buster) EOL: 2024. https://wiki.debian.org/LTS # -# - g++ version 6.3.0 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=g%2B%2B) -# - libc version 2.24 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=libc6) +# - libgcc version 8.3.0 (https://packages.debian.org/search?suite=buster&arch=any&searchon=names&keywords=libgcc1) +# - libc version 2.28 (https://packages.debian.org/search?suite=buster&arch=any&searchon=names&keywords=libc6) # -# Ubuntu 16.04 (Xenial) EOL: 2026. https://wiki.ubuntu.com/Releases +# Ubuntu 18.04 (Bionic) EOL: 2028. https://wiki.ubuntu.com/ReleaseTeam # -# - g++ version 5.3.1 -# - libc version 2.23 +# - libgcc version 8.4.0 (https://packages.ubuntu.com/bionic/libgcc1) +# - libc version 2.27 (https://packages.ubuntu.com/bionic/libc6) # # CentOS Stream 8 EOL: 2024. https://wiki.centos.org/About/Product # -# - g++ version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) +# - libgcc version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) # - libc version 2.28 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) # # See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. @@ -39,32 +35,25 @@ MAX_VERSIONS = { 'GCC': (4,8,0), 'GLIBC': { - lief.ELF.ARCH.i386: (2,18), - lief.ELF.ARCH.x86_64: (2,18), - lief.ELF.ARCH.ARM: (2,18), - lief.ELF.ARCH.AARCH64:(2,18), - lief.ELF.ARCH.PPC64: (2,18), - LIEF_ELF_ARCH_RISCV: (2,27), + lief.ELF.ARCH.x86_64: (2,27), + lief.ELF.ARCH.ARM: (2,27), + lief.ELF.ARCH.AARCH64:(2,27), + lief.ELF.ARCH.PPC64: (2,27), + lief.ELF.ARCH.RISCV: (2,27), }, 'LIBATOMIC': (1,0), 'V': (0,5,0), # xkb (bitcoin-qt only) } -# See here for a description of _IO_stdin_used: -# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { -'_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', -'__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', -'environ', '_environ', '__environ', +'environ', '_environ', '__environ', '_fini', '_init', 'stdin', +'stdout', 'stderr', } # Expected linker-loader names can be found here: # https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = { - lief.ELF.ARCH.i386: { - lief.ENDIANNESS.LITTLE: "/lib/ld-linux.so.2", - }, lief.ELF.ARCH.x86_64: { lief.ENDIANNESS.LITTLE: "/lib64/ld-linux-x86-64.so.2", }, @@ -78,7 +67,7 @@ lief.ENDIANNESS.BIG: "/lib64/ld64.so.1", lief.ENDIANNESS.LITTLE: "/lib64/ld64.so.2", }, - LIEF_ELF_ARCH_RISCV: { + lief.ELF.ARCH.RISCV: { lief.ENDIANNESS.LITTLE: "/lib/ld-linux-riscv64-lp64d.so.1", }, } @@ -200,7 +189,7 @@ def check_exported_symbols(binary) -> bool: if not symbol.exported: continue name = symbol.name - if binary.header.machine_type == LIEF_ELF_ARCH_RISCV or name in IGNORE_EXPORTS: + if binary.header.machine_type == lief.ELF.ARCH.RISCV or name in IGNORE_EXPORTS: continue print(f'{binary.name}: export of symbol {name} not allowed!') ok = False diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index d3d225f3abd..54718fd7a1f 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2021 The Bitcoin Core developers +# Copyright (c) 2015-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' @@ -39,7 +39,7 @@ def call_security_check(cc, source, executable, options): env_flags += filter(None, os.environ.get(var, '').split(' ')) subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) - p = subprocess.run(['./contrib/devtools/security-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.run([os.path.join(os.path.dirname(__file__), 'security-check.py'), executable], stdout=subprocess.PIPE, text=True) return (p.returncode, p.stdout.rstrip()) def get_arch(cc, source, executable): @@ -94,19 +94,19 @@ def test_PE(self): cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--disable-nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--disable-nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE','-fno-stack-protector']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE','-fstack-protector-all', '-lssp']), (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE','-fstack-protector-all', '-lssp']), (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE','-fstack-protector-all', '-lssp']), (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) # -pie -fPIE does nothing unless --dynamicbase is also supplied - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE','-fstack-protector-all', '-lssp']), (1, executable+': failed HIGH_ENTROPY_VA CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE','-fstack-protector-all', '-lssp']), (1, executable+': failed CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full']), + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full','-fstack-protector-all', '-lssp']), (0, '')) clean_files(source, executable) diff --git a/contrib/devtools/test-symbol-check.py b/contrib/devtools/test-symbol-check.py index e1a2ebc491a..e304880140e 100755 --- a/contrib/devtools/test-symbol-check.py +++ b/contrib/devtools/test-symbol-check.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' @@ -23,13 +23,13 @@ def call_symbol_check(cc: List[str], source, executable, options): env_flags += filter(None, os.environ.get(var, '').split(' ')) subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) - p = subprocess.run(['./contrib/devtools/symbol-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.run([os.path.join(os.path.dirname(__file__), 'symbol-check.py'), executable], stdout=subprocess.PIPE, text=True) os.remove(source) os.remove(executable) return (p.returncode, p.stdout.rstrip()) def get_machine(cc: List[str]): - p = subprocess.run([*cc,'-dumpmachine'], stdout=subprocess.PIPE, universal_newlines=True) + p = subprocess.run([*cc,'-dumpmachine'], stdout=subprocess.PIPE, text=True) return p.stdout.rstrip() class TestSymbolChecks(unittest.TestCase): @@ -38,31 +38,6 @@ def test_ELF(self): executable = 'test1' cc = determine_wellknown_cmd('CC', 'gcc') - # there's no way to do this test for RISC-V at the moment; we build for - # RISC-V in a glibc 2.27 envinonment and we allow all symbols from 2.27. - if 'riscv' in get_machine(cc): - self.skipTest("test not available for RISC-V") - - # nextup was introduced in GLIBC 2.24, so is newer than our supported - # glibc (2.18), and available in our release build environment (2.24). - with open(source, 'w', encoding="utf8") as f: - f.write(''' - #define _GNU_SOURCE - #include - - double nextup(double x); - - int main() - { - nextup(3.14); - return 0; - } - ''') - - self.assertEqual(call_symbol_check(cc, source, executable, ['-lm']), - (1, executable + ': symbol nextup from unsupported version GLIBC_2.24(3)\n' + - executable + ': failed IMPORTED_SYMBOLS')) - # -lutil is part of the libc6 package so a safe bet that it's installed # it's also out of context enough that it's unlikely to ever become a real dependency source = 'test2.c' @@ -187,7 +162,7 @@ def test_PE(self): executable = 'test3.exe' with open(source, 'w', encoding="utf8") as f: f.write(''' - #include + #include int main() { diff --git a/contrib/guix/INSTALL.md b/contrib/guix/INSTALL.md index 68aae18731d..bbd88e58f3d 100644 --- a/contrib/guix/INSTALL.md +++ b/contrib/guix/INSTALL.md @@ -72,11 +72,11 @@ writing (July 2021). Guix is expected to be more widely packaged over time. For an up-to-date view on Guix's package status/version across distros, please see: https://repology.org/project/guix/versions -### Debian 11 (Bullseye)/Ubuntu 21.04 (Hirsute Hippo) +### Debian / Ubuntu Guix v1.2.0 is available as a distribution package starting in [Debian 11](https://packages.debian.org/bullseye/guix) and [Ubuntu -21.04](https://packages.ubuntu.com/hirsute/guix). +21.04](https://packages.ubuntu.com/search?keywords=guix). Note that if you intend on using Guix without using any substitutes (more details [here][security-model]), v1.2.0 has a known problem when building GnuTLS @@ -167,6 +167,10 @@ For reference, the graphic below outlines Guix v1.3.0's dependency graph: ![bootstrap map](https://user-images.githubusercontent.com/6399679/125064185-a9a59880-e0b0-11eb-82c1-9b8e5dc9950d.png) +#### Consider /tmp on tmpfs + +If you use an NVME (SSD) drive, you may encounter [cryptic build errors](#coreutils-fail-teststail-2inotify-dir-recreate). Mounting a [tmpfs at /tmp](https://ubuntu.com/blog/data-driven-analysis-tmp-on-tmpfs) should prevent this and may improve performance as a bonus. + #### Guile ##### Choosing a Guile version and sticking to it @@ -334,6 +338,8 @@ packages in Debian at the time of writing. |-----------------------|---------------------| | guile-gcrypt | libgcrypt-dev | | guile-git | libgit2-dev | +| guile-gnutls | (none) | +| guile-json | (none) | | guile-lzlib | liblz-dev | | guile-ssh | libssh-dev | | guile-sqlite3 | libsqlite3-dev | @@ -384,8 +390,9 @@ cd guix ``` You will likely want to build the latest release, however, if the latest release -when you're reading this is still 1.2.0 then you may want to use 95aca29 instead -to avoid a problem in the GnuTLS test suite. +when you're reading this is still 1.3.0 then you may want to use 998eda30 instead +to avoid the issues described in [#25099]( +https://github.com/bitcoin/bitcoin/pull/25099). ``` git branch -a -l 'origin/version-*' # check for the latest release @@ -609,6 +616,8 @@ systemctl enable guix-daemon systemctl start guix-daemon ``` +Remember to set `--no-substitute` in `$libdir/systemd/system/guix-daemon.service` and other customizations if you used them for `guix-daemon-original.service`. + ##### If you installed Guix via the Debian/Ubuntu distribution packages You will need to create a `guix-daemon-latest` service which points to the new @@ -717,6 +726,19 @@ $ bzcat /var/log/guix/drvs/../...-foo-3.6.12.drv.bz2 | less times, it may be `/tmp/...drv-1` or `/tmp/...drv-2`. Always consult the build failure output for the most accurate, up-to-date information. +### openssl-1.1.1l and openssl-1.1.1n + +OpenSSL includes tests that will fail once some certificate has expired. A workaround +is to change your system clock: + +```sh +sudo timedatectl set-ntp no +sudo date --set "28 may 2022 15:00:00" +sudo --login guix build --cores=1 /gnu/store/g9alz81w4q03ncm542487xd001s6akd4-openssl-1.1.1l.drv +sudo --login guix build --cores=1 /gnu/store/mw6ax0gk33gh082anrdrxp2flrbskxv6-openssl-1.1.1n.drv +sudo timedatectl set-ntp yes +``` + ### python(-minimal): [Errno 84] Invalid or incomplete multibyte or wide character This error occurs when your `$TMPDIR` (default: /tmp) exists on a filesystem @@ -774,7 +796,7 @@ The inotify-dir-create test fails on "remote" filesystems such as overlayfs as non-remote. A relatively easy workaround to this is to make sure that a somewhat traditional -filesystem is mounted at `/tmp` (where `guix-daemon` performs its builds). For +filesystem is mounted at `/tmp` (where `guix-daemon` performs its builds), see [/tmp on tmpfs](#consider-tmp-on-tmpfs). For Docker users, this might mean [using a volume][docker/volumes], [binding mounting][docker/bind-mnt] from host, or (for those with enough RAM and swap) [mounting a tmpfs][docker/tmpfs] using the `--tmpfs` flag. @@ -782,7 +804,7 @@ mounting][docker/bind-mnt] from host, or (for those with enough RAM and swap) Please see the following links for more details: - An upstream coreutils bug has been filed: [debbugs#47940](https://debbugs.gnu.org/cgi/bugreport.cgi?bug=47940) -- A Guix bug detailing the underlying problem has been filed: [guix-issues#47935](https://issues.guix.gnu.org/47935) +- A Guix bug detailing the underlying problem has been filed: [guix-issues#47935](https://issues.guix.gnu.org/47935), [guix-issues#49985](https://issues.guix.gnu.org/49985#5) - A commit to skip this test in Guix has been merged into the core-updates branch: [savannah/guix@6ba1058](https://git.savannah.gnu.org/cgit/guix.git/commit/?id=6ba1058df0c4ce5611c2367531ae5c3cdc729ab4) @@ -799,3 +821,39 @@ Please see the following links for more details: [docker/volumes]: https://docs.docker.com/storage/volumes/ [docker/bind-mnt]: https://docs.docker.com/storage/bind-mounts/ [docker/tmpfs]: https://docs.docker.com/storage/tmpfs/ + +# Purging/Uninstalling Guix + +In the extraordinarily rare case where you messed up your Guix installation in +an irreversible way, you may want to completely purge Guix from your system and +start over. + +1. Uninstall Guix itself according to the way you installed it (e.g. `sudo apt + purge guix` for Ubuntu packaging, `sudo make uninstall` for a build from source). +2. Remove all build users and groups + + You may check for relevant users and groups using: + + ``` + getent passwd | grep guix + getent group | grep guix + ``` + + Then, you may remove users and groups using: + + ``` + sudo userdel + sudo groupdel + ``` + +3. Remove all possible Guix-related directories + - `/var/guix/` + - `/var/log/guix/` + - `/gnu/` + - `/etc/guix/` + - `/home/*/.config/guix/` + - `/home/*/.cache/guix/` + - `/home/*/.guix-profile/` + - `/root/.config/guix/` + - `/root/.cache/guix/` + - `/root/.guix-profile/` diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 90289f9d40e..c0feb486ff2 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -75,7 +75,7 @@ crucial differences: 1. Since only Windows and macOS build outputs require codesigning, the `HOSTS` environment variable will have a sane default value of `x86_64-w64-mingw32 - x86_64-apple-darwin` instead of all the platforms. + x86_64-apple-darwin arm64-apple-darwin` instead of all the platforms. 2. The `guix-codesign` command ***requires*** a `DETACHED_SIGS_REPO` flag. * _**DETACHED_SIGS_REPO**_ @@ -382,7 +382,7 @@ https://ci.guix.gnu.org is automatically used unless the `--no-substitutes` flag is supplied. This default list of substitute servers is overridable both on a `guix-daemon` level and when you invoke `guix` commands. See examples below for the various ways of adding dongcarl's substitute server after having [authorized -his signing key](#authorize-the-signing-keys). +his signing key](#step-1-authorize-the-signing-keys). Change the **default list** of substitute servers by starting `guix-daemon` with the `--substitute-urls` option (you will likely need to edit your init script): @@ -430,55 +430,6 @@ used. If you start `guix-daemon` using an init script, you can edit said script to supply this flag. - -# Purging/Uninstalling Guix - -In the extraordinarily rare case where you messed up your Guix installation in -an irreversible way, you may want to completely purge Guix from your system and -start over. - -1. Uninstall Guix itself according to the way you installed it (e.g. `sudo apt - purge guix` for Ubuntu packaging, `sudo make uninstall` for a build from source). -2. Remove all build users and groups - - You may check for relevant users and groups using: - - ``` - getent passwd | grep guix - getent group | grep guix - ``` - - Then, you may remove users and groups using: - - ``` - sudo userdel - sudo groupdel - ``` - -3. Remove all possible Guix-related directories - - `/var/guix/` - - `/var/log/guix/` - - `/gnu/` - - `/etc/guix/` - - `/home/*/.config/guix/` - - `/home/*/.cache/guix/` - - `/home/*/.guix-profile/` - - `/root/.config/guix/` - - `/root/.cache/guix/` - - `/root/.guix-profile/` - [b17e]: https://bootstrappable.org/ [r12e/source-date-epoch]: https://reproducible-builds.org/docs/source-date-epoch/ - -[guix/install.sh]: https://git.savannah.gnu.org/cgit/guix.git/plain/etc/guix-install.sh -[guix/bin-install]: https://www.gnu.org/software/guix/manual/en/html_node/Binary-Installation.html -[guix/env-setup]: https://www.gnu.org/software/guix/manual/en/html_node/Build-Environment-Setup.html -[guix/substitutes]: https://www.gnu.org/software/guix/manual/en/html_node/Substitutes.html -[guix/substitute-server-auth]: https://www.gnu.org/software/guix/manual/en/html_node/Substitute-Server-Authorization.html -[guix/time-machine]: https://guix.gnu.org/manual/en/html_node/Invoking-guix-time_002dmachine.html - -[debian/guix-bullseye]: https://packages.debian.org/bullseye/guix -[ubuntu/guix-hirsute]: https://packages.ubuntu.com/hirsute/guix -[fanquake/guix-docker]: https://github.com/fanquake/core-review/tree/master/guix - [env-vars-list]: #recognized-environment-variables diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest index 6e12cbead73..b0ef28dc3f9 100755 --- a/contrib/guix/guix-attest +++ b/contrib/guix/guix-attest @@ -19,8 +19,16 @@ source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" ################ check_tools cat env basename mkdir diff sort + if [ -z "$NO_SIGN" ]; then - check_tools gpg + # make it possible to override the gpg binary + GPG=${GPG:-gpg} + + # $GPG can contain extra arguments passed to the binary + # so let's check only the existence of arg[0] + # shellcheck disable=SC2206 + GPG_ARRAY=($GPG) + check_tools "${GPG_ARRAY[0]}" fi ################ @@ -90,7 +98,7 @@ if [ -z "${signer_name}" ]; then signer_name="$gpg_key_name" fi -if [ -z "$NO_SIGN" ] && ! gpg --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then +if [ -z "$NO_SIGN" ] && ! ${GPG} --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then echo "ERR: GPG can't seem to find any key named '${gpg_key_name}'" exit 1 fi @@ -239,11 +247,11 @@ mkdir -p "$outsigdir" echo "Signing SHA256SUMS to produce SHA256SUMS.asc" for i in *.SHA256SUMS; do if [ ! -e "$i".asc ]; then - gpg --detach-sign \ - --digest-algo sha256 \ - --local-user "$gpg_key_name" \ - --armor \ - --output "$i".asc "$i" + ${GPG} --detach-sign \ + --digest-algo sha256 \ + --local-user "$gpg_key_name" \ + --armor \ + --output "$i".asc "$i" else echo "Signature already there" fi diff --git a/contrib/guix/guix-build b/contrib/guix/guix-build index b38f91f3c7b..74b24b96120 100755 --- a/contrib/guix/guix-build +++ b/contrib/guix/guix-build @@ -121,7 +121,7 @@ else fi ################ -# When building for darwin, the macOS SDK should exists +# When building for darwin, the macOS SDK should exist ################ for host in $HOSTS; do @@ -130,8 +130,9 @@ for host in $HOSTS; do OSX_SDK="$(make -C "${PWD}/depends" --no-print-directory HOST="$host" print-OSX_SDK | sed 's@^[^=]\+=@@g')" if [ -e "$OSX_SDK" ]; then echo "Found macOS SDK at '${OSX_SDK}', using..." + break else - echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, exiting..." + echo "macOS SDK does not exist at '${OSX_SDK}', please place the extracted, untarred SDK there to perform darwin builds, or define SDK_PATH environment variable. Exiting..." exit 1 fi ;; @@ -234,21 +235,6 @@ host_to_commonname() { # Determine the reference time used for determinism (overridable by environment) SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" -# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility -# across time. -time-machine() { - # shellcheck disable=SC2086 - guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=ae03f401381e956c4c41b4cf495cbde964fa43d0 \ - --cores="$JOBS" \ - --keep-failed \ - --fallback \ - ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ - ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ - -- "$@" -} - - # Precious directories are those which should not be cleaned between successive # guix builds depends_precious_dir_names='SOURCES_PATH BASE_CACHE SDK_PATH' diff --git a/contrib/guix/guix-codesign b/contrib/guix/guix-codesign index 4b272f30847..3279d431aaf 100755 --- a/contrib/guix/guix-codesign +++ b/contrib/guix/guix-codesign @@ -222,20 +222,6 @@ JOBS="${JOBS:-$(nproc)}" # Determine the reference time used for determinism (overridable by environment) SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git -c log.showSignature=false log --format=%at -1)}" -# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility -# across time. -time-machine() { - # shellcheck disable=SC2086 - guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ - --commit=ae03f401381e956c4c41b4cf495cbde964fa43d0 \ - --cores="$JOBS" \ - --keep-failed \ - --fallback \ - ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ - ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ - -- "$@" -} - # Make sure an output directory exists for our builds OUTDIR_BASE="${OUTDIR_BASE:-${VERSION_BASE}/output}" mkdir -p "$OUTDIR_BASE" diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 7340b0625fa..e0bd15493f8 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C @@ -69,29 +69,12 @@ unset CPLUS_INCLUDE_PATH unset OBJC_INCLUDE_PATH unset OBJCPLUS_INCLUDE_PATH -export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC}/lib64:${NATIVE_GCC_STATIC}/lib:${NATIVE_GCC_STATIC}/lib64" +export LIBRARY_PATH="${NATIVE_GCC}/lib:${NATIVE_GCC_STATIC}/lib" export C_INCLUDE_PATH="${NATIVE_GCC}/include" export CPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" export OBJC_INCLUDE_PATH="${NATIVE_GCC}/include" export OBJCPLUS_INCLUDE_PATH="${NATIVE_GCC}/include/c++:${NATIVE_GCC}/include" -prepend_to_search_env_var() { - export "${1}=${2}${!1:+:}${!1}" -} - -case "$HOST" in - *darwin*) - # When targeting darwin, zlib is required by native_libdmg-hfsplus. - zlib_store_path=$(store_path "zlib") - zlib_static_store_path=$(store_path "zlib" static) - - prepend_to_search_env_var LIBRARY_PATH "${zlib_static_store_path}/lib:${zlib_store_path}/lib" - prepend_to_search_env_var C_INCLUDE_PATH "${zlib_store_path}/include" - prepend_to_search_env_var CPLUS_INCLUDE_PATH "${zlib_store_path}/include" - prepend_to_search_env_var OBJC_INCLUDE_PATH "${zlib_store_path}/include" - prepend_to_search_env_var OBJCPLUS_INCLUDE_PATH "${zlib_store_path}/include" -esac - # Set environment variables to point the CROSS toolchain to the right # includes/libs for $HOST case "$HOST" in @@ -167,7 +150,6 @@ case "$HOST" in *linux*) glibc_dynamic_linker=$( case "$HOST" in - i686-linux-gnu) echo /lib/ld-linux.so.2 ;; x86_64-linux-gnu) echo /lib64/ld-linux-x86-64.so.2 ;; arm-linux-gnueabihf) echo /lib/ld-linux-armhf.so.3 ;; aarch64-linux-gnu) echo /lib/ld-linux-aarch64.so.1 ;; @@ -204,20 +186,12 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ ${SOURCES_PATH+SOURCES_PATH="$SOURCES_PATH"} \ ${BASE_CACHE+BASE_CACHE="$BASE_CACHE"} \ ${SDK_PATH+SDK_PATH="$SDK_PATH"} \ - i686_linux_CC=i686-linux-gnu-gcc \ - i686_linux_CXX=i686-linux-gnu-g++ \ - i686_linux_AR=i686-linux-gnu-ar \ - i686_linux_RANLIB=i686-linux-gnu-ranlib \ - i686_linux_NM=i686-linux-gnu-nm \ - i686_linux_STRIP=i686-linux-gnu-strip \ x86_64_linux_CC=x86_64-linux-gnu-gcc \ x86_64_linux_CXX=x86_64-linux-gnu-g++ \ - x86_64_linux_AR=x86_64-linux-gnu-ar \ - x86_64_linux_RANLIB=x86_64-linux-gnu-ranlib \ - x86_64_linux_NM=x86_64-linux-gnu-nm \ + x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \ + x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \ + x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \ x86_64_linux_STRIP=x86_64-linux-gnu-strip \ - qt_config_opts_i686_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ - qt_config_opts_x86_64_linux='-platform linux-g++ -xplatform bitcoin-linux-g++' \ FORCE_USE_SYSTEM_CLANG=1 @@ -244,6 +218,7 @@ CONFIGFLAGS="--enable-reduce-exports --disable-bench --disable-gui-tests --disab # CFLAGS HOST_CFLAGS="-O2 -g" +HOST_CFLAGS+=$(find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) case "$HOST" in *linux*) HOST_CFLAGS+=" -ffile-prefix-map=${PWD}=." ;; *mingw*) HOST_CFLAGS+=" -fno-ident" ;; @@ -263,17 +238,6 @@ case "$HOST" in *mingw*) HOST_LDFLAGS="-Wl,--no-insert-timestamp" ;; esac -# Using --no-tls-get-addr-optimize retains compatibility with glibc 2.18, by -# avoiding a PowerPC64 optimisation available in glibc 2.22 and later. -# https://sourceware.org/binutils/docs-2.35/ld/PowerPC64-ELF64.html -case "$HOST" in - *powerpc64*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,--no-tls-get-addr-optimize" ;; -esac - -case "$HOST" in - powerpc64-linux-*|riscv64-linux-*) HOST_LDFLAGS="${HOST_LDFLAGS} -Wl,-z,noexecstack" ;; -esac - # Make $HOST-specific native binaries from depends available in $PATH export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" mkdir -p "$DISTSRC" @@ -340,8 +304,7 @@ mkdir -p "$DISTSRC" mkdir -p "unsigned-app-${HOST}" cp --target-directory="unsigned-app-${HOST}" \ osx_volname \ - contrib/macdeploy/detached-sig-{apply,create}.sh \ - "${BASEPREFIX}/${HOST}"/native/bin/dmg + contrib/macdeploy/detached-sig-create.sh mv --target-directory="unsigned-app-${HOST}" dist ( cd "unsigned-app-${HOST}" @@ -390,6 +353,12 @@ mkdir -p "$DISTSRC" ;; esac + # copy over the example bitcoin.conf file. if contrib/devtools/gen-bitcoin-conf.sh + # has not been run before buildling, this file will be a stub + cp "${DISTSRC}/share/examples/bitcoin.conf" "${DISTNAME}/" + + cp -r "${DISTSRC}/share/rpcauth" "${DISTNAME}/share/" + # Finally, deterministically produce {non-,}debug binary tarballs ready # for release case "$HOST" in diff --git a/contrib/guix/libexec/codesign.sh b/contrib/guix/libexec/codesign.sh index 6ede95f42b9..6ffa0f07b2e 100755 --- a/contrib/guix/libexec/codesign.sh +++ b/contrib/guix/libexec/codesign.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2021 The Bitcoin Core developers +# Copyright (c) 2021-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C @@ -77,6 +77,7 @@ mkdir -p "$DISTSRC" osslsigncode attach-signature \ -in "$infile" \ -out "${OUTDIR}/${infile_base/-unsigned}" \ + -CAfile "$GUIX_ENVIRONMENT/etc/ssl/certs/ca-certificates.crt" \ -sigin codesignatures/win/"$infile_base".pem done ;; @@ -84,14 +85,11 @@ mkdir -p "$DISTSRC" # Apply detached codesignatures to dist/ (in-place) signapple apply dist/Bitcoin-Qt.app codesignatures/osx/dist - # Make an uncompressed DMG from dist/ + # Make a DMG from dist/ xorrisofs -D -l -V "$(< osx_volname)" -no-pad -r -dir-mode 0755 \ - -o uncompressed.dmg \ + -o "${OUTDIR}/${DISTNAME}-${HOST}.dmg" \ dist \ -- -volume_date all_file_dates ="$SOURCE_DATE_EPOCH" - - # Compress uncompressed.dmg and output to OUTDIR - ./dmg dmg uncompressed.dmg "${OUTDIR}/${DISTNAME}-${HOST}.dmg" ;; *) exit 1 diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 882d497a1d4..26a832832e1 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -45,6 +45,22 @@ EOF exit 1 fi +################ +# Execute "$@" in a pinned, possibly older version of Guix, for reproducibility +# across time. +time-machine() { + # shellcheck disable=SC2086 + guix time-machine --url=https://git.savannah.gnu.org/git/guix.git \ + --commit=998eda3067c7d21e0d9bb3310d2f5a14b8f1c681 \ + --cores="$JOBS" \ + --keep-failed \ + --fallback \ + ${SUBSTITUTE_URLS:+--substitute-urls="$SUBSTITUTE_URLS"} \ + ${ADDITIONAL_GUIX_COMMON_FLAGS} ${ADDITIONAL_GUIX_TIMEMACHINE_FLAGS} \ + -- "$@" +} + + ################ # Set common variables ################ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 0dcff1e43ea..24581cb6c1e 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -21,7 +21,6 @@ (gnu packages llvm) (gnu packages mingw) (gnu packages moreutils) - (gnu packages perl) (gnu packages pkg-config) (gnu packages python) (gnu packages python-crypto) @@ -29,6 +28,7 @@ (gnu packages shells) (gnu packages tls) (gnu packages version-control) + (guix build-system cmake) (guix build-system gnu) (guix build-system python) (guix build-system trivial) @@ -135,27 +135,23 @@ chain for " target " development.")) (license (package-license xgcc))))) (define base-gcc gcc-10) +(define base-linux-kernel-headers linux-libre-headers-5.15) -;; Building glibc with stack smashing protector first landed in glibc 2.25, use -;; this function to disable for older glibcs -;; -;; From glibc 2.25 changelog: -;; -;; * Most of glibc can now be built with the stack smashing protector enabled. -;; It is recommended to build glibc with --enable-stack-protector=strong. -;; Implemented by Nick Alcock (Oracle). -(define (make-glibc-without-ssp xglibc) - (package-with-extra-configure-variable - (package-with-extra-configure-variable - xglibc "libc_cv_ssp" "no") - "libc_cv_ssp_strong" "no")) +;; https://gcc.gnu.org/install/configure.html +(define (hardened-gcc gcc) + (package-with-extra-configure-variable ( + package-with-extra-configure-variable ( + package-with-extra-configure-variable gcc + "--enable-initfini-array" "yes") + "--enable-default-ssp" "yes") + "--enable-default-pie" "yes")) (define* (make-bitcoin-cross-toolchain target #:key - (base-gcc-for-libc gcc-7) - (base-kernel-headers linux-libre-headers-4.9) - (base-libc (make-glibc-without-ssp glibc-2.24)) - (base-gcc (make-gcc-rpath-link base-gcc))) + (base-gcc-for-libc base-gcc) + (base-kernel-headers base-linux-kernel-headers) + (base-libc (hardened-glibc glibc-2.27)) + (base-gcc (make-gcc-rpath-link (hardened-gcc base-gcc)))) "Convenience wrapper around MAKE-CROSS-TOOLCHAIN with default values desirable for building Bitcoin Core release binaries." (make-cross-toolchain target @@ -165,11 +161,15 @@ desirable for building Bitcoin Core release binaries." base-gcc)) (define (make-gcc-with-pthreads gcc) - (package-with-extra-configure-variable gcc "--enable-threads" "posix")) + (package-with-extra-configure-variable + (package-with-extra-patches gcc + (search-our-patches "gcc-10-remap-guix-store.patch")) + "--enable-threads" "posix")) -;; Required to support std::filesystem for mingw-w64 target. -(define (make-gcc-without-newlib gcc) - (package-with-extra-configure-variable gcc "--with-newlib" "no")) +(define (make-mingw-w64-cross-gcc cross-gcc) + (package-with-extra-patches cross-gcc + (search-our-patches "vmov-alignment.patch" + "gcc-broken-longjmp.patch"))) (define (make-mingw-w64-cross-gcc cross-gcc) (package-with-extra-patches cross-gcc @@ -182,7 +182,7 @@ desirable for building Bitcoin Core release binaries." (pthreads-xlibc mingw-w64-x86_64-winpthreads) (pthreads-xgcc (make-gcc-with-pthreads (cross-gcc target - #:xgcc (make-gcc-without-newlib (make-ssp-fixed-gcc (make-mingw-w64-cross-gcc base-gcc))) + #:xgcc (make-ssp-fixed-gcc (make-mingw-w64-cross-gcc base-gcc)) #:xbinutils xbinutils #:libc pthreads-xlibc)))) ;; Define a meta-package that propagates the resulting XBINUTILS, XLIBC, and @@ -209,53 +209,62 @@ chain for " target " development.")) (search-our-patches "nsis-gcc-10-memmove.patch" "nsis-disable-installer-reloc.patch"))) -(define-public lief +(define (fix-ppc64-nx-default lief) + (package-with-extra-patches lief + (search-our-patches "lief-fix-ppc64-nx-default.patch"))) + +;; Our python-lief package can be removed once we are using +;; guix 83bfdb409787cb2737e68b093a319b247b7858e6 or later. +;; Note we currently use cmake-minimal. +(define-public python-lief (package - (name "python-lief") - (version "0.11.5") - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/lief-project/LIEF.git") - (commit version))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "0qahjfg1n0x76ps2mbyljvws1l3qhkqvmxqbahps4qgywl2hbdkj")))) - (build-system python-build-system) - (native-inputs - `(("cmake" ,cmake))) - (home-page "https://github.com/lief-project/LIEF") - (synopsis "Library to Instrument Executable Formats") - (description "Python library to to provide a cross platform library which can -parse, modify and abstract ELF, PE and MachO formats.") - (license license:asl2.0))) + (name "python-lief") + (version "0.12.3") + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/lief-project/LIEF") + (commit version))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "11i6hqmcjh56y554kqhl61698n9v66j2qk1c1g63mv2w07h2z661")))) + (build-system python-build-system) + (native-inputs (list cmake-minimal)) + (arguments + (list + #:tests? #f ;needs network + #:phases #~(modify-phases %standard-phases + (replace 'build + (lambda _ + (invoke + "python" "setup.py" "--sdk" "build" + (string-append + "-j" (number->string (parallel-job-count))))))))) + (home-page "https://github.com/lief-project/LIEF") + (synopsis "Library to instrument executable formats") + (description + "@code{python-lief} is a cross platform library which can parse, modify +and abstract ELF, PE and MachO formats.") + (license license:asl2.0))) (define osslsigncode (package (name "osslsigncode") - (version "2.0") + (version "2.5") (source (origin (method url-fetch) (uri (string-append "https://github.com/mtrojnar/" name "/archive/" version ".tar.gz")) (sha256 (base32 - "0byri6xny770wwb2nciq44j5071122l14bvv65axdd70nfjf0q2s")))) - (build-system gnu-build-system) - (native-inputs - `(("pkg-config" ,pkg-config) - ("autoconf" ,autoconf) - ("automake" ,automake) - ("libtool" ,libtool))) + "03by9706gg0an6dn48pljx38vcb76ziv11bgm8ilwsf293x2k4hv")))) + (build-system cmake-build-system) (inputs - `(("openssl" ,openssl))) + `(("openssl", openssl))) (arguments - `(#:configure-flags - `("--without-gsf" - "--without-curl" - "--disable-dependency-tracking"))) + '(#:configure-flags + (list "-DCMAKE_DISABLE_FIND_PACKAGE_CURL=TRUE"))) (home-page "https://github.com/mtrojnar/osslsigncode") (synopsis "Authenticode signing and timestamping tool") (description "osslsigncode is a small tool that implements part of the @@ -265,7 +274,7 @@ thus should be able to compile on most platforms where these exist.") (license license:gpl3+))) ; license is with openssl exception (define-public python-elfesteem - (let ((commit "87bbd79ab7e361004c98cc8601d4e5f029fd8bd5")) + (let ((commit "2eb1e5384ff7a220fd1afacd4a0170acff54fe56")) (package (name "python-elfesteem") (version (git-version "0.1" "1" commit)) @@ -278,8 +287,7 @@ thus should be able to compile on most platforms where these exist.") (file-name (git-file-name name commit)) (sha256 (base32 - "1nyvjisvyxyxnd0023xjf5846xd03lwawp5pfzr8vrky7wwm5maz")) - (patches (search-our-patches "elfsteem-value-error-python-39.patch")))) + "07x6p8clh11z8s1n2kdxrqwqm2almgc5qpkcr9ckb6y5ivjdr5r6")))) (build-system python-build-system) ;; There are no tests, but attempting to run python setup.py test leads to ;; PYTHONPATH problems, just disable the test @@ -363,7 +371,7 @@ thus should be able to compile on most platforms where these exist.") #t))))))) (define-public python-certvalidator - (let ((commit "e5bdb4bfcaa09fa0af355eb8867d00dfeecba08c")) + (let ((commit "a145bf25eb75a9f014b3e7678826132efbba6213")) (package (name "python-certvalidator") (version (git-version "0.1" "1" commit)) @@ -376,7 +384,7 @@ thus should be able to compile on most platforms where these exist.") (file-name (git-file-name name commit)) (sha256 (base32 - "18pvxkvpkfkzgvfylv0kx65pmxfcv1hpsg03cip93krfvrrl4c75")))) + "1qw2k7xis53179lpqdqyylbcmp76lj7sagp883wmxg5i7chhc96k")))) (build-system python-build-system) (propagated-inputs `(("python-asn1crypto" ,python-asn1crypto) @@ -522,8 +530,7 @@ and endian independent.") ("python-certvalidator" ,python-certvalidator) ("python-elfesteem" ,python-elfesteem) ("python-requests" ,python-requests) - ("python-macholib" ,python-macholib) - ("libcrypto" ,openssl))) + ("python-macholib" ,python-macholib))) ;; There are no tests, but attempting to run python setup.py test leads to ;; problems, just disable the test (arguments '(#:tests? #f)) @@ -533,25 +540,18 @@ and endian independent.") inspecting signatures in Mach-O binaries.") (license license:expat)))) -(define-public glibc-2.24 - (package - (inherit glibc-2.31) - (version "2.24") - (source (origin - (method git-fetch) - (uri (git-reference - (url "https://sourceware.org/git/glibc.git") - (commit "0d7f1ed30969886c8dde62fbf7d2c79967d4bace"))) - (file-name (git-file-name "glibc" "0d7f1ed30969886c8dde62fbf7d2c79967d4bace")) - (sha256 - (base32 - "0g5hryia5v1k0qx97qffgwzrz4lr4jw3s5kj04yllhswsxyjbic3")) - (patches (search-our-patches "glibc-ldd-x86_64.patch" - "glibc-versioned-locpath.patch" - "glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch" - "glibc-2.24-no-build-time-cxx-header-run.patch")))))) +;; https://www.gnu.org/software/libc/manual/html_node/Configuring-and-compiling.html +;; We don't use --disable-werror directly, as that would be passed through to bash, +;; and cause it's build to fail. +(define (hardened-glibc glibc) + (package-with-extra-configure-variable ( + package-with-extra-configure-variable ( + package-with-extra-configure-variable glibc + "enable_werror" "no") + "--enable-stack-protector" "all") + "--enable-bind-now" "yes")) -(define-public glibc-2.27/bitcoin-patched +(define-public glibc-2.27 (package (inherit glibc-2.31) (version "2.27") @@ -559,20 +559,23 @@ inspecting signatures in Mach-O binaries.") (method git-fetch) (uri (git-reference (url "https://sourceware.org/git/glibc.git") - (commit "23158b08a0908f381459f273a984c6fd328363cb"))) - (file-name (git-file-name "glibc" "23158b08a0908f381459f273a984c6fd328363cb")) + (commit "73886db6218e613bd6d4edf529f11e008a6c2fa6"))) + (file-name (git-file-name "glibc" "73886db6218e613bd6d4edf529f11e008a6c2fa6")) (sha256 (base32 - "1b2n1gxv9f4fd5yy68qjbnarhf8mf4vmlxk10i3328c1w5pmp0ca")) + "0azpb9cvnbv25zg8019rqz48h8i2257ngyjg566dlnp74ivrs9vq")) (patches (search-our-patches "glibc-ldd-x86_64.patch" - "glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch")))))) + "glibc-versioned-locpath.patch" + "glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch" + "glibc-2.27-fcommon.patch" + "glibc-2.27-guix-prefix.patch")))))) (packages->manifest (append (list ;; The Basics - bash + bash-minimal which - coreutils + coreutils-minimal util-linux ;; File(system) inspection file @@ -589,39 +592,32 @@ inspecting signatures in Mach-O binaries.") bzip2 gzip xz - zlib - (list zlib "static") ;; Build tools gnu-make - libtool + libtool-2.4.7 autoconf-2.71 automake pkg-config bison + ;; Native GCC 10 toolchain + gcc-toolchain-10 + (list gcc-toolchain-10 "static") ;; Scripting - perl - python-3 + python-minimal ;; (3.9) ;; Git - git + git-minimal ;; Tests - lief - ;; Native gcc 7 toolchain - gcc-toolchain-7 - (list gcc-toolchain-7 "static")) + (fix-ppc64-nx-default python-lief)) (let ((target (getenv "HOST"))) (cond ((string-suffix? "-mingw32" target) ;; Windows (list zip (make-mingw-pthreads-cross-toolchain "x86_64-w64-mingw32") (make-nsis-for-gcc-10 nsis-x86_64) + nss-certs osslsigncode)) ((string-contains target "-linux-") - (list (cond ((string-contains target "riscv64-") - (make-bitcoin-cross-toolchain target - #:base-libc glibc-2.27/bitcoin-patched - #:base-kernel-headers linux-libre-headers-4.19)) - (else - (make-bitcoin-cross-toolchain target))))) + (list (make-bitcoin-cross-toolchain target))) ((string-contains target "darwin") - (list clang-toolchain-10 binutils cmake xorriso python-signapple)) + (list clang-toolchain-10 binutils cmake-minimal xorriso python-signapple)) (else '()))))) diff --git a/contrib/guix/patches/elfsteem-value-error-python-39.patch b/contrib/guix/patches/elfsteem-value-error-python-39.patch deleted file mode 100644 index 21e1228afd8..00000000000 --- a/contrib/guix/patches/elfsteem-value-error-python-39.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/examples/otool.py b/examples/otool.py -index 2b8efc0..d797b2e 100755 ---- a/examples/otool.py -+++ b/examples/otool.py -@@ -342,7 +342,7 @@ if __name__ == '__main__': - try: - e = macho_init.MACHO(raw, - parseSymbols = False) -- except ValueError, err: -+ except ValueError as err: - print("%s:" %file) - print(" %s" % err) - continue diff --git a/contrib/guix/patches/gcc-10-remap-guix-store.patch b/contrib/guix/patches/gcc-10-remap-guix-store.patch new file mode 100644 index 00000000000..a47ef7a2df1 --- /dev/null +++ b/contrib/guix/patches/gcc-10-remap-guix-store.patch @@ -0,0 +1,25 @@ +From aad25427e74f387412e8bc9a9d7bbc6c496c792f Mon Sep 17 00:00:00 2001 +From: Andrew Chow +Date: Wed, 6 Jul 2022 16:49:41 -0400 +Subject: [PATCH] guix: remap guix store paths to /usr + +--- + libgcc/Makefile.in | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/libgcc/Makefile.in b/libgcc/Makefile.in +index 851e7657d07..476c2becd1c 100644 +--- a/libgcc/Makefile.in ++++ b/libgcc/Makefile.in +@@ -854,7 +854,7 @@ endif + # libgcc_eh.a, only LIB2ADDEH matters. If we do, only LIB2ADDEHSTATIC and + # LIB2ADDEHSHARED matter. (Usually all three are identical.) + +-c_flags := -fexceptions ++c_flags := -fexceptions $(shell find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;) + + ifeq ($(enable_shared),yes) + +-- +2.37.0 + diff --git a/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch b/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch deleted file mode 100644 index 5c4d0c6ebe1..00000000000 --- a/contrib/guix/patches/glibc-2.24-elfm-loadaddr-dynamic-rewrite.patch +++ /dev/null @@ -1,62 +0,0 @@ -https://sourceware.org/git/?p=glibc.git;a=commit;h=a68ba2f3cd3cbe32c1f31e13c20ed13487727b32 - -commit 6b02af31e9a721bb15a11380cd22d53b621711f8 -Author: Szabolcs Nagy -Date: Wed Oct 18 17:26:23 2017 +0100 - - [AARCH64] Rewrite elf_machine_load_address using _DYNAMIC symbol - - This patch rewrites aarch64 elf_machine_load_address to use special _DYNAMIC - symbol instead of _dl_start. - - The static address of _DYNAMIC symbol is stored in the first GOT entry. - Here is the change which makes this solution work (part of binutils 2.24): - https://sourceware.org/ml/binutils/2013-06/msg00248.html - - i386, x86_64 targets use the same method to do this as well. - - The original implementation relies on a trick that R_AARCH64_ABS32 relocation - being resolved at link time and the static address fits in the 32bits. - However, in LP64, normally, the address is defined to be 64 bit. - - Here is the C version one which should be portable in all cases. - - * sysdeps/aarch64/dl-machine.h (elf_machine_load_address): Use - _DYNAMIC symbol to calculate load address. - -diff --git a/sysdeps/aarch64/dl-machine.h b/sysdeps/aarch64/dl-machine.h -index e86d8b5b63..5a5b8a5de5 100644 ---- a/sysdeps/aarch64/dl-machine.h -+++ b/sysdeps/aarch64/dl-machine.h -@@ -49,26 +49,11 @@ elf_machine_load_address (void) - /* To figure out the load address we use the definition that for any symbol: - dynamic_addr(symbol) = static_addr(symbol) + load_addr - -- The choice of symbol is arbitrary. The static address we obtain -- by constructing a non GOT reference to the symbol, the dynamic -- address of the symbol we compute using adrp/add to compute the -- symbol's address relative to the PC. -- This depends on 32bit relocations being resolved at link time -- and that the static address fits in the 32bits. */ -- -- ElfW(Addr) static_addr; -- ElfW(Addr) dynamic_addr; -- -- asm (" \n" --" adrp %1, _dl_start; \n" --" add %1, %1, #:lo12:_dl_start \n" --" ldr %w0, 1f \n" --" b 2f \n" --"1: \n" --" .word _dl_start \n" --"2: \n" -- : "=r" (static_addr), "=r" (dynamic_addr)); -- return dynamic_addr - static_addr; -+ _DYNAMIC sysmbol is used here as its link-time address stored in -+ the special unrelocated first GOT entry. */ -+ -+ extern ElfW(Dyn) _DYNAMIC[] attribute_hidden; -+ return (ElfW(Addr)) &_DYNAMIC - elf_machine_dynamic (); - } - - /* Set up the loaded object described by L so its unrelocated PLT diff --git a/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch b/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch deleted file mode 100644 index 11fe7fdc99c..00000000000 --- a/contrib/guix/patches/glibc-2.24-no-build-time-cxx-header-run.patch +++ /dev/null @@ -1,100 +0,0 @@ -https://sourceware.org/git/?p=glibc.git;a=commit;h=fc3e1337be1c6935ab58bd13520f97a535cf70cc - -commit dc23a45db566095e83ff0b7a57afc87fb5ca89a1 -Author: Florian Weimer -Date: Wed Sep 21 10:45:32 2016 +0200 - - Avoid running $(CXX) during build to obtain header file paths - - This reduces the build time somewhat and is particularly noticeable - during rebuilds with few code changes. - -diff --git a/Makerules b/Makerules -index 7e4077ee50..c338850de5 100644 ---- a/Makerules -+++ b/Makerules -@@ -121,14 +121,10 @@ ifneq (,$(CXX)) - # will be used instead of /usr/include/stdlib.h and /usr/include/math.h. - before-compile := $(common-objpfx)cstdlib $(common-objpfx)cmath \ - $(before-compile) --cstdlib=$(shell echo "\#include " | $(CXX) -M -MP -x c++ - \ -- | sed -n "/cstdlib:/{s/:$$//;p}") --$(common-objpfx)cstdlib: $(cstdlib) -+$(common-objpfx)cstdlib: $(c++-cstdlib-header) - $(INSTALL_DATA) $< $@T - $(move-if-change) $@T $@ --cmath=$(shell echo "\#include " | $(CXX) -M -MP -x c++ - \ -- | sed -n "/cmath:/{s/:$$//;p}") --$(common-objpfx)cmath: $(cmath) -+$(common-objpfx)cmath: $(c++-cmath-header) - $(INSTALL_DATA) $< $@T - $(move-if-change) $@T $@ - endif -diff --git a/config.make.in b/config.make.in -index 95c6f36876..04a8b3ed7f 100644 ---- a/config.make.in -+++ b/config.make.in -@@ -45,6 +45,8 @@ defines = @DEFINES@ - sysheaders = @sysheaders@ - sysincludes = @SYSINCLUDES@ - c++-sysincludes = @CXX_SYSINCLUDES@ -+c++-cstdlib-header = @CXX_CSTDLIB_HEADER@ -+c++-cmath-header = @CXX_CMATH_HEADER@ - all-warnings = @all_warnings@ - enable-werror = @enable_werror@ - -diff --git a/configure b/configure -index 17625e1041..6ff252744b 100755 ---- a/configure -+++ b/configure -@@ -635,6 +635,8 @@ BISON - INSTALL_INFO - PERL - BASH_SHELL -+CXX_CMATH_HEADER -+CXX_CSTDLIB_HEADER - CXX_SYSINCLUDES - SYSINCLUDES - AUTOCONF -@@ -5054,6 +5056,18 @@ fi - - - -+# Obtain some C++ header file paths. This is used to make a local -+# copy of those headers in Makerules. -+if test -n "$CXX"; then -+ find_cxx_header () { -+ echo "#include <$1>" | $CXX -M -MP -x c++ - | sed -n "/$1:/{s/:\$//;p}" -+ } -+ CXX_CSTDLIB_HEADER="$(find_cxx_header cstdlib)" -+ CXX_CMATH_HEADER="$(find_cxx_header cmath)" -+fi -+ -+ -+ - # Test if LD_LIBRARY_PATH contains the notation for the current directory - # since this would lead to problems installing/building glibc. - # LD_LIBRARY_PATH contains the current directory if one of the following -diff --git a/configure.ac b/configure.ac -index 33bcd62180..9938ab0dc2 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -1039,6 +1039,18 @@ fi - AC_SUBST(SYSINCLUDES) - AC_SUBST(CXX_SYSINCLUDES) - -+# Obtain some C++ header file paths. This is used to make a local -+# copy of those headers in Makerules. -+if test -n "$CXX"; then -+ find_cxx_header () { -+ echo "#include <$1>" | $CXX -M -MP -x c++ - | sed -n "/$1:/{s/:\$//;p}" -+ } -+ CXX_CSTDLIB_HEADER="$(find_cxx_header cstdlib)" -+ CXX_CMATH_HEADER="$(find_cxx_header cmath)" -+fi -+AC_SUBST(CXX_CSTDLIB_HEADER) -+AC_SUBST(CXX_CMATH_HEADER) -+ - # Test if LD_LIBRARY_PATH contains the notation for the current directory - # since this would lead to problems installing/building glibc. - # LD_LIBRARY_PATH contains the current directory if one of the following diff --git a/contrib/guix/patches/glibc-2.27-fcommon.patch b/contrib/guix/patches/glibc-2.27-fcommon.patch new file mode 100644 index 00000000000..f3baacab98e --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-fcommon.patch @@ -0,0 +1,32 @@ +commit 264a4a0dbe1f4369db315080034b500bed66016c +Author: fanquake +Date: Fri May 6 11:03:04 2022 +0100 + + build: use -fcommon to retain legacy behaviour with GCC 10 + + GCC 10 started using -fno-common by default, which causes issues with + the powerpc builds using gibc 2.24. A patch was commited to glibc to fix + the issue, 18363b4f010da9ba459b13310b113ac0647c2fcc but is non-trvial + to backport, and was broken in at least one way, see the followup in + commit 7650321ce037302bfc2f026aa19e0213b8d02fe6. + + For now, retain the legacy GCC behaviour by passing -fcommon when + building glibc. + + https://gcc.gnu.org/onlinedocs/gcc/Code-Gen-Options.html. + https://sourceware.org/git/?p=glibc.git;a=commit;h=18363b4f010da9ba459b13310b113ac0647c2fcc + https://sourceware.org/git/?p=glibc.git;a=commit;h=7650321ce037302bfc2f026aa19e0213b8d02fe6 + +diff --git a/Makeconfig b/Makeconfig +index 86a71e5802..aa2166be60 100644 +--- a/Makeconfig ++++ b/Makeconfig +@@ -896,7 +896,7 @@ ifeq "$(strip $(+cflags))" "" + endif # $(+cflags) == "" + + +cflags += $(cflags-cpu) $(+gccwarn) $(+merge-constants) $(+math-flags) \ +- $(+stack-protector) ++ $(+stack-protector) -fcommon + +gcc-nowarn := -w + + # Don't duplicate options if we inherited variables from the parent. diff --git a/contrib/guix/patches/glibc-2.27-guix-prefix.patch b/contrib/guix/patches/glibc-2.27-guix-prefix.patch new file mode 100644 index 00000000000..6648bc6c053 --- /dev/null +++ b/contrib/guix/patches/glibc-2.27-guix-prefix.patch @@ -0,0 +1,22 @@ +Without ffile-prefix-map, the debug symbols will contain paths for the +guix store which will include the hashes of each package. However, the +hash for the same package will differ when on different architectures. +In order to be reproducible regardless of the architecture used to build +the package, map all guix store prefixes to something fixed, e.g. /usr. + +We might be able to drop this in favour of using --with-nonshared-cflags +when we being using newer versions of glibc. + +--- a/Makeconfig ++++ b/Makeconfig +@@ -992,6 +992,10 @@ object-suffixes := + CPPFLAGS-.o = $(pic-default) + # libc.a must be compiled with -fPIE/-fpie for static PIE. + CFLAGS-.o = $(filter %frame-pointer,$(+cflags)) $(pie-default) ++ ++# Map Guix store paths to /usr ++CFLAGS-.o += `find /gnu/store -maxdepth 1 -mindepth 1 -type d -exec echo -n " -ffile-prefix-map={}=/usr" \;` ++ + libtype.o := lib%.a + object-suffixes += .o + ifeq (yes,$(build-shared)) diff --git a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch similarity index 90% rename from contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch rename to contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch index d6217157ee5..c0f8495c41d 100644 --- a/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include__-to-include-asm-syscalls.h.patch +++ b/contrib/guix/patches/glibc-2.27-riscv64-Use-__has_include-to-include-asm-syscalls.h.patch @@ -1,3 +1,7 @@ +Note that this has been modified from the original commit, to use __has_include +instead of __has_include__, as the later was causing build failures with GCC 10. +See also: http://lists.busybox.net/pipermail/buildroot/2020-July/590376.html. + https://sourceware.org/git/?p=glibc.git;a=commit;h=0b9c84906f653978fb8768c7ebd0ee14a47e662e From 562c52cc81a4e456a62e6455feb32732049e9070 Mon Sep 17 00:00:00 2001 @@ -59,7 +63,7 @@ index d612ef4c6c..0b2042620b 100644 #include #include -#include -+#if __has_include__ () ++#if __has_include () +# include +#else +# include diff --git a/contrib/guix/patches/glibc-ldd-x86_64.patch b/contrib/guix/patches/glibc-ldd-x86_64.patch index b1b6d5a5486..a23b095caa7 100644 --- a/contrib/guix/patches/glibc-ldd-x86_64.patch +++ b/contrib/guix/patches/glibc-ldd-x86_64.patch @@ -1,8 +1,8 @@ By default, 'RTDLLIST' in 'ldd' refers to 'lib64/ld-linux-x86-64.so', whereas it's in 'lib/' for us. This patch fixes that. ---- glibc-2.17/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed 2012-12-25 04:02:13.000000000 +0100 -+++ glibc-2.17/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed 2013-09-15 23:08:03.000000000 +0200 +--- a/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed ++++ b/sysdeps/unix/sysv/linux/x86_64/ldd-rewrite.sed @@ -1,3 +1,3 @@ /LD_TRACE_LOADED_OBJECTS=1/a\ add_env="$add_env LD_LIBRARY_VERSION=\\$verify_out" diff --git a/contrib/guix/patches/lief-fix-ppc64-nx-default.patch b/contrib/guix/patches/lief-fix-ppc64-nx-default.patch new file mode 100644 index 00000000000..101bc1ddc0c --- /dev/null +++ b/contrib/guix/patches/lief-fix-ppc64-nx-default.patch @@ -0,0 +1,29 @@ +Correct default for Binary::has_nx on ppc64 + +From the Linux kernel source: + + * This is the default if a program doesn't have a PT_GNU_STACK + * program header entry. The PPC64 ELF ABI has a non executable stack + * stack by default, so in the absence of a PT_GNU_STACK program header + * we turn execute permission off. + +This patch can be dropped the next time we update LIEF. + +diff --git a/src/ELF/Binary.cpp b/src/ELF/Binary.cpp +index a90be1ab..fd2d9764 100644 +--- a/src/ELF/Binary.cpp ++++ b/src/ELF/Binary.cpp +@@ -1084,7 +1084,12 @@ bool Binary::has_nx() const { + return segment->type() == SEGMENT_TYPES::PT_GNU_STACK; + }); + if (it_stack == std::end(segments_)) { +- return false; ++ if (header().machine_type() == ARCH::EM_PPC64) { ++ // The PPC64 ELF ABI has a non-executable stack by default. ++ return true; ++ } else { ++ return false; ++ } + } + + return !(*it_stack)->has(ELF_SEGMENT_FLAGS::PF_X); diff --git a/contrib/install_db4.sh b/contrib/install_db4.sh deleted file mode 100755 index 2850c4b993e..00000000000 --- a/contrib/install_db4.sh +++ /dev/null @@ -1,260 +0,0 @@ -#!/bin/sh -# Copyright (c) 2017-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -# Install libdb4.8 (Berkeley DB). - -export LC_ALL=C -set -e - -if [ -z "${1}" ]; then - echo "Usage: $0 [ ...]" - echo - echo "Must specify a single argument: the directory in which db4 will be built." - echo "This is probably \`pwd\` if you're at the root of the bitcoin repository." - exit 1 -fi - -expand_path() { - cd "${1}" && pwd -P -} - -BDB_PREFIX="$(expand_path "${1}")/db4"; shift; -BDB_VERSION='db-4.8.30.NC' -BDB_HASH='12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef' -BDB_URL="https://download.oracle.com/berkeley-db/${BDB_VERSION}.tar.gz" - -check_exists() { - command -v "$1" >/dev/null -} - -sha256_check() { - # Args: - # - if check_exists sha256sum; then - echo "${1} ${2}" | sha256sum -c - elif check_exists sha256; then - if [ "$(uname)" = "FreeBSD" ]; then - sha256 -c "${1}" "${2}" - else - echo "${1} ${2}" | sha256 -c - fi - else - echo "${1} ${2}" | shasum -a 256 -c - fi -} - -http_get() { - # Args: - # - # It's acceptable that we don't require SSL here because we manually verify - # content hashes below. - # - if [ -f "${2}" ]; then - echo "File ${2} already exists; not downloading again" - elif check_exists curl; then - curl --insecure --retry 5 "${1}" -o "${2}" - elif check_exists wget; then - wget --no-check-certificate "${1}" -O "${2}" - else - echo "Simple transfer utilities 'curl' and 'wget' not found. Please install one of them and try again." - exit 1 - fi - - sha256_check "${3}" "${2}" -} - -# Ensure the commands we use exist on the system -if ! check_exists patch; then - echo "Command-line tool 'patch' not found. Install patch and try again." - exit 1 -fi - -mkdir -p "${BDB_PREFIX}" -http_get "${BDB_URL}" "${BDB_VERSION}.tar.gz" "${BDB_HASH}" -tar -xzvf ${BDB_VERSION}.tar.gz -C "$BDB_PREFIX" -cd "${BDB_PREFIX}/${BDB_VERSION}/" - -# Apply a patch necessary when building with clang and c++11 (see https://community.oracle.com/thread/3952592) -patch --ignore-whitespace -p1 << 'EOF' -commit 3311d68f11d1697565401eee6efc85c34f022ea7 -Author: fanquake -Date: Mon Aug 17 20:03:56 2020 +0800 - - Fix C++11 compatibility - -diff --git a/dbinc/atomic.h b/dbinc/atomic.h -index 0034dcc..7c11d4a 100644 ---- a/dbinc/atomic.h -+++ b/dbinc/atomic.h -@@ -70,7 +70,7 @@ typedef struct { - * These have no memory barriers; the caller must include them when necessary. - */ - #define atomic_read(p) ((p)->value) --#define atomic_init(p, val) ((p)->value = (val)) -+#define atomic_init_db(p, val) ((p)->value = (val)) - - #ifdef HAVE_ATOMIC_SUPPORT - -@@ -144,7 +144,7 @@ typedef LONG volatile *interlocked_val; - #define atomic_inc(env, p) __atomic_inc(p) - #define atomic_dec(env, p) __atomic_dec(p) - #define atomic_compare_exchange(env, p, o, n) \ -- __atomic_compare_exchange((p), (o), (n)) -+ __atomic_compare_exchange_db((p), (o), (n)) - static inline int __atomic_inc(db_atomic_t *p) - { - int temp; -@@ -176,7 +176,7 @@ static inline int __atomic_dec(db_atomic_t *p) - * http://gcc.gnu.org/onlinedocs/gcc-4.1.0/gcc/Atomic-Builtins.html - * which configure could be changed to use. - */ --static inline int __atomic_compare_exchange( -+static inline int __atomic_compare_exchange_db( - db_atomic_t *p, atomic_value_t oldval, atomic_value_t newval) - { - atomic_value_t was; -@@ -206,7 +206,7 @@ static inline int __atomic_compare_exchange( - #define atomic_dec(env, p) (--(p)->value) - #define atomic_compare_exchange(env, p, oldval, newval) \ - (DB_ASSERT(env, atomic_read(p) == (oldval)), \ -- atomic_init(p, (newval)), 1) -+ atomic_init_db(p, (newval)), 1) - #else - #define atomic_inc(env, p) __atomic_inc(env, p) - #define atomic_dec(env, p) __atomic_dec(env, p) -diff --git a/mp/mp_fget.c b/mp/mp_fget.c -index 5fdee5a..0b75f57 100644 ---- a/mp/mp_fget.c -+++ b/mp/mp_fget.c -@@ -617,7 +617,7 @@ alloc: /* Allocate a new buffer header and data space. */ - - /* Initialize enough so we can call __memp_bhfree. */ - alloc_bhp->flags = 0; -- atomic_init(&alloc_bhp->ref, 1); -+ atomic_init_db(&alloc_bhp->ref, 1); - #ifdef DIAGNOSTIC - if ((uintptr_t)alloc_bhp->buf & (sizeof(size_t) - 1)) { - __db_errx(env, -@@ -911,7 +911,7 @@ alloc: /* Allocate a new buffer header and data space. */ - MVCC_MPROTECT(bhp->buf, mfp->stat.st_pagesize, - PROT_READ); - -- atomic_init(&alloc_bhp->ref, 1); -+ atomic_init_db(&alloc_bhp->ref, 1); - MUTEX_LOCK(env, alloc_bhp->mtx_buf); - alloc_bhp->priority = bhp->priority; - alloc_bhp->pgno = bhp->pgno; -diff --git a/mp/mp_mvcc.c b/mp/mp_mvcc.c -index 34467d2..f05aa0c 100644 ---- a/mp/mp_mvcc.c -+++ b/mp/mp_mvcc.c -@@ -276,7 +276,7 @@ __memp_bh_freeze(dbmp, infop, hp, bhp, need_frozenp) - #else - memcpy(frozen_bhp, bhp, SSZA(BH, buf)); - #endif -- atomic_init(&frozen_bhp->ref, 0); -+ atomic_init_db(&frozen_bhp->ref, 0); - if (mutex != MUTEX_INVALID) - frozen_bhp->mtx_buf = mutex; - else if ((ret = __mutex_alloc(env, MTX_MPOOL_BH, -@@ -428,7 +428,7 @@ __memp_bh_thaw(dbmp, infop, hp, frozen_bhp, alloc_bhp) - #endif - alloc_bhp->mtx_buf = mutex; - MUTEX_LOCK(env, alloc_bhp->mtx_buf); -- atomic_init(&alloc_bhp->ref, 1); -+ atomic_init_db(&alloc_bhp->ref, 1); - F_CLR(alloc_bhp, BH_FROZEN); - } - -diff --git a/mp/mp_region.c b/mp/mp_region.c -index e6cece9..ddbe906 100644 ---- a/mp/mp_region.c -+++ b/mp/mp_region.c -@@ -224,7 +224,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg) - MTX_MPOOL_FILE_BUCKET, 0, &htab[i].mtx_hash)) != 0) - return (ret); - SH_TAILQ_INIT(&htab[i].hash_bucket); -- atomic_init(&htab[i].hash_page_dirty, 0); -+ atomic_init_db(&htab[i].hash_page_dirty, 0); - } - - /* -@@ -269,7 +269,7 @@ __memp_init(env, dbmp, reginfo_off, htab_buckets, max_nreg) - hp->mtx_hash = (mtx_base == MUTEX_INVALID) ? MUTEX_INVALID : - mtx_base + i; - SH_TAILQ_INIT(&hp->hash_bucket); -- atomic_init(&hp->hash_page_dirty, 0); -+ atomic_init_db(&hp->hash_page_dirty, 0); - #ifdef HAVE_STATISTICS - hp->hash_io_wait = 0; - hp->hash_frozen = hp->hash_thawed = hp->hash_frozen_freed = 0; -diff --git a/mutex/mut_method.c b/mutex/mut_method.c -index 2588763..5c6d516 100644 ---- a/mutex/mut_method.c -+++ b/mutex/mut_method.c -@@ -426,7 +426,7 @@ atomic_compare_exchange(env, v, oldval, newval) - MUTEX_LOCK(env, mtx); - ret = atomic_read(v) == oldval; - if (ret) -- atomic_init(v, newval); -+ atomic_init_db(v, newval); - MUTEX_UNLOCK(env, mtx); - - return (ret); -diff --git a/mutex/mut_tas.c b/mutex/mut_tas.c -index f3922e0..e40fcdf 100644 ---- a/mutex/mut_tas.c -+++ b/mutex/mut_tas.c -@@ -46,7 +46,7 @@ __db_tas_mutex_init(env, mutex, flags) - - #ifdef HAVE_SHARED_LATCHES - if (F_ISSET(mutexp, DB_MUTEX_SHARED)) -- atomic_init(&mutexp->sharecount, 0); -+ atomic_init_db(&mutexp->sharecount, 0); - else - #endif - if (MUTEX_INIT(&mutexp->tas)) { -@@ -486,7 +486,7 @@ __db_tas_mutex_unlock(env, mutex) - F_CLR(mutexp, DB_MUTEX_LOCKED); - /* Flush flag update before zeroing count */ - MEMBAR_EXIT(); -- atomic_init(&mutexp->sharecount, 0); -+ atomic_init_db(&mutexp->sharecount, 0); - } else { - DB_ASSERT(env, sharecount > 0); - MEMBAR_EXIT(); -EOF - -# The packaged config.guess and config.sub are ancient (2009) and can cause build issues. -# Replace them with modern versions. -# See https://github.com/bitcoin/bitcoin/issues/16064 -CONFIG_GUESS_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=4550d2f15b3a7ce2451c1f29500b9339430c877f' -CONFIG_GUESS_HASH='c8f530e01840719871748a8071113435bdfdf75b74c57e78e47898edea8754ae' -CONFIG_SUB_URL='https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=4550d2f15b3a7ce2451c1f29500b9339430c877f' -CONFIG_SUB_HASH='3969f7d5f6967ccc6f792401b8ef3916a1d1b1d0f0de5a4e354c95addb8b800e' - -rm -f "dist/config.guess" -rm -f "dist/config.sub" - -http_get "${CONFIG_GUESS_URL}" dist/config.guess "${CONFIG_GUESS_HASH}" -http_get "${CONFIG_SUB_URL}" dist/config.sub "${CONFIG_SUB_HASH}" - -cd build_unix/ - -"${BDB_PREFIX}/${BDB_VERSION}/dist/configure" \ - --enable-cxx --disable-shared --disable-replication --with-pic --prefix="${BDB_PREFIX}" \ - "${@}" - -make install - -echo -echo "db4 build complete." -echo -# shellcheck disable=SC2016 -echo 'When compiling bitcoind, run `./configure` in the following way:' -echo -echo " export BDB_PREFIX='${BDB_PREFIX}'" -# shellcheck disable=SC2016 -echo ' ./configure BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" BDB_CFLAGS="-I${BDB_PREFIX}/include" ...' diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index 441b5da764d..24f6b29a26f 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -2,7 +2,7 @@ # # linearize-data.py: Construct a linear, no-fork version of the chain. # -# Copyright (c) 2013-2021 The Bitcoin Core developers +# Copyright (c) 2013-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # @@ -20,49 +20,9 @@ settings = {} -def hex_switchEndian(s): - """ Switches the endianness of a hex string (in pairs of hex chars) """ - pairList = [s[i:i+2].encode() for i in range(0, len(s), 2)] - return b''.join(pairList[::-1]).decode() - -def uint32(x): - return x & 0xffffffff - -def bytereverse(x): - return uint32(( ((x) << 24) | (((x) << 8) & 0x00ff0000) | - (((x) >> 8) & 0x0000ff00) | ((x) >> 24) )) - -def bufreverse(in_buf): - out_words = [] - for i in range(0, len(in_buf), 4): - word = struct.unpack('@I', in_buf[i:i+4])[0] - out_words.append(struct.pack('@I', bytereverse(word))) - return b''.join(out_words) - -def wordreverse(in_buf): - out_words = [] - for i in range(0, len(in_buf), 4): - out_words.append(in_buf[i:i+4]) - out_words.reverse() - return b''.join(out_words) - -def calc_hdr_hash(blk_hdr): - hash1 = hashlib.sha256() - hash1.update(blk_hdr) - hash1_o = hash1.digest() - - hash2 = hashlib.sha256() - hash2.update(hash1_o) - hash2_o = hash2.digest() - - return hash2_o - def calc_hash_str(blk_hdr): - hash = calc_hdr_hash(blk_hdr) - hash = bufreverse(hash) - hash = wordreverse(hash) - hash_str = hash.hex() - return hash_str + blk_hdr_hash = hashlib.sha256(hashlib.sha256(blk_hdr).digest()).digest() + return blk_hdr_hash[::-1].hex() def get_blk_dt(blk_hdr): members = struct.unpack(" " - exit 1 -fi - -if [ -z "$SIGNATURE" ]; then - echo "usage: $0 " - exit 1 -fi - -${SIGNAPPLE} apply "${UNSIGNED}" "${SIGNATURE}" -mv ${ROOTDIR} ${OUTDIR} -echo "Signed: ${OUTDIR}" diff --git a/contrib/macdeploy/detached-sig-create.sh b/contrib/macdeploy/detached-sig-create.sh index 26ba0cb21a8..048c92de2ad 100755 --- a/contrib/macdeploy/detached-sig-create.sh +++ b/contrib/macdeploy/detached-sig-create.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (c) 2014-2021 The Bitcoin Core developers +# Copyright (c) 2014-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk index ebef1d2db0d..6efaaccb8e1 100755 --- a/contrib/macdeploy/gen-sdk +++ b/contrib/macdeploy/gen-sdk @@ -8,6 +8,21 @@ import gzip import os import contextlib +# monkey-patch Python 3.8 and older to fix wrong TAR header handling +# see https://github.com/bitcoin/bitcoin/pull/24534 +# and https://github.com/python/cpython/pull/18080 for more info +if sys.version_info < (3, 9): + _old_create_header = tarfile.TarInfo._create_header + def _create_header(info, format, encoding, errors): + buf = _old_create_header(info, format, encoding, errors) + # replace devmajor/devminor with binary zeroes + buf = buf[:329] + bytes(16) + buf[345:] + # recompute checksum + chksum = tarfile.calc_chksums(buf)[0] + buf = buf[:-364] + bytes("%06o\0" % chksum, "ascii") + buf[-357:] + return buf + tarfile.TarInfo._create_header = staticmethod(_create_header) + @contextlib.contextmanager def cd(path): """Context manager that restores PWD even if an exception was raised.""" @@ -75,14 +90,21 @@ def run(): tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name)) if tarinfo.linkname and tarinfo.linkname.startswith("./"): tarinfo.linkname = str(pathlib.Path(alt_base_dir, tarinfo.linkname)) + # make metadata deterministic + tarinfo.mtime = 0 + tarinfo.uid, tarinfo.uname = 0, '' + tarinfo.gid, tarinfo.gname = 0, '' + # don't use isdir() as there are also executable files present + tarinfo.mode = 0o0755 if tarinfo.mode & 0o0100 else 0o0644 return tarinfo with cd(dir_to_add): + # recursion already adds entries in sorted order tarfp.add(".", recursive=True, filter=change_tarinfo_base) print("Creating output .tar.gz file...") with out_sdktgz_path.open("wb") as fp: with gzip.GzipFile(fileobj=fp, mode='wb', compresslevel=9, mtime=0) as gzf: - with tarfile.open(mode="w", fileobj=gzf) as tarfp: + with tarfile.open(mode="w", fileobj=gzf, format=tarfile.GNU_FORMAT) as tarfp: print("Adding MacOSX SDK {} files...".format(sdk_version)) tarfp_add_with_base_change(tarfp, sdk_dir, out_name) print("Adding libc++ headers...") diff --git a/contrib/macdeploy/macdeployqtplus b/contrib/macdeploy/macdeployqtplus index 2ad717f5292..af30d15242d 100755 --- a/contrib/macdeploy/macdeployqtplus +++ b/contrib/macdeploy/macdeployqtplus @@ -187,7 +187,7 @@ def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]: if verbose: print(f"Inspecting with otool: {binaryPath}") otoolbin=os.getenv("OTOOL", "otool") - otool = run([otoolbin, "-L", binaryPath], stdout=PIPE, stderr=PIPE, universal_newlines=True) + otool = run([otoolbin, "-L", binaryPath], stdout=PIPE, stderr=PIPE, text=True) if otool.returncode != 0: sys.stderr.write(otool.stderr) sys.stderr.flush() @@ -211,7 +211,7 @@ def getFrameworks(binaryPath: str, verbose: int) -> List[FrameworkInfo]: return libraries def runInstallNameTool(action: str, *args): - installnametoolbin=os.getenv("INSTALLNAMETOOL", "install_name_tool") + installnametoolbin=os.getenv("INSTALL_NAME_TOOL", "install_name_tool") run([installnametoolbin, "-"+action] + list(args), check=True) def changeInstallName(oldName: str, newName: str, binaryPath: str, verbose: int): @@ -544,6 +544,22 @@ ds.close() if platform.system() == "Darwin": subprocess.check_call(f"codesign --deep --force --sign - {target}", shell=True) +print("+ Installing background.tiff +") + +bg_path = os.path.join('dist', '.background', 'background.tiff') +os.mkdir(os.path.dirname(bg_path)) + +tiff_path = os.path.join('contrib', 'macdeploy', 'background.tiff') +shutil.copy2(tiff_path, bg_path) + +# ------------------------------------------------ + +print("+ Generating symlink for /Applications +") + +os.symlink("/Applications", os.path.join('dist', "Applications")) + +# ------------------------------------------------ + if config.dmg is not None: print("+ Preparing .dmg disk image +") @@ -561,30 +577,17 @@ if config.dmg is not None: tempname: str = appname + ".temp.dmg" - run(["hdiutil", "create", tempname, "-srcfolder", "dist", "-format", "UDRW", "-size", str(size), "-volname", appname], check=True, universal_newlines=True) + run(["hdiutil", "create", tempname, "-srcfolder", "dist", "-format", "UDRW", "-size", str(size), "-volname", appname], check=True, text=True) if verbose: print("Attaching temp image...") - output = run(["hdiutil", "attach", tempname, "-readwrite"], check=True, universal_newlines=True, stdout=PIPE).stdout - - m = re.search(r"/Volumes/(.+$)", output) - disk_root = m.group(0) - - print("+ Applying fancy settings +") - - bg_path = os.path.join(disk_root, ".background", os.path.basename('background.tiff')) - os.mkdir(os.path.dirname(bg_path)) - if verbose: - print('background.tiff', "->", bg_path) - shutil.copy2('contrib/macdeploy/background.tiff', bg_path) - - os.symlink("/Applications", os.path.join(disk_root, "Applications")) + output = run(["hdiutil", "attach", tempname, "-readwrite"], check=True, text=True, stdout=PIPE).stdout print("+ Finalizing .dmg disk image +") - run(["hdiutil", "detach", f"/Volumes/{appname}"], universal_newlines=True) + run(["hdiutil", "detach", f"/Volumes/{appname}"], text=True) - run(["hdiutil", "convert", tempname, "-format", "UDZO", "-o", appname, "-imagekey", "zlib-level=9"], check=True, universal_newlines=True) + run(["hdiutil", "convert", tempname, "-format", "UDZO", "-o", appname, "-imagekey", "zlib-level=9"], check=True, text=True) os.unlink(tempname) diff --git a/contrib/merge-prs.sh b/contrib/merge-prs.sh index 7e322a0f8be..e3b8e1696ad 100755 --- a/contrib/merge-prs.sh +++ b/contrib/merge-prs.sh @@ -135,7 +135,7 @@ do PR_ID_ALT=$(echo "$line" | cut -d ' ' -f 8 | tr -d :) if [[ "$PR_ID" == "pull" ]]; then - PR_ID="${PR_ID_ALT}" + PR_ID="${PR_ID_ALT}" fi echo -e "$CHAIN PR \e[37m$PR_ID \e[33m$HASH\e[0m on \e[32m$DATE\e[0m " diff --git a/contrib/message-capture/message-capture-parser.py b/contrib/message-capture/message-capture-parser.py index 9988478f1b9..d6ddc1c149a 100755 --- a/contrib/message-capture/message-capture-parser.py +++ b/contrib/message-capture/message-capture-parser.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Parse message capture binary files. To be used in conjunction with -capturemessages.""" @@ -79,7 +79,7 @@ def to_jsonable(obj: Any) -> Any: val = getattr(obj, slot, None) if slot in HASH_INTS and isinstance(val, int): ret[slot] = ser_uint256(val).hex() - elif slot in HASH_INT_VECTORS and isinstance(val[0], int): + elif slot in HASH_INT_VECTORS and all(isinstance(a, int) for a in val): ret[slot] = [ser_uint256(a).hex() for a in val] else: ret[slot] = to_jsonable(val) diff --git a/contrib/seeds/.gitignore b/contrib/seeds/.gitignore index e4a39d60934..d9a2451f70f 100644 --- a/contrib/seeds/.gitignore +++ b/contrib/seeds/.gitignore @@ -1 +1,2 @@ seeds_main.txt +asmap-filled.dat diff --git a/contrib/seeds/README.md b/contrib/seeds/README.md index c53446bfb0c..6db77cbbea5 100644 --- a/contrib/seeds/README.md +++ b/contrib/seeds/README.md @@ -8,21 +8,13 @@ and remove old versions as necessary (at a minimum when GetDesirableServiceFlags changes its default return value, as those are the services which seeds are added to addrman with). -The seeds compiled into the release are created from sipa's DNS seed data, like this: - - curl https://bitcoin.sipa.be/seeds.txt.gz | gzip -dc > seeds_main.txt - python3 makeseeds.py < seeds_main.txt > nodes_main.txt - cat nodes_main_manual.txt >> nodes_main.txt - python3 generate-seeds.py . > ../../src/chainparamsseeds.h - -## Dependencies - -Ubuntu, Debian: - - sudo apt-get install python3-dnspython - -and/or for other operating systems: - - pip install dnspython - -See https://dnspython.readthedocs.io/en/latest/installation.html for more information. +The seeds compiled into the release are created from sipa's DNS seed and AS map +data. Run the following commands from the `/contrib/seeds` directory: + +``` +curl https://bitcoin.sipa.be/seeds.txt.gz | gzip -dc > seeds_main.txt +curl https://bitcoin.sipa.be/asmap-filled.dat > asmap-filled.dat +python3 makeseeds.py -a asmap-filled.dat -s seeds_main.txt > nodes_main.txt +cat nodes_main_manual.txt >> nodes_main.txt +python3 generate-seeds.py . > ../../src/chainparamsseeds.h +``` diff --git a/contrib/seeds/asmap.py b/contrib/seeds/asmap.py new file mode 100644 index 00000000000..e28e5cf532e --- /dev/null +++ b/contrib/seeds/asmap.py @@ -0,0 +1,815 @@ +# Copyright (c) 2022 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file LICENSE or http://www.opensource.org/licenses/mit-license.php. + +""" +This module provides the ASNEntry and ASMap classes. +""" + +import copy +import ipaddress +import random +import unittest +from enum import Enum +from functools import total_ordering +from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union, overload + +def net_to_prefix(net: Union[ipaddress.IPv4Network,ipaddress.IPv6Network]) -> List[bool]: + """ + Convert an IPv4 or IPv6 network to a prefix represented as a list of bits. + + IPv4 ranges are remapped to their IPv4-mapped IPv6 range (::ffff:0:0/96). + """ + num_bits = net.prefixlen + netrange = int.from_bytes(net.network_address.packed, 'big') + + # Map an IPv4 prefix into IPv6 space. + if isinstance(net, ipaddress.IPv4Network): + num_bits += 96 + netrange += 0xffff00000000 + + # Strip unused bottom bits. + assert (netrange & ((1 << (128 - num_bits)) - 1)) == 0 + return [((netrange >> (127 - i)) & 1) != 0 for i in range(num_bits)] + +def prefix_to_net(prefix: List[bool]) -> Union[ipaddress.IPv4Network,ipaddress.IPv6Network]: + """The reverse operation of net_to_prefix.""" + # Convert to number + netrange = sum(b << (127 - i) for i, b in enumerate(prefix)) + num_bits = len(prefix) + assert num_bits <= 128 + + # Return IPv4 range if in ::ffff:0:0/96 + if num_bits >= 96 and (netrange >> 32) == 0xffff: + return ipaddress.IPv4Network((netrange & 0xffffffff, num_bits - 96), True) + + # Return IPv6 range otherwise. + return ipaddress.IPv6Network((netrange, num_bits), True) + +# Shortcut for (prefix, ASN) entries. +ASNEntry = Tuple[List[bool], int] + +# Shortcut for (prefix, old ASN, new ASN) entries. +ASNDiff = Tuple[List[bool], int, int] + +class _VarLenCoder: + """ + A class representing a custom variable-length binary encoder/decoder for + integers. Each object represents a different coder, with different parameters + minval and clsbits. + + The encoding is easiest to describe using an example. Let's say minval=100 and + clsbits=[4,2,2,3]. In that case: + - x in [100..115]: encoded as [0] + [4-bit BE encoding of (x-100)]. + - x in [116..119]: encoded as [1,0] + [2-bit BE encoding of (x-116)]. + - x in [120..123]: encoded as [1,1,0] + [2-bit BE encoding of (x-120)]. + - x in [124..131]: encoded as [1,1,1] + [3-bit BE encoding of (x-124)]. + + In general, every number is encoded as: + - First, k "1"-bits, where k is the class the number falls in (there is one class + per element of clsbits). + - Then, a "0"-bit, unless k is the highest class, in which case there is nothing. + - Lastly, clsbits[k] bits encoding in big endian the position in its class that + number falls into. + - Every class k consists of 2^clsbits[k] consecutive integers. k=0 starts at minval, + other classes start one past the last element of the class before it. + """ + + def __init__(self, minval: int, clsbits: List[int]): + """Construct a new _VarLenCoder.""" + self._minval = minval + self._clsbits = clsbits + self._maxval = minval + sum(1 << b for b in clsbits) - 1 + + def can_encode(self, val: int) -> bool: + """Check whether value val is in the range this coder supports.""" + return self._minval <= val <= self._maxval + + def encode(self, val: int, ret: List[int]) -> None: + """Append encoding of val onto integer list ret.""" + + assert self._minval <= val <= self._maxval + val -= self._minval + bits = 0 + for k, bits in enumerate(self._clsbits): + if val >> bits: + # If the value will not fit in class k, subtract its range from v, + # emit a "1" bit and continue with the next class. + val -= 1 << bits + ret.append(1) + else: + if k + 1 < len(self._clsbits): + # Unless we're in the last class, emit a "0" bit. + ret.append(0) + break + # And then encode v (now the position within the class) in big endian. + ret.extend((val >> (bits - 1 - b)) & 1 for b in range(bits)) + + def encode_size(self, val: int) -> int: + """Compute how many bits are needed to encode val.""" + assert self._minval <= val <= self._maxval + val -= self._minval + ret = 0 + bits = 0 + for k, bits in enumerate(self._clsbits): + if val >> bits: + val -= 1 << bits + ret += 1 + else: + ret += k + 1 < len(self._clsbits) + break + return ret + bits + + def decode(self, stream, bitpos) -> Tuple[int,int]: + """Decode a number starting at bitpos in stream, returning value and new bitpos.""" + val = self._minval + bits = 0 + for k, bits in enumerate(self._clsbits): + bit = 0 + if k + 1 < len(self._clsbits): + bit = stream[bitpos] + bitpos += 1 + if not bit: + break + val += 1 << bits + for i in range(bits): + bit = stream[bitpos] + bitpos += 1 + val += bit << (bits - 1 - i) + return val, bitpos + +# Variable-length encoders used in the binary asmap format. +_CODER_INS = _VarLenCoder(0, [0, 0, 1]) +_CODER_ASN = _VarLenCoder(1, list(range(15, 25))) +_CODER_MATCH = _VarLenCoder(2, list(range(1, 9))) +_CODER_JUMP = _VarLenCoder(17, list(range(5, 31))) + +class _Instruction(Enum): + """One instruction in the binary asmap format.""" + # A return instruction, encoded as [0], returns a constant ASN. It is followed by + # an integer using the ASN encoding. + RETURN = 0 + # A jump instruction, encoded as [1,0] inspects the next unused bit in the input + # and either continues execution (if 0), or skips a specified number of bits (if 1). + # It is followed by an integer, and then two subprograms. The integer uses jump encoding + # and corresponds to the length of the first subprogram (so it can be skipped). + JUMP = 1 + # A match instruction, encoded as [1,1,0] inspects 1 or more of the next unused bits + # in the input with its argument. If they all match, execution continues. If they do + # not, failure is returned. If a default instruction has been executed before, instead + # of failure the default instruction's argument is returned. It is followed by an + # integer in match encoding, and a subprogram. That value is at least 2 bits and at + # most 9 bits. An n-bit value signifies matching (n-1) bits in the input with the lower + # (n-1) bits in the match value. + MATCH = 2 + # A default instruction, encoded as [1,1,1] sets the default variable to its argument, + # and continues execution. It is followed by an integer in ASN encoding, and a subprogram. + DEFAULT = 3 + # Not an actual instruction, but a way to encode the empty program that fails. In the + # encoder, it is used more generally to represent the failure case inside MATCH instructions, + # which may (if used inside the context of a DEFAULT instruction) actually correspond to + # a successful return. In this usage, they're always converted to an actual MATCH or RETURN + # before the top level is reached (see make_default below). + END = 4 + +class _BinNode: + """A class representing a (node of) the parsed binary asmap format.""" + + @overload + def __init__(self, ins: _Instruction): ... + @overload + def __init__(self, ins: _Instruction, arg1: int): ... + @overload + def __init__(self, ins: _Instruction, arg1: "_BinNode", arg2: "_BinNode"): ... + @overload + def __init__(self, ins: _Instruction, arg1: int, arg2: "_BinNode"): ... + + def __init__(self, ins: _Instruction, arg1=None, arg2=None): + """ + Construct a new asmap node. Possibilities are: + - _BinNode(_Instruction.RETURN, asn) + - _BinNode(_Instruction.JUMP, node_0, node_1) + - _BinNode(_Instruction.MATCH, val, node) + - _BinNode(_Instruction.DEFAULT, asn, node) + - _BinNode(_Instruction.END) + """ + self.ins = ins + self.arg1 = arg1 + self.arg2 = arg2 + if ins == _Instruction.RETURN: + assert isinstance(arg1, int) + assert arg2 is None + self.size = _CODER_INS.encode_size(ins.value) + _CODER_ASN.encode_size(arg1) + elif ins == _Instruction.JUMP: + assert isinstance(arg1, _BinNode) + assert isinstance(arg2, _BinNode) + self.size = (_CODER_INS.encode_size(ins.value) + _CODER_JUMP.encode_size(arg1.size) + + arg1.size + arg2.size) + elif ins == _Instruction.DEFAULT: + assert isinstance(arg1, int) + assert isinstance(arg2, _BinNode) + self.size = _CODER_INS.encode_size(ins.value) + _CODER_ASN.encode_size(arg1) + arg2.size + elif ins == _Instruction.MATCH: + assert isinstance(arg1, int) + assert isinstance(arg2, _BinNode) + self.size = (_CODER_INS.encode_size(ins.value) + _CODER_MATCH.encode_size(arg1) + + arg2.size) + elif ins == _Instruction.END: + assert arg1 is None + assert arg2 is None + self.size = 0 + else: + assert False + + @staticmethod + def make_end() -> "_BinNode": + """Constructor for a _BinNode with just an END instruction.""" + return _BinNode(_Instruction.END) + + @staticmethod + def make_leaf(val: int) -> "_BinNode": + """Constructor for a _BinNode of just a RETURN instruction.""" + assert val is not None and val > 0 + return _BinNode(_Instruction.RETURN, val) + + @staticmethod + def make_branch(node0: "_BinNode", node1: "_BinNode") -> "_BinNode": + """ + Construct a _BinNode corresponding to running either the node0 or node1 subprogram, + based on the next input bit. It exploits shortcuts that are possible in the encoding, + and uses either a JUMP, MATCH, or END instruction. + """ + if node0.ins == _Instruction.END and node1.ins == _Instruction.END: + return node0 + if node0.ins == _Instruction.END: + if node1.ins == _Instruction.MATCH and node1.arg1 <= 0xFF: + return _BinNode(node1.ins, node1.arg1 + (1 << node1.arg1.bit_length()), node1.arg2) + return _BinNode(_Instruction.MATCH, 3, node1) + if node1.ins == _Instruction.END: + if node0.ins == _Instruction.MATCH and node0.arg1 <= 0xFF: + return _BinNode(node0.ins, node0.arg1 + (1 << (node0.arg1.bit_length() - 1)), + node0.arg2) + return _BinNode(_Instruction.MATCH, 2, node0) + return _BinNode(_Instruction.JUMP, node0, node1) + + @staticmethod + def make_default(val: int, sub: "_BinNode") -> "_BinNode": + """ + Construct a _BinNode that corresponds to the specified subprogram, with the specified + default value. It exploits shortcuts that are possible in the encoding, and will use + either a DEFAULT or a RETURN instruction.""" + assert val is not None and val > 0 + if sub.ins == _Instruction.END: + return _BinNode(_Instruction.RETURN, val) + if sub.ins in (_Instruction.RETURN, _Instruction.DEFAULT): + return sub + return _BinNode(_Instruction.DEFAULT, val, sub) + +@total_ordering +class ASMap: + """ + A class whose objects represent a mapping from subnets to ASNs. + + Internally the mapping is stored as a binary trie, but can be converted + from/to a list of ASNEntry objects, and from/to the binary asmap file format. + + In the trie representation, nodes are represented as bare lists for efficiency + and ease of manipulation: + - [0] means an unassigned subnet (no ASN mapping for it is present) + - [int] means a subnet mapped entirely to the specified ASN. + - [node,node] means a subnet whose lower half and upper half have different + - mappings, represented by new trie nodes. + """ + + def update(self, prefix: List[bool], asn: int) -> None: + """Update this ASMap object to map prefix to the specified asn.""" + assert asn == 0 or _CODER_ASN.can_encode(asn) + + def recurse(node: List, offset: int) -> None: + if offset == len(prefix): + # Reached the end of prefix; overwrite this node. + node.clear() + node.append(asn) + return + if len(node) == 1: + # Need to descend into a leaf node; split it up. + oldasn = node[0] + node.clear() + node.append([oldasn]) + node.append([oldasn]) + # Descend into the node. + recurse(node[prefix[offset]], offset + 1) + # If the result is two identical leaf children, merge them. + if len(node[0]) == 1 and len(node[1]) == 1 and node[0] == node[1]: + oldasn = node[0][0] + node.clear() + node.append(oldasn) + recurse(self._trie, 0) + + def update_multi(self, entries: List[Tuple[List[bool], int]]) -> None: + """Apply multiple update operations, where longer prefixes take precedence.""" + entries.sort(key=lambda entry: len(entry[0])) + for prefix, asn in entries: + self.update(prefix, asn) + + def _set_trie(self, trie) -> None: + """Set trie directly. Internal use only.""" + def recurse(node: List) -> None: + if len(node) < 2: + return + recurse(node[0]) + recurse(node[1]) + if len(node[0]) == 2: + return + if node[0] == node[1]: + if len(node[0]) == 0: + node.clear() + else: + asn = node[0][0] + node.clear() + node.append(asn) + recurse(trie) + self._trie = trie + + def __init__(self, entries: Optional[Iterable[ASNEntry]] = None) -> None: + """Construct an ASMap object from an optional list of entries.""" + self._trie = [0] + if entries is not None: + def entry_key(entry): + """Sort function that places shorter prefixes first.""" + prefix, asn = entry + return len(prefix), prefix, asn + for prefix, asn in sorted(entries, key=entry_key): + self.update(prefix, asn) + + def lookup(self, prefix: List[bool]) -> Optional[int]: + """Look up a prefix. Returns ASN, or 0 if unassigned, or None if indeterminate.""" + node = self._trie + for bit in prefix: + if len(node) == 1: + break + node = node[bit] + if len(node) == 1: + return node[0] + return None + + def _to_entries_flat(self, fill: bool = False) -> List[ASNEntry]: + """Convert an ASMap object to a list of non-overlapping (prefix, asn) objects.""" + prefix : List[bool] = [] + + def recurse(node: List) -> List[ASNEntry]: + ret = [] + if len(node) == 1: + if node[0] > 0: + ret = [(list(prefix), node[0])] + elif len(node) == 2: + prefix.append(False) + ret = recurse(node[0]) + prefix[-1] = True + ret += recurse(node[1]) + prefix.pop() + if fill and len(ret) > 1: + asns = set(x[1] for x in ret) + if len(asns) == 1: + ret = [(list(prefix), list(asns)[0])] + return ret + return recurse(self._trie) + + def _to_entries_minimal(self, fill: bool = False) -> List[ASNEntry]: + """Convert a trie to a minimal list of ASNEntry objects, exploiting overlap.""" + prefix : List[bool] = [] + + def recurse(node: List) -> (Tuple[Dict[Optional[int], List[ASNEntry]], bool]): + if len(node) == 1 and node[0] == 0: + return {None if fill else 0: []}, True + if len(node) == 1: + return {node[0]: [], None: [(list(prefix), node[0])]}, False + ret: Dict[Optional[int], List[ASNEntry]] = {} + prefix.append(False) + left, lhole = recurse(node[0]) + prefix[-1] = True + right, rhole = recurse(node[1]) + prefix.pop() + hole = not fill and (lhole or rhole) + def candidate(ctx: Optional[int], res0: Optional[List[ASNEntry]], + res1: Optional[List[ASNEntry]]): + if res0 is not None and res1 is not None: + if ctx not in ret or len(res0) + len(res1) < len(ret[ctx]): + ret[ctx] = res0 + res1 + for ctx in set(left) | set(right): + candidate(ctx, left.get(ctx), right.get(ctx)) + candidate(ctx, left.get(None), right.get(ctx)) + candidate(ctx, left.get(ctx), right.get(None)) + if not hole: + for ctx in list(ret): + if ctx is not None: + candidate(None, [(list(prefix), ctx)], ret[ctx]) + if None in ret: + ret = {ctx:entries for ctx, entries in ret.items() + if ctx is None or len(entries) < len(ret[None])} + if hole: + ret = {ctx:entries for ctx, entries in ret.items() if ctx is None or ctx == 0} + return ret, hole + res, _ = recurse(self._trie) + return res[0] if 0 in res else res[None] + + def __str__(self) -> str: + """Convert this ASMap object to a string containing Python code constructing it.""" + return f"ASMap({self._trie})" + + def to_entries(self, overlapping: bool = True, fill: bool = False) -> List[ASNEntry]: + """ + Convert the mappings in this ASMap object to a list of ASNEntry objects. + + Arguments: + overlapping: Permit the subnets in the resulting ASNEntry to overlap. + Setting this can result in a shorter list. + fill: Permit the resulting ASNEntry objects to cover subnets that + are unassigned in this ASMap object. Setting this can + result in a shorter list. + """ + if overlapping: + return self._to_entries_minimal(fill) + return self._to_entries_flat(fill) + + @staticmethod + def from_random(num_leaves: int = 10, max_asn: int = 6, + unassigned_prob: float = 0.5) -> "ASMap": + """ + Construct a random ASMap object, with specified: + - Number of leaves in its trie (at least 1) + - Maximum ASN value (at least 1) + - Probability for leaf nodes to be unassigned + + The number of leaves in the resulting object may be less than what is + requested. This method is mostly intended for testing. + """ + assert num_leaves >= 1 + assert max_asn >= 1 or unassigned_prob == 1 + assert _CODER_ASN.can_encode(max_asn) + assert 0.0 <= unassigned_prob <= 1.0 + trie: List = [] + leaves = [trie] + ret = ASMap() + for i in range(1, num_leaves): + idx = random.randrange(i) + leaf = leaves[idx] + lastleaf = leaves.pop() + if idx + 1 < i: + leaves[idx] = lastleaf + leaf.append([]) + leaf.append([]) + leaves.append(leaf[0]) + leaves.append(leaf[1]) + for leaf in leaves: + if random.random() >= unassigned_prob: + leaf.append(random.randrange(1, max_asn + 1)) + else: + leaf.append(0) + #pylint: disable=protected-access + ret._set_trie(trie) + return ret + + def _to_binnode(self, fill: bool = False) -> _BinNode: + """Convert a trie to a _BinNode object.""" + def recurse(node: List) -> Tuple[Dict[Optional[int], _BinNode], bool]: + if len(node) == 1 and node[0] == 0: + return {(None if fill else 0): _BinNode.make_end()}, True + if len(node) == 1: + return {None: _BinNode.make_leaf(node[0]), node[0]: _BinNode.make_end()}, False + ret: Dict[Optional[int], _BinNode] = {} + left, lhole = recurse(node[0]) + right, rhole = recurse(node[1]) + hole = (lhole or rhole) and not fill + + def candidate(ctx: Optional[int], arg1, arg2, func: Callable): + if arg1 is not None and arg2 is not None: + cand = func(arg1, arg2) + if ctx not in ret or cand.size < ret[ctx].size: + ret[ctx] = cand + + for ctx in set(left) | set(right): + candidate(ctx, left.get(ctx), right.get(ctx), _BinNode.make_branch) + candidate(ctx, left.get(None), right.get(ctx), _BinNode.make_branch) + candidate(ctx, left.get(ctx), right.get(None), _BinNode.make_branch) + if not hole: + for ctx in set(ret) - set([None]): + candidate(None, ctx, ret[ctx], _BinNode.make_default) + if None in ret: + ret = {ctx:enc for ctx, enc in ret.items() + if ctx is None or enc.size < ret[None].size} + if hole: + ret = {ctx:enc for ctx, enc in ret.items() if ctx is None or ctx == 0} + return ret, hole + res, _ = recurse(self._trie) + return res[0] if 0 in res else res[None] + + @staticmethod + def _from_binnode(binnode: _BinNode) -> "ASMap": + """Construct an ASMap object from a _BinNode. Internal use only.""" + def recurse(node: _BinNode, default: int) -> List: + if node.ins == _Instruction.RETURN: + return [node.arg1] + if node.ins == _Instruction.JUMP: + return [recurse(node.arg1, default), recurse(node.arg2, default)] + if node.ins == _Instruction.MATCH: + val = node.arg1 + sub = recurse(node.arg2, default) + while val >= 2: + bit = val & 1 + val >>= 1 + if bit: + sub = [[default], sub] + else: + sub = [sub, [default]] + return sub + assert node.ins == _Instruction.DEFAULT + return recurse(node.arg2, node.arg1) + ret = ASMap() + if binnode.ins != _Instruction.END: + #pylint: disable=protected-access + ret._set_trie(recurse(binnode, 0)) + return ret + + def to_binary(self, fill: bool = False) -> bytes: + """ + Convert this ASMap object to binary. + + Argument: + fill: permit the resulting binary encoder to contain mappers for + unassigned subnets in this ASMap object. Doing so may + reduce the size of the encoding. + Returns: + A bytes object with the encoding of this ASMap object. + """ + bits: List[int] = [] + + def recurse(node: _BinNode) -> None: + _CODER_INS.encode(node.ins.value, bits) + if node.ins == _Instruction.RETURN: + _CODER_ASN.encode(node.arg1, bits) + elif node.ins == _Instruction.JUMP: + _CODER_JUMP.encode(node.arg1.size, bits) + recurse(node.arg1) + recurse(node.arg2) + elif node.ins == _Instruction.DEFAULT: + _CODER_ASN.encode(node.arg1, bits) + recurse(node.arg2) + else: + assert node.ins == _Instruction.MATCH + _CODER_MATCH.encode(node.arg1, bits) + recurse(node.arg2) + + binnode = self._to_binnode(fill) + if binnode.ins != _Instruction.END: + recurse(binnode) + + val = 0 + nbits = 0 + ret = [] + for bit in bits: + val += (bit << nbits) + nbits += 1 + if nbits == 8: + ret.append(val) + val = 0 + nbits = 0 + if nbits: + ret.append(val) + return bytes(ret) + + @staticmethod + def from_binary(bindata: bytes) -> Optional["ASMap"]: + """Decode an ASMap object from the provided binary encoding.""" + + bits: List[int] = [] + for byte in bindata: + bits.extend((byte >> i) & 1 for i in range(8)) + + def recurse(bitpos: int) -> Tuple[_BinNode, int]: + insval, bitpos = _CODER_INS.decode(bits, bitpos) + ins = _Instruction(insval) + if ins == _Instruction.RETURN: + asn, bitpos = _CODER_ASN.decode(bits, bitpos) + return _BinNode(ins, asn), bitpos + if ins == _Instruction.JUMP: + jump, bitpos = _CODER_JUMP.decode(bits, bitpos) + left, bitpos1 = recurse(bitpos) + if bitpos1 != bitpos + jump: + raise ValueError("Inconsistent jump") + right, bitpos = recurse(bitpos1) + return _BinNode(ins, left, right), bitpos + if ins == _Instruction.MATCH: + match, bitpos = _CODER_MATCH.decode(bits, bitpos) + sub, bitpos = recurse(bitpos) + return _BinNode(ins, match, sub), bitpos + assert ins == _Instruction.DEFAULT + asn, bitpos = _CODER_ASN.decode(bits, bitpos) + sub, bitpos = recurse(bitpos) + return _BinNode(ins, asn, sub), bitpos + + if len(bits) == 0: + binnode = _BinNode(_Instruction.END) + else: + try: + binnode, bitpos = recurse(0) + except (ValueError, IndexError): + return None + if bitpos < len(bits) - 7: + return None + if not all(bit == 0 for bit in bits[bitpos:]): + return None + + return ASMap._from_binnode(binnode) + + def __lt__(self, other: "ASMap") -> bool: + return self._trie < other._trie + + def __eq__(self, other: object) -> bool: + if isinstance(other, ASMap): + return self._trie == other._trie + return False + + def extends(self, req: "ASMap") -> bool: + """Determine whether this matches req for all subranges where req is assigned.""" + def recurse(actual: List, require: List) -> bool: + if len(require) == 1 and require[0] == 0: + return True + if len(require) == 1: + if len(actual) == 1: + return bool(require[0] == actual[0]) + return recurse(actual[0], require) and recurse(actual[1], require) + if len(actual) == 2: + return recurse(actual[0], require[0]) and recurse(actual[1], require[1]) + return recurse(actual, require[0]) and recurse(actual, require[1]) + assert isinstance(req, ASMap) + #pylint: disable=protected-access + return recurse(self._trie, req._trie) + + def diff(self, other: "ASMap") -> List[ASNDiff]: + """Compute the diff from self to other.""" + prefix: List[bool] = [] + ret: List[ASNDiff] = [] + + def recurse(old_node: List, new_node: List): + if len(old_node) == 1 and len(new_node) == 1: + if old_node[0] != new_node[0]: + ret.append((list(prefix), old_node[0], new_node[0])) + else: + old_left: List = old_node if len(old_node) == 1 else old_node[0] + old_right: List = old_node if len(old_node) == 1 else old_node[1] + new_left: List = new_node if len(new_node) == 1 else new_node[0] + new_right: List = new_node if len(new_node) == 1 else new_node[1] + prefix.append(False) + recurse(old_left, new_left) + prefix[-1] = True + recurse(old_right, new_right) + prefix.pop() + assert isinstance(other, ASMap) + #pylint: disable=protected-access + recurse(self._trie, other._trie) + return ret + + def __copy__(self) -> "ASMap": + """Construct a copy of this ASMap object. Its state will not be shared.""" + ret = ASMap() + #pylint: disable=protected-access + ret._set_trie(copy.deepcopy(self._trie)) + return ret + + def __deepcopy__(self, _) -> "ASMap": + # ASMap objects do not allow sharing of the _trie member, so we don't need the memoization. + return self.__copy__() + + +class TestASMap(unittest.TestCase): + """Unit tests for this module.""" + + def test_ipv6_prefix_roundtrips(self) -> None: + """Test that random IPv6 network ranges roundtrip through prefix encoding.""" + for _ in range(20): + net_bits = random.getrandbits(128) + for prefix_len in range(0, 129): + masked_bits = (net_bits >> (128 - prefix_len)) << (128 - prefix_len) + net = ipaddress.IPv6Network((masked_bits.to_bytes(16, 'big'), prefix_len)) + prefix = net_to_prefix(net) + self.assertTrue(len(prefix) <= 128) + net2 = prefix_to_net(prefix) + self.assertEqual(net, net2) + + def test_ipv4_prefix_roundtrips(self) -> None: + """Test that random IPv4 network ranges roundtrip through prefix encoding.""" + for _ in range(100): + net_bits = random.getrandbits(32) + for prefix_len in range(0, 33): + masked_bits = (net_bits >> (32 - prefix_len)) << (32 - prefix_len) + net = ipaddress.IPv4Network((masked_bits.to_bytes(4, 'big'), prefix_len)) + prefix = net_to_prefix(net) + self.assertTrue(32 <= len(prefix) <= 128) + net2 = prefix_to_net(prefix) + self.assertEqual(net, net2) + + def test_asmap_roundtrips(self) -> None: + """Test case that verifies random ASMap objects roundtrip to/from entries/binary.""" + # Iterate over the number of leaves the random test ASMap objects have. + for leaves in range(1, 20): + # Iterate over the number of bits in the AS numbers used. + for asnbits in range(0, 24): + # Iterate over the probability that leaves are unassigned. + for pct in range(101): + # Construct a random ASMap object according to the above parameters. + asmap = ASMap.from_random(num_leaves=leaves, max_asn=1 + (1 << asnbits), + unassigned_prob=0.01 * pct) + # Run tests for to_entries and construction from those entries, both + # for overlapping and non-overlapping ones. + for overlapping in [False, True]: + entries = asmap.to_entries(overlapping=overlapping, fill=False) + random.shuffle(entries) + asmap2 = ASMap(entries) + assert asmap2 is not None + self.assertEqual(asmap2, asmap) + entries = asmap.to_entries(overlapping=overlapping, fill=True) + random.shuffle(entries) + asmap2 = ASMap(entries) + assert asmap2 is not None + self.assertTrue(asmap2.extends(asmap)) + + # Run tests for to_binary and construction from binary. + enc = asmap.to_binary(fill=False) + asmap3 = ASMap.from_binary(enc) + assert asmap3 is not None + self.assertEqual(asmap3, asmap) + enc = asmap.to_binary(fill=True) + asmap3 = ASMap.from_binary(enc) + assert asmap3 is not None + self.assertTrue(asmap3.extends(asmap)) + + def test_patching(self) -> None: + """Test behavior of update, lookup, extends, and diff.""" + #pylint: disable=too-many-locals,too-many-nested-blocks + # Iterate over the number of leaves the random test ASMap objects have. + for leaves in range(1, 20): + # Iterate over the number of bits in the AS numbers used. + for asnbits in range(0, 10): + # Iterate over the probability that leaves are unassigned. + for pct in range(0, 101): + # Construct a random ASMap object according to the above parameters. + asmap = ASMap.from_random(num_leaves=leaves, max_asn=1 + (1 << asnbits), + unassigned_prob=0.01 * pct) + # Make a copy of that asmap object to which patches will be applied. + # It starts off being equal to asmap. + patched = copy.copy(asmap) + # Keep a list of patches performed. + patches: List[ASNEntry] = [] + # Initially there cannot be any difference. + self.assertEqual(asmap.diff(patched), []) + # Make 5 patches, each building on top of the previous ones. + for _ in range(0, 5): + # Construct a random path and new ASN to assign it to, apply it to patched, + # and remember it in patches. + pathlen = random.randrange(5) + path = [random.getrandbits(1) != 0 for _ in range(pathlen)] + newasn = random.randrange(1 + (1 << asnbits)) + patched.update(path, newasn) + patches = [(path, newasn)] + patches + + # Compute the diff, and whether asmap extends patched, and the other way + # around. + diff = asmap.diff(patched) + self.assertEqual(asmap == patched, len(diff) == 0) + extends = asmap.extends(patched) + back_extends = patched.extends(asmap) + # Determine whether those extends results are consistent with the diff + # result. + self.assertEqual(extends, all(d[2] == 0 for d in diff)) + self.assertEqual(back_extends, all(d[1] == 0 for d in diff)) + # For every diff found: + for path, old_asn, new_asn in diff: + # Verify asmap and patched actually differ there. + self.assertTrue(old_asn != new_asn) + self.assertEqual(asmap.lookup(path), old_asn) + self.assertEqual(patched.lookup(path), new_asn) + for _ in range(2): + # Extend the path far enough that it's smaller than any mapped + # range, and check the lookup holds there too. + spec_path = list(path) + while len(spec_path) < 32: + spec_path.append(random.getrandbits(1) != 0) + self.assertEqual(asmap.lookup(spec_path), old_asn) + self.assertEqual(patched.lookup(spec_path), new_asn) + # Search through the list of performed patches to find the last one + # applying to the extended path (note that patches is in reverse + # order, so the first match should work). + found = False + for patch_path, patch_asn in patches: + if spec_path[:len(patch_path)] == patch_path: + # When found, it must match whatever the result was patched + # to. + self.assertEqual(new_asn, patch_asn) + found = True + break + # And such a patch must exist. + self.assertTrue(found) + +if __name__ == '__main__': + unittest.main() diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py index 44345e39874..e921757802a 100755 --- a/contrib/seeds/generate-seeds.py +++ b/contrib/seeds/generate-seeds.py @@ -3,7 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' -Script to generate list of seed nodes for chainparams.cpp. +Script to generate list of seed nodes for kernel/chainparams.cpp. This script expects two text files in the directory that is passed as an argument: @@ -70,13 +70,13 @@ def name_to_bip155(addr): if i == 0 or i == (len(addr)-1): # skip empty component at beginning or end continue x += 1 # :: skips to suffix - assert(x < 2) + assert x < 2 else: # two bytes per component val = int(comp, 16) sub[x].append(val >> 8) sub[x].append(val & 0xff) nullbytes = 16 - len(sub[0]) - len(sub[1]) - assert((x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0)) + assert (x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0) addr_bytes = bytes(sub[0] + ([0] * nullbytes) + sub[1]) if addr_bytes[0] == 0xfc: # Assume that seeds with fc00::/8 addresses belong to CJDNS, diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py index 2b377f6c012..af408c2df53 100755 --- a/contrib/seeds/makeseeds.py +++ b/contrib/seeds/makeseeds.py @@ -1,27 +1,28 @@ #!/usr/bin/env python3 -# Copyright (c) 2013-2020 The Bitcoin Core developers +# Copyright (c) 2013-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # # Generate seeds.txt from Pieter's DNS seeder # +import argparse +import collections +import ipaddress import re import sys -import dns.resolver -import collections +from typing import List, Dict, Union -NSEEDS=512 +from asmap import ASMap, net_to_prefix -MAX_SEEDS_PER_ASN=2 - -MIN_BLOCKS = 337600 +NSEEDS=512 -# These are hosts that have been observed to be behaving strangely (e.g. -# aggressively connecting to every node). -with open("suspicious_hosts.txt", mode="r", encoding="utf-8") as f: - SUSPICIOUS_HOSTS = {s.strip() for s in f if s.strip()} +MAX_SEEDS_PER_ASN = { + 'ipv4': 2, + 'ipv6': 10, +} +MIN_BLOCKS = 730000 PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$") PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$") @@ -36,14 +37,26 @@ r"0.19.(0|1|2|99)|" r"0.20.(0|1|2|99)|" r"0.21.(0|1|2|99)|" - r"22.(0|99)|" - r"23.99" + r"22.(0|1|99)|" + r"23.(0|1|99)|" + r"24.(0|1|99)|" + r"25.99" r")") -def parseline(line): +def parseline(line: str) -> Union[dict, None]: + """ Parses a line from `seeds_main.txt` into a dictionary of details for that line. + or `None`, if the line could not be parsed. + """ + if line.startswith('#'): + # Ignore line that starts with comment + return None sline = line.split() if len(sline) < 11: - return None + # line too short to be valid, skip it. + return None + # Skip bad results. + if int(sline[1]) == 0: + return None m = PATTERN_IPV4.match(sline[0]) sortkey = None ip = None @@ -77,9 +90,6 @@ def parseline(line): sortkey = ip ipstr = m.group(1) port = int(m.group(6)) - # Skip bad results. - if sline[1] == 0: - return None # Extract uptime %. uptime30 = float(sline[7][:-1]) # Extract Unix timestamp of last success. @@ -107,98 +117,97 @@ def parseline(line): 'sortkey': sortkey, } -def dedup(ips): - '''deduplicate by address,port''' +def dedup(ips: List[Dict]) -> List[Dict]: + """ Remove duplicates from `ips` where multiple ips share address and port. """ d = {} for ip in ips: d[ip['ip'],ip['port']] = ip return list(d.values()) -def filtermultiport(ips): - '''Filter out hosts with more nodes per IP''' +def filtermultiport(ips: List[Dict]) -> List[Dict]: + """ Filter out hosts with more nodes per IP""" hist = collections.defaultdict(list) for ip in ips: hist[ip['sortkey']].append(ip) return [value[0] for (key,value) in list(hist.items()) if len(value)==1] -def lookup_asn(net, ip): - ''' - Look up the asn for an IP (4 or 6) address by querying cymru.com, or None - if it could not be found. - ''' - try: - if net == 'ipv4': - ipaddr = ip - prefix = '.origin' - else: # http://www.team-cymru.com/IP-ASN-mapping.html - res = str() # 2001:4860:b002:23::68 - for nb in ip.split(':')[:4]: # pick the first 4 nibbles - for c in nb.zfill(4): # right padded with '0' - res += c + '.' # 2001 4860 b002 0023 - ipaddr = res.rstrip('.') # 2.0.0.1.4.8.6.0.b.0.0.2.0.0.2.3 - prefix = '.origin6' - - asn = int([x.to_text() for x in dns.resolver.resolve('.'.join( - reversed(ipaddr.split('.'))) + prefix + '.asn.cymru.com', - 'TXT').response.answer][0].split('\"')[1].split(' ')[0]) - return asn - except Exception as e: - sys.stderr.write(f'ERR: Could not resolve ASN for "{ip}": {e}\n') - return None - # Based on Greg Maxwell's seed_filter.py -def filterbyasn(ips, max_per_asn, max_per_net): +def filterbyasn(asmap: ASMap, ips: List[Dict], max_per_asn: Dict, max_per_net: int) -> List[Dict]: + """ Prunes `ips` by + (a) trimming ips to have at most `max_per_net` ips from each net (e.g. ipv4, ipv6); and + (b) trimming ips to have at most `max_per_asn` ips from each asn in each net. + """ # Sift out ips by type ips_ipv46 = [ip for ip in ips if ip['net'] in ['ipv4', 'ipv6']] ips_onion = [ip for ip in ips if ip['net'] == 'onion'] # Filter IPv46 by ASN, and limit to max_per_net per network result = [] - net_count = collections.defaultdict(int) - asn_count = collections.defaultdict(int) - for ip in ips_ipv46: + net_count: Dict[str, int] = collections.defaultdict(int) + asn_count: Dict[int, int] = collections.defaultdict(int) + + for i, ip in enumerate(ips_ipv46): if net_count[ip['net']] == max_per_net: + # do not add this ip as we already too many + # ips from this network continue - asn = lookup_asn(ip['net'], ip['ip']) - if asn is None or asn_count[asn] == max_per_asn: + asn = asmap.lookup(net_to_prefix(ipaddress.ip_network(ip['ip']))) + if not asn or asn_count[ip['net'], asn] == max_per_asn[ip['net']]: + # do not add this ip as we already have too many + # ips from this ASN on this network continue - asn_count[asn] += 1 + asn_count[ip['net'], asn] += 1 net_count[ip['net']] += 1 + ip['asn'] = asn result.append(ip) # Add back Onions (up to max_per_net) result.extend(ips_onion[0:max_per_net]) return result -def ip_stats(ips): - hist = collections.defaultdict(int) +def ip_stats(ips: List[Dict]) -> str: + """ Format and return pretty string from `ips`. """ + hist: Dict[str, int] = collections.defaultdict(int) for ip in ips: if ip is not None: hist[ip['net']] += 1 - return '%6d %6d %6d' % (hist['ipv4'], hist['ipv6'], hist['onion']) + return f"{hist['ipv4']:6d} {hist['ipv6']:6d} {hist['onion']:6d}" + +def parse_args(): + argparser = argparse.ArgumentParser(description='Generate a list of bitcoin node seed ip addresses.') + argparser.add_argument("-a","--asmap", help='the location of the asmap asn database file (required)', required=True) + argparser.add_argument("-s","--seeds", help='the location of the DNS seeds file (required)', required=True) + return argparser.parse_args() def main(): - lines = sys.stdin.readlines() + args = parse_args() + + print(f'Loading asmap database "{args.asmap}"…', end='', file=sys.stderr, flush=True) + with open(args.asmap, 'rb') as f: + asmap = ASMap.from_binary(f.read()) + print('Done.', file=sys.stderr) + + print('Loading and parsing DNS seeds…', end='', file=sys.stderr, flush=True) + with open(args.seeds, 'r', encoding='utf8') as f: + lines = f.readlines() ips = [parseline(line) for line in lines] + print('Done.', file=sys.stderr) print('\x1b[7m IPv4 IPv6 Onion Pass \x1b[0m', file=sys.stderr) - print('%s Initial' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Initial', file=sys.stderr) # Skip entries with invalid address. ips = [ip for ip in ips if ip is not None] - print('%s Skip entries with invalid address' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Skip entries with invalid address', file=sys.stderr) # Skip duplicates (in case multiple seeds files were concatenated) ips = dedup(ips) - print('%s After removing duplicates' % (ip_stats(ips)), file=sys.stderr) - # Skip entries from suspicious hosts. - ips = [ip for ip in ips if ip['ip'] not in SUSPICIOUS_HOSTS] - print('%s Skip entries from suspicious hosts' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} After removing duplicates', file=sys.stderr) # Enforce minimal number of blocks. ips = [ip for ip in ips if ip['blocks'] >= MIN_BLOCKS] - print('%s Enforce minimal number of blocks' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Enforce minimal number of blocks', file=sys.stderr) # Require service bit 1. ips = [ip for ip in ips if (ip['service'] & 1) == 1] - print('%s Require service bit 1' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Require service bit 1', file=sys.stderr) # Require at least 50% 30-day uptime for clearnet, 10% for onion. req_uptime = { 'ipv4': 50, @@ -206,25 +215,28 @@ def main(): 'onion': 10, } ips = [ip for ip in ips if ip['uptime'] > req_uptime[ip['net']]] - print('%s Require minimum uptime' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Require minimum uptime', file=sys.stderr) # Require a known and recent user agent. ips = [ip for ip in ips if PATTERN_AGENT.match(ip['agent'])] - print('%s Require a known and recent user agent' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Require a known and recent user agent', file=sys.stderr) # Sort by availability (and use last success as tie breaker) ips.sort(key=lambda x: (x['uptime'], x['lastsuccess'], x['ip']), reverse=True) # Filter out hosts with multiple bitcoin ports, these are likely abusive ips = filtermultiport(ips) - print('%s Filter out hosts with multiple bitcoin ports' % (ip_stats(ips)), file=sys.stderr) + print(f'{ip_stats(ips):s} Filter out hosts with multiple bitcoin ports', file=sys.stderr) # Look up ASNs and limit results, both per ASN and globally. - ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS) - print('%s Look up ASNs and limit results per ASN and per net' % (ip_stats(ips)), file=sys.stderr) + ips = filterbyasn(asmap, ips, MAX_SEEDS_PER_ASN, NSEEDS) + print(f'{ip_stats(ips):s} Look up ASNs and limit results per ASN and per net', file=sys.stderr) # Sort the results by IP address (for deterministic output). ips.sort(key=lambda x: (x['net'], x['sortkey'])) for ip in ips: if ip['net'] == 'ipv6': - print('[%s]:%i' % (ip['ip'], ip['port'])) + print(f"[{ip['ip']}]:{ip['port']}", end="") else: - print('%s:%i' % (ip['ip'], ip['port'])) + print(f"{ip['ip']}:{ip['port']}", end="") + if 'asn' in ip: + print(f" # AS{ip['asn']}", end="") + print() if __name__ == '__main__': main() diff --git a/contrib/seeds/nodes_main.txt b/contrib/seeds/nodes_main.txt index d8e34bdb602..100fe09685f 100644 --- a/contrib/seeds/nodes_main.txt +++ b/contrib/seeds/nodes_main.txt @@ -1,689 +1,896 @@ -2.37.30.144:8777 -2.138.174.158:8333 -2.152.78.124:8333 -5.8.18.154:8333 -5.45.74.50:8333 -5.79.123.3:8333 -5.102.168.217:22220 -5.103.137.146:9333 -5.128.87.126:8333 -5.172.132.200:8333 -5.188.62.18:8333 -5.254.101.226:8334 -8.210.18.56:8333 -8.210.92.32:8333 -14.13.34.225:16181 -14.39.151.167:8333 -18.196.79.108:8333 -18.218.139.58:48333 -20.184.15.116:8433 -23.175.0.220:8333 -23.233.107.21:8333 -24.35.68.229:8333 -24.37.3.26:8333 -24.102.91.203:8333 -24.116.153.115:8333 -24.134.6.165:8333 -24.155.218.13:8333 -24.160.137.173:8333 -24.177.106.85:8333 -24.184.0.146:8333 -24.194.222.116:8333 -24.205.215.192:8333 -27.124.108.19:8333 -31.14.40.64:8333 -31.47.202.112:8333 -31.165.115.7:8333 -34.65.45.157:8333 -34.78.48.104:8333 -34.80.134.68:8333 -34.101.132.198:8333 -34.227.68.216:8333 -35.137.212.22:8333 -35.231.190.134:8333 -37.1.217.35:8333 -37.15.62.32:8333 -37.143.118.174:8333 -37.200.59.67:8333 -37.205.9.165:8333 -38.23.180.228:8333 -38.65.119.26:8333 -38.141.134.140:8333 -39.109.122.127:8444 -41.79.70.146:8333 -41.193.122.191:8333 -43.225.62.107:8333 -45.35.73.152:8333 -45.43.97.103:8333 -45.63.10.52:20008 -45.84.153.40:8333 -45.95.64.225:8333 -45.129.180.214:8333 -45.154.255.162:8333 -45.226.80.102:8333 -46.6.10.230:8333 -46.23.87.218:8333 -46.32.50.98:8333 -46.47.84.85:8333 -46.48.126.58:8333 -46.146.248.89:8333 -46.165.221.209:9333 -46.166.142.2:8333 -46.166.162.45:20001 -46.173.50.58:8333 -46.175.178.3:8333 -46.188.30.118:8333 -46.219.120.59:3673 -46.229.238.187:8333 -47.93.230.171:8333 -47.100.162.210:18332 -47.144.106.249:8333 -47.188.70.205:8333 -47.227.226.242:8333 -50.2.13.164:8333 -50.5.46.195:8333 -50.45.128.28:8333 -51.148.153.60:8333 -51.154.62.103:8333 -51.154.131.18:8333 -51.158.150.155:8333 -51.159.2.218:8333 -54.198.19.34:8333 -58.105.168.41:8333 -58.158.0.86:8333 -60.251.129.61:8336 -61.239.91.250:8333 -62.28.190.194:8333 -62.152.58.16:9421 -62.171.129.32:8333 -62.251.54.163:8333 -63.247.147.166:8333 -64.33.68.176:8333 -64.156.192.61:8333 -64.187.175.226:8333 -64.233.245.39:8333 -64.237.82.149:8333 -65.101.247.26:8333 -66.29.129.218:8333 -66.49.204.11:8333 -66.58.243.215:8333 -66.85.234.129:8333 -66.130.120.52:8333 -67.10.121.145:8333 -67.210.228.203:8333 -67.213.87.21:8333 -68.181.4.12:8333 -69.7.124.146:8333 -69.8.175.201:8333 -69.59.18.22:8333 -69.119.193.9:8333 -69.130.201.27:8333 -69.131.101.176:8333 -70.15.194.32:8333 -70.64.27.12:8333 -72.29.170.151:8333 -72.74.34.99:8333 -72.133.177.119:8333 -73.166.84.222:8333 -74.67.240.204:8333 -74.91.115.229:8333 -74.118.137.119:8333 -74.213.251.203:8333 -74.220.255.190:8333 -76.11.60.155:8333 -76.66.144.127:8333 -77.70.16.245:8333 -77.85.204.149:8333 -77.105.87.97:8333 -77.120.113.69:8433 -77.120.113.71:8433 -77.120.122.116:8433 -77.120.122.118:8433 -77.162.190.90:8333 -77.167.245.239:55544 -77.232.41.189:8333 -78.20.227.249:8333 -78.21.167.8:8333 -78.27.139.13:8333 -78.43.208.25:8333 -78.63.28.146:8333 -78.72.228.239:8333 -78.108.102.8:8333 -78.129.0.39:8333 -78.129.169.69:8333 -79.77.182.180:8333 -79.77.182.183:8333 -79.107.178.59:8333 -80.55.225.158:8333 -80.64.211.102:8333 -80.64.211.103:8333 -80.71.57.50:8333 -80.81.3.27:8333 -80.82.55.43:8333 -80.88.172.227:64264 -80.89.203.172:8001 -80.93.213.246:8333 -80.147.82.165:8333 -80.229.28.60:8333 -80.247.233.40:8333 -80.255.8.93:8333 -81.7.17.202:8333 -81.10.241.165:8333 -81.21.86.157:8333 -81.171.22.143:8333 -81.237.206.224:8343 -82.69.23.195:8333 -82.96.96.40:8333 -82.116.50.101:8333 -82.136.99.122:8333 -82.149.97.25:17567 -82.154.24.209:8333 -82.165.241.50:8333 -82.197.218.253:8333 -82.202.68.231:8333 -83.137.41.10:8333 -83.208.6.211:8333 -83.217.8.31:44420 -83.220.110.48:8333 -83.222.138.85:8333 -83.243.191.199:8333 -84.22.139.57:8333 -84.27.155.17:8333 -84.75.28.247:8333 -84.112.60.16:8333 -84.211.7.56:8333 -84.237.7.249:8333 -85.23.51.177:8333 -85.24.145.198:8333 -85.184.138.108:8333 -85.194.238.134:8333 -85.195.54.110:8333 -85.208.71.36:8333 -85.208.71.39:8333 -85.214.136.45:8333 -85.214.161.252:8333 -85.227.245.128:8333 -86.18.34.243:8333 -86.20.50.170:8333 -86.49.105.90:8333 -86.76.7.132:8333 -86.100.26.188:8333 -86.106.143.143:55373 -86.120.58.66:8333 -86.133.251.239:8901 -86.149.8.23:8901 -87.78.197.234:8333 -87.120.8.5:20008 -87.121.37.156:8333 -88.82.181.44:8333 -88.87.93.52:1691 -88.98.235.134:8333 -88.136.187.214:8333 -88.147.244.250:8333 -88.148.153.148:8333 -88.212.45.166:8333 -88.212.55.138:8333 -89.38.96.153:9273 -89.47.161.135:8333 -89.88.62.190:8333 -89.158.32.44:8333 -89.163.145.240:8333 -89.163.249.234:3673 -89.176.196.80:8333 -89.216.21.96:8333 -90.84.227.255:8333 -90.146.130.214:8333 -90.250.9.1:8333 -91.93.194.154:8333 -91.106.188.229:8333 -91.126.40.109:8333 -91.137.127.123:8333 -91.147.232.98:8333 -91.152.123.18:8333 -91.178.17.120:8333 -91.204.99.178:8333 -91.223.175.14:8333 -92.42.110.242:8333 -92.53.90.84:8333 -92.221.155.228:8333 -93.57.81.162:8333 -93.95.88.13:8333 -93.103.13.1:8333 -93.123.180.164:8333 -93.190.117.26:8333 -94.105.125.240:8333 -94.110.23.215:8333 -94.154.159.99:8333 -94.189.161.119:8333 -94.203.255.70:8333 -94.232.173.93:8333 -95.79.122.99:8333 -95.80.1.110:8333 -95.83.73.31:8333 -95.110.133.223:8333 -95.110.234.93:8333 -95.164.65.194:8333 -95.165.8.182:8333 -95.174.219.101:8333 -95.191.130.100:8333 -95.214.53.154:8333 -95.215.205.180:8333 -96.43.130.234:8333 -98.25.201.31:8333 -98.128.247.182:8333 -98.171.21.129:8333 -99.147.135.161:8333 -101.100.163.118:8327 -102.132.245.16:8333 -102.182.204.96:8333 -102.182.235.245:8333 -103.14.245.250:8333 -103.47.192.15:8333 -103.84.84.250:8335 -103.99.168.130:8333 -103.99.168.140:8333 -103.198.192.14:20008 -103.232.104.227:8333 -104.143.2.195:8333 -104.172.235.227:8333 -104.238.220.199:8333 -107.11.115.68:8333 -107.173.166.43:8333 -108.4.212.83:8333 -109.136.73.97:8333 -109.173.98.23:8333 -109.190.68.116:8333 -109.235.246.60:8333 -109.248.206.13:8333 -110.12.64.96:8333 -111.90.140.46:8333 -111.90.159.184:50001 -113.107.201.131:8333 -115.47.141.250:8885 -116.58.171.67:8333 -116.87.57.218:8333 -116.202.161.56:8333 -117.51.159.130:8333 -118.103.126.140:28333 -121.45.190.210:8333 -121.99.193.25:8333 -122.112.148.153:8339 -122.148.135.234:8333 -128.0.190.26:8333 -128.65.194.136:8333 -129.126.172.115:8333 -129.226.125.10:8333 -131.188.40.191:8333 -134.195.185.52:8333 -135.180.44.61:8333 -136.52.114.123:8333 -136.56.170.96:8333 -137.116.213.143:8333 -137.226.34.46:8333 -138.43.233.57:8333 -139.130.41.82:8333 -140.190.12.129:8333 -142.4.105.77:8333 -142.54.181.218:8333 -143.177.231.247:8333 -143.178.64.10:8333 -144.34.161.65:18333 -146.4.124.134:8333 -146.83.56.69:8333 -146.90.193.68:8333 -146.196.55.156:28833 -148.66.50.50:8335 -148.251.1.20:8343 -151.48.95.212:8333 -151.252.193.245:8333 -152.44.137.83:8333 -152.115.191.196:8333 -154.221.31.86:8333 -156.17.103.2:8088 -157.138.20.22:8333 -158.58.188.37:8333 -158.140.209.79:8333 -159.89.230.128:8333 -159.246.25.52:8333 -160.20.59.250:8433 -162.0.234.190:8333 -162.62.26.218:8333 -162.250.188.194:8333 -162.251.70.82:8333 -163.158.206.255:8333 -164.68.105.105:8333 -165.228.174.117:8333 -166.62.82.103:32771 -166.70.49.26:8333 -166.78.241.9:8333 -166.78.241.25:8333 -167.71.73.244:8333 -167.179.147.155:8333 -168.91.238.8:8333 -172.105.21.216:8333 -172.117.105.95:8333 -173.23.103.30:8000 -173.205.92.151:54805 -173.205.92.154:54805 -173.205.92.157:54805 -173.208.152.218:8333 -173.241.227.243:8333 -174.3.4.232:8333 -174.17.11.22:8333 -174.88.241.167:8333 -174.114.102.41:8333 -174.114.250.86:8333 -174.142.191.136:8333 -175.39.72.87:8333 -176.12.16.135:8333 -176.37.23.30:8333 -176.62.179.221:8333 -176.74.136.237:8333 -176.99.6.226:8333 -176.212.185.153:8333 -177.81.236.117:8333 -178.19.106.26:8333 -178.21.118.178:8333 -178.33.232.69:8333 -178.79.84.139:8333 -178.124.162.209:8333 -178.132.2.246:8333 -178.150.96.46:8333 -178.162.212.44:8333 -178.193.226.120:8333 -178.236.137.63:8333 -180.150.46.187:8333 -181.164.210.228:8530 -183.110.220.210:30301 -184.95.58.166:8336 -184.164.147.82:41333 -184.171.208.109:8333 -185.17.143.220:8333 -185.21.217.49:8333 -185.25.48.184:8333 -185.28.96.16:8333 -185.31.136.246:8333 -185.64.116.15:8333 -185.68.249.91:8333 -185.108.247.190:8333 -185.141.60.36:8333 -185.148.3.227:8333 -185.148.145.74:8333 -185.159.20.143:8333 -185.167.113.59:8333 -185.185.26.141:8111 -185.189.132.178:57780 -185.204.197.112:8333 -185.209.70.17:8333 -185.220.156.193:8333 -185.238.129.113:8333 -185.239.221.5:8333 -185.244.217.39:8333 -185.254.97.164:8333 -186.33.167.11:8333 -188.32.14.31:8334 -188.42.40.234:18333 -188.134.8.36:8333 -188.138.88.14:8333 -188.156.110.239:8333 -188.165.244.143:8333 -188.213.68.38:8333 -188.214.129.65:20012 -188.242.15.74:8333 -188.244.4.78:8333 -189.39.6.82:8333 -189.207.46.32:8333 -189.212.121.74:8333 -192.3.11.20:8333 -192.65.170.15:8333 -192.146.137.44:8333 -192.182.157.119:8333 -192.187.109.141:8333 -192.227.80.83:8333 -193.10.203.23:8334 -193.32.127.160:58477 -193.32.127.162:58477 -193.58.196.212:8333 -193.106.29.106:8333 -193.138.154.43:8333 -193.178.170.232:8333 -193.196.37.62:8333 -193.222.130.14:8333 -193.234.50.227:8333 -194.14.246.205:8333 -194.135.135.69:8333 -194.147.113.201:8333 -194.165.30.20:8333 -194.219.62.23:8333 -195.56.63.4:8333 -195.134.183.188:8333 -195.208.103.30:8444 -195.208.103.31:8444 -198.1.231.6:8333 -198.12.14.136:8333 -198.84.237.70:8333 -198.178.120.5:8112 -199.48.92.184:8333 -199.68.199.19:8333 -199.182.184.204:8333 -199.189.242.141:8333 -199.247.7.208:8333 -200.122.181.37:8333 -201.191.6.103:8333 -202.107.219.130:8333 -202.108.211.135:8333 -203.94.33.112:8333 -203.130.48.117:8885 -203.132.94.196:8333 -203.162.13.181:8332 -204.191.201.43:8333 -204.229.10.90:8333 -205.178.41.124:8333 -206.55.178.157:8333 -206.126.203.8:8333 -206.174.115.96:8333 -206.223.153.52:8333 -207.188.159.25:8333 -207.229.46.80:8333 -209.58.145.157:8333 -209.126.81.147:8333 -209.145.63.150:8333 -209.209.10.30:8333 -209.237.127.227:8333 -212.99.226.36:9020 -212.185.86.84:8333 -212.227.211.87:8333 -213.5.36.58:8333 -213.89.236.219:8333 -213.93.145.183:8333 -213.214.66.182:8333 -216.41.249.178:8333 -216.146.251.8:8333 -216.249.70.22:8333 -217.11.240.4:8333 -217.15.178.7:8333 -217.24.233.116:8333 -217.64.148.98:51401 -217.113.121.169:8333 -217.170.124.170:8333 -220.132.135.54:8333 -220.221.58.25:8333 -220.233.178.199:8333 -221.219.97.105:2001 -[2001:1608:1b:f9::1]:26491 -[2001:1620:510::2]:8333 -[2001:1bc0:c1::2000]:8333 -[2001:470:1f0a:89a::2]:8333 -[2001:470:de5a::ec]:9333 -[2001:4b98:dc0:45:216:3eff:fea2:95cd]:8333 -[2001:4dd0:3564:0:fd76:c1d3:1854:5bd9]:8333 -[2001:4de8:b1b2:1:0:dead:beef:7]:8333 -[2001:638:a000:4140::ffff:191]:8333 -[2001:648:2800:131:4b1f:f6fc:20f7:f99f]:8333 -[2001:678:cc8::1:10:88]:20008 -[2001:67c:26b4:ff00::44]:8333 -[2001:67c:2db8:13::92]:8333 -[2001:7c0:2310:0:f816:3eff:fe6c:4f58]:8333 -[2001:818:ea1b:7600:f053:aade:f47b:b701]:8333 -[2001:8f1:1404:3700:8e49:715a:2e09:b634]:9444 -[2001:985:55a0:1::2]:8333 -[2001:999:270:2c2c:c8b:3a20:3f2f:318f]:8333 -[2001:b07:ac9:442b:79d6:bbbe:b37c:a783]:8333 -[2002:2f5b:a5f9::2f5b:a5f9]:8885 -[2002:b6ff:3dca::b6ff:3dca]:28364 -[2400:2410:cea2:d00:41bc:c9ea:861b:51ee]:8333 -[2400:3b00:20:c:bacb:29ff:feab:8886]:8333 -[2401:d002:3902:700:d72c:5e22:4e95:389d]:8333 -[2403:6200:88a0:fb17:f5f2:d8b5:b7ba:f4d3]:8333 -[2405:9800:b910:5f8e:1830:f630:2cc6:88fb]:8333 -[2405:9800:b970:c64c:109f:74e7:ae5f:87c7]:8333 -[2405:aa00:2::40]:8333 -[2407:8800:bc61:2202:d63d:7eff:fe6c:dc36]:8333 -[2408:8248:7004:f831::83c]:8333 -[2409:10:ca20:1df0:224:e8ff:fe1f:60d9]:8333 -[240b:11:43a1:bd00:e589:f8a7:49b:3b86]:8333 -[240d:1a:791:3400:d65d:64ff:fe28:927e]:8333 -[240d:1a:791:3400:d681:d7ff:fef6:a21e]:10050 -[2600:1700:5b2b:5f::8040]:8333 -[2600:2104:1003:c5ab:dc5e:90ff:fe18:1d08]:8333 -[2600:3c00:e002:2e32::1:14]:8333 -[2600:8805:2400:14e:12dd:b1ff:fef2:3013]:8333 -[2602:ffb8::208:72:57:200]:8333 -[2603:301f:1ebf:e000:e23f:49ff:fee7:7431]:8333 -[2603:6081:1800:6600:16dd:a9ff:feee:b2f3]:8333 -[2604:1380:1000:7400::1]:8333 -[2604:4500::2e06]:8112 -[2604:5500:c134:4000:7285:c2ff:fe4a:e143]:32797 -[2604:5500:c134:4000::3fc]:32797 -[2604:7c00:120:4b::eb24]:8333 -[2605:6400:30:f220::]:8333 -[2605:6f80:0:7:fc1b:ccff:fe8a:d822]:8333 -[2605:ae00:203::203]:8333 -[2605:c000:2a0a:1::102]:8333 -[2605:f700:c0:827:225:90ff:fee3:34a6]:8333 -[2607:9280:b:73b:250:56ff:fe14:25b5]:8333 -[2607:f2f8:ad40:bc1::1]:8333 -[2607:fa18:3a01::20]:8333 -[2620:11c:5001:1118:d267:e5ff:fee9:e673]:8333 -[2620:11c:5001:2199:d267:e5ff:fee9:e673]:8333 -[2620:6:2003:105:2d8:61ff:fe0f:853]:8333 -[2620:6e:a000:1:42:42:42:42]:8333 -[2803:cf00:af8:f200:b89e:cf34:92c7:2d26]:8333 -[2804:14c:65d1:402c:bc53:bf5d:68a:2136]:8333 -[2804:7f1:e783:d401:661c:67ff:feba:5547]:8333 -[2804:d57:5537:4800:21e:67ff:fea8:d798]:8333 -[2804:d57:5537:4800:3615:9eff:fe23:d610]:8333 -[2806:2f0:2080:62a:86f:1a01:c44f:1794]:8333 -[2a00:1028:8382:bf22:5f7f:b78f:2737:7739]:8333 -[2a00:12e0:101:99:20c:29ff:fe29:d03f]:8333 -[2a00:1328:e101:c00::163]:8333 -[2a00:1630:10:1003:0:b19:b00b:babe]:8333 -[2a00:1768:2001:27::ef6a]:8333 -[2a00:1828:a004:2::666]:8333 -[2a00:1838:2a:1400:92e2:baff:fe4a:c416]:8333 -[2a00:1c10:2:709::217]:22220 -[2a00:1f40:5001:108:5d17:7703:b0f5:4133]:8333 -[2a00:6020:15dd:ee00:c8c2:2c77:1749:35db]:8333 -[2a00:6020:b482:9200:491a:358c:d8f7:1da]:8333 -[2a00:7145:c1:1:ae29:727:2b87:f64]:5141 -[2a00:8a60:e012:a00::21]:8333 -[2a00:a040:100:f3:45a5:ac0:fea3:71e1]:8333 -[2a01:488:2000:9801::d]:8333 -[2a01:490:16:301::2]:8333 -[2a01:5200:6c:6162:7a61:746b:6f2e:736b]:8333 -[2a01:6380:fffe:73:4e3:b3cc:a871:36d1]:8333 -[2a01:7a0:2:137c::3]:8333 -[2a01:7c8:aac9:c9:5054:ff:fedf:ff95]:8333 -[2a01:7c8:d001:1c1:5054:ff:feee:3e1a]:8333 -[2a01:8740:1:ffc5::8c6a]:8333 -[2a01:cb00:d3d:7700:227:eff:fe28:c565]:8333 -[2a01:d0:0:1c::253]:8333 -[2a01:d0:bef2::12]:8333 -[2a01:e0a:9fb:b0e0:54f8:1901:6e83:62c1]:8333 -[2a01:e0a:aa7:c8c0:9679:affa:b6e5:efc7]:8333 -[2a02:13b8:f000:101::a]:8333 -[2a02:168:6328:0:2a8:2cff:fe68:e32c]:8333 -[2a02:2780:9000:70::7]:8333 -[2a02:2e02:3900:5400:a099:e1ff:feb6:d0e]:8333 -[2a02:390:9000:0:aaa1:59ff:fe43:b57b]:8333 -[2a02:58:97:7d20::60]:8333 -[2a02:6d40:305e:601:dea6:32ff:fe44:4b25]:8333 -[2a02:7a01::91:228:45:130]:8333 -[2a02:7aa0:1619::adc:8de0]:8333 -[2a02:7b40:3e4d:998d::1]:8333 -[2a02:7b40:592f:a187::1]:8333 -[2a02:8388:e5c6:d380:201:2eff:fe82:b3cc]:8333 -[2a02:9a0:102::110]:8333 -[2a02:a311:8143:8c00::4]:8353 -[2a02:af8:fab0:808:85:234:145:132]:8333 -[2a02:e00:fff0:506::1]:8444 -[2a02:e00:fff0:506::a]:8444 -[2a02:e98:20:1504::1]:8333 -[2a03:4000:47:f1::1]:8333 -[2a03:6000:870:0:46:23:87:218]:8333 -[2a03:7380:3015:524:afc5:d3bc:7c66:8f94]:8333 -[2a03:ec0:0:928:8c00:93ff:fe84:a007]:8333 -[2a03:ec0:0:928::701]:8333 -[2a04:2180:0:2::aa]:8333 -[2a04:52c0:101:29e::]:8333 -[2a04:52c0:103:c455::1]:8333 -[2a04:bc40:1dc3:8d::2:1001]:8333 -[2a05:1500:702:0:1c00:40ff:fe00:c]:8333 -[2a06:dd00:10:3:225:90ff:fe32:64cc]:8333 -[2a06:dd00:1:22:225:90ff:fe0e:bd48]:8333 -[2a07:6b47:100:464::9357:ffda]:8333 -[2a07:a880:4601:1062:b4b4:bd2a:39d4:7acf]:51401 -[2a07:abc4::1:946]:8333 -[2a07:abc4::89:234:180:194]:8333 -[2a09:2681:102::210]:8333 -[2a0a:c801:1:7::183]:8333 -[2a0b:f300:2:6::2]:8333 -[2a0c:59c0:18::a20e]:57658 -[2a0d:5600:24:a8e::a91e]:55373 -[2a0d:eb00:8005:1::13]:8333 -[2a10:4740:45:1:a013:d1ff:fe85:36e3]:8333 -[2a10:8b40:1::103]:8335 -[2c0f:f8f0:da51:0:70c3:eea9:9717:9579]:8333 +1.65.195.98:8333 # AS4760 +2.59.236.56:8333 # AS24904 +2.83.114.20:8333 # AS8657 +2.248.194.16:8333 # AS3301 +5.2.154.6:8333 # AS8708 +5.101.140.30:8333 # AS42831 +5.128.87.126:8333 # AS31200 +5.144.21.49:8333 # AS15600 +5.172.132.104:8333 # AS15600 +5.188.62.18:8333 # AS34665 +5.200.2.180:8333 # AS49544 +8.129.184.255:8333 # AS37963 +8.209.105.138:8333 # AS45102 +12.34.98.148:8333 # AS7018 +14.199.102.151:8333 # AS9269 +18.27.79.17:8333 # AS3 +18.27.124.231:8333 # AS3 +18.216.249.151:8333 # AS16509 +23.88.155.58:8333 # AS10242 +23.93.101.158:8333 # AS46375 +23.109.156.76:8333 # AS7979 +23.175.0.220:8333 # AS395502 +23.175.0.222:8333 # AS395502 +24.232.36.225:8333 # AS7303 +27.124.108.19:8333 # AS58511 +27.148.206.140:8333 # AS4134 +31.7.70.195:8333 # AS49666 +31.25.98.16:8333 # AS48635 +31.41.23.249:8333 # AS31287 +31.47.102.92:8333 # AS8251 +31.47.202.112:8333 # AS34385 +31.165.78.146:8333 # AS6730 +31.165.228.138:8333 # AS6730 +34.64.101.4:8333 # AS139070 +34.105.19.97:8333 # AS15169 +34.126.107.179:8333 # AS396982 +34.126.115.35:8333 # AS396982 +35.245.186.117:8333 # AS15169 +37.15.60.144:8333 # AS12479 +37.16.105.63:8333 # AS20904 +37.120.155.34:8333 # AS9009 +37.120.179.29:8333 # AS47147 +37.139.102.73:8333 # AS35816 +37.193.227.16:8333 # AS31200 +37.220.135.151:8333 # AS41206 +38.53.129.67:8333 # AS40237 +38.54.14.89:8333 # AS138915 +38.141.134.140:8333 # AS174 +38.145.151.150:8333 # AS40545 +41.72.154.66:8333 # AS37153 +43.143.203.198:8333 # AS45090 +45.15.124.117:8333 # AS35913 +45.43.97.103:8333 # AS26827 +45.44.213.116:8333 # AS54198 +45.58.187.101:8333 # AS46844 +45.79.192.236:8333 # AS63949 +45.81.241.97:8333 # AS30823 +45.83.220.102:8333 # AS39351 +45.83.241.46:8333 # AS206238 +45.87.106.57:8333 # AS39238 +45.129.38.5:8333 # AS49666 +45.130.20.177:8333 # AS3214 +45.134.142.40:8333 # AS60068 +45.135.4.143:8333 # AS25596 +45.135.92.127:8333 # AS12555 +45.145.188.112:8333 # AS206805 +46.23.87.218:8333 # AS51088 +46.32.50.98:8333 # AS39642 +46.32.78.17:8333 # AS48416 +46.59.40.91:8333 # AS8473 +46.138.246.77:8333 # AS8359 +46.166.142.2:8333 # AS43350 +46.166.162.59:8333 # AS16125 +46.175.178.3:8333 # AS28725 +46.188.15.6:8333 # AS39153 +46.188.30.118:8333 # AS39153 +46.223.223.216:8333 # AS51185 +46.226.18.135:8333 # AS52176 +47.88.86.79:8333 # AS45102 +47.148.7.69:8333 # AS5650 +47.198.223.60:8333 # AS5650 +50.2.13.164:8333 # AS62904 +50.4.135.84:8333 # AS12083 +50.53.39.237:8333 # AS20055 +50.53.250.162:8333 # AS20055 +50.68.121.44:8333 # AS6327 +50.117.132.178:8333 # AS577 +51.154.62.103:8333 # AS15796 +51.158.150.155:8333 # AS12876 +51.250.46.215:8333 # AS200350 +54.176.63.16:8333 # AS16509 +58.158.0.86:8333 # AS2519 +60.205.205.119:8333 # AS37963 +61.74.99.193:8333 # AS4766 +61.92.59.104:8333 # AS9269 +62.122.173.171:8333 # AS50245 +62.171.129.32:8333 # AS51167 +62.178.27.239:8333 # AS8412 +62.209.210.3:8333 # AS6855 +62.215.127.73:8333 # AS21050 +62.238.148.104:8333 # AS15435 +62.245.153.8:8333 # AS8767 +64.146.136.45:8333 # AS16713 +65.21.134.184:8333 # AS24940 +66.18.13.146:8333 # AS13767 +66.23.233.43:8333 # AS19318 +66.27.98.216:8333 # AS20001 +66.29.129.218:8333 # AS22612 +66.38.94.13:8333 # AS11979 +66.45.141.46:8333 # AS11232 +66.58.243.215:8333 # AS8047 +66.114.33.49:8333 # AS23175 +66.198.211.167:8333 # AS10835 +66.208.64.128:8333 # AS10352 +66.219.196.170:8333 # AS29933 +67.210.228.203:8333 # AS7819 +68.183.75.251:8333 # AS14061 +68.194.125.140:8333 # AS6128 +68.199.120.17:8333 # AS6128 +69.4.94.226:8333 # AS36352 +69.8.175.201:8333 # AS21766 +69.59.18.22:8333 # AS397444 +69.196.152.33:8333 # AS5645 +69.228.219.124:8333 # AS7018 +70.64.27.12:8333 # AS6327 +70.160.240.132:8333 # AS22773 +71.79.109.128:8333 # AS7843 +71.184.193.75:8333 # AS701 +72.15.59.173:8333 # AS21949 +72.48.253.168:8333 # AS7459 +72.207.171.210:8333 # AS22773 +73.117.132.138:8333 # AS7922 +73.212.226.59:8333 # AS7922 +74.76.151.110:8333 # AS7843 +74.91.115.229:8333 # AS14586 +74.118.137.119:8333 # AS20326 +74.213.175.108:8333 # AS21949 +74.213.251.239:8333 # AS14978 +74.220.255.190:8333 # AS23175 +74.221.189.109:8333 # AS26827 +75.83.203.225:8333 # AS20001 +75.172.52.186:8333 # AS209 +76.24.143.22:8333 # AS1351 +76.69.202.247:8333 # AS577 +76.73.198.242:8333 # AS12083 +76.119.248.240:8333 # AS1351 +77.20.48.144:8333 # AS3209 +77.22.152.239:8333 # AS204028 +77.37.224.222:8333 # AS42610 +77.48.196.234:8333 # AS16019 +77.70.16.245:8333 # AS8717 +77.162.190.90:8333 # AS1136 +78.20.227.249:8333 # AS6848 +78.21.167.8:8333 # AS6848 +78.35.147.203:8333 # AS8422 +78.108.108.25:8333 # AS8251 +78.154.237.60:8333 # AS9155 +79.11.31.76:8333 # AS3269 +79.87.88.235:8333 # AS15557 +79.101.1.25:8333 # AS8400 +79.124.7.241:8333 # AS203380 +79.124.7.253:8333 # AS203380 +79.150.68.42:8333 # AS3352 +79.249.10.53:8333 # AS3320 +80.82.21.77:8333 # AS42927 +80.82.76.59:8333 # AS202425 +80.88.172.227:8333 # AS31263 +80.93.213.246:8333 # AS42910 +80.111.142.213:8333 # AS6830 +80.208.227.134:8333 # AS62282 +80.208.228.9:8333 # AS62282 +80.209.64.86:8333 # AS31027 +80.229.28.60:8333 # AS2856 +81.7.16.182:8333 # AS35366 +81.19.10.2:8333 # AS24641 +81.162.196.43:8333 # AS34955 +81.171.22.143:8333 # AS60781 +81.172.221.4:8333 # AS12430 +81.224.44.164:8333 # AS3301 +81.245.96.36:8333 # AS5432 +82.1.68.54:8333 # AS5089 +82.66.10.11:8333 # AS12322 +82.66.211.31:8333 # AS12322 +82.71.4.154:8333 # AS13037 +82.96.96.40:8333 # AS29686 +82.116.50.101:8333 # AS30936 +82.136.98.249:8333 # AS8821 +82.195.237.253:8333 # AS1836 +83.137.41.10:8333 # AS31394 +83.171.175.5:8333 # AS8767 +83.208.193.242:8333 # AS5610 +83.233.76.165:8333 # AS29518 +83.240.89.196:8333 # AS31246 +84.38.3.249:8333 # AS196691 +84.54.23.48:8333 # AS35913 +84.126.216.77:8333 # AS12430 +84.211.187.211:8333 # AS41164 +84.246.200.122:8333 # AS42455 +84.255.244.61:8333 # AS34779 +85.165.42.115:8333 # AS2119 +85.194.238.134:8333 # AS47605 +85.208.69.21:8333 # AS25091 +85.208.71.36:8333 # AS42275 +85.209.240.91:8333 # AS205581 +85.214.118.71:8333 # AS6724 +85.214.161.252:8333 # AS6724 +85.236.190.252:8333 # AS35032 +85.243.115.136:8333 # AS8657 +86.22.20.13:8333 # AS5089 +86.49.34.92:8333 # AS16019 +86.95.8.249:8333 # AS1136 +86.104.228.10:8333 # AS31638 +86.104.228.23:8333 # AS31638 +87.79.94.221:8333 # AS8422 +88.10.89.23:8333 # AS3352 +88.84.223.30:8333 # AS21453 +88.86.125.50:8333 # AS39392 +88.90.77.100:8333 # AS2119 +88.97.40.50:8333 # AS13037 +88.137.109.62:8333 # AS15557 +88.147.244.250:8333 # AS12389 +88.208.115.70:8333 # AS29208 +88.212.53.246:8333 # AS42841 +89.35.142.168:8333 # AS34977 +89.78.111.197:8333 # AS6830 +89.117.59.129:8333 # AS1239 +89.147.108.200:8333 # AS44735 +89.163.132.180:8333 # AS24961 +89.165.232.242:8333 # AS48161 +89.216.21.96:8333 # AS31042 +90.50.172.182:8333 # AS3215 +90.146.130.214:8333 # AS12605 +90.146.208.162:8333 # AS12605 +90.156.26.148:8333 # AS12741 +90.163.172.139:8333 # AS12479 +90.177.163.77:8333 # AS5610 +91.67.145.110:8333 # AS3209 +91.93.194.154:8333 # AS34984 +91.123.182.164:8333 # AS51648 +91.123.183.219:8333 # AS51792 +91.135.0.187:8333 # AS12496 +91.147.232.98:8333 # AS5483 +91.184.168.249:8333 # AS9063 +91.193.237.116:8333 # AS42916 +91.199.41.45:8333 # AS6866 +91.204.149.5:8333 # AS42765 +91.215.91.254:8333 # AS48078 +91.219.25.232:8333 # AS50448 +91.237.88.218:8333 # AS56813 +92.27.150.46:8333 # AS13285 +92.27.150.47:8333 # AS13285 +92.221.20.232:8333 # AS29695 +92.221.126.65:8333 # AS29695 +93.33.192.204:8333 # AS12874 +93.41.237.78:8333 # AS12874 +93.95.88.13:8333 # AS35434 +93.95.227.125:8333 # AS44735 +93.103.13.1:8333 # AS34779 +93.115.86.239:8333 # AS3223 +93.123.180.164:8333 # AS35539 +93.186.201.173:8333 # AS24961 +93.190.117.26:8333 # AS196881 +94.19.7.55:8333 # AS35807 +94.23.21.80:8333 # AS16276 +94.23.205.110:8333 # AS16276 +94.131.0.73:8333 # AS29632 +94.142.237.4:8333 # AS48926 +94.154.159.99:8333 # AS62240 +94.202.50.200:8333 # AS15802 +94.231.253.18:8333 # AS35224 +95.42.140.142:8333 # AS8866 +95.67.18.100:8333 # AS34867 +95.70.238.176:8333 # AS12735 +95.83.73.31:8333 # AS8359 +95.90.128.3:8333 # AS204028 +95.110.234.93:8333 # AS31034 +95.161.12.45:8333 # AS39598 +95.172.62.167:8333 # AS201826 +95.179.128.87:8333 # AS20473 +95.191.130.100:8333 # AS12389 +95.214.53.154:8333 # AS201814 +96.3.53.254:8333 # AS11232 +97.75.145.12:8333 # AS22709 +97.81.198.180:8333 # AS20115 +97.87.216.110:8333 # AS20115 +99.229.210.111:8333 # AS812 +99.246.87.2:8333 # AS812 +101.43.124.195:8333 # AS45090 +102.132.192.141:8333 # AS37680 +103.21.3.89:8333 # AS38195 +103.35.121.72:8333 # AS9498 +103.99.168.100:8333 # AS6939 +103.99.168.140:8333 # AS6939 +103.99.170.210:8333 # AS54415 +103.99.170.220:8333 # AS54415 +103.105.202.50:8333 # AS137764 +104.238.220.199:8333 # AS23470 +104.243.33.165:8333 # AS23470 +104.244.73.6:8333 # AS53667 +108.26.125.214:8333 # AS701 +109.86.60.33:8333 # AS13188 +109.99.63.159:8333 # AS9050 +109.120.194.136:8333 # AS34569 +109.123.233.138:8333 # AS15685 +109.123.240.53:8333 # AS15685 +109.153.94.35:8333 # AS2856 +109.173.126.157:8333 # AS42610 +109.193.76.200:8333 # AS51185 +109.221.229.197:8333 # AS3215 +109.236.90.117:8333 # AS49981 +109.248.206.13:8333 # AS203493 +111.90.140.23:8333 # AS45839 +111.90.140.46:8333 # AS45839 +111.90.145.37:8333 # AS18106 +114.173.159.209:8333 # AS4713 +116.58.171.67:8333 # AS2514 +119.31.179.202:8333 # AS17408 +119.42.55.203:8333 # AS133159 +122.222.160.190:8333 # AS2519 +123.60.213.192:8333 # AS55990 +124.197.54.113:8333 # AS9790 +125.168.140.108:8333 # AS4826 +128.0.190.26:8333 # AS30764 +128.65.194.136:8333 # AS29222 +129.13.189.212:8333 # AS34878 +129.13.189.215:8333 # AS34878 +129.226.216.148:8333 # AS132203 +131.188.40.191:8333 # AS680 +134.65.9.63:8333 # AS19653 +134.122.200.160:8333 # AS64050 +134.195.185.52:8333 # AS13536 +135.19.253.101:8333 # AS5769 +136.29.109.58:8333 # AS19165 +136.32.238.6:8333 # AS16591 +136.49.201.24:8333 # AS16591 +137.226.34.46:8333 # AS680 +138.207.211.189:8333 # AS11776 +139.130.41.82:8333 # AS1221 +140.238.220.99:8333 # AS31898 +142.54.181.218:8333 # AS32097 +142.166.19.23:8333 # AS855 +142.254.87.115:8333 # AS46375 +143.177.229.149:8333 # AS50266 +144.2.101.21:8333 # AS3303 +144.24.236.64:8333 # AS31898 +145.40.51.52:8333 # AS49808 +146.71.69.103:8333 # AS7782 +146.120.241.173:8333 # AS208515 +147.50.238.53:8333 # AS45265 +148.103.101.132:8333 # AS28118 +149.75.48.92:8333 # AS6079 +152.44.137.83:8333 # AS11404 +154.0.3.194:8333 # AS37680 +154.26.137.105:8333 # AS174 +154.26.154.73:8333 # AS1299 +154.57.5.11:8333 # AS200736 +155.4.55.21:8333 # AS8473 +156.146.137.142:8333 # AS1448 +156.146.177.221:8333 # AS1448 +157.22.72.175:8333 # AS397379 +157.97.0.118:8333 # AS43571 +158.140.141.69:8333 # AS132132 +158.181.132.84:8333 # AS41750 +159.2.215.98:8333 # AS855 +159.196.3.239:8333 # AS4764 +159.224.189.250:8333 # AS13188 +160.80.12.16:8333 # AS137 +161.230.38.160:8333 # AS12353 +161.246.11.230:8333 # AS9486 +162.0.210.152:8333 # AS22612 +162.62.18.226:8333 # AS132203 +162.254.118.20:8333 # AS6130 +163.158.168.181:8333 # AS15435 +165.173.19.33:8333 # AS132132 +165.228.174.117:8333 # AS1221 +165.255.241.184:8333 # AS327693 +167.88.11.203:8333 # AS20278 +167.179.147.155:8333 # AS4764 +170.17.151.235:8333 # AS3303 +170.64.174.230:8333 # AS15108 +172.92.102.115:8333 # AS11404 +172.105.21.216:8333 # AS63949 +172.111.176.244:8333 # AS46562 +172.255.98.108:8333 # AS7979 +173.82.5.202:8333 # AS35916 +173.181.35.50:8333 # AS395570 +173.212.253.137:8333 # AS51167 +173.235.73.87:8333 # AS11272 +174.30.29.85:8333 # AS209 +174.141.209.40:8333 # AS6461 +176.9.17.121:8333 # AS24940 +176.12.16.135:8333 # AS8717 +176.74.136.237:8333 # AS35613 +176.74.139.120:8333 # AS35613 +176.122.122.134:8333 # AS50581 +176.126.167.10:8333 # AS8449 +176.151.244.130:8333 # AS5410 +176.186.19.106:8333 # AS5410 +176.212.185.153:8333 # AS9049 +177.142.146.193:8333 # AS4230 +178.21.118.178:8333 # AS49544 +178.61.141.198:8333 # AS21050 +178.124.162.209:8333 # AS6697 +178.143.25.194:8333 # AS15962 +178.154.233.197:8333 # AS200350 +178.159.98.133:8333 # AS202390 +178.232.186.191:8333 # AS41164 +178.236.137.63:8333 # AS44843 +179.60.149.4:8333 # AS395839 +184.160.110.104:8333 # AS5769 +184.174.37.139:8333 # AS1239 +185.8.104.179:8333 # AS16125 +185.14.30.25:8333 # AS21100 +185.25.48.184:8333 # AS61272 +185.52.93.45:8333 # AS39449 +185.64.116.15:8333 # AS31736 +185.69.105.117:8333 # AS6855 +185.98.54.20:8333 # AS39572 +185.107.83.55:8333 # AS43350 +185.132.109.122:8333 # AS38919 +185.135.81.50:8333 # AS57494 +185.140.253.169:8333 # AS200735 +185.148.3.227:8333 # AS47605 +185.154.2.3:8333 # AS29119 +185.162.92.36:8333 # AS41722 +185.163.44.36:8333 # AS39798 +185.165.170.19:8333 # AS3223 +185.167.113.59:8333 # AS207054 +185.185.59.12:8333 # AS48614 +185.203.41.148:8333 # AS9009 +185.209.12.76:8333 # AS212323 +185.209.70.17:8333 # AS204568 +185.210.125.33:8333 # AS205671 +185.233.189.210:8333 # AS61303 +185.238.131.19:8333 # AS206238 +185.239.220.210:8333 # AS61282 +185.239.221.5:8333 # AS61282 +185.250.90.246:8333 # AS61955 +186.249.217.25:8333 # AS7195 +186.250.95.132:8333 # AS262967 +188.35.167.14:8333 # AS34123 +188.68.53.44:8333 # AS47147 +188.120.255.115:8333 # AS29182 +189.6.195.111:8333 # AS28573 +190.2.130.44:8333 # AS49981 +190.13.122.89:8333 # AS33576 +190.123.27.11:8333 # AS52468 +190.145.127.254:8333 # AS14080 +191.220.156.64:8333 # AS8167 +192.31.136.90:8333 # AS54098 +192.69.53.43:8333 # AS11142 +192.146.137.44:8333 # AS25376 +192.174.121.33:8333 # AS11492 +192.222.147.175:8333 # AS1403 +193.198.34.24:8333 # AS2108 +193.222.130.14:8333 # AS29208 +194.35.185.167:8333 # AS9063 +194.54.83.234:8333 # AS41018 +194.233.84.100:8333 # AS141995 +195.2.73.88:8333 # AS48282 +195.48.12.8:8333 # AS1836 +195.154.200.157:8333 # AS12876 +197.211.133.15:8333 # AS51265 +198.84.146.8:8333 # AS5645 +198.98.55.86:8333 # AS53667 +199.247.7.208:8333 # AS20473 +200.116.154.131:8333 # AS13489 +201.191.6.103:8333 # AS11830 +201.221.234.200:8333 # AS27928 +202.47.225.242:8333 # AS9931 +202.107.219.130:8333 # AS4134 +202.108.211.135:8333 # AS4837 +202.138.13.122:8333 # AS4826 +203.86.195.32:8333 # AS23655 +203.184.52.247:8333 # AS9790 +204.111.163.114:8333 # AS4922 +205.178.41.124:8333 # AS11039 +206.192.203.0:8333 # AS7029 +207.229.46.80:8333 # AS852 +207.244.248.81:8333 # AS40021 +207.255.193.47:8333 # AS11776 +208.59.133.63:8333 # AS11039 +209.58.145.157:8333 # AS394380 +209.97.189.249:8333 # AS14061 +209.177.138.245:8333 # AS7832 +209.237.133.54:8333 # AS53859 +210.54.37.190:8333 # AS4648 +210.54.39.238:8333 # AS4648 +212.34.225.118:8333 # AS44395 +212.41.9.30:8333 # AS49505 +212.51.132.176:8333 # AS13030 +212.69.60.77:8333 # AS12496 +212.86.32.106:8333 # AS15366 +213.47.64.105:8333 # AS8412 +213.141.154.201:8333 # AS12714 +213.142.148.169:8333 # AS6762 +213.184.244.24:8333 # AS60280 +213.227.147.244:8333 # AS60781 +213.250.21.112:8333 # AS5603 +216.146.251.8:8333 # AS54579 +216.232.157.104:8333 # AS395570 +217.15.178.11:8333 # AS25534 +217.26.32.10:8333 # AS197312 +217.64.47.200:8333 # AS39324 +217.76.51.25:8333 # AS39597 +217.92.55.246:8333 # AS3320 +217.170.124.170:8333 # AS35401 +217.180.221.162:8333 # AS30600 +217.180.238.137:8333 # AS30600 +220.84.232.46:8333 # AS4766 +220.133.39.61:8333 # AS3462 +220.233.91.182:8333 # AS38195 +[2001:19f0:1000:1db3:5400:4ff:fe56:5a8d]:8333 # AS20473 +[2001:19f0:5:24da:3eec:efff:feb9:f36e]:8333 # AS20473 +[2001:19f0:5:24da::]:8333 # AS20473 +[2001:19f0:5:4535:3eec:efff:feb9:87e4]:8333 # AS20473 +[2001:19f0:5:4535::]:8333 # AS20473 +[2001:1bc0:c1::2000]:8333 # AS29686 +[2001:1c04:4008:6300:8a5f:2678:114b:a660]:8333 # AS6830 +[2001:41d0:203:3739::]:8333 # AS16276 +[2001:41d0:203:8f49::]:8333 # AS16276 +[2001:41d0:203:bb0a::]:8333 # AS16276 +[2001:41d0:2:bf8f::]:8333 # AS16276 +[2001:41d0:303:de8b::]:8333 # AS16276 +[2001:41d0:403:3d61::]:8333 # AS16276 +[2001:41d0:405:9600::]:8333 # AS16276 +[2001:41d0:8:ed7f::1]:8333 # AS16276 +[2001:41d0:a:69a2::1]:8333 # AS16276 +[2001:41f0::62:6974:636f:696e]:8333 # AS6830 +[2001:470:1b62::]:8333 # AS6939 +[2001:470:1f05:43b:2831:8530:7179:5864]:8333 # AS6939 +[2001:470:1f09:b14::11]:8333 # AS6939 +[2001:470:1f15:106:e2d5:5eff:fe42:7ae5]:8333 # AS6939 +[2001:470:1f1b:365:aa20:66ff:fe3f:1909]:8333 # AS6939 +[2001:470:1f1b:5a6:216:3eff:fe24:1162]:8333 # AS6939 +[2001:470:6a7c::]:8333 # AS6939 +[2001:470:75e9:1::10]:8333 # AS6939 +[2001:470:8ca0:2:4e72:b9ff:fe56:f8b8]:8333 # AS6939 +[2001:470:dbc7:0:1010::100]:8333 # AS6939 +[2001:4ba0:cafe:14cc::1]:8333 # AS24961 +[2001:4ba0:ffff:24::1]:8333 # AS24961 +[2001:4dd0:3564:0:30b7:1d7b:6fec:4c5c]:8333 # AS8422 +[2001:4dd0:3564:0:88e:b4ff:2ad0:699b]:8333 # AS8422 +[2001:4dd0:3564:0:9c1c:cc31:9fe8:5505]:8333 # AS8422 +[2001:4dd0:3564:0:a0c4:d41f:4c4:1bb0]:8333 # AS8422 +[2001:4dd0:3564:1::7676:8090]:8333 # AS8422 +[2001:4dd0:3564:1:b977:bd71:4612:8e40]:8333 # AS8422 +[2001:4dd0:af0e:3564::69:1]:8333 # AS8422 +[2001:4dd0:af0e:3564::69:90]:8333 # AS8422 +[2001:560:441f:1::4]:8333 # AS18530 +[2001:638:a000:4140::ffff:191]:8333 # AS680 +[2001:67c:25dc:91::2]:8333 # AS41018 +[2001:67c:26b4:ff00::44]:8333 # AS25376 +[2001:67c:2db8:6::36]:8333 # AS39798 +[2001:7c0:2310:0:f816:3eff:fe6c:4f58]:8333 # AS34878 +[2001:861:3242:8420::40]:8333 # AS5410 +[2001:8b0:1301:1000::60]:8333 # AS20712 +[2001:b030:2422::208d]:8333 # AS3462 +[2001:b07:2ef:6e4a:3d:974e:784a:684b]:8333 # AS12874 +[2001:b07:5d32:b142:8f77:3c7d:a2fd:ed2e]:8333 # AS12874 +[2001:b07:6461:7811:489:d2da:e07:1af7]:8333 # AS12874 +[2001:b07:646b:8074:32e8:9243:a337:e60a]:8333 # AS12874 +[2001:b07:646b:8074:4cc6:79a5:3af7:7132]:8333 # AS12874 +[2001:b07:ad4:ca4b:7dd5:8471:50c3:5363]:8333 # AS12874 +[2001:bc8:1201:71a:2e59:e5ff:fe42:52f4]:8333 # AS12876 +[2001:bc8:1600:0:208:a2ff:fe0c:8a2e]:8333 # AS12876 +[2001:bc8:323c:ff:a634:384f:1849:f4bc]:8333 # AS12876 +[2001:bc8:323c:ff:d217:c2ff:fe07:2cd9]:8333 # AS12876 +[2001:bc8:700:2b14::1]:8333 # AS12876 +[2001:bc8:700:8d16::1]:8333 # AS12876 +[2001:e68:5400:58d0:bd15:ea8c:5b20:7523]:8333 # AS4788 +[2400:2411:a3e1:4900:7298:f550:67e7:b99b]:8333 # AS17676 +[2400:8901::f03c:93ff:fe2b:5c0b]:8333 # AS63949 +[2400:8901::f03c:93ff:fe5a:685c]:8333 # AS63949 +[2401:b140:1::100:210]:8333 # AS54415 +[2401:b140:1::100:220]:8333 # AS54415 +[2401:d002:3902:700:d72c:5e22:4e95:389d]:8333 # AS38195 +[2404:4408:63a4:a01::250]:8333 # AS9790 +[2406:3400:216:8b00:211:32ff:feca:336b]:8333 # AS10143 +[2406:8c00:0:3422:133:18:228:108]:8333 # AS24282 +[2406:da11:169:b03:32b5:f901:9f7c:3e4b]:8333 # AS16509 +[2406:da18:9f1:f301:7d2e:c256:c112:f2be]:8333 # AS16509 +[2406:da18:9f1:f303:c1c9:c569:b799:2057]:8333 # AS16509 +[2406:da1e:a4e:8a00:20db:dd8d:3670:28f0]:8333 # AS16509 +[2406:da1e:a4e:8a03:2aad:496b:768d:e497]:8333 # AS16509 +[2407:3640:2107:1278::1]:8333 # AS141995 +[2407:3640:3010:4012::1]:8333 # AS141995 +[2407:8800:bc61:2202:d63d:7eff:fe6c:dc36]:8333 # AS7545 +[2600:1700:5c5b:b0:aaa1:59ff:fe5f:615a]:8333 # AS7018 +[2600:1700:ec7b:5730::48]:8333 # AS7018 +[2600:1900:4000:4cc4:0:1::]:8333 # AS15169 +[2600:1900:4000:4cc4:0:2::]:8333 # AS15169 +[2600:1900:4000:4cc4:0:3::]:8333 # AS15169 +[2600:1900:4000:4cc4::]:8333 # AS15169 +[2600:1900:4030:a25e::]:8333 # AS15169 +[2600:1f14:40e:e301:afdd:ad00:e568:d220]:8333 # AS16509 +[2600:1f1c:2d3:2400:f15e:2f2a:760d:a33d]:8333 # AS16509 +[2600:2104:1003:c5ab:dc5e:90ff:fe18:1d08]:8333 # AS11404 +[2600:3c00::f03c:92ff:fe92:2745]:8333 # AS63949 +[2600:3c00::f03c:92ff:fecf:61b6]:8333 # AS63949 +[2600:3c00:e002:2e32::1:14]:8333 # AS63949 +[2600:3c01::f03c:93ff:fe2a:5266]:8333 # AS63949 +[2600:3c01::f03c:93ff:fe74:5f59]:8333 # AS63949 +[2600:3c01::f03c:93ff:fee6:2146]:8333 # AS63949 +[2600:3c02::f03c:92ff:fe5d:9fb]:8333 # AS63949 +[2600:4040:2004:3201:459f:8fe8:444d:baf1]:8333 # AS13786 +[2600:4040:4541:4900:4e1:b58a:8438:450e]:8333 # AS13786 +[2600:6c54:7100:1ad1:c92e:36d:651:bd18]:8333 # AS20115 +[2600:8801:2f80:477::141c]:8333 # AS22773 +[2600:8801:8d00:3eb0:20c:29ff:fec3:d799]:8333 # AS22773 +[2600:8805:2400:14e:12dd:b1ff:fef2:3013]:8333 # AS22773 +[2601:184:300:156c:ba4c:30:9da:6c06]:8333 # AS7922 +[2601:346:d7f:fff7:18c6:4856:ef75:744c]:8333 # AS7922 +[2601:405:4a00:876:c8d3:f081:2ce8:ba8e]:8333 # AS7922 +[2602:24c:b8f:cd90::7840]:8333 # AS46375 +[2602:fec3:0:1::69]:8333 # AS62563 +[2602:ff16:1:0:1:412:0:1]:8333 # AS29802 +[2603:3001:2618:c000:2ec1:df1f:a463:9119]:8333 # AS7922 +[2603:3003:11b:e100:20c:29ff:fe38:bbc0]:8333 # AS7922 +[2603:3004:6a1:3800:851f:584d:7aba:affb]:8333 # AS7922 +[2603:3004:6a1:3800::7bba]:8333 # AS7922 +[2603:3004:6a1:3800::f667]:8333 # AS7922 +[2603:3024:1606:1400::29ec]:8333 # AS7922 +[2603:3024:18ee:8000:20e:c4ff:fed1:ef15]:8333 # AS7922 +[2603:6000:a400:9300::2000]:8333 # AS7843 +[2603:6010:7001:4830::2:1]:8333 # AS7843 +[2603:8080:1f07:6fdd:7de2:d969:78c9:b7ea]:8333 # AS7843 +[2603:8080:d600:1800:7ce1:74a2:6a8a:4643]:8333 # AS7843 +[2603:8081:6c00:306e:215:5dff:fe02:150a]:8333 # AS7843 +[2604:3d09:7182:8700:bba9:cde6:5b37:a8df]:8333 # AS6327 +[2604:4080:1036:80b1::3be]:8333 # AS11404 +[2604:a00:3:1223:216:3eff:fe27:76e0]:8333 # AS19318 +[2604:a880:400:d0::261f:6001]:8333 # AS14061 +[2604:a880:4:1d0::13e:f000]:8333 # AS14061 +[2604:a880:4:1d0::17a:7000]:8333 # AS14061 +[2604:a880:4:1d0::c1:3000]:8333 # AS14061 +[2604:a880:4:1d0::e5:b000]:8333 # AS14061 +[2605:4a80:a302:7940:7254:1ed4:90d7:4f39]:8333 # AS11232 +[2605:4a80:a302:7940::2]:8333 # AS11232 +[2605:6400:30:f220::]:8333 # AS53667 +[2605:a140:3010:4014::1]:8333 # AS40021 +[2605:ae00:203::203]:8333 # AS7819 +[2605:b40:14d0:5b00:7988:eb8:6bb6:66e2]:8333 # AS174 +[2605:c000:2a0a:1::102]:8333 # AS7393 +[2607:5300:61:854::1]:8333 # AS16276 +[2607:9280:b:73b:250:56ff:fe14:25b5]:8333 # AS395502 +[2607:9280:b:73b:250:56ff:fe21:9c2f]:8333 # AS395502 +[2607:9280:b:73b:250:56ff:fe21:bf32]:8333 # AS395502 +[2607:9280:b:73b:250:56ff:fe33:4d1b]:8333 # AS395502 +[2607:9280:b:73b:250:56ff:fe3d:401]:8333 # AS395502 +[2620:6e:a000:1:42:42:42:42]:8333 # AS397444 +[2620:a6:2000:1:1:0:5:1601]:8333 # AS27566 +[2620:a6:2000:1:2:0:9:900b]:8333 # AS27566 +[2620:a6:2000:1:2:0:b:300e]:8333 # AS27566 +[2800:150:11d:d2f:bdac:7807:2f5:4aa0]:8333 # AS22047 +[2803:9800:a007:82ba:650b:82b8:8377:d0]:8333 # AS19037 +[2804:14c:155:45e0:1e86:15a3:efd9:7287]:8333 # AS28573 +[2804:14c:657d:4030:28b4:eff:fe9b:8894]:8333 # AS28573 +[2804:14d:1087:9434::1002]:8333 # AS4230 +[2804:954:24:2:b390:d83b:358a:db53]:8333 # AS263073 +[2804:d57:554d:de00:3e7c:3fff:fe7b:80aa]:8333 # AS8167 +[2a00:1028:838c:563a:fd25:87b6:5a54:811]:8333 # AS5610 +[2a00:1298:8001::6542]:8333 # AS5578 +[2a00:1398:4:2a03:215:5dff:fed6:1033]:8333 # AS34878 +[2a00:1398:4:2a03::bc03]:8333 # AS34878 +[2a00:1768:2001:27::ef6a]:8333 # AS43350 +[2a00:1f40:5001:108:5d17:7703:b0f5:4133]:8333 # AS42864 +[2a00:23c5:fe80:7301:d6ae:52ff:fed5:56a5]:8333 # AS2856 +[2a00:6020:13dc:bc00:5559:258:27d:b52b]:8333 # AS60294 +[2a00:6020:4503:3700:20c:29ff:fe61:4a4c]:8333 # AS60294 +[2a00:6020:b434:eb00:dea6:32ff:fe0d:a5c0]:8333 # AS60294 +[2a00:6020:b489:2000:5054:ff:fefc:5ed8]:8333 # AS60294 +[2a00:7c80:0:10c::2]:8333 # AS49981 +[2a00:7c80:0:25::e37a]:8333 # AS49981 +[2a00:8a60:e012:a00::21]:8333 # AS680 +[2a00:bbe0:cc:0:5a11:22ff:feb4:8f5c]:8333 # AS47605 +[2a00:bbe0:cc:0:62a4:4cff:fe23:7510]:8333 # AS47605 +[2a00:ca8:a15:9a5b:8b42:a886:7d48:7a21]:8333 # AS30764 +[2a00:ca8:a1f:f9b7:cb55:5766:524b:acaa]:8333 # AS30764 +[2a00:d4e0:ff:fc02:5e55:4a7c:b83b:e5a1]:8333 # AS15600 +[2a00:d520:9:9300:420b:544e:8019:6d3a]:8333 # AS15600 +[2a00:d880:5:c2::d329]:8333 # AS198203 +[2a00:ee2:1200:1900:8d3:d2ff:feb1:bc58]:8333 # AS5603 +[2a01:4f8:173:230a::2]:8333 # AS24940 +[2a01:4f8:200:7222::2]:8333 # AS24940 +[2a01:4f8:202:3e6::2]:8333 # AS24940 +[2a01:4f8:221:44d7::2]:8333 # AS24940 +[2a01:4f8:231:915::2]:8333 # AS24940 +[2a01:4f8:261:2bcd::2]:8333 # AS24940 +[2a01:4f8:261:3cae::2]:8333 # AS24940 +[2a01:4f8:261:420c::2]:8333 # AS24940 +[2a01:4f9:2b:29a::2]:8333 # AS24940 +[2a01:4f9:3a:2dd2::2]:8333 # AS24940 +[2a01:7a7:2:2804:ae1f:6bff:fe9d:6c94]:8333 # AS20773 +[2a01:7c8:aac2:180:5054:ff:fe56:8d10]:8333 # AS20857 +[2a01:7c8:aac9:c9:5054:ff:fedf:ff95]:8333 # AS20857 +[2a01:7e01::f03c:93ff:fe49:2f5b]:8333 # AS63949 +[2a01:8740:1:753::e5cb]:8333 # AS57344 +[2a01:8740:1:ffc5::8c6a]:8333 # AS57344 +[2a01:cb00:b63:c000:227:eff:fe28:c565]:8333 # AS3215 +[2a01:cb19:688:e900:aa60:b6ff:fe29:bbae]:8333 # AS3215 +[2a01:e0a:163:c0b0:9da5:1690:a12b:bede]:8333 # AS12322 +[2a01:e0a:282:67b0:b4f4:aaff:fe7c:44a6]:8333 # AS12322 +[2a01:e0a:301:7010:b87d:e14b:cea9:b998]:8333 # AS12322 +[2a01:e0a:320:39a0:325a:3aff:fe02:3180]:8333 # AS12322 +[2a01:e0a:351:9fb0:6bf2:95d6:b7bd:b846]:8333 # AS12322 +[2a01:e0a:5fa:a0a0:ca1f:66ff:fece:b8a2]:8333 # AS12322 +[2a01:e0a:83d:dd30:3676:5d8e:8a6f:115a]:8333 # AS12322 +[2a01:e0a:9e9:c240:7b44:f32a:6ec0:a8af]:8333 # AS12322 +[2a01:e0a:b5:7f50:c257:a55b:4846:97e1]:8333 # AS12322 +[2a01:e11:100c:70:cbc8:9e31:4b77:1626]:8333 # AS12322 +[2a02:1210:2cdf:4600:2bc:e03e:43e8:4718]:8333 # AS3303 +[2a02:1210:86bf:f100:a9ac:d041:1f8e:6925]:8333 # AS3303 +[2a02:1210:94c3:3400:d8c3:743c:90f6:a48a]:8333 # AS3303 +[2a02:168:2000:96::12]:8333 # AS13030 +[2a02:168:420b:a::20]:8333 # AS13030 +[2a02:168:676e:0:e65f:1ff:fe09:3591]:8333 # AS13030 +[2a02:1748:f39f:5872:216:3eff:fe21:266]:8333 # AS51184 +[2a02:180:1:1::517:10b6]:8333 # AS35366 +[2a02:2780:9000:70::7]:8333 # AS35434 +[2a02:2780:9000:70::f]:8333 # AS35434 +[2a02:2780::e01a]:8333 # AS35434 +[2a02:2f05:6008:ce00::1]:8333 # AS48571 +[2a02:390:9000:0:aaa1:59ff:fe43:b57b]:8333 # AS12496 +[2a02:578:85ce:1600:1e1b:dff:fee3:774b]:8333 # AS9031 +[2a02:768:f92b:db46:5e46:772b:71d:29b7]:8333 # AS44489 +[2a02:7a01::91:228:45:130]:8333 # AS16019 +[2a02:7b40:50d0:e386::1]:8333 # AS62282 +[2a02:7b40:50d1:e35b::1]:8333 # AS62282 +[2a02:7b40:5928:89::1]:8333 # AS62282 +[2a02:7b40:b945:344d::1]:8333 # AS62282 +[2a02:7b40:d418:6d9a::1]:8333 # AS62282 +[2a02:8070:b84:6ae0:f9c6:fbb9:1c41:81aa]:8333 # AS51185 +[2a02:8070:f186:38e0::d5a6]:8333 # AS51185 +[2a02:8084:103:6810:1e69:7aff:fea2:1acc]:8333 # AS6830 +[2a02:8308:8081:f300:3b8:7ec0:2837:1b57]:8333 # AS16019 +[2a02:8388:e302:7980:6f85:a0b3:4b4d:8b0f]:8333 # AS8412 +[2a02:8388:e5c3:4a80:201:2eff:fe82:b3cc]:8333 # AS8412 +[2a02:842a:1df:8a01:1e1b:dff:fe0b:236d]:8333 # AS15557 +[2a02:a210:28be:5f80::111]:8333 # AS6830 +[2a02:a44b:5cf9:1:b62e:99ff:fe49:d492]:8333 # AS1136 +[2a02:a44d:14d6:1:2c0:8ff:fe8f:b3b2]:8333 # AS1136 +[2a02:a45a:94cd:f00d::1]:8333 # AS1136 +[2a02:a45f:3b9d:31::199]:8333 # AS1136 +[2a02:a464:3d6b::1:2]:8333 # AS1136 +[2a02:a46c:7f8e:1:35bf:3aeb:137c:1d35]:8333 # AS1136 +[2a02:a46d:36f:1:20d:b9ff:fe4e:6398]:8333 # AS1136 +[2a02:c205:2021:4216::1]:8333 # AS51167 +[2a02:c206:2044:9826::1]:8333 # AS51167 +[2a02:c206:2075:3351::1]:8333 # AS51167 +[2a02:c207:0:3829::1]:8333 # AS51167 +[2a02:c207:2014:4199::1]:8333 # AS51167 +[2a02:c207:2014:8757::1]:8333 # AS51167 +[2a02:c207:2026:6682::1]:8333 # AS51167 +[2a02:c207:2034:7358::1]:8333 # AS51167 +[2a02:c207:3002:7468::1]:8333 # AS51167 +[2a02:c207:3008:4592::1]:8333 # AS51167 +[2a02:cb43:4000::178]:8333 # AS33891 +[2a02:e5e:1:10::27]:8333 # AS25057 +[2a02:e98:20:1504::1]:8333 # AS24641 +[2a03:4000:28:68:7411:53ff:fe4c:21d]:8333 # AS47147 +[2a03:4000:65:fdc:3462:66ff:fe05:ec5c]:8333 # AS47147 +[2a03:6000:870:0:46:23:87:218]:8333 # AS51088 +[2a03:94e0:ffff:185:243:218:0:19]:8333 # AS56655 +[2a03:b0c0:1:e0::397:6001]:8333 # AS14061 +[2a03:b0c0:1:e0::794:9001]:8333 # AS14061 +[2a03:b0c0:2:f0::288:c001]:8333 # AS14061 +[2a03:b0c0:2:f0::30c:1]:8333 # AS14061 +[2a03:b0c0:3:d0::e3b:5001]:8333 # AS14061 +[2a03:cfc0:8000:7::5fd6:3557]:8333 # AS201814 +[2a04:2180:dc05:2::3b]:8333 # AS61272 +[2a04:2180:ffff:fffe::d]:8333 # AS61272 +[2a04:52c0:103:c455::1]:8333 # AS60404 +[2a04:bc40:1dc3:8d::2:1001]:8333 # AS35277 +[2a05:3580:dc0b:1600:def4:5a62:de42:324a]:8333 # AS20764 +[2a05:d014:a55:4000:8dde:69f:4ac7:b26]:8333 # AS16509 +[2a05:d016:98f:5201:6be0:a4de:80c7:32d5]:8333 # AS16509 +[2a05:d018:a75:6c03:75b:2c73:8caa:414b]:8333 # AS16509 +[2a05:f480:1800:697:5400:2ff:feb6:c36d]:8333 # AS20473 +[2a06:e040:7603:2918:c6ef:464e:9fe5:73ec]:8333 # AS198507 +[2a07:abc4::89:234:180:194]:8333 # AS62000 +[2a07:d884::127e]:8333 # AS6762 +[2a09:2681:1010:10::5]:8333 # AS61282 +[2a09:2681:102::210]:8333 # AS61282 +[2a0b:f300:2:6::2]:8333 # AS62240 +[2a0d:8340:24::2]:8333 # AS50113 +[2a0e:8f02:21d1:144::101]:8333 # AS20473 +[2a0e:b780::55d1:f05b]:8333 # AS205581 +[2a10:3781:2c19::1]:8333 # AS206238 +[2a10:d200:1:33:a6bf:1ff:fe6a:46a9]:8333 # AS212323 +[2a12:8e40:5668:e40a::1]:8333 # AS34465 +[2a12:8e40:5668:e40b::1]:8333 # AS34465 +[2a12:8e40:5668:e40c::1]:8333 # AS34465 +[2a12:8e40:5668:e40d::1]:8333 # AS34465 +[2a12:8e40:5668:e40e::1]:8333 # AS34465 +[2a12:8e40:5668:e40f::1]:8333 # AS34465 +[2a12:8e40:5668:e410::1]:8333 # AS34465 +[2a12:8e40:5668:e411::1]:8333 # AS34465 +[2a12:8e40:5668:e412::1]:8333 # AS34465 +[2a12:8e40:5668:e417::1]:8333 # AS34465 +[2c0f:f8f0:da51:0:3a45:fc57:5e30:2593]:8333 # AS30844 -# manually added 2021-03 for minimal torv3 bootstrap support -2g5qfdkn2vvcbqhzcyvyiitg4ceukybxklraxjnu7atlhd22gdwywaid.onion:8333 -2jmtxvyup3ijr7u6uvu7ijtnojx4g5wodvaedivbv74w4vzntxbrhvad.onion:8333 -37m62wn7dz3uqpathpc4qfmgrbupachj52nt3jbtbjugpbu54kbud7yd.onion:8333 +# manually updated 2023-04 for minimal torv3 bootstrap support + +2bqghnldu6mcug4pikzprwhtjjnsyederctvci6klcwzepnjd46ikjyd.onion:8333 +4lr3w2iyyl5u5l6tosizclykf5v3smqroqdn2i4h3kq6pfbbjb2xytad.onion:8333 5g72ppm3krkorsfopcm2bi7wlv4ohhs4u4mlseymasn7g7zhdcyjpfid.onion:8333 -7cgwjuwi5ehvcay4tazy7ya6463bndjk6xzrttw5t3xbpq4p22q6fyid.onion:8333 -7pyrpvqdhmayxggpcyqn5l3m5vqkw3qubnmgwlpya2mdo6x7pih7r7id.onion:8333 +5sbmcl4m5api5tqafi4gcckrn3y52sz5mskxf3t6iw4bp7erwiptrgqd.onion:8333 +776aegl7tfhg6oiqqy76jnwrwbvcytsx2qegcgh2mjqujll4376ohlid.onion:8333 +77mdte42srl42shdh2mhtjr7nf7dmedqrw6bkcdekhdvmnld6ojyyiad.onion:8333 +azbpsh4arqlm6442wfimy7qr65bmha2zhgjg7wbaji6vvaug53hur2qd.onion:8333 b64xcbleqmwgq2u46bh4hegnlrzzvxntyzbmucn3zt7cssm7y4ubv3id.onion:8333 -ejxefzf5fpst4mg2rib7grksvscl7p6fvjp6agzgfc2yglxnjtxc3aid.onion:8333 -fjdyxicpm4o42xmedlwl3uvk5gmqdfs5j37wir52327vncjzvtpfv7yd.onion:8333 +bsqbtcparrfihlwolt4xgjbf4cgqckvrvsfyvy6vhiqrnh4w6ghixoid.onion:8333 +bsqbtctulf2g4jtjsdfgl2ed7qs6zz5wqx27qnyiik7laockryvszqqd.onion:8333 +cwi3ekrwhig47dhhzfenr5hbvckj7fzaojygvazi2lucsenwbzwoyiqd.onion:8333 +devinbtcmwkuitvxl3tfi5of4zau46ymeannkjv6fpnylkgf3q5fa3id.onion:8333 +devinbtctu7uctl7hly2juu3thbgeivfnvw3ckj3phy6nyvpnx66yeyd.onion:8333 +devinbtcyk643iruzfpaxw3on2jket7rbjmwygm42dmdyub3ietrbmid.onion:8333 +dtql5vci4iaml4anmueftqr7bfgzqlauzfy4rc2tfgulldd3ekyijjyd.onion:8333 +emzybtc25oddoa2prol2znpz2axnrg6k77xwgirmhv7igoiucddsxiad.onion:8333 +emzybtc3ewh7zihpkdvuwlgxrhzcxy2p5fvjggp7ngjbxcytxvt4rjid.onion:8333 +emzybtc454ewbviqnmgtgx3rgublsgkk23r4onbhidcv36wremue4kqd.onion:8333 +emzybtc5bnpb2o6gh54oquiox54o4r7yn4a2wiiwzrjonlouaibm2zid.onion:8333 fpz6r5ppsakkwypjcglz6gcnwt7ytfhxskkfhzu62tnylcknh3eq6pad.onion:8333 -fzhn4uoxfbfss7h7d6ffbn266ca432ekbbzvqtsdd55ylgxn4jucm5qd.onion:8333 -gxo5anvfnffnftfy5frkgvplq3rpga2ie3tcblo2vl754fvnhgorn5yd.onion:8333 -ifdu5qvbofrt4ekui2iyb3kbcyzcsglazhx2hn4wfskkrx2v24qxriid.onion:8333 -itz3oxsihs62muvknc237xabl5f6w6rfznfhbpayrslv2j2ubels47yd.onion:8333 -lrjh6fywjqttmlifuemq3puhvmshxzzyhoqx7uoufali57eypuenzzid.onion:8333 +hanvo3hzqbhcqm5vahhi5a3czxxdwc7vt56p5gr7bifcvelaqurv6iid.onion:8333 +hz7oqntvj4adrwtqappcgaxfribg5u4rvfkpwlo3xup5fcuyvylkxlqd.onion:8333 +ityrxhidvjnjnf6imzyuqqnkkwridjnebkbokx25so3suq3fzezmksid.onion:8333 +jto2jfbsxhb6yvhcrrjddrgbakte6tgsy3c3z3prss64gndgvovvosyd.onion:8333 +k7nb3r7hxi5exvr4xmvnilhfw6hei7sw4rwz2t6onh4py6wbora6tuyd.onion:8333 +kpgvmscirrdqpekbqjsvw5teanhatztpp2gl6eee4zkowvwfxwenqaid.onion:8333 +l7kw3vjs4cf5mnuejjgqcxrw6wwsjmabllq3h3amy4f5q33d6cgo2kyd.onion:8333 m7cbpjolo662uel7rpaid46as2otcj44vvwg3gccodnvaeuwbm3anbyd.onion:8333 -opnyfyeiibe5qo5a3wbxzbb4xdiagc32bbce46owmertdknta5mi7uyd.onion:8333 -owjsdxmzla6d7lrwkbmetywqym5cyswpihciesfl5qdv2vrmwsgy4uqd.onion:8333 -q7kgmd7n7h27ds4fg7wocgniuqb3oe2zxp4nfe4skd5da6wyipibqzqd.onion:8333 -rp7k2go3s5lyj3fnj6zn62ktarlrsft2ohlsxkyd7v3e3idqyptvread.onion:8333 -sys54sv4xv3hn3sdiv3oadmzqpgyhd4u4xphv4xqk64ckvaxzm57a7yd.onion:8333 -tddeij4qigtjr6jfnrmq6btnirmq5msgwcsdpcdjr7atftm7cxlqztid.onion:8333 -vi5bnbxkleeqi6hfccjochnn65lcxlfqs4uwgmhudph554zibiusqnad.onion:8333 -xqt25cobm5zqucac3634zfght72he6u3eagfyej5ellbhcdgos7t2had.onion:8333 +mowb2qwpjgs2a6q3yj3xa7nxklfssul4w7ynonyycw3uyopfu3x6ujad.onion:8333 +mwmfluek4au6mxxpw6fy7sjhkm65bdfc7izc7lpz3trewfdghyrzsbid.onion:8333 +rfqmn3qe36uaptkxhdvi74p4hyrzhir6vhmzb2hqryxodig4gue2zbyd.onion:8333 +rsgwtnousfc7zyg4qsm3gvczjx7cihh2njyjbjl3qvcj3xg7wmvhddqd.onion:8333 +s2d52bbttuwcl3pdrwzhxpmhtxn3jg23havjqg5eygwhtiw6lgyelpqd.onion:8333 +upvthy74hgvgbqi6w3zd2mlchoi5tvvw7b5hpmmhcddd5fnnwrixneid.onion:8333 +who3qs4eqlqzoxhqqgan4mg54ua5uz3mk4lj33ag53ei4orvnznrjbad.onion:8333 +wizbit5555bsslwv4ctronnsgk5vh2w2pdx7v7eyuivlyuoteejk7lid.onion:8333 +yrmedr35tt4wqfnwgilltxh5bnukeukxjpgg3jzmmsyld5lgsn5amvyd.onion:8333 -# manually added 2021-08 for minimal i2p bootstrap support +# manually updated 2023-04 for minimal i2p bootstrap support +255fhcp6ajvftnyo7bwz3an3t4a4brhopm3bamyh2iu5r3gnr2rq.b32.i2p:0 +27yrtht5b5bzom2w5ajb27najuqvuydtzb7bavlak25wkufec5mq.b32.i2p:0 +3gocb7wc4zvbmmebktet7gujccuux4ifk3kqilnxnj5wpdpqx2hq.b32.i2p:0 +4fcc23wt3hyjk3csfzcdyjz5pcwg5dzhdqgma6bch2qyiakcbboa.b32.i2p:0 +4osyqeknhx5qf3a73jeimexwclmt42cju6xdp7icja4ixxguu2hq.b32.i2p:0 +4umsi4nlmgyp4rckosg4vegd2ysljvid47zu7pqsollkaszcbpqq.b32.i2p:0 +6j2ezegd3e2e2x3o3pox335f5vxfthrrigkdrbgfbdjchm5h4awa.b32.i2p:0 +6n36ljyr55szci5ygidmxqer64qr24f4qmnymnbvgehz7qinxnla.b32.i2p:0 +72yjs6mvlby3ky6mgpvvlemmwq5pfcznrzd34jkhclgrishqdxva.b32.i2p:0 a5qsnv3maw77mlmmzlcglu6twje6ttctd3fhpbfwcbpmewx6fczq.b32.i2p:0 -bitcornrd36coazsbzsz4pdebyzvaplmsalq4kpoljmn6cg6x5zq.b32.i2p:0 +aovep2pco7v2k4rheofrgytbgk23eg22dczpsjqgqtxcqqvmxk6a.b32.i2p:0 +bitcoi656nll5hu6u7ddzrmzysdtwtnzcnrjd4rfdqbeey7dmn5a.b32.i2p:0 +brifkruhlkgrj65hffybrjrjqcgdgqs2r7siizb5b2232nruik3a.b32.i2p:0 c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p:0 -dhtq2p76tyhi442aidb3vd2bv7yxxjuddpb2jydnnrl2ons5bhha.b32.i2p:0 -h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p:0 -hnbbyjpxx54623l555sta7pocy3se4sdgmuebi5k6reesz5rjp6q.b32.i2p:0 +day3hgxyrtwjslt54sikevbhxxs4qzo7d6vi72ipmscqtq3qmijq.b32.i2p:0 +du5kydummi23bjfp6bd7owsvrijgt7zhvxmz5h5f5spcioeoetwq.b32.i2p:0 +e55k6wu46rzp4pg5pk5npgbr3zz45bc3ihtzu2xcye5vwnzdy7pq.b32.i2p:0 +eciohu5nq7vsvwjjc52epskuk75d24iccgzmhbzrwonw6lx4gdva.b32.i2p:0 +ejlnngarmhqvune74ko7kk55xtgbz5i5ncs4vmnvjpy3l7y63xaa.b32.i2p:0 +fhzlp3xroabohnmjonu5iqazwhlbbwh5cpujvw2azcu3srqdceja.b32.i2p:0 +fx6np3oheacr3t7gluftrqo2qxldbbatgw4hepp7ulb4j5ry57ca.b32.i2p:0 +gehtac45oaghz54ypyopim64mql7oad2bqclla74l6tfeolzmodq.b32.i2p:0 +hhfi4yqkg2twqiwezrfksftjjofbyx3ojkmlnfmcwntgnrjjhkya.b32.i2p:0 jz3s4eurm5vzjresf4mwo7oni4bk36daolwxh4iqtewakylgkxmq.b32.i2p:0 -kokkmpquqlkptu5hkmzqlttsmtwxicldr4so7wqsufk6bwf32nma.b32.i2p:0 +liu75cvktv4icbctg72w7nxbk4eibt7wamizfdii4omz7gcke5vq.b32.i2p:0 +lrah7acdsgopybg43shadwwiv6igezaw64i6jb5muqdg7dmhj3la.b32.i2p:0 +lzuu6mjtu7vd55d2biphicihufipoa7vyym6xfnkmmlra3tiziia.b32.i2p:0 +m6bpynxkv2ktwxkg6p2gyudjfhdupb6kuzabeqdnckkdkf4kxjla.b32.i2p:0 +m6v454xd6p3bt5swujgmveklsp7lzbkqlqqfc2p36cjlwv5dbucq.b32.i2p:0 +mlgeizrroynuhpxbzeosajt5u4ddcvynxfmcbm6kwjpaufilxigq.b32.i2p:0 +o6t4fr5ayfadzieutstgwcllvwxeuzjlxmzsmpj3hpkvefhzfaea.b32.i2p:0 +ofubxr2ir7u2guzjwyrvujicivzmvinwa36nuzlrg7tnsmebal7a.b32.i2p:0 +oz2ia3flpm3du2tyusulrn7h7e2eo3juzkrmn34bvnrlcrugv7ia.b32.i2p:0 +pohfcrfc7prn4bvn4xstw6nt3e7hjmb7kuj4djtsfqsskwhmhnna.b32.i2p:0 +qd6jlsevsexww3wefpqs7iglxb3f63y4e6ydulfzrvwflpicmdqa.b32.i2p:0 +rfjkzdzv4cwpxo6hzuncicvuyui76wxqx3a23lynq72ktwqs7aja.b32.i2p:0 +rizfinyses2r3or4iubs5wx66gdy6mpf73w7uobfacm2l5cral3q.b32.i2p:0 sedndhv5vpcgdmykyi5st4yqhdxl3hpdtglta4do435wupahhx6q.b32.i2p:0 +tugq6wa2ls2bv27pr2iy3da3k5ow3fzefbcvjcr22uc7w5vmevja.b32.i2p:0 +usztavbib756k5vqggzgkyswoj6mttihjvp3c2pa642t2mb4pvsa.b32.i2p:0 +vgu6llqbyjphml25umd5ztvyxrxuplz2g74fzbx75g3kkaetoyiq.b32.i2p:0 +wjrul5jwwb4vqdmkkrjbmly7osj6amecdpsac5xvaoqrti4nb3ha.b32.i2p:0 wwbw7nqr3ahkqv62cuqfwgtneekvvpnuc4i4f6yo7tpoqjswvcwa.b32.i2p:0 +xfkarmvk43vfkfvhkehy7ioj2b6wtfdlezvmlakblz3q4r7mccfq.b32.i2p:0 +yc4xwin5ujenvcr6ynwkz7lnmmq3nmzxvfguele6ovqqpxgjvonq.b32.i2p:0 +zdoabsg7ugzothyawodjhq54nvlofa746rxfkxpnjzj6nukmha6a.b32.i2p:0 zsxwyo6qcn3chqzwxnseusqgsnuw3maqnztkiypyfxtya4snkoka.b32.i2p:0 +zysrlpii5ftrzivfcyhdrwpeyyqddbrdefnfu5q6otk5gtugmh2a.b32.i2p:0 -# manually added 2022-01 for minimal cjdns bootstrap support +# manually updated 2023-04 for minimal cjdns bootstrap support [fc32:17ea:e415:c3bf:9808:149d:b5a2:c9aa]:8333 [fcc7:be49:ccd1:dc91:3125:f0da:457d:8ce]:8333 +[fcdc:73ae:b1a9:1bf8:d4c2:811:a4c7:c34e]:8333 diff --git a/contrib/seeds/nodes_main_manual.txt b/contrib/seeds/nodes_main_manual.txt index a6e0b8763a9..91a64c6e515 100644 --- a/contrib/seeds/nodes_main_manual.txt +++ b/contrib/seeds/nodes_main_manual.txt @@ -1,43 +1,95 @@ -# manually added 2021-03 for minimal torv3 bootstrap support -2g5qfdkn2vvcbqhzcyvyiitg4ceukybxklraxjnu7atlhd22gdwywaid.onion:8333 -2jmtxvyup3ijr7u6uvu7ijtnojx4g5wodvaedivbv74w4vzntxbrhvad.onion:8333 -37m62wn7dz3uqpathpc4qfmgrbupachj52nt3jbtbjugpbu54kbud7yd.onion:8333 +# manually updated 2023-04 for minimal torv3 bootstrap support + +2bqghnldu6mcug4pikzprwhtjjnsyederctvci6klcwzepnjd46ikjyd.onion:8333 +4lr3w2iyyl5u5l6tosizclykf5v3smqroqdn2i4h3kq6pfbbjb2xytad.onion:8333 5g72ppm3krkorsfopcm2bi7wlv4ohhs4u4mlseymasn7g7zhdcyjpfid.onion:8333 -7cgwjuwi5ehvcay4tazy7ya6463bndjk6xzrttw5t3xbpq4p22q6fyid.onion:8333 -7pyrpvqdhmayxggpcyqn5l3m5vqkw3qubnmgwlpya2mdo6x7pih7r7id.onion:8333 +5sbmcl4m5api5tqafi4gcckrn3y52sz5mskxf3t6iw4bp7erwiptrgqd.onion:8333 +776aegl7tfhg6oiqqy76jnwrwbvcytsx2qegcgh2mjqujll4376ohlid.onion:8333 +77mdte42srl42shdh2mhtjr7nf7dmedqrw6bkcdekhdvmnld6ojyyiad.onion:8333 +azbpsh4arqlm6442wfimy7qr65bmha2zhgjg7wbaji6vvaug53hur2qd.onion:8333 b64xcbleqmwgq2u46bh4hegnlrzzvxntyzbmucn3zt7cssm7y4ubv3id.onion:8333 -ejxefzf5fpst4mg2rib7grksvscl7p6fvjp6agzgfc2yglxnjtxc3aid.onion:8333 -fjdyxicpm4o42xmedlwl3uvk5gmqdfs5j37wir52327vncjzvtpfv7yd.onion:8333 +bsqbtcparrfihlwolt4xgjbf4cgqckvrvsfyvy6vhiqrnh4w6ghixoid.onion:8333 +bsqbtctulf2g4jtjsdfgl2ed7qs6zz5wqx27qnyiik7laockryvszqqd.onion:8333 +cwi3ekrwhig47dhhzfenr5hbvckj7fzaojygvazi2lucsenwbzwoyiqd.onion:8333 +devinbtcmwkuitvxl3tfi5of4zau46ymeannkjv6fpnylkgf3q5fa3id.onion:8333 +devinbtctu7uctl7hly2juu3thbgeivfnvw3ckj3phy6nyvpnx66yeyd.onion:8333 +devinbtcyk643iruzfpaxw3on2jket7rbjmwygm42dmdyub3ietrbmid.onion:8333 +dtql5vci4iaml4anmueftqr7bfgzqlauzfy4rc2tfgulldd3ekyijjyd.onion:8333 +emzybtc25oddoa2prol2znpz2axnrg6k77xwgirmhv7igoiucddsxiad.onion:8333 +emzybtc3ewh7zihpkdvuwlgxrhzcxy2p5fvjggp7ngjbxcytxvt4rjid.onion:8333 +emzybtc454ewbviqnmgtgx3rgublsgkk23r4onbhidcv36wremue4kqd.onion:8333 +emzybtc5bnpb2o6gh54oquiox54o4r7yn4a2wiiwzrjonlouaibm2zid.onion:8333 fpz6r5ppsakkwypjcglz6gcnwt7ytfhxskkfhzu62tnylcknh3eq6pad.onion:8333 -fzhn4uoxfbfss7h7d6ffbn266ca432ekbbzvqtsdd55ylgxn4jucm5qd.onion:8333 -gxo5anvfnffnftfy5frkgvplq3rpga2ie3tcblo2vl754fvnhgorn5yd.onion:8333 -ifdu5qvbofrt4ekui2iyb3kbcyzcsglazhx2hn4wfskkrx2v24qxriid.onion:8333 -itz3oxsihs62muvknc237xabl5f6w6rfznfhbpayrslv2j2ubels47yd.onion:8333 -lrjh6fywjqttmlifuemq3puhvmshxzzyhoqx7uoufali57eypuenzzid.onion:8333 +hanvo3hzqbhcqm5vahhi5a3czxxdwc7vt56p5gr7bifcvelaqurv6iid.onion:8333 +hz7oqntvj4adrwtqappcgaxfribg5u4rvfkpwlo3xup5fcuyvylkxlqd.onion:8333 +ityrxhidvjnjnf6imzyuqqnkkwridjnebkbokx25so3suq3fzezmksid.onion:8333 +jto2jfbsxhb6yvhcrrjddrgbakte6tgsy3c3z3prss64gndgvovvosyd.onion:8333 +k7nb3r7hxi5exvr4xmvnilhfw6hei7sw4rwz2t6onh4py6wbora6tuyd.onion:8333 +kpgvmscirrdqpekbqjsvw5teanhatztpp2gl6eee4zkowvwfxwenqaid.onion:8333 +l7kw3vjs4cf5mnuejjgqcxrw6wwsjmabllq3h3amy4f5q33d6cgo2kyd.onion:8333 m7cbpjolo662uel7rpaid46as2otcj44vvwg3gccodnvaeuwbm3anbyd.onion:8333 -opnyfyeiibe5qo5a3wbxzbb4xdiagc32bbce46owmertdknta5mi7uyd.onion:8333 -owjsdxmzla6d7lrwkbmetywqym5cyswpihciesfl5qdv2vrmwsgy4uqd.onion:8333 -q7kgmd7n7h27ds4fg7wocgniuqb3oe2zxp4nfe4skd5da6wyipibqzqd.onion:8333 -rp7k2go3s5lyj3fnj6zn62ktarlrsft2ohlsxkyd7v3e3idqyptvread.onion:8333 -sys54sv4xv3hn3sdiv3oadmzqpgyhd4u4xphv4xqk64ckvaxzm57a7yd.onion:8333 -tddeij4qigtjr6jfnrmq6btnirmq5msgwcsdpcdjr7atftm7cxlqztid.onion:8333 -vi5bnbxkleeqi6hfccjochnn65lcxlfqs4uwgmhudph554zibiusqnad.onion:8333 -xqt25cobm5zqucac3634zfght72he6u3eagfyej5ellbhcdgos7t2had.onion:8333 +mowb2qwpjgs2a6q3yj3xa7nxklfssul4w7ynonyycw3uyopfu3x6ujad.onion:8333 +mwmfluek4au6mxxpw6fy7sjhkm65bdfc7izc7lpz3trewfdghyrzsbid.onion:8333 +rfqmn3qe36uaptkxhdvi74p4hyrzhir6vhmzb2hqryxodig4gue2zbyd.onion:8333 +rsgwtnousfc7zyg4qsm3gvczjx7cihh2njyjbjl3qvcj3xg7wmvhddqd.onion:8333 +s2d52bbttuwcl3pdrwzhxpmhtxn3jg23havjqg5eygwhtiw6lgyelpqd.onion:8333 +upvthy74hgvgbqi6w3zd2mlchoi5tvvw7b5hpmmhcddd5fnnwrixneid.onion:8333 +who3qs4eqlqzoxhqqgan4mg54ua5uz3mk4lj33ag53ei4orvnznrjbad.onion:8333 +wizbit5555bsslwv4ctronnsgk5vh2w2pdx7v7eyuivlyuoteejk7lid.onion:8333 +yrmedr35tt4wqfnwgilltxh5bnukeukxjpgg3jzmmsyld5lgsn5amvyd.onion:8333 -# manually added 2021-08 for minimal i2p bootstrap support +# manually updated 2023-04 for minimal i2p bootstrap support +255fhcp6ajvftnyo7bwz3an3t4a4brhopm3bamyh2iu5r3gnr2rq.b32.i2p:0 +27yrtht5b5bzom2w5ajb27najuqvuydtzb7bavlak25wkufec5mq.b32.i2p:0 +3gocb7wc4zvbmmebktet7gujccuux4ifk3kqilnxnj5wpdpqx2hq.b32.i2p:0 +4fcc23wt3hyjk3csfzcdyjz5pcwg5dzhdqgma6bch2qyiakcbboa.b32.i2p:0 +4osyqeknhx5qf3a73jeimexwclmt42cju6xdp7icja4ixxguu2hq.b32.i2p:0 +4umsi4nlmgyp4rckosg4vegd2ysljvid47zu7pqsollkaszcbpqq.b32.i2p:0 +6j2ezegd3e2e2x3o3pox335f5vxfthrrigkdrbgfbdjchm5h4awa.b32.i2p:0 +6n36ljyr55szci5ygidmxqer64qr24f4qmnymnbvgehz7qinxnla.b32.i2p:0 +72yjs6mvlby3ky6mgpvvlemmwq5pfcznrzd34jkhclgrishqdxva.b32.i2p:0 a5qsnv3maw77mlmmzlcglu6twje6ttctd3fhpbfwcbpmewx6fczq.b32.i2p:0 -bitcornrd36coazsbzsz4pdebyzvaplmsalq4kpoljmn6cg6x5zq.b32.i2p:0 +aovep2pco7v2k4rheofrgytbgk23eg22dczpsjqgqtxcqqvmxk6a.b32.i2p:0 +bitcoi656nll5hu6u7ddzrmzysdtwtnzcnrjd4rfdqbeey7dmn5a.b32.i2p:0 +brifkruhlkgrj65hffybrjrjqcgdgqs2r7siizb5b2232nruik3a.b32.i2p:0 c4gfnttsuwqomiygupdqqqyy5y5emnk5c73hrfvatri67prd7vyq.b32.i2p:0 -dhtq2p76tyhi442aidb3vd2bv7yxxjuddpb2jydnnrl2ons5bhha.b32.i2p:0 -h3r6bkn46qxftwja53pxiykntegfyfjqtnzbm6iv6r5mungmqgmq.b32.i2p:0 -hnbbyjpxx54623l555sta7pocy3se4sdgmuebi5k6reesz5rjp6q.b32.i2p:0 +day3hgxyrtwjslt54sikevbhxxs4qzo7d6vi72ipmscqtq3qmijq.b32.i2p:0 +du5kydummi23bjfp6bd7owsvrijgt7zhvxmz5h5f5spcioeoetwq.b32.i2p:0 +e55k6wu46rzp4pg5pk5npgbr3zz45bc3ihtzu2xcye5vwnzdy7pq.b32.i2p:0 +eciohu5nq7vsvwjjc52epskuk75d24iccgzmhbzrwonw6lx4gdva.b32.i2p:0 +ejlnngarmhqvune74ko7kk55xtgbz5i5ncs4vmnvjpy3l7y63xaa.b32.i2p:0 +fhzlp3xroabohnmjonu5iqazwhlbbwh5cpujvw2azcu3srqdceja.b32.i2p:0 +fx6np3oheacr3t7gluftrqo2qxldbbatgw4hepp7ulb4j5ry57ca.b32.i2p:0 +gehtac45oaghz54ypyopim64mql7oad2bqclla74l6tfeolzmodq.b32.i2p:0 +hhfi4yqkg2twqiwezrfksftjjofbyx3ojkmlnfmcwntgnrjjhkya.b32.i2p:0 jz3s4eurm5vzjresf4mwo7oni4bk36daolwxh4iqtewakylgkxmq.b32.i2p:0 -kokkmpquqlkptu5hkmzqlttsmtwxicldr4so7wqsufk6bwf32nma.b32.i2p:0 +liu75cvktv4icbctg72w7nxbk4eibt7wamizfdii4omz7gcke5vq.b32.i2p:0 +lrah7acdsgopybg43shadwwiv6igezaw64i6jb5muqdg7dmhj3la.b32.i2p:0 +lzuu6mjtu7vd55d2biphicihufipoa7vyym6xfnkmmlra3tiziia.b32.i2p:0 +m6bpynxkv2ktwxkg6p2gyudjfhdupb6kuzabeqdnckkdkf4kxjla.b32.i2p:0 +m6v454xd6p3bt5swujgmveklsp7lzbkqlqqfc2p36cjlwv5dbucq.b32.i2p:0 +mlgeizrroynuhpxbzeosajt5u4ddcvynxfmcbm6kwjpaufilxigq.b32.i2p:0 +o6t4fr5ayfadzieutstgwcllvwxeuzjlxmzsmpj3hpkvefhzfaea.b32.i2p:0 +ofubxr2ir7u2guzjwyrvujicivzmvinwa36nuzlrg7tnsmebal7a.b32.i2p:0 +oz2ia3flpm3du2tyusulrn7h7e2eo3juzkrmn34bvnrlcrugv7ia.b32.i2p:0 +pohfcrfc7prn4bvn4xstw6nt3e7hjmb7kuj4djtsfqsskwhmhnna.b32.i2p:0 +qd6jlsevsexww3wefpqs7iglxb3f63y4e6ydulfzrvwflpicmdqa.b32.i2p:0 +rfjkzdzv4cwpxo6hzuncicvuyui76wxqx3a23lynq72ktwqs7aja.b32.i2p:0 +rizfinyses2r3or4iubs5wx66gdy6mpf73w7uobfacm2l5cral3q.b32.i2p:0 sedndhv5vpcgdmykyi5st4yqhdxl3hpdtglta4do435wupahhx6q.b32.i2p:0 +tugq6wa2ls2bv27pr2iy3da3k5ow3fzefbcvjcr22uc7w5vmevja.b32.i2p:0 +usztavbib756k5vqggzgkyswoj6mttihjvp3c2pa642t2mb4pvsa.b32.i2p:0 +vgu6llqbyjphml25umd5ztvyxrxuplz2g74fzbx75g3kkaetoyiq.b32.i2p:0 +wjrul5jwwb4vqdmkkrjbmly7osj6amecdpsac5xvaoqrti4nb3ha.b32.i2p:0 wwbw7nqr3ahkqv62cuqfwgtneekvvpnuc4i4f6yo7tpoqjswvcwa.b32.i2p:0 +xfkarmvk43vfkfvhkehy7ioj2b6wtfdlezvmlakblz3q4r7mccfq.b32.i2p:0 +yc4xwin5ujenvcr6ynwkz7lnmmq3nmzxvfguele6ovqqpxgjvonq.b32.i2p:0 +zdoabsg7ugzothyawodjhq54nvlofa746rxfkxpnjzj6nukmha6a.b32.i2p:0 zsxwyo6qcn3chqzwxnseusqgsnuw3maqnztkiypyfxtya4snkoka.b32.i2p:0 +zysrlpii5ftrzivfcyhdrwpeyyqddbrdefnfu5q6otk5gtugmh2a.b32.i2p:0 -# manually added 2022-01 for minimal cjdns bootstrap support +# manually updated 2023-04 for minimal cjdns bootstrap support [fc32:17ea:e415:c3bf:9808:149d:b5a2:c9aa]:8333 [fcc7:be49:ccd1:dc91:3125:f0da:457d:8ce]:8333 +[fcdc:73ae:b1a9:1bf8:d4c2:811:a4c7:c34e]:8333 diff --git a/contrib/seeds/nodes_test.txt b/contrib/seeds/nodes_test.txt index 118bec280e2..5b04791d603 100644 --- a/contrib/seeds/nodes_test.txt +++ b/contrib/seeds/nodes_test.txt @@ -1,16 +1,89 @@ # List of fixed seed nodes for testnet -# Onion nodes -35k2va6vyw4oo5ly2quvcszgdqr56kcnfgcqpnpcffut4jn3mhhwgbid.onion:18333 -blo2esfvk2rr7sr4jspmu3vt2vpgr5rigflsj645fnku7v4qmljurtid.onion:18333 -fuckcswupr5rmlvx2kqqrrosxvjyong4hatmuvxsvtcwe4dsh5rus7qd.onion:18333 -gblylyacjlitd2ywdmo2qqylwtdky7kgeqfvlhiw4zdag4x62tx54hyd.onion:18333 -gzwpduv33l7yze3bcdzj3inebiyjwddjnwvnjhh5wvnv4me76mjt2kad.onion:18333 -h3rphzofxzq52tb63mg5f6kc4my3fkcrgh3m5qryeatts43iljbawiid.onion:18333 -kf4qlhek34b3kgyxyodlmvgm4bxfrjsbjtgayyaiuyhr2eoyfgtm3bad.onion:18333 +# Onion nodes, last verified 2022-08 for minimal torv3 bootstrap support +24j74ahq6ed4wmfrghdwroyfzimlkhnrb7zh4zw3vl2allzxbjrhaqid.onion:18333 +2fy74te65gm3c3gv3u5mhwdudvbdfh6k5fdz4gduimrltjjrxftbxrqd.onion:18333 +2lsncqdflwk272dhydrxf7ikfy23ppnmm54dnynyxiym6lqf3wowrmqd.onion:18333 +33o6qaidta7s2pmltet6vynd337vamgcifhh44rehwwxqpflcjt2njid.onion:18333 +3oo6bsc5mvf6a6ypmoaikilta6ka7mbdhdwhrnqhuhjlbaxyedvfvaqd.onion:18333 +3pe3fyklipy4sppkkgnhc22kcxtt57uler5kv72t676bbrwmcseo5qad.onion:18333 +4u4mcz2sfvxs7pwcwncswgmmcdzqtzjx7ztfo332jv4pqucb22ikdhad.onion:18333 +5v3i2kfqiqwp75gznjoptss7qgrcgseceqxpzpqkd34qeqzrg726i7id.onion:18333 +5zlrxk6q24t4vz5k4ie7gtuasdjavhoelhinzimxbfhc77u7vafipsid.onion:18333 +67s3af64ehw7xnxv422axm7tns4d6kutrftc6bjq375n74q3kj4pp7ad.onion:18333 +6a4ony53julvnufo632ktgmwvhupz63wbdwx7n7qudjy32qyq6gm3bqd.onion:18333 +6ftyg3nhc6tn2hyzls6zfdsfbroczhkxtdqumqb5q4yafhy5rdpapbid.onion:18333 +7554uw5djruh34j5ddx3iprzgqgzypcjtptwoldymfbgoywqcw2wiwyd.onion:18333 +766lozlabxaqjpbqsvt6sn3c65n6gkwwhoxyvggj7nfwnmw4cpaoccad.onion:18333 +7blv5abnytdf47yvbhxmykprmvjryqob65i2jmdwq3rrajcn2iiysbqd.onion:18333 +7v2ja4igx4v5y2jr6jrr6gaxohjhlzhvgwe4avlraxchozf7ea3kruqd.onion:18333 +7zgbmtzxow2oevd5aaqtsormw7ujv4zprl3oi2355immhq4gk7cyw5ad.onion:18333 +adstabjz7ec2y3jt4w2dvummowzv7g6m2f3kajeejffuaz7ojwj6epqd.onion:18333 +aesy6tfufadkut6flu2bsqgnw2422ur2ynjalguxlzuzuktg3zehttqd.onion:18333 +alxo32b5edi3bn2e224qrgytgxxpic4knyipvpdvctfsrvcaiq5lgeyd.onion:18333 +aoeart34umoonvd2kbqr3bc4sweu6a4msh2gp4skyqvei3shzcxbgmyd.onion:18333 +aprzvj7hgctsde4mkj3ewq35gvykspjvkqiygg7bpnw5tkvse2n7rhid.onion:18333 +awpk6z3xghx6ozouhodcydaqtr6uzzbnw4creuix7mkupxoxlmhhspad.onion:18333 +ayynqazucyh2jd5rehcfggmhunqpdwzlbhzbqgy6lj4ctz2ocj7chpid.onion:18333 +b2ika53aqckv4gs7wmog3byrea2vfzm5p7ye33digcsmvvnpbyqmzoyd.onion:18333 +be7zx3hh6dlahorlvsrrgqm4oahfrgqm2tbwnbd4u53ntu5f765n6hyd.onion:18333 +bluk62wj24bsvdwh47muo54hhwsatkftiqxevt5kba7hstjoex6ueeyd.onion:18333 +bubm6fiopfzkxqrfx6vqpioe5ahlhyubz57ogsqqy4ha5pnngiqlh6id.onion:18333 +d3czabzjj57lgrsr5gawkjd7v3gznrqa7zyizqmk4lryascavmipnyad.onion:18333 +ddj4cuvb32ve5chtp6jattcdnnmxmpoofjthzi7thgxxht7yqoetj3yd.onion:18333 +dqhhlssfwmh3g6zhwxpcfbw64xz5rfikcglinbhoxv5ajv4qzicjyeid.onion:18333 +drthcyb4x4rdfekw5g7xjogxi7aqoluilgulbgwvsme3nw3oibvchbad.onion:18333 +dwb47cmqa2tjpmvjaear7gdcars2lez6niefhi4qf22qehtyta6577qd.onion:18333 +e7tkrf54ng3q5vcn5gn77zwjwm74lkfav4mwdux3pvon6yvqg3tf46qd.onion:18333 +etuymy47s3quepvdaoo72i5e5mc7uovrzu5m4jf5q6mwlwizoxy4xgid.onion:18333 +fbimesnyhzubbzqc3uaufzkbyfmnkxvypoxaveaub7rzpzh2foxrn2yd.onion:18333 +fzbrwmgwmko7quelrhfuskt3ijabac76zx7g52dfrevmhdkj6ivh7qyd.onion:18333 +gy6nih4pmp5esyvvnhlj6qvk7zkbjuoswkxffyiip3dbkvsfxwz5zcqd.onion:18333 +ha62ziqzqdogd75zg7lfh4fqrg3bim3cpqzyupo43w5pw4fen6nr2pyd.onion:18333 +hacjjgj2mbqqrthzimmi6anvin7dljjhfl3ik6ebg3w3nmgsvr3ymmqd.onion:18333 +hbkp5xwpqo4qm75kpglfrclyiuuvdgv7mtiqfys7oqks4dmpqgpeoeid.onion:18333 +hqgoy62hoqjmz37brdfvoeov3cix5fixbqjoert4ydr6herg5oc3iwyd.onion:18333 +hvbmmzvqrpgps2x5u4ip4ksf3e5m2fneac754gtnhjn2rsevni6cz3ad.onion:18333 +hw3vzp32w4h6giplue6ix445oi6wt7gmeksrznb7tdfwhkgit7gnbbad.onion:18333 +iddr66ewkhenivapgianudjkwqcp6dxtssg7ixrdot5az6uh7m5tmjqd.onion:18333 +imya36iexiiiqrkwuxxcehnv4kg5shtirwd2vg4cnjy6lfjlph3fusqd.onion:18333 +iuhhuocns7entrzlxsxktyz2ibs7hqgiggv6sauzqkzka6laslwz7oqd.onion:18333 +ji5wmshokuc63eiulzlwj2zdvnligvrwfvvc76bice3tu43wfzvpmkyd.onion:18333 +jjfuyj7krgzkmpxvn3b2j2hwlzkmze3ezy3ifwk7dnswwawgmzqhjrqd.onion:18333 +jn2p4sgfphkxpow7kjrubrbqat77kkibzqkvuwhxyalcrazwmcqeaqyd.onion:18333 +jrveyz4us6sog6e6czsvr5mvvhgzjgv4idbe4idrolmqeudvt5a2dgid.onion:18333 +jsc4frvvnl2d3bhzyofsc72xpztgm23nl4fnb4dwkzsxr6fhij2q5iyd.onion:18333 +klymxdvje7kccv3tznabo3udopsftkmjemkbi2urqxjm4hefaudejjyd.onion:18333 +kwjxlauwjtecjfsiwopbl5pvn5n6z5rz76uk6osmlurd3uyuymcw7aid.onion:18333 +lc7upz2srw2yhpcvwg4afy64ylcoo6mfwlttqj5ovuglqnhnohpi5iqd.onion:18333 +lf3mpxfyjuovcqdvinl52pvdmmda6xqyfeiarlfamdjpgy3ouzmmlbyd.onion:18333 mc7k47ndjvvhcgs54wmjzxvate4rtuybbjoryikdssjhcxlx27psbyqd.onion:18333 -mrhiniicugfo7mgrwv3wtolk3tptlcw2uq7ih6sq43fa4k4zbilut3yd.onion:18333 -uiudyws3qizgmepfoh7wwjmsoxoxut4qrmotjjhrn247xnjopr7sfcid.onion:18333 -zc2wvoqcezcrf64trji6jmhtss34a5ds5ntzdhqegzvex3ynrd7nxcad.onion:18333 -zd5m3dgdn46naj36pxvvcalfw2paecle6sdxq64ptwxtxjomkywpklqd.onion:18333 - +mjbg3ggeuelmc7ixty3zjccyo2urg2uyherfqe7ytkm2ejkwlec7h6ad.onion:18333 +nkyqozv6kdwi423s7s2mezzguf5bafot2a3hv4ed2dbvtblisdmad4qd.onion:18333 +nvvqo4xxiwgb3y246jmcbuuveurfdq2zs3a5y7veqkeqv5jfhang7gyd.onion:18333 +o6vfovqxz3oxszfppczpjejwouobztjrgvfojc3emvhan3bkyskzhuad.onion:18333 +oaiw2lnhzgp5ry7ivzneuufmh7lfploquu2rjv5rozmlbefedsnxe5qd.onion:18333 +oln7ybci53wk4g5n42nipyixvyjxbludsbrfsmhnirb6tk7ovlikd5id.onion:18333 +otmfnhc6wrrbf2tpdy6zkisqc3r3urnsuowsnmatoto6yixaocnkseid.onion:18333 +ovc6sajbqfcbwv3wrq7ylklu6q6prvisz4jr4lyycn4kgukzjfe4mjad.onion:18333 +pm57didyzg5ljuvn5ufr5uun2iencuk3af2gzqc5zvgfh452c3rxtjyd.onion:18333 +pmismhpwug34gnqzbutranvx2wjwbshyqj4un2dyzyuvak2eh55psfyd.onion:18333 +polarisultijjhaku6z6u7jyboho5epdsg44ttebfaxmgau2z5sqolad.onion:18333 +qe2jbe447he6panfvpyqhyntf7346gmuf55bxrmdzggmgwyjsyknhxyd.onion:18333 +qz6yd5lsgdajcteoareeptwnipxsezyx5kks6ukpk5tvqinffzunqmyd.onion:18333 +rp6pn3b3oesyr2giolbysbjhqeugxntsu7crnkth4y33ok4zvcl7yrqd.onion:18333 +ujdchuw3hz5gkbouiv4p6pwbfdn7v4k6gluwvd4wiukqc7y7ow754uad.onion:18333 +vctlwaqgmu53eutz2hewuakcipfgtyljsd7czut4dd62xr3rp6fqezad.onion:18333 +vf5ur53tzmdtotvkndcgochklnuav7quqjvkc6mctqfvef6wnmn26mid.onion:18333 +wnxgjgjgplv5iu4mssyuunycvku4qnqr5t4q6cfdt47k7uwrfifuirad.onion:18333 +wpkbkdr7clw7zk3jkwiult6bf422j54u77ml4rgig2xq7icogyrcspid.onion:18333 +wzpdt24tdark26eugredddorik3tqwcj5ialtt2yim4ceiuiq7phkyqd.onion:18333 +xgapnikkbldoggjh5ewxkyauhuwnvf3xkspxroe3ojvfrk4lswkyx5yd.onion:18333 +xkvzdhcirontixbq6pjhru57bf4sgtqylvphk25csfrsy5p5ay3oc3yd.onion:18333 +xnipauenw5wnjb2zbx6v6umgvbb3g6xhf5kjo7pnyn5tdzvzaxtzicid.onion:18333 +yda7kwpii33j2qpq32ftf6lp22znknswipjwaccvsqj7l337jvfesnid.onion:18333 +z3j5foswuhpmtrg3kb56stkzmuoaesvd5jz3eztq46c4cidapglcyuad.onion:18333 +zcep44k7unwjm2wxty4ijh2e4fv5zgbrvwlctzyaqnrqhltjfzrtodad.onion:18333 +zmvizz7fd5hdue6wt3lwqumd6qwt4ijymmmotfzh75curq3mzjm53hyd.onion:18333 +zoaa3x7quyuijggii5zl4uyeioodudsgtr2uyv2qtdsslac5ukiwlxid.onion:18333 +zovauxlorl5eswumbsoxv2m5y3sm3qlk7657dcpr2uld7xf35en46sqd.onion:18333 diff --git a/contrib/seeds/suspicious_hosts.txt b/contrib/seeds/suspicious_hosts.txt deleted file mode 100644 index 13385cc8162..00000000000 --- a/contrib/seeds/suspicious_hosts.txt +++ /dev/null @@ -1,16 +0,0 @@ -130.211.129.106 -148.251.238.178 -176.9.46.6 -178.63.107.226 -54.173.72.127 -54.174.10.182 -54.183.64.54 -54.194.231.211 -54.66.214.167 -54.66.220.137 -54.67.33.14 -54.77.251.214 -54.94.195.96 -54.94.200.247 -83.81.130.26 -88.198.17.7 \ No newline at end of file diff --git a/contrib/signet/getcoins.py b/contrib/signet/getcoins.py index 147d12600df..19751ae2695 100755 --- a/contrib/signet/getcoins.py +++ b/contrib/signet/getcoins.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -142,7 +142,7 @@ def bitcoin_cli(rpc_command_and_params): try: res = session.post(args.faucet, data=data) -except: +except Exception: raise SystemExit(f"Unexpected error when contacting faucet: {sys.exc_info()[0]}") # Display the output as per the returned status code diff --git a/contrib/signet/miner b/contrib/signet/miner index b366b98e2d8..61d9f62be71 100755 --- a/contrib/signet/miner +++ b/contrib/signet/miner @@ -4,7 +4,6 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. import argparse -import base64 import json import logging import math @@ -15,14 +14,13 @@ import sys import time import subprocess -from io import BytesIO - PATH_BASE_CONTRIB_SIGNET = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) PATH_BASE_TEST_FUNCTIONAL = os.path.abspath(os.path.join(PATH_BASE_CONTRIB_SIGNET, "..", "..", "test", "functional")) sys.path.insert(0, PATH_BASE_TEST_FUNCTIONAL) -from test_framework.blocktools import WITNESS_COMMITMENT_HEADER, script_BIP34_coinbase_height # noqa: E402 -from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, from_hex, deser_string, hash256, ser_compact_size, ser_string, ser_uint256, tx_from_hex, uint256_from_str # noqa: E402 +from test_framework.blocktools import get_witness_script, script_BIP34_coinbase_height # noqa: E402 +from test_framework.messages import CBlock, CBlockHeader, COutPoint, CTransaction, CTxIn, CTxInWitness, CTxOut, from_binary, from_hex, ser_string, ser_uint256, tx_from_hex # noqa: E402 +from test_framework.psbt import PSBT, PSBTMap, PSBT_GLOBAL_UNSIGNED_TX, PSBT_IN_FINAL_SCRIPTSIG, PSBT_IN_FINAL_SCRIPTWITNESS, PSBT_IN_NON_WITNESS_UTXO, PSBT_IN_SIGHASH_TYPE # noqa: E402 from test_framework.script import CScriptOp # noqa: E402 logging.basicConfig( @@ -34,99 +32,12 @@ SIGNET_HEADER = b"\xec\xc7\xda\xa2" PSBT_SIGNET_BLOCK = b"\xfc\x06signetb" # proprietary PSBT global field holding the block being signed RE_MULTIMINER = re.compile("^(\d+)(-(\d+))?/(\d+)$") -# #### some helpers that could go into test_framework - -# like from_hex, but without the hex part -def FromBinary(cls, stream): - """deserialize a binary stream (or bytes object) into an object""" - # handle bytes object by turning it into a stream - was_bytes = isinstance(stream, bytes) - if was_bytes: - stream = BytesIO(stream) - obj = cls() - obj.deserialize(stream) - if was_bytes: - assert len(stream.read()) == 0 - return obj - -class PSBTMap: - """Class for serializing and deserializing PSBT maps""" - - def __init__(self, map=None): - self.map = map if map is not None else {} - - def deserialize(self, f): - m = {} - while True: - k = deser_string(f) - if len(k) == 0: - break - v = deser_string(f) - if len(k) == 1: - k = k[0] - assert k not in m - m[k] = v - self.map = m - - def serialize(self): - m = b"" - for k,v in self.map.items(): - if isinstance(k, int) and 0 <= k and k <= 255: - k = bytes([k]) - m += ser_compact_size(len(k)) + k - m += ser_compact_size(len(v)) + v - m += b"\x00" - return m - -class PSBT: - """Class for serializing and deserializing PSBTs""" - - def __init__(self): - self.g = PSBTMap() - self.i = [] - self.o = [] - self.tx = None - - def deserialize(self, f): - assert f.read(5) == b"psbt\xff" - self.g = FromBinary(PSBTMap, f) - assert 0 in self.g.map - self.tx = FromBinary(CTransaction, self.g.map[0]) - self.i = [FromBinary(PSBTMap, f) for _ in self.tx.vin] - self.o = [FromBinary(PSBTMap, f) for _ in self.tx.vout] - return self - - def serialize(self): - assert isinstance(self.g, PSBTMap) - assert isinstance(self.i, list) and all(isinstance(x, PSBTMap) for x in self.i) - assert isinstance(self.o, list) and all(isinstance(x, PSBTMap) for x in self.o) - assert 0 in self.g.map - tx = FromBinary(CTransaction, self.g.map[0]) - assert len(tx.vin) == len(self.i) - assert len(tx.vout) == len(self.o) - - psbt = [x.serialize() for x in [self.g] + self.i + self.o] - return b"psbt\xff" + b"".join(psbt) - - def to_base64(self): - return base64.b64encode(self.serialize()).decode("utf8") - - @classmethod - def from_base64(cls, b64psbt): - return FromBinary(cls, base64.b64decode(b64psbt)) - -# ##### - def create_coinbase(height, value, spk): cb = CTransaction() cb.vin = [CTxIn(COutPoint(0, 0xffffffff), script_BIP34_coinbase_height(height), 0xffffffff)] cb.vout = [CTxOut(value, spk)] return cb -def get_witness_script(witness_root, witness_nonce): - commitment = uint256_from_str(hash256(ser_uint256(witness_root) + ser_uint256(witness_nonce))) - return b"\x6a" + CScriptOp.encode_op_pushdata(WITNESS_COMMITMENT_HEADER + ser_uint256(commitment)) - def signet_txs(block, challenge): # assumes signet solution has not been added yet so does not need # to be removed @@ -163,11 +74,11 @@ def signet_txs(block, challenge): def do_createpsbt(block, signme, spendme): psbt = PSBT() - psbt.g = PSBTMap( {0: signme.serialize(), + psbt.g = PSBTMap( {PSBT_GLOBAL_UNSIGNED_TX: signme.serialize(), PSBT_SIGNET_BLOCK: block.serialize() } ) - psbt.i = [ PSBTMap( {0: spendme.serialize(), - 3: bytes([1,0,0,0])}) + psbt.i = [ PSBTMap( {PSBT_IN_NON_WITNESS_UTXO: spendme.serialize(), + PSBT_IN_SIGHASH_TYPE: bytes([1,0,0,0])}) ] psbt.o = [ PSBTMap() ] return psbt.to_base64() @@ -179,10 +90,10 @@ def do_decode_psbt(b64psbt): assert len(psbt.tx.vout) == 1 assert PSBT_SIGNET_BLOCK in psbt.g.map - scriptSig = psbt.i[0].map.get(7, b"") - scriptWitness = psbt.i[0].map.get(8, b"\x00") + scriptSig = psbt.i[0].map.get(PSBT_IN_FINAL_SCRIPTSIG, b"") + scriptWitness = psbt.i[0].map.get(PSBT_IN_FINAL_SCRIPTWITNESS, b"\x00") - return FromBinary(CBlock, psbt.g.map[PSBT_SIGNET_BLOCK]), ser_string(scriptSig) + scriptWitness + return from_binary(CBlock, psbt.g.map[PSBT_SIGNET_BLOCK]), ser_string(scriptSig) + scriptWitness def finish_block(block, signet_solution, grind_cmd): block.vtx[0].vout[-1].scriptPubKey += CScriptOp.encode_op_pushdata(SIGNET_HEADER + signet_solution) @@ -222,7 +133,7 @@ def generate_psbt(tmpl, reward_spk, *, blocktime=None): cbwit = CTxInWitness() cbwit.scriptWitness.stack = [ser_uint256(witnonce)] block.vtx[0].wit.vtxinwit = [cbwit] - block.vtx[0].vout.append(CTxOut(0, get_witness_script(witroot, witnonce))) + block.vtx[0].vout.append(CTxOut(0, bytes(get_witness_script(witroot, witnonce)))) signme, spendme = signet_txs(block, signet_spk_bin) @@ -314,7 +225,7 @@ def seconds_to_hms(s): out = "-" + out return out -def next_block_delta(last_nbits, last_hash, ultimate_target, do_poisson): +def next_block_delta(last_nbits, last_hash, ultimate_target, do_poisson, max_interval): # strategy: # 1) work out how far off our desired target we are # 2) cap it to a factor of 4 since that's the best we can do in a single retarget period @@ -337,7 +248,7 @@ def next_block_delta(last_nbits, last_hash, ultimate_target, do_poisson): this_interval_variance = 1 this_interval = avg_interval * this_interval_variance - this_interval = max(1, min(this_interval, 3600)) + this_interval = max(1, min(this_interval, max_interval)) return this_interval @@ -397,6 +308,10 @@ def do_generate(args): return 1 my_blocks = (start-1, stop, total) + if args.max_interval < 960: + logging.error("--max-interval must be at least 960 (16 minutes)") + return 1 + ultimate_target = nbits_to_target(int(args.nbits,16)) mined_blocks = 0 @@ -413,7 +328,7 @@ def do_generate(args): if lastheader is None: lastheader = bestheader["hash"] elif bestheader["hash"] != lastheader: - next_delta = next_block_delta(int(bestheader["bits"], 16), bestheader["hash"], ultimate_target, args.poisson) + next_delta = next_block_delta(int(bestheader["bits"], 16), bestheader["hash"], ultimate_target, args.poisson, args.max_interval) next_delta += bestheader["time"] - time.time() next_is_mine = next_block_is_mine(bestheader["hash"], my_blocks) logging.info("Received new block at height %d; next in %s (%s)", bestheader["height"], seconds_to_hms(next_delta), ("mine" if next_is_mine else "backup")) @@ -427,14 +342,14 @@ def do_generate(args): action_time = now is_mine = True elif bestheader["height"] == 0: - time_delta = next_block_delta(int(bestheader["bits"], 16), bci["bestblockhash"], ultimate_target, args.poisson) + time_delta = next_block_delta(int(bestheader["bits"], 16), bci["bestblockhash"], ultimate_target, args.poisson, args.max_interval) time_delta *= 100 # 100 blocks logging.info("Backdating time for first block to %d minutes ago" % (time_delta/60)) mine_time = now - time_delta action_time = now is_mine = True else: - time_delta = next_block_delta(int(bestheader["bits"], 16), bci["bestblockhash"], ultimate_target, args.poisson) + time_delta = next_block_delta(int(bestheader["bits"], 16), bci["bestblockhash"], ultimate_target, args.poisson, args.max_interval) mine_time = bestheader["time"] + time_delta is_mine = next_block_is_mine(bci["bestblockhash"], my_blocks) @@ -508,7 +423,7 @@ def do_generate(args): # report bstr = "block" if is_mine else "backup block" - next_delta = next_block_delta(block.nBits, block.hash, ultimate_target, args.poisson) + next_delta = next_block_delta(block.nBits, block.hash, ultimate_target, args.poisson, args.max_interval) next_delta += block.nTime - time.time() next_is_mine = next_block_is_mine(block.hash, my_blocks) @@ -586,6 +501,7 @@ def main(): generate.add_argument("--multiminer", default=None, type=str, help="Specify which set of blocks to mine (eg: 1-40/100 for the first 40%%, 2/3 for the second 3rd)") generate.add_argument("--backup-delay", default=300, type=int, help="Seconds to delay before mining blocks reserved for other miners (default=300)") generate.add_argument("--standby-delay", default=0, type=int, help="Seconds to delay before mining blocks (default=0)") + generate.add_argument("--max-interval", default=1800, type=int, help="Maximum interblock interval (seconds)") calibrate = cmds.add_parser("calibrate", help="Calibrate difficulty") calibrate.set_defaults(fn=do_calibrate) @@ -627,5 +543,3 @@ def main(): if __name__ == "__main__": main() - - diff --git a/contrib/testgen/README.md b/contrib/testgen/README.md index fcc5a378e2c..2f0288df165 100644 --- a/contrib/testgen/README.md +++ b/contrib/testgen/README.md @@ -2,7 +2,7 @@ Utilities to generate test vectors for the data-driven Bitcoin tests. -Usage: +To use inside a scripted-diff (or just execute directly): - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json + ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json + ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json diff --git a/contrib/testgen/base58.py b/contrib/testgen/base58.py deleted file mode 100644 index 87341ccf96d..00000000000 --- a/contrib/testgen/base58.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (c) 2012-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -''' -Bitcoin base58 encoding and decoding. - -Based on https://bitcointalk.org/index.php?topic=1026.0 (public domain) -''' -import hashlib - -# for compatibility with following code... -class SHA256: - new = hashlib.sha256 - -if str != bytes: - # Python 3.x - def ord(c): - return c - def chr(n): - return bytes( (n,) ) - -__b58chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' -__b58base = len(__b58chars) -b58chars = __b58chars - -def b58encode(v): - """ encode v, which is a string of bytes, to base58. - """ - long_value = 0 - for (i, c) in enumerate(v[::-1]): - if isinstance(c, str): - c = ord(c) - long_value += (256**i) * c - - result = '' - while long_value >= __b58base: - div, mod = divmod(long_value, __b58base) - result = __b58chars[mod] + result - long_value = div - result = __b58chars[long_value] + result - - # Bitcoin does a little leading-zero-compression: - # leading 0-bytes in the input become leading-1s - nPad = 0 - for c in v: - if c == 0: - nPad += 1 - else: - break - - return (__b58chars[0]*nPad) + result - -def b58decode(v, length = None): - """ decode v into a string of len bytes - """ - long_value = 0 - for i, c in enumerate(v[::-1]): - pos = __b58chars.find(c) - assert pos != -1 - long_value += pos * (__b58base**i) - - result = bytes() - while long_value >= 256: - div, mod = divmod(long_value, 256) - result = chr(mod) + result - long_value = div - result = chr(long_value) + result - - nPad = 0 - for c in v: - if c == __b58chars[0]: - nPad += 1 - continue - break - - result = bytes(nPad) + result - if length is not None and len(result) != length: - return None - - return result - -def checksum(v): - """Return 32-bit checksum based on SHA256""" - return SHA256.new(SHA256.new(v).digest()).digest()[0:4] - -def b58encode_chk(v): - """b58encode a string, with 32-bit checksum""" - return b58encode(v + checksum(v)) - -def b58decode_chk(v): - """decode a base58 string, check and remove checksum""" - result = b58decode(v) - if result is None: - return None - if result[-4:] == checksum(result[:-4]): - return result[:-4] - else: - return None - -def get_bcaddress_version(strAddress): - """ Returns None if strAddress is invalid. Otherwise returns integer version of address. """ - addr = b58decode_chk(strAddress) - if addr is None or len(addr)!=21: - return None - version = addr[0] - return ord(version) - -if __name__ == '__main__': - # Test case (from http://gitorious.org/bitcoin/python-base58.git) - assert get_bcaddress_version('15VjRaDX9zpbA8LVnbrCAFzrVzN7ixHNsC') == 0 - _ohai = 'o hai'.encode('ascii') - _tmp = b58encode(_ohai) - assert _tmp == 'DYB3oMS' - assert b58decode(_tmp, 5) == _ohai - print("Tests passed") diff --git a/contrib/testgen/gen_key_io_test_vectors.py b/contrib/testgen/gen_key_io_test_vectors.py index eafaaaeceb6..7bfb1d76a8b 100755 --- a/contrib/testgen/gen_key_io_test_vectors.py +++ b/contrib/testgen/gen_key_io_test_vectors.py @@ -1,21 +1,21 @@ #!/usr/bin/env python3 -# Copyright (c) 2012-2021 The Bitcoin Core developers +# Copyright (c) 2012-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Generate valid and invalid base58/bech32(m) address and private key test vectors. - -Usage: - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py valid 70 > ../../src/test/data/key_io_valid.json - PYTHONPATH=../../test/functional/test_framework ./gen_key_io_test_vectors.py invalid 70 > ../../src/test/data/key_io_invalid.json ''' -# 2012 Wladimir J. van der Laan -# Released under MIT License -import os + from itertools import islice -from base58 import b58encode_chk, b58decode_chk, b58chars +import os import random -from segwit_addr import bech32_encode, decode_segwit_address, convertbits, CHARSET, Encoding +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), '../../test/functional')) + +from test_framework.address import base58_to_byte, byte_to_base58, b58chars # noqa: E402 +from test_framework.script import OP_0, OP_1, OP_2, OP_3, OP_16, OP_DUP, OP_EQUAL, OP_EQUALVERIFY, OP_HASH160, OP_CHECKSIG # noqa: E402 +from test_framework.segwit_addr import bech32_encode, decode_segwit_address, convertbits, CHARSET, Encoding # noqa: E402 # key types PUBKEY_ADDRESS = 0 @@ -29,16 +29,6 @@ PRIVKEY_REGTEST = 239 # script -OP_0 = 0x00 -OP_1 = 0x51 -OP_2 = 0x52 -OP_3 = 0x53 -OP_16 = 0x60 -OP_DUP = 0x76 -OP_EQUAL = 0x87 -OP_EQUALVERIFY = 0x88 -OP_HASH160 = 0xa9 -OP_CHECKSIG = 0xac pubkey_prefix = (OP_DUP, OP_HASH160, 20) pubkey_suffix = (OP_EQUALVERIFY, OP_CHECKSIG) script_prefix = (OP_HASH160, 20) @@ -114,8 +104,10 @@ def is_valid(v): '''Check vector v for validity''' if len(set(v) - set(b58chars)) > 0: return is_valid_bech32(v) - result = b58decode_chk(v) - if result is None: + try: + payload, version = base58_to_byte(v) + result = bytes([version]) + payload + except ValueError: # thrown if checksum doesn't match return is_valid_bech32(v) for template in templates: prefix = bytearray(template[0]) @@ -135,18 +127,19 @@ def is_valid_bech32(v): def gen_valid_base58_vector(template): '''Generate valid base58 vector''' prefix = bytearray(template[0]) - payload = bytearray(os.urandom(template[1])) + payload = rand_bytes(size=template[1]) suffix = bytearray(template[2]) dst_prefix = bytearray(template[4]) dst_suffix = bytearray(template[5]) - rv = b58encode_chk(prefix + payload + suffix) + assert len(prefix) == 1 + rv = byte_to_base58(payload + suffix, prefix[0]) return rv, dst_prefix + payload + dst_suffix def gen_valid_bech32_vector(template): '''Generate valid bech32 vector''' hrp = template[0] witver = template[1] - witprog = bytearray(os.urandom(template[2])) + witprog = rand_bytes(size=template[2]) encoding = template[4] dst_prefix = bytearray(template[5]) rv = bech32_encode(encoding, hrp, [witver] + convertbits(witprog, 8, 5)) @@ -176,21 +169,22 @@ def gen_invalid_base58_vector(template): corrupt_suffix = randbool(0.2) if corrupt_prefix: - prefix = os.urandom(1) + prefix = rand_bytes(size=1) else: prefix = bytearray(template[0]) if randomize_payload_size: - payload = os.urandom(max(int(random.expovariate(0.5)), 50)) + payload = rand_bytes(size=max(int(random.expovariate(0.5)), 50)) else: - payload = os.urandom(template[1]) + payload = rand_bytes(size=template[1]) if corrupt_suffix: - suffix = os.urandom(len(template[2])) + suffix = rand_bytes(size=len(template[2])) else: suffix = bytearray(template[2]) - val = b58encode_chk(prefix + payload + suffix) + assert len(prefix) == 1 + val = byte_to_base58(payload + suffix, prefix[0]) if random.randint(0,10)<1: # line corruption if randbool(): # add random character to end val += random.choice(b58chars) @@ -206,7 +200,7 @@ def gen_invalid_bech32_vector(template): to_upper = randbool(0.1) hrp = template[0] witver = template[1] - witprog = bytearray(os.urandom(template[2])) + witprog = rand_bytes(size=template[2]) encoding = template[3] if no_data: @@ -236,6 +230,9 @@ def randbool(p = 0.5): '''Return True with P(p)''' return random.random() < p +def rand_bytes(*, size): + return bytearray(random.getrandbits(8) for _ in range(size)) + def gen_invalid_vectors(): '''Generate invalid test vectors''' # start with some manual edge-cases @@ -250,9 +247,9 @@ def gen_invalid_vectors(): yield val, if __name__ == '__main__': - import sys import json iters = {'valid':gen_valid_vectors, 'invalid':gen_invalid_vectors} + random.seed(42) try: uiter = iters[sys.argv[1]] except IndexError: diff --git a/contrib/tracing/README.md b/contrib/tracing/README.md index a409a23ef8d..206bec16478 100644 --- a/contrib/tracing/README.md +++ b/contrib/tracing/README.md @@ -286,3 +286,53 @@ Spent a05880b8c77971ed0b9f73062c7c4cdb0ff3856ab14cbf8bc481ed571cd34b83:1 Added eb689865f7d957938978d6207918748f74e6aa074f47874724327089445b0960:0 5589696005 2094513 No Added eb689865f7d957938978d6207918748f74e6aa074f47874724327089445b0960:1 1565556 2094513 No ``` + +### mempool_monitor.py + +A BCC Python script producing mempool statistics and an event log. Based on the +`mempool:added`, `mempool:removed`, `mempool:replaced`, and `mempool:rejected` +tracepoints. + +Statistics include incidence and rate for each event type since the script was +started (`total`) as well as during the last minute (`1 min`) and ten minutes +(`10 min`). The event log shows mempool events in real time, each entry +comprising a timestamp along with all event data available via the event's +tracepoint. + +```console +$ python3 contrib/tracing/mempool_monitor.py ./src/bitcoind +``` + +``` + Mempool Monitor + Press CTRL-C to stop. + + ┌─Event count───────────────────────┐ ┌─Event rate──────────────────────────┐ + │ Event total 1 min 10 min │ │ Event total 1 min 10 min │ + │ added 1425tx 201tx 1425tx │ │ added 4.7tx/s 3.4tx/s 4.7tx/s │ + │ removed 35tx 4tx 35tx │ │ removed 0.1tx/s 0.1tx/s 0.1tx/s │ + │ replaced 35tx 4tx 35tx │ │ replaced 0.1tx/s 0.1tx/s 0.1tx/s │ + │ rejected 0tx 0tx 0tx │ │ rejected 0.0tx/s 0.0tx/s 0.0tx/s │ + └───────────────────────────────────┘ └─────────────────────────────────────┘ + + ┌─Event log────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ 13:10:30Z added f9064ca5bfc87cdd191faa42bf697217cd920b2b94838c1f1192e4f06c4fd217 with feerate 8.92 sat/vB (981 sat, 110 vbytes) │ + │ 13:10:30Z added 53ffa3afbe57b1bfe423e1755ca2b52c5b6cb4aa91b8b7ee9cb694953f47f234 with feerate 5.00 sat/vB (550 sat, 110 vbytes) │ + │ 13:10:30Z added 4177df5e19465eb5e53c3f8b6830a293f57474921bc6c2ae89375e0986e1f0f9 with feerate 2.98 sat/vB (429 sat, 144 vbytes) │ + │ 13:10:30Z added 931a10d83f0a268768da75dc4b9e199f2f055f12979ae5491cc304ee10f890ea with feerate 3.55 sat/vB (500 sat, 141 vbytes) │ + │ 13:10:30Z added 4cf32b295723cc4ab73f2a2e51d4bb276c0042760a4c00a3eb9595b8ebb24721 with feerate 89.21 sat/vB (12668 sat, 142 vbytes) │ + │ 13:10:31Z replaced d1eecf9d662121322f4f31f0c2267a752d14bb3956e6016ba96e87f47890e1db with feerate 27.12 sat/vB received 23.3 seconds ago (7213 sat, 266 vbytes) with c412db908│ + │ 9b7ed53f3e5e36d2819dd291278b59ccaabaeb17fd37c3d87fdcd57 with feerate 28.12 sat/vB (8351 sat, 297 vbytes) │ + │ 13:10:31Z added c412db9089b7ed53f3e5e36d2819dd291278b59ccaabaeb17fd37c3d87fdcd57 with feerate 28.12 sat/vB (8351 sat, 297 vbytes) │ + │ 13:10:31Z added b8388a5bdc421b11460bdf477d5a85a1a39c2784e7dd7bffabe688740424ea57 with feerate 25.21 sat/vB (3554 sat, 141 vbytes) │ + │ 13:10:31Z added 4ddb88bc90a122cd9eae8a664e73bdf5bebe75f3ef901241b4a251245854a98e with feerate 24.15 sat/vB (5072 sat, 210 vbytes) │ + │ 13:10:31Z added 19101e4161bca5271ad5d03e7747f2faec7793b274dc2f3c4cf516b7cef1aac3 with feerate 7.06 sat/vB (1080 sat, 153 vbytes) │ + │ 13:10:31Z removed d1eecf9d662121322f4f31f0c2267a752d14bb3956e6016ba96e87f47890e1db with feerate 27.12 sat/vB (7213 sat, 266 vbytes): replaced │ + │ 13:10:31Z added 6c511c60d9b95b9eff81df6ecba5c86780f513fe62ce3ad6be2c5340d957025a with feerate 4.00 sat/vB (440 sat, 110 vbytes) │ + │ 13:10:31Z added 44d66f7f004bd52c46be4dff3067cab700e51c7866a84282bd8aab560a5bfb79 with feerate 3.15 sat/vB (448 sat, 142 vbytes) │ + │ 13:10:31Z added b17b7c9ec5acfbbf12f0eeef8e29826fad3105bb95eef7a47d2f1f22b4784643 with feerate 4.10 sat/vB (1348 sat, 329 vbytes) │ + │ 13:10:31Z added b7a4ad93554e57454e8a8049bfc0bd803fa962bd3f0a08926aa72e7cb23e2276 with feerate 1.01 sat/vB (205 sat, 202 vbytes) │ + │ 13:10:32Z added c78e87be86c828137a6e7e00a177c03b52202ce4c39029b99904c2a094b9da87 with feerate 11.00 sat/vB (1562 sat, 142 vbytes) │ + │ │ + └──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` diff --git a/contrib/tracing/log_utxocache_flush.py b/contrib/tracing/log_utxocache_flush.py index 8c073bea0d1..6c568998e99 100755 --- a/contrib/tracing/log_utxocache_flush.py +++ b/contrib/tracing/log_utxocache_flush.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2021 The Bitcoin Core developers +# Copyright (c) 2021-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/contrib/tracing/mempool_monitor.py b/contrib/tracing/mempool_monitor.py new file mode 100755 index 00000000000..9d427d4632a --- /dev/null +++ b/contrib/tracing/mempool_monitor.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" Example logging Bitcoin Core mempool events using the mempool:added, + mempool:removed, mempool:replaced, and mempool:rejected tracepoints. """ + +import curses +import sys +from datetime import datetime, timezone + +from bcc import BPF, USDT + +# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into +# a sandboxed Linux kernel VM. +PROGRAM = """ +# include + +// The longest rejection reason is 118 chars and is generated in case of SCRIPT_ERR_EVAL_FALSE by +// strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError())) +#define MAX_REJECT_REASON_LENGTH 118 +// The longest string returned by RemovalReasonToString() is 'sizelimit' +#define MAX_REMOVAL_REASON_LENGTH 9 +#define HASH_LENGTH 32 + +struct added_event +{ + u8 hash[HASH_LENGTH]; + u64 vsize; + s64 fee; +}; + +struct removed_event +{ + u8 hash[HASH_LENGTH]; + char reason[MAX_REMOVAL_REASON_LENGTH]; + u64 vsize; + s64 fee; + u64 entry_time; +}; + +struct rejected_event +{ + u8 hash[HASH_LENGTH]; + char reason[MAX_REJECT_REASON_LENGTH]; +}; + +struct replaced_event +{ + u8 replaced_hash[HASH_LENGTH]; + u64 replaced_vsize; + s64 replaced_fee; + u64 replaced_entry_time; + u8 replacement_hash[HASH_LENGTH]; + u64 replacement_vsize; + s64 replacement_fee; +}; + +// BPF perf buffer to push the data to user space. +BPF_PERF_OUTPUT(added_events); +BPF_PERF_OUTPUT(removed_events); +BPF_PERF_OUTPUT(rejected_events); +BPF_PERF_OUTPUT(replaced_events); + +int trace_added(struct pt_regs *ctx) { + struct added_event added = {}; + + bpf_usdt_readarg_p(1, ctx, &added.hash, HASH_LENGTH); + bpf_usdt_readarg(2, ctx, &added.vsize); + bpf_usdt_readarg(3, ctx, &added.fee); + + added_events.perf_submit(ctx, &added, sizeof(added)); + return 0; +} + +int trace_removed(struct pt_regs *ctx) { + struct removed_event removed = {}; + + bpf_usdt_readarg_p(1, ctx, &removed.hash, HASH_LENGTH); + bpf_usdt_readarg_p(2, ctx, &removed.reason, MAX_REMOVAL_REASON_LENGTH); + bpf_usdt_readarg(3, ctx, &removed.vsize); + bpf_usdt_readarg(4, ctx, &removed.fee); + bpf_usdt_readarg(5, ctx, &removed.entry_time); + + removed_events.perf_submit(ctx, &removed, sizeof(removed)); + return 0; +} + +int trace_rejected(struct pt_regs *ctx) { + struct rejected_event rejected = {}; + + bpf_usdt_readarg_p(1, ctx, &rejected.hash, HASH_LENGTH); + bpf_usdt_readarg_p(2, ctx, &rejected.reason, MAX_REJECT_REASON_LENGTH); + + rejected_events.perf_submit(ctx, &rejected, sizeof(rejected)); + return 0; +} + +int trace_replaced(struct pt_regs *ctx) { + struct replaced_event replaced = {}; + + bpf_usdt_readarg_p(1, ctx, &replaced.replaced_hash, HASH_LENGTH); + bpf_usdt_readarg(2, ctx, &replaced.replaced_vsize); + bpf_usdt_readarg(3, ctx, &replaced.replaced_fee); + bpf_usdt_readarg(4, ctx, &replaced.replaced_entry_time); + bpf_usdt_readarg_p(5, ctx, &replaced.replacement_hash, HASH_LENGTH); + bpf_usdt_readarg(6, ctx, &replaced.replacement_vsize); + bpf_usdt_readarg(7, ctx, &replaced.replacement_fee); + + replaced_events.perf_submit(ctx, &replaced, sizeof(replaced)); + return 0; +} +""" + + +def main(bitcoind_path): + bitcoind_with_usdts = USDT(path=str(bitcoind_path)) + + # attaching the trace functions defined in the BPF program + # to the tracepoints + bitcoind_with_usdts.enable_probe(probe="mempool:added", fn_name="trace_added") + bitcoind_with_usdts.enable_probe(probe="mempool:removed", fn_name="trace_removed") + bitcoind_with_usdts.enable_probe(probe="mempool:replaced", fn_name="trace_replaced") + bitcoind_with_usdts.enable_probe(probe="mempool:rejected", fn_name="trace_rejected") + bpf = BPF(text=PROGRAM, usdt_contexts=[bitcoind_with_usdts]) + + events = [] + + def get_timestamp(): + return datetime.now(timezone.utc) + + def handle_added(_, data, size): + event = bpf["added_events"].event(data) + events.append((get_timestamp(), "added", event)) + + def handle_removed(_, data, size): + event = bpf["removed_events"].event(data) + events.append((get_timestamp(), "removed", event)) + + def handle_rejected(_, data, size): + event = bpf["rejected_events"].event(data) + events.append((get_timestamp(), "rejected", event)) + + def handle_replaced(_, data, size): + event = bpf["replaced_events"].event(data) + events.append((get_timestamp(), "replaced", event)) + + bpf["added_events"].open_perf_buffer(handle_added) + # By default, open_perf_buffer uses eight pages for a buffer, making for a total + # buffer size of 32k on most machines. In practice, this size is insufficient: + # Each `mempool:removed` event takes up 57 bytes in the buffer (32 bytes for txid, + # 9 bytes for removal reason, and 8 bytes each for vsize and fee). Full blocks + # contain around 2k transactions, requiring a buffer size of around 114kB. To cover + # this amount, 32 4k pages are required. + bpf["removed_events"].open_perf_buffer(handle_removed, page_cnt=32) + bpf["rejected_events"].open_perf_buffer(handle_rejected) + bpf["replaced_events"].open_perf_buffer(handle_replaced) + + curses.wrapper(loop, bpf, events) + + +def loop(screen, bpf, events): + dashboard = Dashboard(screen) + while True: + try: + bpf.perf_buffer_poll(timeout=50) + dashboard.render(events) + except KeyboardInterrupt: + exit() + + +class Dashboard: + """Visualization of mempool state using ncurses.""" + + INFO_WIN_HEIGHT = 2 + EVENT_WIN_HEIGHT = 7 + + def __init__(self, screen): + screen.nodelay(True) + curses.curs_set(False) + self._screen = screen + self._time_started = datetime.now(timezone.utc) + self._timestamps = {"added": [], "removed": [], "rejected": [], "replaced": []} + self._event_history = {"added": 0, "removed": 0, "rejected": 0, "replaced": 0} + self._init_windows() + + def _init_windows(self): + """Initialize all windows.""" + self._init_info_win() + self._init_event_count_win() + self._init_event_rate_win() + self._init_event_log_win() + + @staticmethod + def create_win(x, y, height, width, title=None): + """Helper function to create generic windows and decorate them with box and title if requested.""" + win = curses.newwin(height, width, x, y) + if title: + win.box() + win.addstr(0, 2, title, curses.A_BOLD) + return win + + def _init_info_win(self): + """Create and populate the info window.""" + self._info_win = Dashboard.create_win( + x=0, y=1, height=Dashboard.INFO_WIN_HEIGHT, width=22 + ) + self._info_win.addstr(0, 0, "Mempool Monitor", curses.A_REVERSE) + self._info_win.addstr(1, 0, "Press CTRL-C to stop.", curses.A_NORMAL) + self._info_win.refresh() + + def _init_event_count_win(self): + """Create and populate the event count window.""" + self._event_count_win = Dashboard.create_win( + x=3, y=1, height=Dashboard.EVENT_WIN_HEIGHT, width=37, title="Event count" + ) + header = " {:<8} {:>8} {:>7} {:>7} " + self._event_count_win.addstr( + 1, 1, header.format("Event", "total", "1 min", "10 min"), curses.A_UNDERLINE + ) + self._event_count_win.refresh() + + def _init_event_rate_win(self): + """Create and populate the event rate window.""" + self._event_rate_win = Dashboard.create_win( + x=3, y=40, height=Dashboard.EVENT_WIN_HEIGHT, width=42, title="Event rate" + ) + header = " {:<8} {:>9} {:>9} {:>9} " + self._event_rate_win.addstr( + 1, 1, header.format("Event", "total", "1 min", "10 min"), curses.A_UNDERLINE + ) + self._event_rate_win.refresh() + + def _init_event_log_win(self): + """Create windows showing event log. This comprises a dummy boxed window and an + inset window so line breaks don't overwrite box.""" + # dummy boxed window + num_rows, num_cols = self._screen.getmaxyx() + space_above = Dashboard.INFO_WIN_HEIGHT + 1 + Dashboard.EVENT_WIN_HEIGHT + 1 + box_win_height = num_rows - space_above + box_win_width = num_cols - 2 + win_box = Dashboard.create_win( + x=space_above, + y=1, + height=box_win_height, + width=box_win_width, + title="Event log", + ) + # actual logging window + log_lines = box_win_height - 2 # top and bottom box lines + log_line_len = box_win_width - 2 - 1 # box lines and left padding + win = win_box.derwin(log_lines, log_line_len, 1, 2) + win.idlok(True) + win.scrollok(True) + win_box.refresh() + win.refresh() + self._event_log_win_box = win_box + self._event_log_win = win + + def calculate_metrics(self, events): + """Calculate count and rate metrics.""" + count, rate = {}, {} + for event_ts, event_type, event_data in events: + self._timestamps[event_type].append(event_ts) + for event_type, ts in self._timestamps.items(): + # remove timestamps older than ten minutes but keep track of their + # count for the 'total' metric + # + self._event_history[event_type] += len( + [t for t in ts if Dashboard.timestamp_age(t) >= 600] + ) + ts = [t for t in ts if Dashboard.timestamp_age(t) < 600] + self._timestamps[event_type] = ts + # count metric + count_1m = len([t for t in ts if Dashboard.timestamp_age(t) < 60]) + count_10m = len(ts) + count_total = self._event_history[event_type] + len(ts) + count[event_type] = (count_total, count_1m, count_10m) + # rate metric + runtime = Dashboard.timestamp_age(self._time_started) + rate_1m = count_1m / min(60, runtime) + rate_10m = count_10m / min(600, runtime) + rate_total = count_total / runtime + rate[event_type] = (rate_total, rate_1m, rate_10m) + return count, rate + + def _update_event_count(self, count): + """Update the event count window.""" + w = self._event_count_win + row_format = " {:<8} {:>6}tx {:>5}tx {:>5}tx " + for line, metric in enumerate(["added", "removed", "replaced", "rejected"]): + w.addstr(2 + line, 1, row_format.format(metric, *count[metric])) + w.refresh() + + def _update_event_rate(self, rate): + """Update the event rate window.""" + w = self._event_rate_win + row_format = " {:<8} {:>5.1f}tx/s {:>5.1f}tx/s {:>5.1f}tx/s " + for line, metric in enumerate(["added", "removed", "replaced", "rejected"]): + w.addstr(2 + line, 1, row_format.format(metric, *rate[metric])) + w.refresh() + + def _update_event_log(self, events): + """Update the event log window.""" + w = self._event_log_win + for event in events: + w.addstr(Dashboard.parse_event(event) + "\n") + w.refresh() + + def render(self, events): + """Render the dashboard.""" + count, rate = self.calculate_metrics(events) + self._update_event_count(count) + self._update_event_rate(rate) + self._update_event_log(events) + events.clear() + + @staticmethod + def parse_event(event): + """Converts events into human-readable messages""" + + ts_dt, type_, data = event + ts = ts_dt.strftime("%H:%M:%SZ") + if type_ == "added": + return ( + f"{ts} added {bytes(data.hash)[::-1].hex()}" + f" with feerate {data.fee/data.vsize:.2f} sat/vB" + f" ({data.fee} sat, {data.vsize} vbytes)" + ) + + if type_ == "removed": + return ( + f"{ts} removed {bytes(data.hash)[::-1].hex()}" + f" with feerate {data.fee/data.vsize:.2f} sat/vB" + f" ({data.fee} sat, {data.vsize} vbytes)" + f" received {ts_dt.timestamp()-data.entry_time:.1f} seconds ago" + f": {data.reason.decode('UTF-8')}" + ) + + if type_ == "rejected": + return ( + f"{ts} rejected {bytes(data.hash)[::-1].hex()}" + f": {data.reason.decode('UTF-8')}" + ) + + if type_ == "replaced": + return ( + f"{ts} replaced {bytes(data.replaced_hash)[::-1].hex()}" + f" with feerate {data.replaced_fee/data.replaced_vsize:.2f} sat/vB" + f" received {ts_dt.timestamp()-data.replaced_entry_time:.1f} seconds ago" + f" ({data.replaced_fee} sat, {data.replaced_vsize} vbytes)" + f" with {bytes(data.replacement_hash)[::-1].hex()}" + f" with feerate {data.replacement_fee/data.replacement_vsize:.2f} sat/vB" + f" ({data.replacement_fee} sat, {data.replacement_vsize} vbytes)" + ) + + raise NotImplementedError("Unsupported event type: {type_}") + + @staticmethod + def timestamp_age(timestamp): + """Return age of timestamp in seconds.""" + return (datetime.now(timezone.utc) - timestamp).total_seconds() + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("USAGE: ", sys.argv[0], "path/to/bitcoind") + exit(1) + + path = sys.argv[1] + main(path) diff --git a/contrib/valgrind.supp b/contrib/valgrind.supp index 99ca305fe77..faba1f6ae6c 100644 --- a/contrib/valgrind.supp +++ b/contrib/valgrind.supp @@ -13,19 +13,8 @@ # # Note that suppressions may depend on OS and/or library versions. # Tested on: -# * aarch64 (Ubuntu 22.04 system libs, clang, without gui) -# * x86_64 (Ubuntu 22.04 system libs, clang, without gui) -{ - Suppress libstdc++ warning - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - obj:*/libstdc++.* - fun:call_init.part.0 - fun:call_init - fun:_dl_init - obj:*/ld-*.so -} +# * aarch64 (Debian Bookworm system libs, clang, without gui) +# * x86_64 (Debian Bookworm system libs, clang, without gui) { Suppress libdb warning - https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=662917 Memcheck:Cond @@ -64,39 +53,12 @@ ... obj:*/libdb_cxx-*.so } -{ - Suppress leaks on init - Memcheck:Leak - ... - fun:_Z11AppInitMainR11NodeContext -} { Suppress leaks on shutdown Memcheck:Leak ... fun:_Z8ShutdownR11NodeContext } -{ - Ignore GUI warning - Memcheck:Leak - ... - obj:/usr/lib64/libgdk-3.so.0.2404.7 -} -{ - Suppress leveldb warning (leveldb::InitModule()) - https://github.com/google/leveldb/issues/113 - Memcheck:Leak - match-leak-kinds: reachable - fun:_Znwm - fun:_ZN7leveldbL10InitModuleEv -} -{ - Suppress leveldb warning (leveldb::Env::Default()) - https://github.com/google/leveldb/issues/113 - Memcheck:Leak - match-leak-kinds: reachable - fun:_Znwm - ... - fun:_ZN7leveldbL14InitDefaultEnvEv -} { Suppress leveldb leak Memcheck:Leak @@ -112,16 +74,6 @@ ... fun:GetCoin } -{ - Suppress boost warning - Memcheck:Leak - fun:_Znwm - ... - fun:_ZN5boost9unit_test9framework5state17execute_test_treeEmjPKNS2_23random_generator_helperE - fun:_ZN5boost9unit_test9framework3runEmb - fun:_ZN5boost9unit_test14unit_test_mainEPFbvEiPPc - fun:main -} { Suppress LogInstance still reachable memory warning Memcheck:Leak @@ -129,14 +81,6 @@ fun:_Znwm fun:_Z11LogInstancev } -{ - Suppress secp256k1_context_create still reachable memory warning - Memcheck:Leak - match-leak-kinds: reachable - fun:malloc - ... - fun:secp256k1_context_create -} { Suppress BCLog::Logger::StartLogging() still reachable memory warning Memcheck:Leak diff --git a/contrib/verify-binaries/README.md b/contrib/verify-binaries/README.md new file mode 100644 index 00000000000..c62d760e1a7 --- /dev/null +++ b/contrib/verify-binaries/README.md @@ -0,0 +1,88 @@ +### Verify Binaries + +#### Preparation + +As of Bitcoin Core v22.0, releases are signed by a number of public keys on the basis +of the [guix.sigs repository](https://github.com/bitcoin-core/guix.sigs/). When +verifying binary downloads, you (the end user) decide which of these public keys you +trust and then use that trust model to evaluate the signature on a file that contains +hashes of the release binaries. The downloaded binaries are then hashed and compared to +the signed checksum file. + +First, you have to figure out which public keys to recognize. Browse the [list of frequent +builder-keys](https://github.com/bitcoin-core/guix.sigs/tree/main/builder-keys) and +decide which of these keys you would like to trust. For each key you want to trust, you +must obtain that key for your local GPG installation. + +You can obtain these keys by + - through a browser using a key server (e.g. keyserver.ubuntu.com), + - manually using the `gpg --keyserver --recv-keys ` command, or + - you can run the packaged `verify.py ... --import-keys` script to + have it automatically retrieve unrecognized keys. + +#### Usage + +This script attempts to download the checksum file (`SHA256SUMS`) and corresponding +signature file `SHA256SUMS.asc` from https://bitcoincore.org and https://bitcoin.org. + +It first checks if the checksum file is valid based upon a plurality of signatures, and +then downloads the release files specified in the checksum file, and checks if the +hashes of the release files are as expected. + +If we encounter pubkeys in the signature file that we do not recognize, the script +can prompt the user as to whether they'd like to download the pubkeys. To enable +this behavior, use the `--import-keys` flag. + +The script returns 0 if everything passes the checks. It returns 1 if either the +signature check or the hash check doesn't pass. An exit code of >2 indicates an error. + +See the `Config` object for various options. + +#### Examples + +Validate releases with default settings: +```sh +./contrib/verify-binaries/verify.py pub 22.0 +./contrib/verify-binaries/verify.py pub 22.0-rc3 +``` + +Get JSON output and don't prompt for user input (no auto key import): + +```sh +./contrib/verify-binaries/verify.py --json pub 22.0-x86 +``` + +Rely only on local GPG state and manually specified keys, while requiring a +threshold of at least 10 trusted signatures: +```sh +./contrib/verify-binaries/verify.py \ + --trusted-keys 74E2DEF5D77260B98BC19438099BAD163C70FBFA,9D3CC86A72F8494342EA5FD10A41BDC3F4FAFF1C \ + --min-good-sigs 10 pub 22.0-x86 +``` + +If you only want to download the binaries for a certain platform, add the corresponding suffix, e.g.: + +```sh +./contrib/verify-binaries/verify.py pub 24.0.1-darwin +./contrib/verify-binaries/verify.py pub 23.1-rc1-win64 +``` + +If you do not want to keep the downloaded binaries, specify the cleanup option. + +```sh +./contrib/verify-binaries/verify.py pub --cleanup 22.0 +``` + +Use the bin subcommand to verify all files listed in a local checksum file + +```sh +./contrib/verify-binaries/verify.py bin SHA256SUMS +``` + +Verify only a subset of the files listed in a local checksum file + +```sh +./contrib/verify-binaries/verify.py bin ~/Downloads/SHA256SUMS \ + ~/Downloads/bitcoin-24.0.1-x86_64-linux-gnu.tar.gz \ + ~/Downloads/bitcoin-24.0.1-arm-linux-gnueabihf.tar.gz +``` diff --git a/contrib/verify-binaries/test.py b/contrib/verify-binaries/test.py new file mode 100755 index 00000000000..22d718ece33 --- /dev/null +++ b/contrib/verify-binaries/test.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +import json +import sys +import subprocess +from pathlib import Path + + +def main(): + """Tests ordered roughly from faster to slower.""" + expect_code(run_verify("", "pub", '0.32'), 4, "Nonexistent version should fail") + expect_code(run_verify("", "pub", '0.32.awefa.12f9h'), 11, "Malformed version should fail") + expect_code(run_verify('--min-good-sigs 20', "pub", "22.0"), 9, "--min-good-sigs 20 should fail") + + print("- testing verification (22.0)", flush=True) + _220 = run_verify("--json", "pub", "22.0") + try: + result = json.loads(_220.stdout.decode()) + except Exception: + print("failed on 22.0 --json:") + print_process_failure(_220) + raise + + expect_code(_220, 0, "22.0 should succeed") + v = result['verified_binaries'] + assert result['good_trusted_sigs'] + assert v['bitcoin-22.0-aarch64-linux-gnu.tar.gz'] == 'ac718fed08570a81b3587587872ad85a25173afa5f9fbbd0c03ba4d1714cfa3e' + assert v['bitcoin-22.0-osx64.tar.gz'] == '2744d199c3343b2d94faffdfb2c94d75a630ba27301a70e47b0ad30a7e0155e9' + assert v['bitcoin-22.0-x86_64-linux-gnu.tar.gz'] == '59ebd25dd82a51638b7a6bb914586201e67db67b919b2a1ff08925a7936d1b16' + + +def run_verify(global_args: str, command: str, command_args: str) -> subprocess.CompletedProcess: + maybe_here = Path.cwd() / 'verify.py' + path = maybe_here if maybe_here.exists() else Path.cwd() / 'contrib' / 'verify-binaries' / 'verify.py' + + if command == "pub": + command += " --cleanup" + + return subprocess.run( + f"{path} {global_args} {command} {command_args}", + stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + + +def expect_code(completed: subprocess.CompletedProcess, expected_code: int, msg: str): + if completed.returncode != expected_code: + print(f"{msg!r} failed: got code {completed.returncode}, expected {expected_code}") + print_process_failure(completed) + sys.exit(1) + else: + print(f"✓ {msg!r} passed") + + +def print_process_failure(completed: subprocess.CompletedProcess): + print(f"stdout:\n{completed.stdout.decode()}") + print(f"stderr:\n{completed.stderr.decode()}") + + +if __name__ == '__main__': + main() diff --git a/contrib/verify-binaries/verify.py b/contrib/verify-binaries/verify.py new file mode 100755 index 00000000000..d0749f503f8 --- /dev/null +++ b/contrib/verify-binaries/verify.py @@ -0,0 +1,713 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020-2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Script for verifying Bitcoin Core release binaries. + +This script attempts to download the sum file SHA256SUMS and corresponding +signature file SHA256SUMS.asc from bitcoincore.org and bitcoin.org and +compares them. + +The sum-signature file is signed by a number of builder keys. This script +ensures that there is a minimum threshold of signatures from pubkeys that +we trust. This trust is articulated on the basis of configuration options +here, but by default is based upon local GPG trust settings. + +The builder keys are available in the guix.sigs repo: + + https://github.com/bitcoin-core/guix.sigs/tree/main/builder-keys + +If a minimum good, trusted signature threshold is met on the sum file, we then +download the files specified in SHA256SUMS, and check if the hashes of these +files match those that are specified. The script returns 0 if everything passes +the checks. It returns 1 if either the signature check or the hash check +doesn't pass. If an error occurs the return value is >= 2. + +Logging output goes to stderr and final binary verification data goes to stdout. + +JSON output can by obtained by setting env BINVERIFY_JSON=1. +""" +import argparse +import difflib +import json +import logging +import os +import subprocess +import typing as t +import re +import sys +import shutil +import tempfile +import textwrap +import urllib.request +import urllib.error +import enum +from hashlib import sha256 +from pathlib import PurePath, Path + +# The primary host; this will fail if we can't retrieve files from here. +HOST1 = "https://bitcoincore.org" +HOST2 = "https://bitcoin.org" +VERSIONPREFIX = "bitcoin-core-" +SUMS_FILENAME = 'SHA256SUMS' +SIGNATUREFILENAME = f"{SUMS_FILENAME}.asc" + + +class ReturnCode(enum.IntEnum): + SUCCESS = 0 + INTEGRITY_FAILURE = 1 + FILE_GET_FAILED = 4 + FILE_MISSING_FROM_ONE_HOST = 5 + FILES_NOT_EQUAL = 6 + NO_BINARIES_MATCH = 7 + NOT_ENOUGH_GOOD_SIGS = 9 + BINARY_DOWNLOAD_FAILED = 10 + BAD_VERSION = 11 + + +def set_up_logger(is_verbose: bool = True) -> logging.Logger: + """Set up a logger that writes to stderr.""" + log = logging.getLogger(__name__) + log.setLevel(logging.INFO if is_verbose else logging.WARNING) + console = logging.StreamHandler(sys.stderr) # log to stderr + console.setLevel(logging.DEBUG) + formatter = logging.Formatter('[%(levelname)s] %(message)s') + console.setFormatter(formatter) + log.addHandler(console) + return log + + +log = set_up_logger() + + +def indent(output: str) -> str: + return textwrap.indent(output, ' ') + + +def bool_from_env(key, default=False) -> bool: + if key not in os.environ: + return default + raw = os.environ[key] + + if raw.lower() in ('1', 'true'): + return True + elif raw.lower() in ('0', 'false'): + return False + raise ValueError(f"Unrecognized environment value {key}={raw!r}") + + +VERSION_FORMAT = ".[.][-rc[0-9]][-platform]" +VERSION_EXAMPLE = "22.0-x86_64 or 23.1-rc1-darwin" + +def parse_version_string(version_str): + parts = version_str.split('-') + version_base = parts[0] + version_rc = "" + version_os = "" + if len(parts) == 2: # "-rcN" or "version-platform" + if "rc" in parts[1]: + version_rc = parts[1] + else: + version_os = parts[1] + elif len(parts) == 3: # "-rcN-platform" + version_rc = parts[1] + version_os = parts[2] + + return version_base, version_rc, version_os + + +def download_with_wget(remote_file, local_file): + result = subprocess.run(['wget', '-O', local_file, remote_file], + stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + return result.returncode == 0, result.stdout.decode().rstrip() + + +def download_lines_with_urllib(url) -> t.Tuple[bool, t.List[str]]: + """Get (success, text lines of a file) over HTTP.""" + try: + return (True, [ + line.strip().decode() for line in urllib.request.urlopen(url).readlines()]) + except urllib.error.HTTPError as e: + log.warning(f"HTTP request to {url} failed (HTTPError): {e}") + except Exception as e: + log.warning(f"HTTP request to {url} failed ({e})") + return (False, []) + + +def verify_with_gpg( + filename, + signature_filename, + output_filename: t.Optional[str] = None +) -> t.Tuple[int, str]: + with tempfile.NamedTemporaryFile() as status_file: + args = [ + 'gpg', '--yes', '--verify', '--verify-options', 'show-primary-uid-only', "--status-file", status_file.name, + '--output', output_filename if output_filename else '', signature_filename, filename] + + env = dict(os.environ, LANGUAGE='en') + result = subprocess.run(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, env=env) + + gpg_data = status_file.read().decode().rstrip() + + log.debug(f'Result from GPG ({result.returncode}): {result.stdout.decode()}') + log.debug(f"{gpg_data}") + return result.returncode, gpg_data + + +def remove_files(filenames): + for filename in filenames: + os.remove(filename) + + +class SigData: + """GPG signature data as parsed from GPG stdout.""" + def __init__(self): + self.key = None + self.name = "" + self.trusted = False + self.status = "" + + def __bool__(self): + return self.key is not None + + def __repr__(self): + return ( + "SigData(%r, %r, trusted=%s, status=%r)" % + (self.key, self.name, self.trusted, self.status)) + + +def parse_gpg_result( + output: t.List[str] +) -> t.Tuple[t.List[SigData], t.List[SigData], t.List[SigData]]: + """Returns good, unknown, and bad signatures from GPG stdout.""" + good_sigs: t.List[SigData] = [] + unknown_sigs: t.List[SigData] = [] + bad_sigs: t.List[SigData] = [] + total_resolved_sigs = 0 + + # Ensure that all lines we match on include a prefix that prevents malicious input + # from fooling the parser. + def line_begins_with(patt: str, line: str) -> t.Optional[re.Match]: + return re.match(r'^(\[GNUPG:\])\s+' + patt, line) + + curr_sigs = unknown_sigs + curr_sigdata = SigData() + + for line in output: + if line_begins_with(r"NEWSIG(?:\s|$)", line): + total_resolved_sigs += 1 + if curr_sigdata: + curr_sigs.append(curr_sigdata) + curr_sigdata = SigData() + newsig_split = line.split() + if len(newsig_split) == 3: + curr_sigdata.name = newsig_split[2] + + elif line_begins_with(r"GOODSIG(?:\s|$)", line): + curr_sigdata.key, curr_sigdata.name = line.split(maxsplit=3)[2:4] + curr_sigs = good_sigs + + elif line_begins_with(r"EXPKEYSIG(?:\s|$)", line): + curr_sigdata.key, curr_sigdata.name = line.split(maxsplit=3)[2:4] + curr_sigs = good_sigs + curr_sigdata.status = "expired" + + elif line_begins_with(r"REVKEYSIG(?:\s|$)", line): + curr_sigdata.key, curr_sigdata.name = line.split(maxsplit=3)[2:4] + curr_sigs = good_sigs + curr_sigdata.status = "revoked" + + elif line_begins_with(r"BADSIG(?:\s|$)", line): + curr_sigdata.key, curr_sigdata.name = line.split(maxsplit=3)[2:4] + curr_sigs = bad_sigs + + elif line_begins_with(r"ERRSIG(?:\s|$)", line): + curr_sigdata.key, _, _, _, _, _ = line.split()[2:8] + curr_sigs = unknown_sigs + + elif line_begins_with(r"TRUST_(UNDEFINED|NEVER)(?:\s|$)", line): + curr_sigdata.trusted = False + + elif line_begins_with(r"TRUST_(MARGINAL|FULLY|ULTIMATE)(?:\s|$)", line): + curr_sigdata.trusted = True + + # The last one won't have been added, so add it now + assert curr_sigdata + curr_sigs.append(curr_sigdata) + + all_found = len(good_sigs + bad_sigs + unknown_sigs) + if all_found != total_resolved_sigs: + raise RuntimeError( + f"failed to evaluate all signatures: found {all_found} " + f"but expected {total_resolved_sigs}") + + return (good_sigs, unknown_sigs, bad_sigs) + + +def files_are_equal(filename1, filename2): + with open(filename1, 'rb') as file1: + contents1 = file1.read() + with open(filename2, 'rb') as file2: + contents2 = file2.read() + eq = contents1 == contents2 + + if not eq: + with open(filename1, 'r', encoding='utf-8') as f1, \ + open(filename2, 'r', encoding='utf-8') as f2: + f1lines = f1.readlines() + f2lines = f2.readlines() + + diff = indent( + ''.join(difflib.unified_diff(f1lines, f2lines))) + log.warning(f"found diff in files ({filename1}, {filename2}):\n{diff}\n") + + return eq + + +def get_files_from_hosts_and_compare( + hosts: t.List[str], path: str, filename: str, require_all: bool = False +) -> ReturnCode: + """ + Retrieve the same file from a number of hosts and ensure they have the same contents. + The first host given will be treated as the "primary" host, and is required to succeed. + + Args: + filename: for writing the file locally. + """ + assert len(hosts) > 1 + primary_host = hosts[0] + other_hosts = hosts[1:] + got_files = [] + + def join_url(host: str) -> str: + return host.rstrip('/') + '/' + path.lstrip('/') + + url = join_url(primary_host) + success, output = download_with_wget(url, filename) + if not success: + log.error( + f"couldn't fetch file ({url}). " + "Have you specified the version number in the following format?\n" + f"{VERSION_FORMAT} " + f"(example: {VERSION_EXAMPLE})\n" + f"wget output:\n{indent(output)}") + return ReturnCode.FILE_GET_FAILED + else: + log.info(f"got file {url} as {filename}") + got_files.append(filename) + + for i, host in enumerate(other_hosts): + url = join_url(host) + fname = filename + f'.{i + 2}' + success, output = download_with_wget(url, fname) + + if require_all and not success: + log.error( + f"{host} failed to provide file ({url}), but {primary_host} did?\n" + f"wget output:\n{indent(output)}") + return ReturnCode.FILE_MISSING_FROM_ONE_HOST + elif not success: + log.warning( + f"{host} failed to provide file ({url}). " + f"Continuing based solely upon {primary_host}.") + else: + log.info(f"got file {url} as {fname}") + got_files.append(fname) + + for i, got_file in enumerate(got_files): + if got_file == got_files[-1]: + break # break on last file, nothing after it to compare to + + compare_to = got_files[i + 1] + if not files_are_equal(got_file, compare_to): + log.error(f"files not equal: {got_file} and {compare_to}") + return ReturnCode.FILES_NOT_EQUAL + + return ReturnCode.SUCCESS + + +def check_multisig(sums_file: str, sigfilename: str, args: argparse.Namespace) -> t.Tuple[int, str, t.List[SigData], t.List[SigData], t.List[SigData]]: + # check signature + # + # We don't write output to a file because this command will almost certainly + # fail with GPG exit code '2' (and so not writing to --output) because of the + # likely presence of multiple untrusted signatures. + retval, output = verify_with_gpg(sums_file, sigfilename) + + if args.verbose: + log.info(f"gpg output:\n{indent(output)}") + + good, unknown, bad = parse_gpg_result(output.splitlines()) + + if unknown and args.import_keys: + # Retrieve unknown keys and then try GPG again. + for unsig in unknown: + if prompt_yn(f" ? Retrieve key {unsig.key} ({unsig.name})? (y/N) "): + ran = subprocess.run( + ["gpg", "--keyserver", args.keyserver, "--recv-keys", unsig.key]) + + if ran.returncode != 0: + log.warning(f"failed to retrieve key {unsig.key}") + + # Reparse the GPG output now that we have more keys + retval, output = verify_with_gpg(sums_file, sigfilename) + good, unknown, bad = parse_gpg_result(output.splitlines()) + + return retval, output, good, unknown, bad + + +def prompt_yn(prompt) -> bool: + """Return true if the user inputs 'y'.""" + got = '' + while got not in ['y', 'n']: + got = input(prompt).lower() + return got == 'y' + +def verify_shasums_signature( + signature_file_path: str, sums_file_path: str, args: argparse.Namespace +) -> t.Tuple[ + ReturnCode, t.List[SigData], t.List[SigData], t.List[SigData], t.List[SigData] +]: + min_good_sigs = args.min_good_sigs + gpg_allowed_codes = [0, 2] # 2 is returned when untrusted signatures are present. + + gpg_retval, gpg_output, good, unknown, bad = check_multisig(sums_file_path, signature_file_path, args) + + if gpg_retval not in gpg_allowed_codes: + if gpg_retval == 1: + log.critical(f"Bad signature (code: {gpg_retval}).") + else: + log.critical(f"unexpected GPG exit code ({gpg_retval})") + + log.error(f"gpg output:\n{indent(gpg_output)}") + return (ReturnCode.INTEGRITY_FAILURE, [], [], [], []) + + # Decide which keys we trust, though not "trust" in the GPG sense, but rather + # which pubkeys convince us that this sums file is legitimate. In other words, + # which pubkeys within the Bitcoin community do we trust for the purposes of + # binary verification? + trusted_keys = set() + if args.trusted_keys: + trusted_keys |= set(args.trusted_keys.split(',')) + + # Tally signatures and make sure we have enough goods to fulfill + # our threshold. + good_trusted = [sig for sig in good if sig.trusted or sig.key in trusted_keys] + good_untrusted = [sig for sig in good if sig not in good_trusted] + num_trusted = len(good_trusted) + len(good_untrusted) + log.info(f"got {num_trusted} good signatures") + + if num_trusted < min_good_sigs: + log.info("Maybe you need to import " + f"(`gpg --keyserver {args.keyserver} --recv-keys `) " + "some of the following keys: ") + log.info('') + for sig in unknown: + log.info(f" {sig.key} ({sig.name})") + log.info('') + log.error( + "not enough trusted sigs to meet threshold " + f"({num_trusted} vs. {min_good_sigs})") + + return (ReturnCode.NOT_ENOUGH_GOOD_SIGS, [], [], [], []) + + for sig in good_trusted: + log.info(f"GOOD SIGNATURE: {sig}") + + for sig in good_untrusted: + log.info(f"GOOD SIGNATURE (untrusted): {sig}") + + for sig in [sig for sig in good if sig.status == 'expired']: + log.warning(f"key {sig.key} for {sig.name} is expired") + + for sig in bad: + log.warning(f"BAD SIGNATURE: {sig}") + + for sig in unknown: + log.warning(f"UNKNOWN SIGNATURE: {sig}") + + return (ReturnCode.SUCCESS, good_trusted, good_untrusted, unknown, bad) + + +def parse_sums_file(sums_file_path: str, filename_filter: t.List[str]) -> t.List[t.List[str]]: + # extract hashes/filenames of binaries to verify from hash file; + # each line has the following format: " " + with open(sums_file_path, 'r', encoding='utf8') as hash_file: + return [line.split()[:2] for line in hash_file if len(filename_filter) == 0 or any(f in line for f in filename_filter)] + + +def verify_binary_hashes(hashes_to_verify: t.List[t.List[str]]) -> t.Tuple[ReturnCode, t.Dict[str, str]]: + offending_files = [] + files_to_hashes = {} + + for hash_expected, binary_filename in hashes_to_verify: + with open(binary_filename, 'rb') as binary_file: + hash_calculated = sha256(binary_file.read()).hexdigest() + if hash_calculated != hash_expected: + offending_files.append(binary_filename) + else: + files_to_hashes[binary_filename] = hash_calculated + + if offending_files: + joined_files = '\n'.join(offending_files) + log.critical( + "Hashes don't match.\n" + f"Offending files:\n{joined_files}") + return (ReturnCode.INTEGRITY_FAILURE, files_to_hashes) + + return (ReturnCode.SUCCESS, files_to_hashes) + + +def verify_published_handler(args: argparse.Namespace) -> ReturnCode: + WORKINGDIR = Path(tempfile.gettempdir()) / f"bitcoin_verify_binaries.{args.version}" + + def cleanup(): + log.info("cleaning up files") + os.chdir(Path.home()) + shutil.rmtree(WORKINGDIR) + + # determine remote dir dependent on provided version string + try: + version_base, version_rc, os_filter = parse_version_string(args.version) + version_tuple = [int(i) for i in version_base.split('.')] + except Exception as e: + log.debug(e) + log.error(f"unable to parse version; expected format is {VERSION_FORMAT}") + log.error(f" e.g. {VERSION_EXAMPLE}") + return ReturnCode.BAD_VERSION + + remote_dir = f"/bin/{VERSIONPREFIX}{version_base}/" + if version_rc: + remote_dir += f"test.{version_rc}/" + remote_sigs_path = remote_dir + SIGNATUREFILENAME + remote_sums_path = remote_dir + SUMS_FILENAME + + # create working directory + os.makedirs(WORKINGDIR, exist_ok=True) + os.chdir(WORKINGDIR) + + hosts = [HOST1, HOST2] + + got_sig_status = get_files_from_hosts_and_compare( + hosts, remote_sigs_path, SIGNATUREFILENAME, args.require_all_hosts) + if got_sig_status != ReturnCode.SUCCESS: + return got_sig_status + + # Multi-sig verification is available after 22.0. + if version_tuple[0] < 22: + log.error("Version too old - single sig not supported. Use a previous " + "version of this script from the repo.") + return ReturnCode.BAD_VERSION + + got_sums_status = get_files_from_hosts_and_compare( + hosts, remote_sums_path, SUMS_FILENAME, args.require_all_hosts) + if got_sums_status != ReturnCode.SUCCESS: + return got_sums_status + + # Verify the signature on the SHA256SUMS file + sigs_status, good_trusted, good_untrusted, unknown, bad = verify_shasums_signature(SIGNATUREFILENAME, SUMS_FILENAME, args) + if sigs_status != ReturnCode.SUCCESS: + if sigs_status == ReturnCode.INTEGRITY_FAILURE: + cleanup() + return sigs_status + + # Extract hashes and filenames + hashes_to_verify = parse_sums_file(SUMS_FILENAME, [os_filter]) + if not hashes_to_verify: + log.error("no files matched the platform specified") + return ReturnCode.NO_BINARIES_MATCH + + # remove binaries that are known not to be hosted by bitcoincore.org + fragments_to_remove = ['-unsigned', '-debug', '-codesignatures'] + for fragment in fragments_to_remove: + nobinaries = [i for i in hashes_to_verify if fragment in i[1]] + if nobinaries: + remove_str = ', '.join(i[1] for i in nobinaries) + log.info( + f"removing *{fragment} binaries ({remove_str}) from verification " + f"since {HOST1} does not host *{fragment} binaries") + hashes_to_verify = [i for i in hashes_to_verify if fragment not in i[1]] + + # download binaries + for _, binary_filename in hashes_to_verify: + log.info(f"downloading {binary_filename} to {WORKINGDIR}") + success, output = download_with_wget( + HOST1 + remote_dir + binary_filename, binary_filename) + + if not success: + log.error( + f"failed to download {binary_filename}\n" + f"wget output:\n{indent(output)}") + return ReturnCode.BINARY_DOWNLOAD_FAILED + + # verify hashes + hashes_status, files_to_hashes = verify_binary_hashes(hashes_to_verify) + if hashes_status != ReturnCode.SUCCESS: + return hashes_status + + + if args.cleanup: + cleanup() + else: + log.info(f"did not clean up {WORKINGDIR}") + + if args.json: + output = { + 'good_trusted_sigs': [str(s) for s in good_trusted], + 'good_untrusted_sigs': [str(s) for s in good_untrusted], + 'unknown_sigs': [str(s) for s in unknown], + 'bad_sigs': [str(s) for s in bad], + 'verified_binaries': files_to_hashes, + } + print(json.dumps(output, indent=2)) + else: + for filename in files_to_hashes: + print(f"VERIFIED: {filename}") + + return ReturnCode.SUCCESS + + +def verify_binaries_handler(args: argparse.Namespace) -> ReturnCode: + binary_to_basename = {} + for file in args.binary: + binary_to_basename[PurePath(file).name] = file + + sums_sig_path = None + if args.sums_sig_file: + sums_sig_path = Path(args.sums_sig_file) + else: + log.info(f"No signature file specified, assuming it is {args.sums_file}.asc") + sums_sig_path = Path(args.sums_file).with_suffix(".asc") + + # Verify the signature on the SHA256SUMS file + sigs_status, good_trusted, good_untrusted, unknown, bad = verify_shasums_signature(str(sums_sig_path), args.sums_file, args) + if sigs_status != ReturnCode.SUCCESS: + return sigs_status + + # Extract hashes and filenames + hashes_to_verify = parse_sums_file(args.sums_file, [k for k, n in binary_to_basename.items()]) + if not hashes_to_verify: + log.error(f"No files in {args.sums_file} match the specified binaries") + return ReturnCode.NO_BINARIES_MATCH + + # Make sure all files are accounted for + sums_file_path = Path(args.sums_file) + missing_files = [] + files_to_hash = [] + if len(binary_to_basename) > 0: + for file_hash, file in hashes_to_verify: + files_to_hash.append([file_hash, binary_to_basename[file]]) + del binary_to_basename[file] + if len(binary_to_basename) > 0: + log.error(f"Not all specified binaries are in {args.sums_file}") + return ReturnCode.NO_BINARIES_MATCH + else: + log.info(f"No binaries specified, assuming all files specified in {args.sums_file} are located relatively") + for file_hash, file in hashes_to_verify: + file_path = Path(sums_file_path.parent.joinpath(file)) + if file_path.exists(): + files_to_hash.append([file_hash, str(file_path)]) + else: + missing_files.append(file) + + # verify hashes + hashes_status, files_to_hashes = verify_binary_hashes(files_to_hash) + if hashes_status != ReturnCode.SUCCESS: + return hashes_status + + if args.json: + output = { + 'good_trusted_sigs': [str(s) for s in good_trusted], + 'good_untrusted_sigs': [str(s) for s in good_untrusted], + 'unknown_sigs': [str(s) for s in unknown], + 'bad_sigs': [str(s) for s in bad], + 'verified_binaries': files_to_hashes, + "missing_binaries": missing_files, + } + print(json.dumps(output, indent=2)) + else: + for filename in files_to_hashes: + print(f"VERIFIED: {filename}") + for filename in missing_files: + print(f"MISSING: {filename}") + + return ReturnCode.SUCCESS + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '-v', '--verbose', action='store_true', + default=bool_from_env('BINVERIFY_VERBOSE'), + ) + parser.add_argument( + '-q', '--quiet', action='store_true', + default=bool_from_env('BINVERIFY_QUIET'), + ) + parser.add_argument( + '--import-keys', action='store_true', + default=bool_from_env('BINVERIFY_IMPORTKEYS'), + help='if specified, ask to import each unknown builder key' + ) + parser.add_argument( + '--min-good-sigs', type=int, action='store', nargs='?', + default=int(os.environ.get('BINVERIFY_MIN_GOOD_SIGS', 3)), + help=( + 'The minimum number of good signatures to require successful termination.'), + ) + parser.add_argument( + '--keyserver', action='store', nargs='?', + default=os.environ.get('BINVERIFY_KEYSERVER', 'hkps://keys.openpgp.org'), + help='which keyserver to use', + ) + parser.add_argument( + '--trusted-keys', action='store', nargs='?', + default=os.environ.get('BINVERIFY_TRUSTED_KEYS', ''), + help='A list of trusted signer GPG keys, separated by commas. Not "trusted keys" in the GPG sense.', + ) + parser.add_argument( + '--json', action='store_true', + default=bool_from_env('BINVERIFY_JSON'), + help='If set, output the result as JSON', + ) + + subparsers = parser.add_subparsers(title="Commands", required=True, dest="command") + + pub_parser = subparsers.add_parser("pub", help="Verify a published release.") + pub_parser.set_defaults(func=verify_published_handler) + pub_parser.add_argument( + 'version', type=str, help=( + f'version of the bitcoin release to download; of the format ' + f'{VERSION_FORMAT}. Example: {VERSION_EXAMPLE}') + ) + pub_parser.add_argument( + '--cleanup', action='store_true', + default=bool_from_env('BINVERIFY_CLEANUP'), + help='if specified, clean up files afterwards' + ) + pub_parser.add_argument( + '--require-all-hosts', action='store_true', + default=bool_from_env('BINVERIFY_REQUIRE_ALL_HOSTS'), + help=( + f'If set, require all hosts ({HOST1}, {HOST2}) to provide signatures. ' + '(Sometimes bitcoin.org lags behind bitcoincore.org.)') + ) + + bin_parser = subparsers.add_parser("bin", help="Verify local binaries.") + bin_parser.set_defaults(func=verify_binaries_handler) + bin_parser.add_argument("--sums-sig-file", "-s", help="Path to the SHA256SUMS.asc file to verify") + bin_parser.add_argument("sums_file", help="Path to the SHA256SUMS file to verify") + bin_parser.add_argument( + "binary", nargs="*", + help="Path to a binary distribution file to verify. Can be specified multiple times for multiple files to verify." + ) + + args = parser.parse_args() + if args.quiet: + log.setLevel(logging.WARNING) + + return args.func(args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/contrib/verify-commits/README.md b/contrib/verify-commits/README.md index b8b15280bae..020890c3660 100644 --- a/contrib/verify-commits/README.md +++ b/contrib/verify-commits/README.md @@ -27,6 +27,10 @@ Note that the above isn't a good UI/UX yet, and needs significant improvements to make it more convenient and reduce the chance of errors; pull-reqs improving this process would be much appreciated. +Unless `--clean-merge 0` is specified, `verify-commits.py` will attempt to verify that +each merge commit applies cleanly (with some exceptions). This requires using at least +git v2.38.0. + Configuration files ------------------- diff --git a/contrib/verify-commits/allow-incorrect-sha512-commits b/contrib/verify-commits/allow-incorrect-sha512-commits index c572806f261..e69de29bb2d 100644 --- a/contrib/verify-commits/allow-incorrect-sha512-commits +++ b/contrib/verify-commits/allow-incorrect-sha512-commits @@ -1,2 +0,0 @@ -f8feaa4636260b599294c7285bcf1c8b7737f74e -8040ae6fc576e9504186f2ae3ff2c8125de1095c diff --git a/contrib/verify-commits/allow-revsig-commits b/contrib/verify-commits/allow-revsig-commits index 0bb299b8fae..e69de29bb2d 100644 --- a/contrib/verify-commits/allow-revsig-commits +++ b/contrib/verify-commits/allow-revsig-commits @@ -1,645 +0,0 @@ -a06ede9a138d0fb86b0de17c42b936d9fe6e2158 -923dc447eaa8e017985b2afbbb12dd1283fbea0e -71148b8947fe8b4d756822420a7f31c380159425 -6696b4635ceb9b47aaa63244bff9032fa7b08354 -812714fd80e96e28cd288c553c83838cecbfc2d9 -8a445c5651edb9a1f51497055b7ddf4402be9188 -e126d0c12ca66278d9e7b12187c5ff4fc02a7e6c -3908fc4728059719bed0e1c7b1c8b388c2d4a8da -8b66bf74e2a349e71eaa183af81fa63eaee76ad2 -05950427d310654774031764a7141a1a4fd9c6e4 -07fd147b9f12e9205afd66a624edce357977d615 -12e31127948fa4bb01c3bddc1b8c85b432f7465b -8c87f175d335e9d9e93f987d871ae9f05f6a10a7 -46b249e578e8a3dfbe85bc7253a12e82ef4b658b -a55716abe5662ec74c2f8af93023f1e7cca901fc -f646275b90b1de93bc62b4c4d045d75ac0b96eee -c252685aa5867631e9a5ef07ccae7c7c25cae8ff -a7d55c93385359952d85decd5037843ac70ba3d4 -7dac1e5e9e887f5f6ff146e812a05bd3bf281eae -2a524b8e8fe69ef487fd8ea1b4f7a03f473ed201 -ce5c1f4acae43477989cdf9a82ed33703919cda2 -2db4cbcc437f51f5dac82cc4de46f383b92e6f11 -7aa700424cbda387536373d8dfec88aee43f950e -b99a093afed880f23fb279c443cc6ae5e379cc43 -b83264d9c7a8ddb79f64bd9540caddc8632ef31f -57e337d40e94ba33d8cd265c134d6ef857b32b59 -a1dcf2e1087beaf3981739fd2bb74f35ecad630a -d38b0d7a6b6056cba26999b702815775e2437d87 -815640ec6af9a38d6a2da4a4400056e2f4105080 -09c4fd157c5b88df2d97fad4826c79b094db90c9 -2efcfa5acfacb958973d9e8125e1d81f102e2dfd -dc6dee41f7cf2ba93fcd0fea7c157e4b2775d439 -ad826b3df9f763b49f1e3e3d50c4efdd438c7547 -c1a52276848d8caa9a9789dff176408c1aa6b1ed -3bf06e9bac57b5b5a746677b75e297a7b154bdbd -72ae6f8cf0224370e8121d6769b21e612ca15d6f -a143b88dbd4971ecfdd1d39a494489c8f2db0344 -76fec09d878d6dbf214bdb6228d480bd9195db4c -93566e0c37c5ae104095474fea89f00dcb40f551 -407d9232ef5cb1ebf6cff21f3d13e07ea4158eeb -9346f8429957e356d21c665bab59fe45bcf1f74e -6eeac6e30d65f9a972067c1ea8c49978c8e631ac -dc6b9406bdfab2af8c86cb080cb3e6cf8f2385d8 -9f554e03ebe5701c1b75ff03b3d6152095c0cad3 -05009935f9ac070197113954d680bc2c9150b9b3 -508404de98a8a5435f52916cef8f328e82651961 -ed0cc50afed146c27f6d8129c683c225fb940093 -6429cfa8a70308241c576aeb92ffe3db5203b2ef -6898213409811b140843c3d89af43328c3b22fad -5b2ea29cf4fd298346437bb16a54407f8c1f9dca -e2a1a1ee895149c544d4ae295466611f0cec3094 -e82fb872ff5cc8fd22d43327c1ee3e755f61c562 -19b0f33de0efd9da788e8e4f3fdc2a9e159abdb1 -89de1538ce1f8c00f80e8d11f43e1b77e24d7dea -de07fdcf77e97b8613091285e4d0a734f5de7492 -01680195f8aa586c55c44767397380def3a23b54 -05e1c85fb687c82ae477c72d4a7e2d6b0c692167 -c072b8fd95cd4fa84f08189a0cd8b173ea2dbb8e -9a0ed08b40b15ae2b791aa8549b53e69934b4ea7 -53f8f226bd1d627c4a6dec5862a1d4ea5a933e45 -9d0f43b7ca7241d8a018fd35dd3bc01555235ec6 -f12d2b5a8ac397e4bcaefcc19898f8ff5705dea5 -8250de13587ed05ca45df3e12c5dc9bcb1500e2c -d727f77e390426e9e463336bda08d50c451c7086 -484312bda2d43e3ea60047be076332299463adf8 -c7e05b35ab0a791c7a8e2d863e716fdec6f3f671 -b9c1cd81848da9de1baf9c2f29c19c50e549de13 -8ea7d31e384975019733b5778feabbd9955c79d8 -f798b891bcecea9548eedacae70eeb9906c1ddbf -ebefe7a00b46579cdd1e033a8c7fd8ce9aa578e4 -ad087638ee4864d6244ec9381ff764bfa6ee5086 -66db2d62d59817320c9182fc18e75a93b76828ea -7ce9ac5c83b1844a518ef2e12e87aae3cacdfe58 -4286f43025149cf44207c3ad98e4a1f068520ada -cd0c5135ab2291aaa5410ac919bad3fc87249a4a -66ed450d771a8fc01c159a8402648ebd1c35eb4c -a82f03393a32842d49236e8666ee57805ca701f8 -f972b04d63eb8af79ff3cec1dc561ed13dfa6053 -ec45cc5e27668171b55271b0c735194c70e7da41 -715e9fd7454f7a48d7adba7d42f662c20a3e3367 -2e0a99037dcc35bc63ba0d54371bc678af737c8e -7fa8d758598407f3bf0beb0118dc122ea5340736 -6a22373771edbc3c7513cacb9355f880c73c2cbf -b89ef131147f71a96152a7b5c4374266cdf539b2 -01d8359983e2f77b5118fede3ffa947072c666c8 -58f0c929a3d70a4bff79cc200f1c186f71ef1675 -950be19727a581970591d8f8138dfe4725750382 -425278d17bd0edf8a3a7cc81e55016f7fd8e7726 -c028c7b7557da2baff7af8840108e8be4db8e0c6 -47a7cfb0aa2498f6801026d258a59f9de48f60b0 -f6b7df3155ddb4cedfbcf5d3eb3383d4614b3a85 -d72098038f3b55a714ed8adb34fab547b15eb0d5 -c49c825bd9f4764536b45df5a684d97173673fc7 -33799afe83eec4200ff140e9bf5eae83701a4d7f -5c3f8ddcaa1164079105c452429fccf8127b01b6 -1f01443567b03ac75a91c810f1733f5c21b5699d -b3e42b6d02e8d19658a9135e427ebceab5367779 -69b3a6dd9d9a0adf5506c8b9fde42187356bd4a8 -bafd075c5e6a1088ef0f1aa0b0b224e026a3d3e0 -7daa3adb242d9c8728fdb15c6af6596aaad5502f -514993554c370f4cf30a109ac28d5d64893dbf0a -c8d2473e6cb042e7275a10c49d3f6a4a91bf0166 -386f4385ab04b0b2c3d47bddc0dc0f2de7354964 -9f33dba05c01ecc5c56eb1284ab7d64d42f55171 -7466a26cab5d66665991433947964a638f5b957e -b43aba89e356ff95b706e80d4802f60fc46a569a -02b7e8319aef2a870264ad4fa2e3bb18664dcc36 -f686002a8eba820a40ac2f34a6e8f57b2b5cc54c -2b1c50b9352ab1dc40b0f877db23c1fa4048fae3 -2405ce1df043f778b8efb9205009500cbc17313a -4ad3b3c72c73d61e0a0cab541dca20acf651320d -4ba3d4f4393d81148422d24d222fe7ed00130194 -8ee5c7b747171e335793c74cd9d2f7491da58164 -872c921c0a208b04bd0713758e52fcab5b7c1684 -00d1680498c5550e7db1f359202d3433a092fafd -585db41e9ab7a6fb262c8bad7f427cdbdc497188 -18462960c0f13bd07d8f52b61e7d7bc17e991eea -0630974647dacaf25e7fcb7f9cbb785bb078ede6 -0f58d7f3d62f012f2584f5e781fc73de4763dd9e -3d16f581538b0974853e820508e8b3093269d2fd -66e91420ab233cf1dac64504e0dc129019bf8c0d -d8d9162f5bad39b2720dd2b2da237c6159e4755f -29fad97c320c892ab6a480c81e2078ec22ab354b -791c3ea61b4e49fd46a1a71b84ca99ddf69d2ff7 -a312e201ba56742499a5480b5f2115f01505c217 -ce56fdd2e8cdf94fd0ab76d71adbfa755e23ce7d -480f42630cbd598c04fa59ee0e406f56904ecffb -6012f1caf744ac9b53383d7d10a8f1b70ca2c0e1 -ded6a2afa549f693dcabb430ce0862f8631360c8 -07090c5339436f856e79a8036d1c85deeb453803 -0e265916d1c6a63e4a3821dab9db597b5ec64b46 -e4ffcacc2187d3419c8ea12b82fb06d82d8751d2 -e117cfe45eee9169409e74a44ef4a866be25bc35 -dcfe218626b05204e9fbc95ba5d95ca0eb72ec9b -23481fa50301201ef5a60675ef899aa6ce94ca03 -27c59dc502f29cf1d76290556c21e366145e3b2e -4a62ddd01873d18dbca96c81d756be1020249b45 -a233fb4f1d037e68ff70eef3a9f5b7bf1d631918 -b2089c51cc4af2f7e1c0ec75be9449ee222b1d69 -c997f8808256521397f1c003bb1e9896fee6eaa0 -5dc00f68c49c46a380a98d06233f90528b8e2557 -fe53d5f3636aed064823bc220d828c7ff08d1d52 -935eb8de039dec65669a96a1c3b86f4b03a1b86c -0277173b1defb63216d40a8d8805ae6d5d563c26 -2a30e67d20f76bbcd9a7d445f616f005316e0a1a -d32528e733f2711b34dbc41fbb2bb0f153bf7e9a -4cad91663df381d0dff8526f3b4aa74569dfb626 -1b06ed136f17b526360617a70026aed5ded5746c -895fbd768f0c89cea3f78acac58b233d4e3a145e -f0295becbf3ef1fb78095306408789253fe0c114 -8d573198638e52e2dbd9abc609861430f9d2bcc3 -9d9c4185fadaf243bb97c226e2fef16b65299699 -eebe4580bc8d6484d79ecb24dd87412221cf2ea7 -9cf6393a4f82b9c81d3b4b468a17a89db10531a2 -598a9c4e4dcd03c6d80fba005de729a6a3aeba7e -6970b30c6f1d2be7947295fe18f2390649b17a4b -f359afcc410432ed5d30001acda0c66741ee8935 -126000ba9e7ff16271be2f4eef3df99ade8d624f -b5e4b9b5100ec15217d43edb5f4149439f4b20a5 -b987ca4ee495a7fff82f0ac14ef0753bfb7586e2 -b03013396cb2f4bf25746388b3982a2c3616e16b -9a97f39afaa890caa7987c6bc001b9a66e3e74e8 -cad504bf4c302f7a72e0a0e191f3fdbafda7340f -45cf8a03cb57b8639a8d47323bde46ba22d9eeaf -b7450cdbd89a1c862f4d4d8bf093f8a0b5448f9c -0910cbe4ef31eb95fd76c7c2f820419fe64a3150 -92a810d04b906722c9efe60e3997243c71ff3d4c -45173fa6fca9537abb0a0554f731d14b9f89c456 -fd4ca17360e6fc0c9bb76bf6b5b07c9102c12728 -ddff3447f29b62d79a33f728791f42fa9436216e -36a5a4404836da323c755523fbd27563a8e84f94 -c991b304dee368f506cfee27ddaa333f1f82c518 -d38d1a3e75aa97ffa8755ddd431754a6d0942964 -a332a7d5a15214015f9553fdb2bcf80a1a4b8dc0 -604e08c83cf58ca7e7cda2ab284c1ace7bb12977 -18a1bbad98bd4321f15e7921d9aec91661499d90 -8049241e226c16bd07b029c0cb4b62ac40f0c923 -797441ee995aac59f55d59a93ecb55e8ecbe7dbc -62fdf9b07087b80d2142799bdd2324f61483359d -f60b4ad57912b78a96af08046a503f7905610a8c -13e31dd6548d64a5992f439e74bb424bf88aca04 -fbce66a982679b5409a295be5c99a2eef429cabf -9f2c2dba21855b8cb9b193b1819be73fa4a23a99 -a89221873a3ee2451c73b41bbe2d99d36f439d31 -3d6ad407770e13958e157bf026cae0bfb9254899 -901ba3e3819405306414628306746552b0aa1d28 -7a43fbb959c38e025e558e472ad57de357539894 -0d89fa0877930c6c8a539a656c1009ad8ab6755b -54aedc013744c86b11157423fa3cffc9a51eef02 -f0c1f8abb0182da557d07372b938f3a0a4bb906f -4ed818060ecf4a38a02c8cb48f6cbc78d2ee7708 -3bdf242fc68a8d767932c6214455d4d413effbc9 -5e468994fbb349e8eefc996954a31a67a34aaa15 -41aa9c4a801a01eca1fad22a7095372d23dace60 -2adbddb03840ad71e843c6c4a207a13e871cd1d4 -13e352dc53dec0127c5f94a60055d0ca829420dc -95e14dc81dd30ee0d396ad08dca9a6980d16eee1 -61fb80660f73e5aa5b69302ecc7ac33da206ba5a -05a761932edd05cf94ffe938908baf058f38632a -ee92243e66f2df03b3a759a8ffb75dc06f0cea0d -22cdf93c062eeaa0f8f9d6220f01b67240073dfb -76b33491596736ca804e3a29bd8398d7a1516ab7 -6e4e98ee8ce2da3cca2e2fd210e9e8dbc9b1c936 -c838283ecdfb9490425bb071b7c22e542de46c7c -5e3f5e4f25b65b583d3bfefac9e1148035781089 -f7388e93d3dd91a90239aedac4ec58404f103a2e -0a2f46b0158b6fc7244a585913b0925c0acf707f -dd561667cb7ccbbfed3134b05a565971ef6f5873 -6f01dcf63873a5e42798635ab4026c9a5f9fa213 -70fec9e36bcd1a3d93df019be084aaf89cecd7d7 -f9b74ef3fc74fd7d2aa94560820341f03cda8e12 -998c3046fab2b52bc9f141cfb588a18c05506a86 -89cc4f905e30b913ca20e4192d538cc5cbe2c38d -87d90efd69b64f769116956a5db89e536e9e3714 -5aeaa9ccd1568a77e075dbe2bd2435bd60c87c91 -bfb270acfa30713dc8c968bb9ee40cf5a2360359 -1b8c88451b0554502435d3883c528ad0aad1b09b -57ee73990f1ce29916adfd99f93eae1ccea1a43b -808c84f89d0edcef9ddaab0b849a382719f6ec9e -14b860bf64020451ced823b859da8cb912278ab9 -c63364610f4a041df1c1bd81d01b1f6856160749 -92eadc395071876d77f3babddc056b4325bdbabc -e93fff1463ae906fc986bf98c3b118c82f171546 -9ccafb1d7bdd172a9b963444072a844da379c4f7 -b4a509a3f817121c3df98ddfd96b2769e18a3e5a -dbc4ae03963014ab4b7957d62ba59dbd8f938c33 -8ddf60db7ad636b6a31b590251c671ded635fa1d -f199b8a33d9443a258a1f49a1a29674cd9ee9a20 -e542728cde676f218c552d841d0af29b92f9800b -763231051596b8e3455b839911ad6a3a1f1c3c74 -ff4cd6075b12fb32b9a906deea3ed033e3f9560a -9c3c9cdae3e20b5bdea91a0631edac5116bbc89f -93d20a734d2ee873832bed8ca5c05cf8e539c53c -ef8340d25f7c5dd5682bdecea97ce84cfce1493c -69c7ecef405d168f658a9cc7996da84c17f61e66 -4ce2f3d0d33346e9f0e96851689ee6550b2a72e3 -44e1fd926cfb0df0fbd8c41de8cd65ed8d5d6e18 -d6d2c8503c4039b682196d83a67dc28359c10c5c -ae233c4ec3d14a97c6195059f52873cdba2b4755 -0f399a9ff227896265cafab9b2e9fab6cdb9b5b9 -f4ed44ab4a8f9a87ba678d5fd1449fbf636103dc -7fcd61b2613c211bb042a82a889655178be6a212 -42973f834445d7735738bdba8847812ba3c34d95 -8df48b36ed3201d938b9974ecbee455d7dc2fb84 -96ac26e56627f0c24213fcd3a1cce9fc95f1f661 -cce94c518a46b7b0006f984bbe4d69e8749182d2 -801dd40666d1e6009920ad3ff755c7bb993b2a62 -ce829855cfca103dde55661fa1524e66b139d063 -b148803b181e30213e8a7f3bd89c8239e9dcb866 -c377feaad87f8109f85da6caf62602b30c20effc -b37cab65c63e051ebc5b491da9bd687581df94df -16e41844e7d6c5876d2caaeef6010656950c6ec5 -ee50c9e48786dea0d9df2e45805c25565c100fe3 -11dacc6154c42bc6fe3ba94c1823f8a46e4fe81a -791a0e6ddade27d1b69f4861a6640de60b9553cf -638e6c59da4fad987c437592174b188510193b2e -52f8877525d5238f3440e73710507be889d14127 -2a56baf395bf11835d784c4f8634f4525deed6a1 -bc561b4b7d6a3f71649d37d5eb9047c29efa2b13 -31809d6f8514c4a8d5677e947e3f1ebb0db210b9 -a31e9ad4f027955d43c04a05517244647e250161 -777519bd96f68c18150a0f5942f8f97a91937f5e -4eb1f39d421024d9666cec61deaf96715ffae4c6 -50fae68d416b4b8ec4ca192923dfd5ae9ea42773 -ce665863b137ac4a7470cf006a92aa7694faca71 -81f8c0378b2ab5ea0d7b65635cb529bd3c69127c -108222b9c323a05cc9339368f10ddd0859f62b43 -28f788e47e58f2b462351d6989348a4e1a241b2b -d81dccf191a48a6b59c3747d7b4ccbe3535dde40 -a90e6d2bffc422ddcdb771c53aac0bceb970a2c4 -91e49c51f1aecc9e1d75457f4920d52a4b0a133c -60dd9cc470584960431de425e2a9ffbed0e8034a -ede386c2193fc31351e193b3a8cf30030d6be62c -a084767b40c0d3ba8fa8f8d60f1e8d99a9dc3457 -3f726c99f819f97f2ab21b94d34c6b3129cd883a -77fc469fc78cdd87c29f398d46ac58dbb9ef62c0 -4ae6d0fbef60ccbecf8f23bb482e201b3678f7a3 -8858b6ddd3bce9daa08da6e05de3ca863a399c15 -22e301a3d56dc9e6878380ee92c7d19ca43119d2 -c484ec6c9b85ca4e331e395c564ae232fd0681dd -a46a671e253528e450bd57645c400bf761da07ab -655970d9c60ae6850daf452457e14e21047c0e1b -b6a48914c50631914192aa11b19205436a9c664d -7db65c363a0cc6ca7cdb04de9a973ab70013baad -6366941275344dac7e2130b0c972e90117d37ed0 -4fb2586661471a1572c2df2a5a091011d45eb7c4 -d7be7b39fa1021ec4518186afe145ee948e12a94 -85aec87b11ec41295558175c63f1f5a849460fdf -aeb31756276034dd506fdf97c8aaade0e7e584f5 -ac016e17d20253129a0287cee7e1d06b7ef15966 -bf74d377fb8e20140da6eac1407414928384bcea -2c811e08db651a4aed6ea0f7c1972d60de6de8ab -e5d26e47c7a482c072a7fe47bb84c56854734184 -96a63a3e0cefe920819bd42add0041837b1214a1 -e526ca6284b9e13be1b912b80dd73a34e739b539 -ecd21357f16106e541e9c2854ead2a906659b938 -4b5a7ce0c301ad971f383eb60f61bf9b4026efda -929fd7276c0f0c30b9416f61a6f5f35d763d81e4 -fa8a0639f7b0ce04030b72b4d5be4f0aa36fc5cb -f1f1605c22a6283bbfd757055fcf2b584a857709 -0c173a15ca1bf20999f74987988985508c9de463 -df0793f324e33066cc746c0cb1d053d35733d626 -2b0179d8a9b75397937126b36114df0dddeab40c -bf0a08be281dc42241e7f264c2a20515eb4781bb -3895e25a77363ae8b49358fb793f50fa8b271e2d -1fc783fc08bc078239537535f174ab8a489772c0 -1d4805ce04645f3203b0cfd3d66ea710e7433eb4 -d3b58704d1d325875fc605580c1c02b825c1bbcc -ed88e3194c4bc43aeafef929da7b419d03dea1ad -dd07f47b79628668e29cc0143b21e790100ee445 -65cc7aacfbfc7b747926375280a1d839e88d576b -080ec5209172ac9605f1434559dbb3c1e012b10a -416af3edf5b5ab265acf95568f2bc9eabd3d96de -e0a7801223fd573863939e76cb633f1dcc2d22c4 -4bc853b50fd9127687eb9e4f3b679dd261a4fa96 -c68a9a69278aa194fed96bd9733d32af3690a11e -c38f540298f0e188df5ed68fd56c623b9ac8331b -643fa0b22d70e459d7f7ec3d728ae4811dc5158f -e053e05c130549f43953f1d70e724dc9ce3e1b85 -75e898c094eea533d1dfaf141c6afccc3072c49f -2805d606bc46bf5589093a1b92d3542c13ce50c2 -32751807c9c06011eb689cba56b401a6302699c0 -30853e16d332816752dafcfca92147c7ffef5b54 -bea5b00cfe95cd37832305c0f93c339a22a7d79d -c871f323b418fac27bf834843ca26985010df53f -329fc1dce7a1c372c8b10c2f2f8732b2c60daff0 -1aefc94dd78d6e0c9209cb09fc16f53dedf42108 -8e5725666b519b61fcdc3141da5c6a57c1959909 -a4ca0b042365061020627a8c045cddacea3312ec -8bd16ee12fc8ef6723e0572c29b979c15b92b4f4 -87abe20fc118721cc5efdbd94a8462468cd1da2b -4b766fcdd4ca16399075d1e081a321b3b05ce516 -f6241b3e420e19f3f0507cbbc872fe9218916a02 -7ee523604851af62c0a47c07ee023a8710ef32f7 -776ba233e939fe41a74c6b2632b93a0679a32c71 -6a796b2b53fe542e0f340f250f4f20d69efed8d0 -23d78c4dd01bc74ba35db3e3df95280f6f1b2e22 -f4b15e2de97c4f8cdbb40bef4c9d0ab2807974d9 -fff72de5bf8ac7b70208e655f237b80e70e18851 -170bc2c381f86a523de2fc8b71d62ade66303c0d -314ebdfcb38d4b4c977579f787d5e1a20d068c94 -e9274839bf316b1972d80d28e45759f898edbf86 -75171f099e82e3527d7c3469b15891bd92227ec2 -3c5e6c94caf40395e031fbde44a0cca46fdd76ec -dc8fc0c73bebbc1c48ac5540026030c9cc00ec23 -492d22f92919d8d9d59568318c26c1e2ac4890cc -80c3a734298e824f9321c4efdd446086a3baad89 -47535d7c3ec79c5978cdcc03a5351ddbbb22538d -1b25b6df0f08f7474228c5b6ed13b58682e1e440 -c530c15180631cea95e9c292cf7fabde9dca9db3 -2723bcdce3248417e98e6c43207bef74d34076c1 -ed22eb4a62bd8d5369aaec87d4cbdc03c9f16368 -9111df9673beb6d6616d491a5478f09b5f14d040 -d86bb075bf6d1e78c1e4f3dd38b0ea828ef5ecfe -50a1cc0f0aef1514b917a5a3f4476967170b429d -6ce733747e160ca699711f2c47e686284ca9aa07 -b44adf92342ad4f9c343ba29c081a91687932936 -88799ea1b1c08f4bc1a487c9e3c2effd5e1650ae -080d7c700fc3291560d79fc590e05b8e2bad984f -12af74b289f8cdc6caf850dc6c802f9936b1e8b3 -8e4f7e72410df3ba430082c7cf385f26fd75b033 -8ac80412867118172dc4172494304e19969e9489 -f2734c2828f69d9cfd535e5eab0592a7674b2b61 -0b9fb682890b8fe10cec54072b809a5efe57d33d -5b029aaedb5fcf7cadd249607dd28eb3f233ab8c -79af9fbd8c3c0e54702a9c92b171f134bd4466c8 -c412fd805ddf3282dc2e1f28e30f51ffcb1f1da2 -111849345bb5140f86b48e730ceab4bff45fa2e9 -a0b1e57b20a17177ed5a9a54e4a8aab597a546b4 -ca209230c8e73745cf8cfc79f500c9c46e103306 -a230b0588788dbe1ac84622aea169c577b381241 -dfef6b6af08097f0676a2323085558fbbd3c48c6 -3192e5278abca7c1f3b4a2a7f77a0ce941c73985 -7c7ddd9ead99a8b5033a1a5d4698032c9e2b3a92 -10b930dde8f14e9cb661810e97a33bbf144fc55c -9225de2cf652fe2bf6e50636824cdb641546f57d -598ef9c44b3ea2cc142c175f077b493f39f5ba22 -c49355c7170a64bdd7864cc3ba9a64916b67fe7c -857d1e171e051b254a617f27b39f6a551054cee2 -21833f9456f6ad5bc06321ad6d9590f42ce0195c -8910b4717e5bb946ee6988f7fe9fd461f53a5935 -5703dff0939f05c7457cebd6fc61d88ab13afe41 -8bfa13b15b84cb372950fb7b25a1080173060b6a -ac23a7c1f19b3d8c326ffe75c8e13edf285f90fe -19be26afe3d04783a92d032b55bf3fb1e2ae63cc -f7ec7cfd38b543ba81ac7bed5b77f9a19739460b -36afd4db4442c45d4078b1a7ad16a1872b5bee0d -88c2ae3ed2bb5d367dd408c9255cd8f1e7a36c7d -a13a417cdcfdfd1f1b3bf997bb6ffe6e69b096b9 -d6064a89ac97dc0d2ce9da3982e1a4e25afaeda8 -7146d96de3e15a80cafbab2af48ff6f65d8e41bb -5628c70f2a44567695e5331fe2293c5b7f35b629 -7ff4a538a8682cdf02a4bcd6f15499c841001b73 -aa5fa642b0e7ce2ea55e2298886f212f11a8894e -8efd1c820b9a782d8608d54d924658536178295c -50a226563cd8d7c0a5e8448e87fede0eb72a8354 -b860915f8b0dae98e57a254d11575ea41f5c5a79 -d304fef3746039183f51b3ac8f4774dcf3a64f59 -53ab12d9318d5d195ccc77028b0e3ae66dc6e1fd -668de70be039a4f1ffcf20aeae2a22ee71fc55a8 -0fea960ca917b73aff853fe88476174c8a313863 -f89502306dcf6393a2c7b0efbb0fa728fc582137 -ff58b1c3bdff5e5f687f10f9e40ce495ca49674e -0b96abc35f1a9d46a27eeddd7df418d107c29c57 -b0b57a17306a7e963a4fe463f84e2b150a00a859 -4105cb6fd964ad13099ca83b1fdf3d35f3961f74 -23281a4dc3afc42a001346caec4dbb8193f0bb53 -8daf103fa138f9a184448ebf1c2e03b9dbd96f21 -02e5308c1b9f3771bbe49bc5036215fa2bd66aa9 -a65ced1a66575c652baf5084644b8647f531be8c -2456a835f0bc7796d9ff71f64837fa6790e2b7cc -9ec1330b455c1ab2eb6b89f8a2ab885677d4ae8a -0b738075bd43fbd4410e30a51e0498cbfd2b7513 -98c80e374b84e5a9c2d5c36889a0b1ebed5b814b -25720fc394e27a951bcad26095fb5a711bfacb8f -4cfd57d2e38207d78722ce8c9274ba8dd700d1cc -0fc1c31a878e93d938c67db3f958e82e3c39659f -df1ab5b4d67b46b5e9e840b1fbe0ff02520831f9 -5bc3b6cede8dabdf3f4f27ddb03723cbb7cde51a -c2ea1e6561caba3abffce361abc800822b9e0efe -caa2f106d704ec3ade63498031dd58d34510bc76 -dce853ef76ef90c46d84294225088d595467d08c -dbc8a8c86ae50059fddb2d6834fa5f0c9bbf9b71 -0f921e6a0492c4e9f037a9ed91f474885032d68c -041331e1da23e4136fd046ed870cdcc177464176 -e6ba5068f107ac234576e77cedbd748b665369c2 -76fcd9d5034143a5b041766552670d19f926097d -72bf1b3d0962304850a3ef5fe375db4bff1d0a39 -919db037f1f5cc73cdcaef92dd9cb0e7f5c8dec3 -c36229b0b2e9d4554053f5c9fc451ac29a493b1f -9e4bb312e6958d2baa309ba670e5eed1523c6f47 -d7ba4a233bd5a6f8fadee681c68a995e23fe36d7 -98514988a3d3e8b7dbf0463884a5c38f5ed5562d -5412c08c3cf13577566064edd04da021c37b7cbe -31bcc667863f368157efa1143a78623a5db8f0d1 -7bd1aa566fb4a4fe194f209085649f2c722b0cff -c4522e71c7e1d8ecfd70112e9375b9d00d6733a8 -e22f409f18881b63a8e747036584a71217f40e6e -97ec6e5c9098a1240655cfcab05b6cd5eedb6cd1 -bc121b0eb19713ec72002b5be03ba5ac35903a17 -c98f6b3d93a2cc1b49a6db425ea2b661089d0f9e -0de7fd36de57a68e543b4c1f184fba192c398c73 -e662d281b837c25b2b70525aa8fe8af894339823 -44adf683ad232db8ce0cb89b3e236a1f5944cfb0 -cb2ed300a89ebf9f0654da869ced665ed8b2abe7 -0a6d48d9ed60b0b02177059ab116f8f46d2cbed3 -b42291334651fff46dbfe5947a726f65cb9d7dfe -e5364991daecb73aca3bb5ac37f2619d7a89211b -4a2b170c075ce703cbdc82519a48016a9ee3f99c -924de0bd75a7f75df65d7d15f9d1587a2e794abf -1253f8692fc3a11be9430685cd405236a68df6c3 -2b799ae9e1e0a540f9a5971ddf27d83254668279 -c9bdf9a75f9fde8cd011e4aa94be4ed4347078a3 -3d69ecb4edeb80003a1a41442e320898a30dbd9c -f08222e882b18c1f279308636e03beceece2dbf1 -23e03f8d26d7bd03273a5dcbdcfe3905dfb49ffb -03dd707dc027fbf6f24120213f8eb66571600374 -d0754799698de2c032abcb8198ee5d5401063213 -072116fceb2294b97d1c40f79305f2e3ff71812b -e66cc1d58e16bf1650dd6479fed64ecaca8c6098 -f137753a2dcd8229f89d1d1ac28039364e5850b4 -61d191fbf953700ba8aeadc9c8cf4c195efbd10c -76f3c02fb01a6df98fbd8c16ac21d159d4649d37 -6013c73b3312e11b447ed387426749014716f820 -6faffb8a83db3f209a303a4464dbdd597faad5a4 -cc9e8aca5f950c78dcfeff63c441ba993c1fe12f -8ca69a2a88a77eb06149fa049ab1a7e6de38b321 -2f71490d21796594ca6f55e375558944de9db5a0 -08cc5fd666456cb476467473ed1880c90c92dedb -e31a43c725ebe641d7c219c3886eee18eebf0bb8 -52b5a8785de760a204b2b0aab19dfaf79c2c3ff0 -483e8e4f4875a1a621ec9e9df2880d3037d95ed7 -1e5799c52535a3fc20e885916f1e7ed33ecc7f46 -a82e5d8220bbc8b5d786bed99b0876f530b9b7cc -7fe6c5c993706e8395cdaf7977bee793c06f48f3 -2a0836f6d5e7c1d7e97bedb0e0ea33dcaf981f77 -ddc308068d69c6c9aa629ee3c4ce75e1d1cf08b5 -ec139a5621a9c9f03e1988391a3c7c6c5d849776 -c01a6c48b982d625fd9f4f69005878781d3d56fa -95a983d56dbda457e3bf8766d59bac74c7aa5699 -760741a00833876976389ed7a6b73f36ee5b4c13 -6e5e5abba6f8bbbe61c22795df440dfafcfdc378 -cf2cecb18779ce83de9adebf382dff1c19b12840 -af9b7a9f2f73b1a2f9728106774dd13e8d1cdd8d -115735d547fdeade822f547eb3e8c8f9961a9b07 -c2c69edf37b5c02aafa01d0407dadbf5ef8751b5 -a072d1a83787e786d074a4b5871b0b961781f7c6 -ed2cd59e258f756b2eaed7909a60956ade6ef7ee -ae5575ba41c8a782805afb1c08730343cfc22397 -6ff2c8d29f6b5a5c2ce63f0a16f3bb0dbd049451 -a80de15113166354cdf208e3d8b6e25f4511a591 -06bd4f637f15e769f088d9051a5af94bbb0217a3 -6700cc993cc07fb0f5b8b577ff8c4afcf0b18274 -37f9a1f627c0995d89b62923e75cd092600894f9 -8844ef15ded02d5ed86fb95aaf251235fcef2396 -1b87e5b5b184a0a6c683eda23b36393822b57f03 -e2bf830bb6c1bfa038c943dd6f5d92a406bd723f -423ca302a3ee87000530da3c105f269b8fabece7 -4e14afe42fdd468d5de11df8cc13defdcb8e83f8 -3e90fe6534206412ea22beaa445cf20d28fbe718 -88b77c7da0a672c89e24df37ea6e9085b4e2a05c -0ad104190465d8d65c2344bbe10dcf3df025d86c -5c7df7022bcd360e6af00b9458b1a3fd54e1cc9a -59ad56851a342d2c62f6b38bf15002b23ab439e1 -d8cd7b137fb075616f31d2b43b85fa2e27ea7477 -655937ebcbf681ededf86b1f0f60aac45c73393d -abdfd2d0e3ebec7dbead89317ee9192189a35809 -e439aeb30c0439001a781c5979aec41e1fc2aa50 -b9b26d9c3615d15669ae0a049c1dede39a9e59a9 -fdf146f3293c487afdc4d6d9f6b64099aa8bd28a -16e3b175781caacee403a2dc40cd6c70448e12ef -b30c62d4b954df05bf404cfbeb5b728282201496 -b3f377daaa86cd7755a552fa3adfeb195835f58e -0a8f519a0626d7cd385114ce610d23215c051b3f -544f3234384b2f6c290e987ad18576e1b50d7db9 -91482e5bf22d283d32c9f83c8057f10971848107 -e754c6e33194e9ed69ba5350c5139b0423b645fe -dc1e54206d76e5fa378d28a18ae1fb2bcf714485 -b2863c0685a5c12f829095cbabaf26ccc49e46ec -b14db5abab405a708f0166293f1ea12222a6bf03 -8010ded6da56842c09b14665343cd189d7e08401 -d387507aeca652a5569825af65243536f2ce26ea -27bf14f6f3e0fb1f348f13c1b54fc6b67b3bef6e -f8d470e24606297dab95e30b1d39ff664fbda31d -b25a4c2284babdf1e8cf0ec3b1402200dd25f33f -1329ef1f00e4fad83937ddd8721d0292ccfe7808 -9a1ad2c5cbdfa3114d05df57103c34f72e087f26 -1e90862f5d0b5f7dcc18fb018b2bbaa323dcca1d -ad552a54c56a420be84b47154882c3e4c76f9bdd -90b1c7e5c50551f39d4983008d1d5ab3b085803e -d6b2235ca45e072961e25a35e6a159e97c9e556b -2643fa50869f22672cbc72ac497d9c30234075b8 -01f909828d126d5443bc28758f51781eccdf5848 -f54f3738c8ce839c413d7b6b719be2ff341536ca -418ae49ee1eac2c9d6cd4ba83c036a41f1afe922 -5a666428b0f11d62af2002bd54a45ff2f79f30cd -a07e8caa5d5000286604458e6887f57fec7fdcbb -8b262eb2d80bfa27ae8501078ce47bc1407e9c55 -5df84de583c900e00fef63bedaef32786f205a33 -4ba6da55743a55189164e29e45ac9e73a074d808 -88430cbab4dca36b6a867364cab319cde6a9ebca -e0f7515f5500968c86e5a9f4912d83d4abc5b2b9 -9b8b1079ddab64ac955766536c38d23dc57bc499 -af20f9b1d485582b8c8aa8294bac4f2c540246d2 -7be9a9a570c1140048f8781ced1111e1d930e517 -2bac3e484114c30548e286972525dd799dbd0a5b -df529dcc65e8037c5a3a3ad74545be294a770f07 -6acd8700bc0ee1d10207a362c1e07372ba274041 -ffc6e48b2983189dc0ce7de0a038e5329bc07b1b -252ae7111cbff09a4cbc5caee9e02b6ed3580476 -b3ecb7bab6074377d87c700bf0c5d351e5d3174f -d9fdac130a5ed1d96fcac6bb87c10bec9d596b17 -a07e8caa5d5000286604458e6887f57fec7fdcbb -8b262eb2d80bfa27ae8501078ce47bc1407e9c55 -5df84de583c900e00fef63bedaef32786f205a33 -4ba6da55743a55189164e29e45ac9e73a074d808 -5bea05bc1d17aa43cbdf3a3413241f8132790d93 -c17f11f7b43ad3bd9e242c67db1f3679558a0581 -5ea932a51083837cdd27715e10a3a0d5d553af24 -033c78671b91b12d589ebff6c5ede8d94d7500f8 -ef8a634358848847e006c43ce621bc17a612fd1f -ba216b5fa63e7e6cae847d1e3621f5c54840f898 -26fee4f6bd9aec62c6caa60683ad66574cf16aa6 -6ab0e4cf49549640b903bf5fce0e6035b8116397 -326a5652e0d25fdb60c337ef4f1c98a63e0748f0 -424be03305143cbe5da5d5adb54d73d3dc3747b6 -38c201f47c0bc388a05cdb35d6137150fa90193e -12ed800ab870e0fc527a84d6e4584b10c8d239f5 -aeed345c9bade5d52a3fbf0a943203f6c82e6344 -c6223b3daab0328ca742b1cc3c15e89e698630bb -877678710800a4d78afc12519424f232f1a583d3 -6c4fecfaf7beefad0d1c3f8520bf50bb515a0716 -98212745c8acb5cc4e688bbb3979bfd46b25f98a -b9bceaf1c081a84d9fcc680372614e797b168a9e -1afc22a7667a7a5c66b4b5d7f50832356dd5ec12 -3255d6347b1f9eccbec3d6d93d4a424087a3b35b -ec20f01ba0945a3113797ac98a6b3500e24603d4 -75b5643c47c3b382ed97a9f5e2bdc883a0f98709 -fee0d803fb55c8d85b5cd1ff69d799c5ad522e18 -565494619d809655fa94d274bb2202d25553e485 -ad6fce67b9bb6eee864c8431ad3291aebaa2e5d2 -99c7db8731cc77f143b52f544b3fdd93033ed20d -b4d03be3cac04da8b5d5fa17e29c5220b75d970b -ef37f2033c4ae104585cd980141262f95d33166e -5cfdda2503c995cdd563b1a2a29162ac298d173d -c5904e871479514b2e2e18b4fdbbe468c4e5ec8e -10b22e3141a603ec891d2cfc7100c29c7409aabe -afd2fca911c4a5e3a4d1f0993a226d40f250aff4 -505955052e60e0681865f3064e005ca0d3aa90bf -8fdd23a224ba236874ef662c4ca311b002dbcab3 -1c011ff430106b5f727f2eaa0f7f4883cd2122a3 -ec8a50b8d786a8cd1192e692ab19b46979add582 -f90603ac6d24f5263649675d51233f1fce8b2ecd -b7d6623c76e1468f2a93db5a3120580e2784d74a -66270a416edb1610f276124483feceef9cba93ff -e4fcbf797ed3b472d352ac3794ec82f581209c50 -479afa0f8486146a35f1fb96be1826061ecbcf23 -2a09a3891fde052a585dc019eea9fba26d42445d -90a002ea647dcea57a2ed4294eab77897168ba1d -30c21306c17165c3925fea4ac9d1a4763c6d2a99 -b3eb0d6485510f2bdf36d256ab60ce29b8213744 -efbcf2b1d5ff4ee7132eae9c9e203d2b875c545b -b33ca14f594e2cf2a16ef27778169deb7cc9f4dc -d636f3943d39ec893dab2d2546f77f3f2607769d -cafe24f039e117d53288387c2720f44f27deecd0 -de8db47b7ff351f3287c5efb85102ba8836058d6 -d76e84a21416ef77e78138e326d4d249454e79dc -7a74f88a26cf251ba36b26f604f1ac9940fd9c92 -1ad3d4e1261f4a444d982a1470c257c78233bda3 -8d9f45ea6a5e4220e44d34139438eea75a07530b -c98ebf1bfb29a8203b5090412afeb333384213cd -f18bb49547095020a30e81b648075bc7e707515c -76f268b9bd1b69eb7784c5324abbb67f3e395b97 -e801084decf4542d57cf5ddb95820643766a172a -be3e042c20e2f3449b7b55d1cab0a80b0c6f00af -400fdd08cc95f1e85afafd07ddd9c0bed11483ea -098b01dc58ff555c473ae58c92c34b03a77eda5f -7cc2c670e3d7cf26454ac8547a94ec2c8ca90b34 -1088b02f0ccd7358d2b7076bb9e122d59d502d02 -f94b7d5bfa911ea7125920589723ee63a3eec9f0 -b4b057a3e0712dd16b50cbcfe7d613e4413ffa1c -b40ceed98a112f4f0d07351ce07270d9ff2bf796 -4cb8757aae1ae31e5519d81e854f44ed062d9836 -f2f7e97e8cc24cf7a2b7954cb74ecfd0f91a95ad -ae786098bc58b1ca92f596a698b23aeade9cd2cd -c33652576ce21694b33a94832378f737dd6959fb -e317c0d19201ff75fa7afedf93a9d1cd2c560af2 -bee35299716cc72cb7d5bd4daa9fddca05c58378 -318ea50a1c2f612e750a93e36620dd0c4531e9cf -b6ee855b411ee9bc39f935d0da3298a773a2ed37 -daf3e7def7b9e5db7a32f5a20b5c4e09e3f0dd18 -bc64b5aa0fc543fe8fd3dbaec275f89df44dc409 -3f57c55dba6ef1fda2bdf6fd9abd8ca7eb6828e4 -431a548faaf51c7a5fc89b6e479187a1c0e29805 -e4bbd3d230f22401ba0a0a72c8ed41ee1bd098a0 -c45da32047cac54afc99cf9b8a539389c577ded1 -ab1f1d32469180b3d011e9625d67c86a22b55903 -a550f6e415fd8aec8c45d4704712a408c37ecd18 -c73af5416b66f09cec0eb106f5a10f9bb6ef9cb1 -a077a90da88f12d9f10c8b85840bdb847a98b0a0 -c5e9e428a9198c8c4076f239b5eaa8dc95e7985b -b7365f0545b1a6862e3277b2b2139ee0d5aee1cf -4bd0e9b90a39c5c6a016b83882ae44cb4d28f1f8 -7438ceac716fdfe6621728c05e718eaa89dd89aa -4e3efd47e0d50c6cd1dc81ccc9669a5b2658f495 diff --git a/contrib/verify-commits/allow-unclean-merge-commits b/contrib/verify-commits/allow-unclean-merge-commits index 7aab274b9a8..e69de29bb2d 100644 --- a/contrib/verify-commits/allow-unclean-merge-commits +++ b/contrib/verify-commits/allow-unclean-merge-commits @@ -1,4 +0,0 @@ -6052d509105790a26b3ad5df43dd61e7f1b24a12 -3798e5de334c3deb5f71302b782f6b8fbd5087f1 -326ffed09bfcc209a2efd6a2ebc69edf6bd200b5 -97d83739db0631be5d4ba86af3616014652c00ec diff --git a/contrib/verify-commits/gpg.sh b/contrib/verify-commits/gpg.sh index db5bfce208e..cfd68e45b8d 100755 --- a/contrib/verify-commits/gpg.sh +++ b/contrib/verify-commits/gpg.sh @@ -5,12 +5,9 @@ export LC_ALL=C INPUT=$(cat /dev/stdin) -VALID=false -REVSIG=false -IFS=' -' if [ "$BITCOIN_VERIFY_COMMITS_ALLOW_SHA1" = 1 ]; then - GPG_RES="$(printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" + printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null + exit $? else # Note how we've disabled SHA1 with the --weak-digest option, disabling # signatures - including selfsigs - that use SHA1. While you might think that @@ -20,12 +17,12 @@ else # an attacker could construct a pull-req that results in a commit object that # they've created a collision for. Not the most likely attack, but preventing # it is pretty easy so we do so as a "belt-and-suspenders" measure. - GPG_RES="" for LINE in $(gpg --version); do case "$LINE" in "gpg (GnuPG) 1.4.1"*|"gpg (GnuPG) 2.0."*) echo "Please upgrade to at least gpg 2.1.10 to check for weak signatures" > /dev/stderr - GPG_RES="$(printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)" + printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null + exit $? ;; # We assume if you're running 2.1+, you're probably running 2.1.10+ # gpg will fail otherwise @@ -33,33 +30,6 @@ else # gpg will fail otherwise esac done - [ "$GPG_RES" = "" ] && GPG_RES="$(printf '%s\n' "$INPUT" | gpg --trust-model always --weak-digest sha1 "$@" 2>/dev/null)" -fi -for LINE in $GPG_RES; do - case "$LINE" in - "[GNUPG:] VALIDSIG "*) - while read KEY; do - [ "${LINE#?GNUPG:? VALIDSIG * * * * * * * * * }" = "$KEY" ] && VALID=true - done < ./contrib/verify-commits/trusted-keys - ;; - "[GNUPG:] REVKEYSIG "*) - [ "$BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG" != 1 ] && exit 1 - REVSIG=true - GOODREVSIG="[GNUPG:] GOODSIG ${LINE#* * *}" - ;; - "[GNUPG:] EXPKEYSIG "*) - [ "$BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG" != 1 ] && exit 1 - REVSIG=true - GOODREVSIG="[GNUPG:] GOODSIG ${LINE#* * *}" - ;; - esac -done -if ! $VALID; then - exit 1 -fi -if $VALID && $REVSIG; then - printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null | grep "^\[GNUPG:\] \(NEWSIG\|SIG_ID\|VALIDSIG\)" - echo "$GOODREVSIG" -else - printf '%s\n' "$INPUT" | gpg --trust-model always "$@" 2>/dev/null + printf '%s\n' "$INPUT" | gpg --trust-model always --weak-digest sha1 "$@" 2>/dev/null + exit $? fi diff --git a/contrib/verify-commits/trusted-git-root b/contrib/verify-commits/trusted-git-root index c60f8ab695e..7ec318e1ea7 100644 --- a/contrib/verify-commits/trusted-git-root +++ b/contrib/verify-commits/trusted-git-root @@ -1 +1 @@ -82bcf405f6db1d55b684a1f63a4aabad376cdad7 +437dfe1c26e752c280014a30f809e62c684ad99e diff --git a/contrib/verify-commits/verify-commits.py b/contrib/verify-commits/verify-commits.py index 7e46c6fd477..a1fe78a6436 100755 --- a/contrib/verify-commits/verify-commits.py +++ b/contrib/verify-commits/verify-commits.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Verify commits against a trusted keys list.""" @@ -82,13 +82,20 @@ def main(): # get directory of this program and read data files dirname = os.path.dirname(os.path.abspath(__file__)) print("Using verify-commits data from " + dirname) - verified_root = open(dirname + "/trusted-git-root", "r", encoding="utf8").read().splitlines()[0] - verified_sha512_root = open(dirname + "/trusted-sha512-root-commit", "r", encoding="utf8").read().splitlines()[0] - revsig_allowed = open(dirname + "/allow-revsig-commits", "r", encoding="utf-8").read().splitlines() - unclean_merge_allowed = open(dirname + "/allow-unclean-merge-commits", "r", encoding="utf-8").read().splitlines() - incorrect_sha512_allowed = open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf-8").read().splitlines() - - # Set commit and branch and set variables + with open(dirname + "/trusted-git-root", "r", encoding="utf8") as f: + verified_root = f.read().splitlines()[0] + with open(dirname + "/trusted-sha512-root-commit", "r", encoding="utf8") as f: + verified_sha512_root = f.read().splitlines()[0] + with open(dirname + "/allow-revsig-commits", "r", encoding="utf8") as f: + revsig_allowed = f.read().splitlines() + with open(dirname + "/allow-unclean-merge-commits", "r", encoding="utf8") as f: + unclean_merge_allowed = f.read().splitlines() + with open(dirname + "/allow-incorrect-sha512-commits", "r", encoding="utf8") as f: + incorrect_sha512_allowed = f.read().splitlines() + with open(dirname + "/trusted-keys", "r", encoding="utf8") as f: + trusted_keys = f.read().splitlines() + + # Set commit and variables current_commit = args.commit if ' ' in current_commit: print("Commit must not contain spaces", file=sys.stderr) @@ -97,7 +104,6 @@ def main(): no_sha1 = True prev_commit = "" initial_commit = current_commit - branch = subprocess.check_output([GIT, 'show', '-s', '--format=%H', initial_commit]).decode('utf8').splitlines()[0] # Iterate through commits while True: @@ -108,17 +114,41 @@ def main(): if current_commit == verified_root: print('There is a valid path from "{}" to {} where all commits are signed!'.format(initial_commit, verified_root)) sys.exit(0) - if current_commit == verified_sha512_root: - if verify_tree: + else: + # Make sure this commit isn't older than trusted roots + check_root_older_res = subprocess.run([GIT, "merge-base", "--is-ancestor", verified_root, current_commit]) + if check_root_older_res.returncode != 0: + print(f"\"{current_commit}\" predates the trusted root, stopping!") + sys.exit(0) + + if verify_tree: + if current_commit == verified_sha512_root: print("All Tree-SHA512s matched up to {}".format(verified_sha512_root), file=sys.stderr) - verify_tree = False - no_sha1 = False + verify_tree = False + no_sha1 = False + else: + # Skip the tree check if we are older than the trusted root + check_root_older_res = subprocess.run([GIT, "merge-base", "--is-ancestor", verified_sha512_root, current_commit]) + if check_root_older_res.returncode != 0: + print(f"\"{current_commit}\" predates the trusted SHA512 root, disabling tree verification.") + verify_tree = False + no_sha1 = False + os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_SHA1'] = "0" if no_sha1 else "1" - os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG'] = "1" if current_commit in revsig_allowed else "0" + allow_revsig = current_commit in revsig_allowed # Check that the commit (and parents) was signed with a trusted key - if subprocess.call([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', current_commit], stdout=subprocess.DEVNULL): + valid_sig = False + verify_res = subprocess.run([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', "--raw", current_commit], capture_output=True) + for line in verify_res.stderr.decode().splitlines(): + if line.startswith("[GNUPG:] VALIDSIG "): + key = line.split(" ")[-1] + valid_sig = key in trusted_keys + elif (line.startswith("[GNUPG:] REVKEYSIG ") or line.startswith("[GNUPG:] EXPKEYSIG ")) and not allow_revsig: + valid_sig = False + break + if not valid_sig: if prev_commit != "": print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr) print("Parents are:", file=sys.stderr) @@ -148,15 +178,24 @@ def main(): allow_unclean = current_commit in unclean_merge_allowed if len(parents) == 2 and check_merge and not allow_unclean: current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit]).decode('utf8').splitlines()[0] - subprocess.call([GIT, 'checkout', '--force', '--quiet', parents[0]]) - subprocess.call([GIT, 'merge', '--no-ff', '--quiet', '--no-gpg-sign', parents[1]], stdout=subprocess.DEVNULL) - recreated_tree = subprocess.check_output([GIT, 'show', '--format=format:%T', 'HEAD']).decode('utf8').splitlines()[0] + + # This merge-tree functionality requires git >= 2.38. The + # --write-tree option was added in order to opt-in to the new + # behavior. Older versions of git will not recognize the option and + # will instead exit with code 128. + try: + recreated_tree = subprocess.check_output([GIT, "merge-tree", "--write-tree", parents[0], parents[1]]).decode('utf8').splitlines()[0] + except subprocess.CalledProcessError as e: + if e.returncode == 128: + print("git v2.38+ is required for this functionality.", file=sys.stderr) + sys.exit(1) + else: + raise e + if current_tree != recreated_tree: print("Merge commit {} is not clean".format(current_commit), file=sys.stderr) - subprocess.call([GIT, 'diff', current_commit]) - subprocess.call([GIT, 'checkout', '--force', '--quiet', branch]) + subprocess.call([GIT, 'diff', recreated_tree, current_tree]) sys.exit(1) - subprocess.call([GIT, 'checkout', '--force', '--quiet', branch]) prev_commit = current_commit current_commit = parents[0] diff --git a/contrib/verifybinaries/README.md b/contrib/verifybinaries/README.md deleted file mode 100644 index c50d4bef715..00000000000 --- a/contrib/verifybinaries/README.md +++ /dev/null @@ -1,41 +0,0 @@ -### Verify Binaries - -#### Preparation: - -Make sure you obtain the proper release signing key and verify the fingerprint with several independent sources. - -```sh -$ gpg --fingerprint "Bitcoin Core binary release signing key" -pub 4096R/36C2E964 2015-06-24 [expires: YYYY-MM-DD] - Key fingerprint = 01EA 5486 DE18 A882 D4C2 6845 90C8 019E 36C2 E964 -uid Wladimir J. van der Laan (Bitcoin Core binary release signing key) -``` - -#### Usage: - -This script attempts to download the signature file `SHA256SUMS.asc` from https://bitcoin.org. - -It first checks if the signature passes, and then downloads the files specified in the file, and checks if the hashes of these files match those that are specified in the signature file. - -The script returns 0 if everything passes the checks. It returns 1 if either the signature check or the hash check doesn't pass. If an error occurs the return value is 2. - - -```sh -./verify.py bitcoin-core-0.11.2 -./verify.py bitcoin-core-0.12.0 -./verify.py bitcoin-core-0.13.0-rc3 -``` - -If you only want to download the binaries of certain platform, add the corresponding suffix, e.g.: - -```sh -./verify.py bitcoin-core-0.11.2-osx -./verify.py 0.12.0-linux -./verify.py bitcoin-core-0.13.0-rc3-win64 -``` - -If you do not want to keep the downloaded binaries, specify anything as the second parameter. - -```sh -./verify.py bitcoin-core-0.13.0 delete -``` diff --git a/contrib/verifybinaries/verify.py b/contrib/verifybinaries/verify.py deleted file mode 100755 index b5e4f1318b9..00000000000 --- a/contrib/verifybinaries/verify.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2020-2021 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Script for verifying Bitcoin Core release binaries - -This script attempts to download the signature file SHA256SUMS.asc from -bitcoincore.org and bitcoin.org and compares them. -It first checks if the signature passes, and then downloads the files -specified in the file, and checks if the hashes of these files match those -that are specified in the signature file. -The script returns 0 if everything passes the checks. It returns 1 if either -the signature check or the hash check doesn't pass. If an error occurs the -return value is >= 2. -""" -from hashlib import sha256 -import os -import subprocess -import sys -from textwrap import indent - -WORKINGDIR = "/tmp/bitcoin_verify_binaries" -HASHFILE = "hashes.tmp" -HOST1 = "https://bitcoincore.org" -HOST2 = "https://bitcoin.org" -VERSIONPREFIX = "bitcoin-core-" -SIGNATUREFILENAME = "SHA256SUMS.asc" - - -def parse_version_string(version_str): - if version_str.startswith(VERSIONPREFIX): # remove version prefix - version_str = version_str[len(VERSIONPREFIX):] - - parts = version_str.split('-') - version_base = parts[0] - version_rc = "" - version_os = "" - if len(parts) == 2: # "-rcN" or "version-platform" - if "rc" in parts[1]: - version_rc = parts[1] - else: - version_os = parts[1] - elif len(parts) == 3: # "-rcN-platform" - version_rc = parts[1] - version_os = parts[2] - - return version_base, version_rc, version_os - - -def download_with_wget(remote_file, local_file=None): - if local_file: - wget_args = ['wget', '-O', local_file, remote_file] - else: - # use timestamping mechanism if local filename is not explicitly set - wget_args = ['wget', '-N', remote_file] - - result = subprocess.run(wget_args, - stderr=subprocess.STDOUT, stdout=subprocess.PIPE) - return result.returncode == 0, result.stdout.decode().rstrip() - - -def files_are_equal(filename1, filename2): - with open(filename1, 'rb') as file1: - contents1 = file1.read() - with open(filename2, 'rb') as file2: - contents2 = file2.read() - return contents1 == contents2 - - -def verify_with_gpg(signature_filename, output_filename): - result = subprocess.run(['gpg', '--yes', '--decrypt', '--output', - output_filename, signature_filename], - stderr=subprocess.STDOUT, stdout=subprocess.PIPE) - return result.returncode, result.stdout.decode().rstrip() - - -def remove_files(filenames): - for filename in filenames: - os.remove(filename) - - -def main(args): - # sanity check - if len(args) < 1: - print("Error: need to specify a version on the command line") - return 3 - - # determine remote dir dependent on provided version string - version_base, version_rc, os_filter = parse_version_string(args[0]) - remote_dir = f"/bin/{VERSIONPREFIX}{version_base}/" - if version_rc: - remote_dir += f"test.{version_rc}/" - remote_sigfile = remote_dir + SIGNATUREFILENAME - - # create working directory - os.makedirs(WORKINGDIR, exist_ok=True) - os.chdir(WORKINGDIR) - - # fetch first signature file - sigfile1 = SIGNATUREFILENAME - success, output = download_with_wget(HOST1 + remote_sigfile, sigfile1) - if not success: - print("Error: couldn't fetch signature file. " - "Have you specified the version number in the following format?") - print(f"[{VERSIONPREFIX}][-rc[0-9]][-platform] " - f"(example: {VERSIONPREFIX}0.21.0-rc3-osx)") - print("wget output:") - print(indent(output, '\t')) - return 4 - - # fetch second signature file - sigfile2 = SIGNATUREFILENAME + ".2" - success, output = download_with_wget(HOST2 + remote_sigfile, sigfile2) - if not success: - print("bitcoin.org failed to provide signature file, " - "but bitcoincore.org did?") - print("wget output:") - print(indent(output, '\t')) - remove_files([sigfile1]) - return 5 - - # ensure that both signature files are equal - if not files_are_equal(sigfile1, sigfile2): - print("bitcoin.org and bitcoincore.org signature files were not equal?") - print(f"See files {WORKINGDIR}/{sigfile1} and {WORKINGDIR}/{sigfile2}") - return 6 - - # check signature and extract data into file - retval, output = verify_with_gpg(sigfile1, HASHFILE) - if retval != 0: - if retval == 1: - print("Bad signature.") - elif retval == 2: - print("gpg error. Do you have the Bitcoin Core binary release " - "signing key installed?") - print("gpg output:") - print(indent(output, '\t')) - remove_files([sigfile1, sigfile2, HASHFILE]) - return 1 - - # extract hashes/filenames of binaries to verify from hash file; - # each line has the following format: " " - with open(HASHFILE, 'r', encoding='utf8') as hash_file: - hashes_to_verify = [ - line.split()[:2] for line in hash_file if os_filter in line] - remove_files([HASHFILE]) - if not hashes_to_verify: - print("error: no files matched the platform specified") - return 7 - - # download binaries - for _, binary_filename in hashes_to_verify: - print(f"Downloading {binary_filename}") - download_with_wget(HOST1 + remote_dir + binary_filename) - - # verify hashes - offending_files = [] - for hash_expected, binary_filename in hashes_to_verify: - with open(binary_filename, 'rb') as binary_file: - hash_calculated = sha256(binary_file.read()).hexdigest() - if hash_calculated != hash_expected: - offending_files.append(binary_filename) - if offending_files: - print("Hashes don't match.") - print("Offending files:") - print('\n'.join(offending_files)) - return 1 - verified_binaries = [entry[1] for entry in hashes_to_verify] - - # clean up files if desired - if len(args) >= 2: - print("Clean up the binaries") - remove_files([sigfile1, sigfile2] + verified_binaries) - else: - print(f"Keep the binaries in {WORKINGDIR}") - - print("Verified hashes of") - print('\n'.join(verified_binaries)) - return 0 - - -if __name__ == '__main__': - sys.exit(main(sys.argv[1:])) diff --git a/depends/Makefile b/depends/Makefile index 723509c81d6..31691176335 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -32,6 +32,8 @@ SOURCES_PATH ?= $(BASEDIR)/sources WORK_PATH = $(BASEDIR)/work BASE_CACHE ?= $(BASEDIR)/built SDK_PATH ?= $(BASEDIR)/SDKs +NO_BOOST ?= +NO_LIBEVENT ?= NO_QT ?= NO_QR ?= NO_BDB ?= @@ -42,8 +44,13 @@ NO_UPNP ?= NO_USDT ?= NO_NATPMP ?= MULTIPROCESS ?= +LTO ?= +NO_HARDEN ?= FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources +C_STANDARD ?= c11 +CXX_STANDARD ?= c++17 + BUILD = $(shell ./config.guess) HOST ?= $(BUILD) PATCHES_PATH = $(BASEDIR)/patches @@ -140,8 +147,12 @@ include packages/packages.mk # 2. Before including packages/*.mk (excluding packages/packages.mk), since # they rely on the build_id variables # -build_id:=$(shell env CC='$(build_CC)' CXX='$(build_CXX)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') -$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' CXX='$(host_CXX)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') +build_id:=$(shell env CC='$(build_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(build_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(build_AR)' RANLIB='$(build_RANLIB)' STRIP='$(build_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' NO_HARDEN='$(NO_HARDEN)' ./gen_id '$(BUILD_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') +$(host_arch)_$(host_os)_id:=$(shell env CC='$(host_CC)' C_STANDARD='$(C_STANDARD)' CXX='$(host_CXX)' CXX_STANDARD='$(CXX_STANDARD)' AR='$(host_AR)' RANLIB='$(host_RANLIB)' STRIP='$(host_STRIP)' SHA256SUM='$(build_SHA256SUM)' DEBUG='$(DEBUG)' LTO='$(LTO)' NO_HARDEN='$(NO_HARDEN)' ./gen_id '$(HOST_ID_SALT)' 'GUIX_ENVIRONMENT=$(realpath $(GUIX_ENVIRONMENT))') + +boost_packages_$(NO_BOOST) = $(boost_packages) + +libevent_packages_$(NO_LIBEVENT) = $(libevent_packages) qrencode_packages_$(NO_QR) = $(qrencode_$(host_os)_packages) @@ -158,7 +169,7 @@ zmq_packages_$(NO_ZMQ) = $(zmq_packages) multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages) usdt_packages_$(NO_USDT) = $(usdt_$(host_os)_packages) -packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) $(usdt_packages_) +packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(boost_packages_) $(libevent_packages_) $(qt_packages_) $(wallet_packages_) $(upnp_packages_) $(natpmp_packages_) $(usdt_packages_) native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages) ifneq ($(zmq_packages_),) @@ -222,6 +233,9 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@RANLIB@|$(host_RANLIB)|' \ -e 's|@NM@|$(host_NM)|' \ -e 's|@STRIP@|$(host_STRIP)|' \ + -e 's|@OTOOL@|$(host_OTOOL)|' \ + -e 's|@INSTALL_NAME_TOOL@|$(host_INSTALL_NAME_TOOL)|' \ + -e 's|@DSYMUTIL@|$(host_DSYMUTIL)|' \ -e 's|@build_os@|$(build_os)|' \ -e 's|@host_os@|$(host_os)|' \ -e 's|@CFLAGS@|$(strip $(host_CFLAGS) $(host_$(release_type)_CFLAGS))|' \ @@ -239,6 +253,8 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_ -e 's|@no_usdt@|$(NO_USDT)|' \ -e 's|@no_natpmp@|$(NO_NATPMP)|' \ -e 's|@multiprocess@|$(MULTIPROCESS)|' \ + -e 's|@lto@|$(LTO)|' \ + -e 's|@no_harden@|$(NO_HARDEN)|' \ -e 's|@debug@|$(DEBUG)|' \ $< > $@ touch $@ @@ -271,7 +287,7 @@ clean-all: clean @rm -rf $(SOURCES_PATH) x86_64* i686* mips* arm* aarch64* powerpc* riscv32* riscv64* s390x* clean: - @rm -rf $(WORK_PATH) $(BASE_CACHE) $(BUILD) + @rm -rf $(WORK_PATH) $(BASE_CACHE) $(BUILD) *.log install: check-packages $(host_prefix)/share/config.site diff --git a/depends/README.md b/depends/README.md index f7647198c6d..1064b7d18a6 100644 --- a/depends/README.md +++ b/depends/README.md @@ -62,7 +62,7 @@ For more information, see [SDK Extraction](../contrib/macdeploy/README.md#sdk-ex Common linux dependencies: - sudo apt-get install make automake cmake curl g++-multilib libtool binutils-gold bsdmainutils pkg-config python3 patch bison + sudo apt-get install make automake cmake curl g++-multilib libtool binutils bsdmainutils pkg-config python3 patch bison For linux ARM cross compilation: @@ -96,6 +96,10 @@ The following can be set when running make: `make FOO=bar` - `BASE_CACHE`: Built packages will be placed here - `SDK_PATH`: Path where SDKs can be found (used by macOS) - `FALLBACK_DOWNLOAD_PATH`: If a source file can't be fetched, try here before giving up +- `C_STANDARD`: Set the C standard version used. Defaults to `c11`. +- `CXX_STANDARD`: Set the C++ standard version used. Defaults to `c++17`. +- `NO_BOOST`: Don't download/build/cache Boost +- `NO_LIBEVENT`: Don't download/build/cache Libevent - `NO_QT`: Don't download/build/cache Qt and its dependencies - `NO_QR`: Don't download/build/cache packages needed for enabling qrencode - `NO_ZMQ`: Don't download/build/cache packages needed for enabling ZeroMQ @@ -103,7 +107,8 @@ The following can be set when running make: `make FOO=bar` - `NO_BDB`: Don't download/build/cache BerkeleyDB - `NO_SQLITE`: Don't download/build/cache SQLite - `NO_UPNP`: Don't download/build/cache packages needed for enabling UPnP -- `NO_NATPMP`: Don't download/build/cache packages needed for enabling NAT-PMP +- `NO_NATPMP`: Don't download/build/cache packages needed for enabling NAT-PMP +- `NO_USDT`: Don't download/build/cache packages needed for enabling USDT tracepoints - `ALLOW_HOST_PACKAGES`: Packages that are missed in dependencies (due to `NO_*` option or build script logic) are searched for among the host system packages using `pkg-config`. It allows building with packages of other (newer) versions @@ -113,7 +118,12 @@ The following can be set when running make: `make FOO=bar` - `BUILD_ID_SALT`: Optional salt to use when generating build package ids - `FORCE_USE_SYSTEM_CLANG`: (EXPERTS ONLY) When cross-compiling for macOS, use Clang found in the system's `$PATH` rather than the default prebuilt release of Clang - from llvm.org. Clang 8 or later is required. + from llvm.org. Clang 8 or later is required +- `LOG`: Use file-based logging for individual packages. During a package build its log file + resides in the `depends` directory, and the log file is printed out automatically in case + of build error. After successful build log files are moved along with package archives +- `LTO`: Use LTO when building packages. +- `NO_HARDEN=1`: Don't use hardening options when building packages If some packages are not built, for example `make NO_WALLET=1`, the appropriate options will be passed to bitcoin's configure. In this case, `--disable-wallet`. diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index ced01229dda..8ed82b276df 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -6,6 +6,7 @@ build_darwin_STRIP:=$(shell xcrun -f strip) build_darwin_OTOOL:=$(shell xcrun -f otool) build_darwin_NM:=$(shell xcrun -f nm) build_darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) +build_darwin_DSYMUTIL:=$(shell xcrun -f dsymutil) build_darwin_SHA256SUM=shasum -a 256 build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o @@ -19,6 +20,7 @@ darwin_LIBTOOL:=$(shell xcrun -f libtool) darwin_OTOOL:=$(shell xcrun -f otool) darwin_NM:=$(shell xcrun -f nm) darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) +darwin_DSYMUTIL:=$(shell xcrun -f dsymutil) darwin_native_binutils= darwin_native_toolchain= diff --git a/depends/builders/default.mk b/depends/builders/default.mk index 0370fb9acb9..cc6dec66c2b 100644 --- a/depends/builders/default.mk +++ b/depends/builders/default.mk @@ -5,15 +5,13 @@ default_build_TAR = tar default_build_RANLIB = ranlib default_build_STRIP = strip default_build_NM = nm -default_build_OTOOL = otool -default_build_INSTALL_NAME_TOOL = install_name_tool define add_build_tool_func build_$(build_os)_$1 ?= $$(default_build_$1) build_$(build_arch)_$(build_os)_$1 ?= $$(build_$(build_os)_$1) build_$1=$$(build_$(build_arch)_$(build_os)_$1) endef -$(foreach var,CC CXX AR TAR RANLIB NM STRIP SHA256SUM DOWNLOAD OTOOL INSTALL_NAME_TOOL,$(eval $(call add_build_tool_func,$(var)))) +$(foreach var,CC CXX AR TAR RANLIB NM STRIP SHA256SUM DOWNLOAD OTOOL INSTALL_NAME_TOOL DSYMUTIL,$(eval $(call add_build_tool_func,$(var)))) define add_build_flags_func build_$(build_arch)_$(build_os)_$1 += $(build_$(build_os)_$1) build_$1=$$(build_$(build_arch)_$(build_os)_$1) diff --git a/depends/config.site.in b/depends/config.site.in index 95e6ae85cfe..05c2ccbac10 100644 --- a/depends/config.site.in +++ b/depends/config.site.in @@ -78,15 +78,19 @@ if test "@host_os@" = darwin; then BREW=no fi -PATH="${depends_prefix}/native/bin:${PATH}" +if test -z "$enable_lto" && test -n "@lto@"; then + enable_lto=yes +fi + +if test -z "$enable_hardening" && test -n "@no_harden@"; then + enable_hardening=no +fi + PKG_CONFIG="$(which pkg-config) --static" -# These two need to remain exported because pkg-config does not see them -# otherwise. That means they must be unexported at the end of configure.ac to -# avoid ruining the cache. Sigh. -export PKG_CONFIG_PATH="${depends_prefix}/share/pkgconfig:${depends_prefix}/lib/pkgconfig" +PKG_CONFIG_PATH="${depends_prefix}/share/pkgconfig:${depends_prefix}/lib/pkgconfig" if test -z "@allow_host_packages@"; then - export PKG_CONFIG_LIBDIR="${depends_prefix}/lib/pkgconfig" + PKG_CONFIG_LIBDIR="${depends_prefix}/lib/pkgconfig" fi CPPFLAGS="-I${depends_prefix}/include/ ${CPPFLAGS}" @@ -102,7 +106,7 @@ PYTHONPATH="${depends_prefix}/native/lib/python3/dist-packages${PYTHONPATH:+${PA if test -n "@AR@"; then AR="@AR@" - ac_cv_path_ac_pt_AR="${AR}" + ac_cv_path_AR="${AR}" fi if test -n "@RANLIB@"; then @@ -115,6 +119,28 @@ if test -n "@NM@"; then ac_cv_path_ac_pt_NM="${NM}" fi +if test -n "@STRIP@"; then + STRIP="@STRIP@" + ac_cv_path_ac_pt_STRIP="${STRIP}" +fi + +if test "@host_os@" = darwin; then + if test -n "@OTOOL@"; then + OTOOL="@OTOOL@" + ac_cv_path_OTOOL="${OTOOL}" + fi + + if test -n "@INSTALL_NAME_TOOL@"; then + INSTALL_NAME_TOOL="@INSTALL_NAME_TOOL@" + ac_cv_path_INSTALL_NAME_TOOL="${INSTALL_NAME_TOOL}" + fi + + if test -n "@DSYMUTIL@"; then + DSYMUTIL="@DSYMUTIL@" + ac_cv_path_DSYMUTIL="${DSYMUTIL}" + fi +fi + if test -n "@debug@"; then enable_reduce_exports=no fi diff --git a/depends/funcs.mk b/depends/funcs.mk index c4ad109c696..f0bbf4a168a 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -67,6 +67,7 @@ $(1)_cached_checksum:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_bui $(1)_patch_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)/.patches-$($(1)_build_id) $(1)_prefixbin:=$($($(1)_type)_prefix)/bin/ $(1)_cached:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz +$(1)_build_log:=$(BASEDIR)/$(1)-$($(1)_version)-$($(1)_build_id).log $(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources) #stamps @@ -75,7 +76,7 @@ $(1)_extracted=$$($(1)_extract_dir)/.stamp_extracted $(1)_preprocessed=$$($(1)_extract_dir)/.stamp_preprocessed $(1)_cleaned=$$($(1)_extract_dir)/.stamp_cleaned $(1)_built=$$($(1)_build_dir)/.stamp_built -$(1)_configured=$$($(1)_build_dir)/.stamp_configured +$(1)_configured=$(host_prefix)/.$(1)_stamp_configured $(1)_staged=$$($(1)_staging_dir)/.stamp_staged $(1)_postprocessed=$$($(1)_staging_prefix_dir)/.stamp_postprocessed $(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path)) @@ -85,10 +86,10 @@ $(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path)) # The default behavior for tar will try to set ownership when running as uid 0 and may not succeed, --no-same-owner disables this behavior $(1)_fetch_cmds ?= $(call fetch_file,$(1),$(subst \:,:,$$($(1)_download_path_fixed)),$$($(1)_download_file),$($(1)_file_name),$($(1)_sha256_hash)) $(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash) $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && $(build_TAR) --no-same-owner --strip-components=1 -xf $$($(1)_source) -$(1)_preprocess_cmds ?= -$(1)_build_cmds ?= -$(1)_config_cmds ?= -$(1)_stage_cmds ?= +$(1)_preprocess_cmds ?= true +$(1)_build_cmds ?= true +$(1)_config_cmds ?= true +$(1)_stage_cmds ?= true $(1)_set_vars ?= @@ -136,6 +137,7 @@ $(1)_config_env+=$($(1)_config_env_$(host_arch)_$(host_os)) $($(1)_config_env_$( $(1)_config_env+=PKG_CONFIG_LIBDIR=$($($(1)_type)_prefix)/lib/pkgconfig $(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig +$(1)_config_env+=PKG_CONFIG_SYSROOT_DIR=/ $(1)_config_env+=CMAKE_MODULE_PATH=$($($(1)_type)_prefix)/lib/cmake $(1)_config_env+=PATH=$(build_prefix)/bin:$(PATH) $(1)_build_env+=PATH=$(build_prefix)/bin:$(PATH) @@ -174,7 +176,7 @@ $(1)_cmake=env CC="$$($(1)_cc)" \ CXX="$$($(1)_cxx)" \ CXXFLAGS="$$($(1)_cppflags) $$($(1)_cxxflags)" \ LDFLAGS="$$($(1)_ldflags)" \ - cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" + cmake -DCMAKE_INSTALL_PREFIX:PATH="$$($($(1)_type)_prefix)" $$($(1)_cmake_opts) ifeq ($($(1)_type),build) $(1)_cmake += -DCMAKE_INSTALL_RPATH:PATH="$$($($(1)_type)_prefix)/lib" else @@ -187,52 +189,59 @@ endif endef define int_add_cmds +ifneq ($(LOG),) +$(1)_logging = >>$$($(1)_build_log) 2>&1 || { if test -f $$($(1)_build_log); then cat $$($(1)_build_log); fi; exit 1; } +endif + $($(1)_fetched): mkdir -p $$(@D) $(SOURCES_PATH) rm -f $$@ touch $$@ - cd $$(@D); $(call $(1)_fetch_cmds,$(1)) + cd $$(@D); $($(1)_fetch_cmds) cd $($(1)_source_dir); $(foreach source,$($(1)_all_sources),$(build_SHA256SUM) $(source) >> $$(@);) touch $$@ $($(1)_extracted): | $($(1)_fetched) echo Extracting $(1)... mkdir -p $$(@D) - cd $$(@D); $(call $(1)_extract_cmds,$(1)) + cd $$(@D); $($(1)_extract_cmds) touch $$@ $($(1)_preprocessed): | $($(1)_extracted) echo Preprocessing $(1)... mkdir -p $$(@D) $($(1)_patch_dir) $(foreach patch,$($(1)_patches),cd $(PATCHES_PATH)/$(1); cp $(patch) $($(1)_patch_dir) ;) - cd $$(@D); $(call $(1)_preprocess_cmds, $(1)) + { cd $$(@D); $($(1)_preprocess_cmds); } $$($(1)_logging) touch $$@ $($(1)_configured): | $($(1)_dependencies) $($(1)_preprocessed) echo Configuring $(1)... rm -rf $(host_prefix); mkdir -p $(host_prefix)/lib; cd $(host_prefix); $(foreach package,$($(1)_all_dependencies), $(build_TAR) --no-same-owner -xf $($(package)_cached); ) - mkdir -p $$(@D) - +cd $$(@D); $($(1)_config_env) $(call $(1)_config_cmds, $(1)) + mkdir -p $$($(1)_build_dir) + +{ cd $$($(1)_build_dir); export $($(1)_config_env); $($(1)_config_cmds); } $$($(1)_logging) touch $$@ $($(1)_built): | $($(1)_configured) echo Building $(1)... mkdir -p $$(@D) - +cd $$(@D); $($(1)_build_env) $(call $(1)_build_cmds, $(1)) + +{ cd $$(@D); export $($(1)_build_env); $($(1)_build_cmds); } $$($(1)_logging) touch $$@ $($(1)_staged): | $($(1)_built) echo Staging $(1)... mkdir -p $($(1)_staging_dir)/$(host_prefix) - cd $($(1)_build_dir); $($(1)_stage_env) $(call $(1)_stage_cmds, $(1)) + +{ cd $($(1)_build_dir); export $($(1)_stage_env); $($(1)_stage_cmds); } $$($(1)_logging) rm -rf $($(1)_extract_dir) touch $$@ $($(1)_postprocessed): | $($(1)_staged) echo Postprocessing $(1)... - cd $($(1)_staging_prefix_dir); $(call $(1)_postprocess_cmds) + cd $($(1)_staging_prefix_dir); $($(1)_postprocess_cmds) touch $$@ $($(1)_cached): | $($(1)_dependencies) $($(1)_postprocessed) echo Caching $(1)... - cd $$($(1)_staging_dir)/$(host_prefix); find . | sort | $(build_TAR) --no-recursion -czf $$($(1)_staging_dir)/$$(@F) -T - + cd $$($(1)_staging_dir)/$(host_prefix); \ + find . ! -name '.stamp_postprocessed' -print0 | TZ=UTC xargs -0r touch -h -m -t 200001011200; \ + find . ! -name '.stamp_postprocessed' | LC_ALL=C sort | $(build_TAR) --numeric-owner --no-recursion -czf $$($(1)_staging_dir)/$$(@F) -T - mkdir -p $$(@D) rm -rf $$(@D) && mkdir -p $$(@D) mv $$($(1)_staging_dir)/$$(@F) $$(@) rm -rf $($(1)_staging_dir) + if test -f $($(1)_build_log); then mv $($(1)_build_log) $$(@D); fi $($(1)_cached_checksum): $($(1)_cached) cd $$(@D); $(build_SHA256SUM) $$( $$(@) diff --git a/depends/gen_id b/depends/gen_id index ac69ca7ee1f..3341310e460 100755 --- a/depends/gen_id +++ b/depends/gen_id @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Usage: env [ CC=... ] [ CXX=... ] [ AR=... ] [ RANLIB=... ] [ STRIP=... ] \ -# [ DEBUG=... ] ./build-id [ID_SALT]... +# Usage: env [ CC=... ] [ C_STANDARD=...] [ CXX=... ] [CXX_STANDARD=...] \ +# [ AR=... ] [ RANLIB=... ] [ STRIP=... ] [ DEBUG=... ] \ +# [ LTO=... ] [ NO_HARDEN=... ] ./build-id [ID_SALT]... # # Prints to stdout a SHA256 hash representing the current toolset, used by # depends/Makefile as a build id for caching purposes (detecting when the @@ -39,12 +40,14 @@ bash -c "${CC} -v" bash -c "${CC} -v -E -xc -o /dev/null - < /dev/null" bash -c "${CC} -v -E -xobjective-c -o /dev/null - < /dev/null" + echo "C_STANDARD=${C_STANDARD}" echo "END CC" echo "BEGIN CXX" bash -c "${CXX} -v" bash -c "${CXX} -v -E -xc++ -o /dev/null - < /dev/null" bash -c "${CXX} -v -E -xobjective-c++ -o /dev/null - < /dev/null" + echo "CXX_STANDARD=${CXX_STANDARD}" echo "END CXX" echo "BEGIN AR" @@ -63,6 +66,14 @@ env | grep '^STRIP_' echo "END STRIP" + echo "BEGIN LTO" + echo "LTO=${LTO}" + echo "END LTO" + + echo "BEGIN NO_HARDEN" + echo "NO_HARDEN=${NO_HARDEN}" + echo "END NO_HARDEN" + echo "END ALL" ) | if [ -n "$DEBUG" ] && command -v tee > /dev/null 2>&1; then # When debugging and `tee` is available, output the preimage to stderr diff --git a/depends/hosts/android.mk b/depends/hosts/android.mk index fcc1c4f5c3c..b53966dcf8a 100644 --- a/depends/hosts/android.mk +++ b/depends/hosts/android.mk @@ -5,6 +5,15 @@ else android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang++ android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang endif + +android_CFLAGS=-std=$(C_STANDARD) +android_CXXFLAGS=-std=$(CXX_STANDARD) + +ifneq ($(LTO),) +android_CFLAGS += -flto +android_LDFLAGS += -flto +endif + android_AR=$(ANDROID_TOOLCHAIN_BIN)/llvm-ar android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/llvm-ranlib diff --git a/depends/hosts/darwin.mk b/depends/hosts/darwin.mk index 6bf30b499a6..522a6b17efc 100644 --- a/depends/hosts/darwin.mk +++ b/depends/hosts/darwin.mk @@ -17,6 +17,7 @@ darwin_native_toolchain=native_cctools clang_prog=$(build_prefix)/bin/clang clangxx_prog=$(clang_prog)++ +llvm_config_prog=$(build_prefix)/bin/llvm-config clang_resource_dir=$(build_prefix)/lib/clang/$(native_clang_version) else @@ -34,11 +35,13 @@ darwin_native_toolchain= # Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang") clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++") +llvm_config_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-config") clang_resource_dir=$(shell clang -print-resource-dir) +llvm_lib_dir=$(shell $(llvm_config_prog) --libdir) endif -cctools_TOOLS=AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL +cctools_TOOLS=AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL DSYMUTIL # Make-only lowercase function lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1)))))))))))))))))))))))))) @@ -96,8 +99,8 @@ darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ $(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \ -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \ -isysroot$(OSX_SDK) \ - -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ - -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include + -Xclang -internal-externc-isystem -Xclang $(clang_resource_dir)/include \ + -Xclang -internal-externc-isystem -Xclang $(OSX_SDK)/usr/include darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \ -u LIBRARY_PATH \ @@ -106,11 +109,17 @@ darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \ -isysroot$(OSX_SDK) \ -stdlib=libc++ \ -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 \ - -Xclang -internal-externc-isystem$(clang_resource_dir)/include \ - -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include + -Xclang -internal-externc-isystem -Xclang $(clang_resource_dir)/include \ + -Xclang -internal-externc-isystem -Xclang $(OSX_SDK)/usr/include -darwin_CFLAGS=-pipe -darwin_CXXFLAGS=$(darwin_CFLAGS) +darwin_CFLAGS=-pipe -std=$(C_STANDARD) +darwin_CXXFLAGS=-pipe -std=$(CXX_STANDARD) + +ifneq ($(LTO),) +darwin_CFLAGS += -flto +darwin_CXXFLAGS += -flto +darwin_LDFLAGS += -flto +endif darwin_release_CFLAGS=-O2 darwin_release_CXXFLAGS=$(darwin_release_CFLAGS) diff --git a/depends/hosts/default.mk b/depends/hosts/default.mk index 258619a9d05..bad4568bcb1 100644 --- a/depends/hosts/default.mk +++ b/depends/hosts/default.mk @@ -8,9 +8,8 @@ default_host_AR = $(host_toolchain)ar default_host_RANLIB = $(host_toolchain)ranlib default_host_STRIP = $(host_toolchain)strip default_host_LIBTOOL = $(host_toolchain)libtool -default_host_INSTALL_NAME_TOOL = $(host_toolchain)install_name_tool -default_host_OTOOL = $(host_toolchain)otool default_host_NM = $(host_toolchain)nm +default_host_OBJCOPY = $(host_toolchain)objcopy define add_host_tool_func ifneq ($(filter $(origin $1),undefined default),) @@ -29,11 +28,16 @@ host_$1=$$($(host_arch)_$(host_os)_$1) endef define add_host_flags_func +ifeq ($(filter $(origin $1),undefined default),) +$(host_arch)_$(host_os)_$1 = +$(host_arch)_$(host_os)_$(release_type)_$1 = $($1) +else $(host_arch)_$(host_os)_$1 += $($(host_os)_$1) $(host_arch)_$(host_os)_$(release_type)_$1 += $($(host_os)_$(release_type)_$1) +endif host_$1 = $$($(host_arch)_$(host_os)_$1) host_$(release_type)_$1 = $$($(host_arch)_$(host_os)_$(release_type)_$1) endef -$(foreach tool,CC CXX AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL,$(eval $(call add_host_tool_func,$(tool)))) +$(foreach tool,CC CXX AR RANLIB STRIP LIBTOOL NM OBJCOPY OTOOL INSTALL_NAME_TOOL DSYMUTIL,$(eval $(call add_host_tool_func,$(tool)))) $(foreach flags,CFLAGS CXXFLAGS CPPFLAGS LDFLAGS, $(eval $(call add_host_flags_func,$(flags)))) diff --git a/depends/hosts/freebsd.mk b/depends/hosts/freebsd.mk index 0a62347b570..5351d0b9009 100644 --- a/depends/hosts/freebsd.mk +++ b/depends/hosts/freebsd.mk @@ -1,5 +1,11 @@ -freebsd_CFLAGS=-pipe -freebsd_CXXFLAGS=$(freebsd_CFLAGS) +freebsd_CFLAGS=-pipe -std=$(C_STANDARD) +freebsd_CXXFLAGS=-pipe -std=$(CXX_STANDARD) + +ifneq ($(LTO),) +freebsd_CFLAGS += -flto +freebsd_CXXFLAGS += -flto +freebsd_LDFLAGS += -flto +endif freebsd_release_CFLAGS=-O2 freebsd_release_CXXFLAGS=$(freebsd_release_CFLAGS) diff --git a/depends/hosts/linux.mk b/depends/hosts/linux.mk index 07da752492c..0e2496174e3 100644 --- a/depends/hosts/linux.mk +++ b/depends/hosts/linux.mk @@ -1,5 +1,15 @@ -linux_CFLAGS=-pipe -linux_CXXFLAGS=$(linux_CFLAGS) +linux_CFLAGS=-pipe -std=$(C_STANDARD) +linux_CXXFLAGS=-pipe -std=$(CXX_STANDARD) + +ifneq ($(LTO),) +linux_CFLAGS += -flto +linux_CXXFLAGS += -flto +linux_LDFLAGS += -flto + +linux_AR = $(host_toolchain)gcc-ar +linux_NM = $(host_toolchain)gcc-nm +linux_RANLIB = $(host_toolchain)gcc-ranlib +endif linux_release_CFLAGS=-O2 linux_release_CXXFLAGS=$(linux_release_CFLAGS) @@ -7,7 +17,7 @@ linux_release_CXXFLAGS=$(linux_release_CFLAGS) linux_debug_CFLAGS=-O1 linux_debug_CXXFLAGS=$(linux_debug_CFLAGS) -linux_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC -D_LIBCPP_DEBUG=1 +linux_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC -D_LIBCPP_ENABLE_ASSERTIONS=1 ifeq (86,$(findstring 86,$(build_arch))) i686_linux_CC=gcc -m32 diff --git a/depends/hosts/mingw32.mk b/depends/hosts/mingw32.mk index 2f370d2b873..fc1cc1afbe5 100644 --- a/depends/hosts/mingw32.mk +++ b/depends/hosts/mingw32.mk @@ -2,8 +2,18 @@ ifneq ($(shell $(SHELL) $(.SHELLFLAGS) "command -v $(host)-g++-posix"),) mingw32_CXX := $(host)-g++-posix endif -mingw32_CFLAGS=-pipe -mingw32_CXXFLAGS=$(mingw32_CFLAGS) +mingw32_CFLAGS=-pipe -std=$(C_STANDARD) +mingw32_CXXFLAGS=-pipe -std=$(CXX_STANDARD) + +ifneq ($(LTO),) +mingw32_CFLAGS += -flto +mingw32_CXXFLAGS += -flto +mingw32_LDFLAGS += -flto + +mingw32_AR = $(host_toolchain)gcc-ar +mingw32_NM = $(host_toolchain)gcc-nm +mingw32_RANLIB = $(host_toolchain)gcc-ranlib +endif mingw32_release_CFLAGS=-O2 mingw32_release_CXXFLAGS=$(mingw32_release_CFLAGS) @@ -13,4 +23,4 @@ mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS) mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC -mingw_cmake_system=Windows +mingw32_cmake_system=Windows diff --git a/depends/hosts/netbsd.mk b/depends/hosts/netbsd.mk index b3e4545a64e..14121dca20f 100644 --- a/depends/hosts/netbsd.mk +++ b/depends/hosts/netbsd.mk @@ -1,4 +1,16 @@ -netbsd_CFLAGS=-pipe +netbsd_CFLAGS=-pipe -std=$(C_STANDARD) +netbsd_CXXFLAGS=-pipe -std=$(CXX_STANDARD) + +ifneq ($(LTO),) +netbsd_CFLAGS += -flto +netbsd_CXXFLAGS += -flto +netbsd_LDFLAGS += -flto + +netbsd_AR = $(host_toolchain)gcc-ar +netbsd_NM = $(host_toolchain)gcc-nm +netbsd_RANLIB = $(host_toolchain)gcc-ranlib +endif + netbsd_CXXFLAGS=$(netbsd_CFLAGS) netbsd_release_CFLAGS=-O2 diff --git a/depends/hosts/openbsd.mk b/depends/hosts/openbsd.mk index dc8393e04c9..d330e94d2ed 100644 --- a/depends/hosts/openbsd.mk +++ b/depends/hosts/openbsd.mk @@ -1,11 +1,17 @@ -openbsd_CFLAGS=-pipe -openbsd_CFLAGS_CXXFLAGS=$(openbsd_CFLAGS) +openbsd_CFLAGS=-pipe -std=$(C_STANDARD) +openbsd_CXXFLAGS=-pipe -std=$(CXX_STANDARD) -openbsd_CFLAGS_release_CFLAGS=-O2 -openbsd_CFLAGS_release_CXXFLAGS=$(openbsd_release_CFLAGS) +ifneq ($(LTO),) +openbsd_CFLAGS += -flto +openbsd_CXXFLAGS += -flto +openbsd_LDFLAGS += -flto +endif + +openbsd_release_CFLAGS=-O2 +openbsd_release_CXXFLAGS=$(openbsd_release_CFLAGS) -openbsd_CFLAGS_debug_CFLAGS=-O1 -openbsd_CFLAGS_debug_CXXFLAGS=$(openbsd_debug_CFLAGS) +openbsd_debug_CFLAGS=-O1 +openbsd_debug_CXXFLAGS=$(openbsd_debug_CFLAGS) ifeq (86,$(findstring 86,$(build_arch))) i686_openbsd_CC=clang -m32 diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk index b69276cb154..9f5a925015b 100644 --- a/depends/packages/bdb.mk +++ b/depends/packages/bdb.mk @@ -14,8 +14,10 @@ $(package)_config_opts_freebsd=--with-pic $(package)_config_opts_netbsd=--with-pic $(package)_config_opts_openbsd=--with-pic $(package)_config_opts_android=--with-pic -$(package)_cflags+=-Wno-error=implicit-function-declaration -Wno-error=format-security -$(package)_cxxflags+=-std=c++17 +$(package)_cflags+=-Wno-error=implicit-function-declaration -Wno-error=format-security -Wno-error=implicit-int +$(package)_cppflags_freebsd=-D_XOPEN_SOURCE=600 -D__BSD_VISIBLE=1 +$(package)_cppflags_netbsd=-D_XOPEN_SOURCE=600 +$(package)_cppflags_openbsd=-D_XOPEN_SOURCE=600 $(package)_cppflags_mingw32=-DUNICODE -D_UNICODE endef diff --git a/depends/packages/capnp.mk b/depends/packages/capnp.mk index 8a3a14810d6..f4778c1ecdc 100644 --- a/depends/packages/capnp.mk +++ b/depends/packages/capnp.mk @@ -6,8 +6,15 @@ $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) $(package)_dependencies=native_$(package) +define $(package)_set_vars := +$(package)_config_opts := --with-external-capnp +$(package)_config_opts += CAPNP="$$(native_capnp_prefixbin)/capnp" +$(package)_config_opts += CAPNP_CXX="$$(native_capnp_prefixbin)/capnp-c++" +$(package)_config_opts_android := --disable-shared +endef + define $(package)_config_cmds - $($(package)_autoconf) --with-external-capnp + $($(package)_autoconf) endef define $(package)_build_cmds diff --git a/depends/packages/expat.mk b/depends/packages/expat.mk index 50791ebc6e2..bb203d06f84 100644 --- a/depends/packages/expat.mk +++ b/depends/packages/expat.mk @@ -1,14 +1,18 @@ package=expat -$(package)_version=2.4.1 +$(package)_version=2.4.8 $(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_$(subst .,_,$($(package)_version))/ $(package)_file_name=$(package)-$($(package)_version).tar.xz -$(package)_sha256_hash=cf032d0dba9b928636548e32b327a2d66b1aab63c4f4a13dd132c2d1d2f2fb6a +$(package)_sha256_hash=f79b8f904b749e3e0d20afeadecf8249c55b2e32d4ebb089ae378df479dcaf25 +# -D_DEFAULT_SOURCE defines __USE_MISC, which exposes additional +# definitions in endian.h, which are required for a working +# endianess check in configure when building with -flto. define $(package)_set_vars $(package)_config_opts=--disable-shared --without-docbook --without-tests --without-examples $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts += --without-xmlwf $(package)_config_opts_linux=--with-pic + $(package)_cppflags += -D_DEFAULT_SOURCE endef define $(package)_config_cmds diff --git a/depends/packages/fontconfig.mk b/depends/packages/fontconfig.mk index c8b2fc33d57..444acfe36d7 100644 --- a/depends/packages/fontconfig.mk +++ b/depends/packages/fontconfig.mk @@ -9,6 +9,7 @@ $(package)_patches=gperf_header_regen.patch define $(package)_set_vars $(package)_config_opts=--disable-docs --disable-static --disable-libxml2 --disable-iconv $(package)_config_opts += --disable-dependency-tracking --enable-option-checking + $(package)_cflags += -Wno-implicit-function-declaration endef define $(package)_preprocess_cmds diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 748ed510c1a..9650f77db92 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -17,6 +17,10 @@ define $(package)_set_vars $(package)_config_opts_openbsd=--with-pic $(package)_config_opts_android=--with-pic $(package)_cppflags_mingw32=-D_WIN32_WINNT=0x0601 + + ifeq ($(NO_HARDEN),) + $(package)_cppflags+=-D_FORTIFY_SOURCE=3 + endif endef define $(package)_preprocess_cmds @@ -36,5 +40,7 @@ define $(package)_stage_cmds endef define $(package)_postprocess_cmds - rm lib/*.la + rm lib/*.la && \ + rm include/ev*.h && \ + rm include/event2/*_compat.h endef diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk index 40ab3c68ea0..6da5693b3fc 100644 --- a/depends/packages/libmultiprocess.mk +++ b/depends/packages/libmultiprocess.mk @@ -4,9 +4,19 @@ $(package)_download_path=$(native_$(package)_download_path) $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) $(package)_dependencies=native_$(package) capnp +ifneq ($(host),$(build)) +$(package)_dependencies += native_capnp +endif + +define $(package)_set_vars := +ifneq ($(host),$(build)) +$(package)_cmake_opts := -DCAPNP_EXECUTABLE="$$(native_capnp_prefixbin)/capnp" +$(package)_cmake_opts += -DCAPNPC_CXX_EXECUTABLE="$$(native_capnp_prefixbin)/capnpc-c++" +endif +endef define $(package)_config_cmds - $($(package)_cmake) + $($(package)_cmake) . endef define $(package)_build_cmds @@ -14,5 +24,5 @@ define $(package)_build_cmds endef define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install + $(MAKE) DESTDIR=$($(package)_staging_dir) install-lib endef diff --git a/depends/packages/libnatpmp.mk b/depends/packages/libnatpmp.mk index cdcf8c0bf2c..2eddc76d9cf 100644 --- a/depends/packages/libnatpmp.mk +++ b/depends/packages/libnatpmp.mk @@ -1,8 +1,8 @@ package=libnatpmp -$(package)_version=4536032ae32268a45c073a4d5e91bbab4534773a +$(package)_version=07004b97cf691774efebe70404cf22201e4d330d $(package)_download_path=https://github.com/miniupnp/libnatpmp/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=543b460aab26acf91e11d15e17d8798f845304199eea2d76c2f444ec749c5383 +$(package)_sha256_hash=9321953ceb39d07c25463e266e50d0ae7b64676bb3a986d932b18881ed94f1fb define $(package)_set_vars $(package)_build_opts=CC="$($(package)_cc)" diff --git a/depends/packages/libxcb.mk b/depends/packages/libxcb.mk index fa30e80f5c0..036eaf6560a 100644 --- a/depends/packages/libxcb.mk +++ b/depends/packages/libxcb.mk @@ -4,6 +4,7 @@ $(package)_download_path=https://xcb.freedesktop.org/dist $(package)_file_name=$(package)-$($(package)_version).tar.xz $(package)_sha256_hash=a55ed6db98d43469801262d81dc2572ed124edc3db31059d4e9916eb9f844c34 $(package)_dependencies=xcb_proto libXau +$(package)_patches = remove_pthread_stubs.patch define $(package)_set_vars $(package)_config_opts=--disable-static --disable-devel-docs --without-doxygen --without-launchd @@ -20,7 +21,7 @@ endef define $(package)_preprocess_cmds cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub build-aux && \ - sed "s/pthread-stubs//" -i configure + patch -p1 -i $($(package)_patch_dir)/remove_pthread_stubs.patch endef define $(package)_config_cmds diff --git a/depends/packages/miniupnpc.mk b/depends/packages/miniupnpc.mk index 99f5b0a8dbc..7ad2529e470 100644 --- a/depends/packages/miniupnpc.mk +++ b/depends/packages/miniupnpc.mk @@ -3,17 +3,20 @@ $(package)_version=2.2.2 $(package)_download_path=https://miniupnp.tuxfamily.org/files/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=888fb0976ba61518276fe1eda988589c700a3f2a69d71089260d75562afd3687 -$(package)_patches=dont_leak_info.patch +$(package)_patches=dont_leak_info.patch respect_mingw_cflags.patch +# Next time this package is updated, ensure that _WIN32_WINNT is still properly set. +# See discussion in https://github.com/bitcoin/bitcoin/pull/25964. define $(package)_set_vars $(package)_build_opts=CC="$($(package)_cc)" $(package)_build_opts_darwin=LIBTOOL="$($(package)_libtool)" -$(package)_build_opts_mingw32=-f Makefile.mingw +$(package)_build_opts_mingw32=-f Makefile.mingw CFLAGS="$($(package)_cflags) -D_WIN32_WINNT=0x0601" $(package)_build_env+=CFLAGS="$($(package)_cflags) $($(package)_cppflags)" AR="$($(package)_ar)" endef define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/dont_leak_info.patch + patch -p1 < $($(package)_patch_dir)/dont_leak_info.patch && \ + patch -p1 < $($(package)_patch_dir)/respect_mingw_cflags.patch endef define $(package)_build_cmds diff --git a/depends/packages/native_cctools.mk b/depends/packages/native_cctools.mk index d169eb67231..03e9002ecd7 100644 --- a/depends/packages/native_cctools.mk +++ b/depends/packages/native_cctools.mk @@ -7,18 +7,24 @@ $(package)_build_subdir=cctools $(package)_dependencies=native_libtapi define $(package)_set_vars - $(package)_config_opts=--target=$(host) + $(package)_config_opts=--target=$(host) --enable-lto-support + $(package)_config_opts+=--with-llvm-config=$(llvm_config_prog) $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib - ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) - $(package)_config_opts+=--enable-lto-support --with-llvm-config=$(build_prefix)/bin/llvm-config - endif $(package)_cc=$(clang_prog) $(package)_cxx=$(clangxx_prog) endef +ifneq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) define $(package)_preprocess_cmds + mkdir -p $($(package)_staging_prefix_dir)/lib && \ + cp $(llvm_lib_dir)/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools endef +else +define $(package)_preprocess_cmds + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools +endef +endif define $(package)_config_cmds $($(package)_autoconf) diff --git a/depends/packages/native_clang.mk b/depends/packages/native_clang.mk index 245269a9d34..f2712294ab2 100644 --- a/depends/packages/native_clang.mk +++ b/depends/packages/native_clang.mk @@ -16,10 +16,13 @@ endef define $(package)_stage_cmds mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include && \ mkdir -p $($(package)_staging_prefix_dir)/bin && \ + mkdir -p $($(package)_staging_prefix_dir)/include/llvm-c && \ cp bin/clang $($(package)_staging_prefix_dir)/bin/ && \ cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ && \ cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \ cp bin/llvm-config $($(package)_staging_prefix_dir)/bin/ && \ + cp include/llvm-c/ExternC.h $($(package)_staging_prefix_dir)/include/llvm-c && \ + cp include/llvm-c/lto.h $($(package)_staging_prefix_dir)/include/llvm-c && \ cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ cp -r lib/clang/$($(package)_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include/ endef diff --git a/depends/packages/native_ds_store.mk b/depends/packages/native_ds_store.mk index 44108925a4f..51a95f48ef7 100644 --- a/depends/packages/native_ds_store.mk +++ b/depends/packages/native_ds_store.mk @@ -1,6 +1,6 @@ package=native_ds_store $(package)_version=1.3.0 -$(package)_download_path=https://github.com/al45tair/ds_store/archive/ +$(package)_download_path=https://github.com/dmgbuild/ds_store/archive/ $(package)_file_name=v$($(package)_version).tar.gz $(package)_sha256_hash=76b3280cd4e19e5179defa23fb594a9dd32643b0c80d774bd3108361d94fb46d $(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages diff --git a/depends/packages/native_libdmg-hfsplus.mk b/depends/packages/native_libdmg-hfsplus.mk deleted file mode 100644 index c7c8adef415..00000000000 --- a/depends/packages/native_libdmg-hfsplus.mk +++ /dev/null @@ -1,24 +0,0 @@ -package=native_libdmg-hfsplus -$(package)_version=7ac55ec64c96f7800d9818ce64c79670e7f02b67 -$(package)_download_path=https://github.com/planetbeing/libdmg-hfsplus/archive -$(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=56fbdc48ec110966342f0ecddd6f8f89202f4143ed2a3336e42bbf88f940850c -$(package)_build_subdir=build -$(package)_patches=remove-libcrypto-dependency.patch - -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/remove-libcrypto-dependency.patch && \ - mkdir build -endef - -define $(package)_config_cmds - $($(package)_cmake) -DCMAKE_C_FLAGS="$$($(1)_cflags) -Wl,--build-id=none" -DCMAKE_SKIP_RPATH="ON" -DCMAKE_EXE_LINKER_FLAGS="-static" -DCMAKE_FIND_LIBRARY_SUFFIXES=".a" .. -endef - -define $(package)_build_cmds - $(MAKE) -C dmg -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) -C dmg install -endef diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk index 14653ce9fb0..e647afba5f3 100644 --- a/depends/packages/native_libmultiprocess.mk +++ b/depends/packages/native_libmultiprocess.mk @@ -1,12 +1,12 @@ package=native_libmultiprocess -$(package)_version=d576d975debdc9090bd2582f83f49c76c0061698 +$(package)_version=1af83d15239ccfa7e47b8764029320953dd7fdf1 $(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive $(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=9f8b055c8bba755dc32fe799b67c20b91e7b13e67cadafbc54c0f1def057a370 +$(package)_sha256_hash=e5587d3feedc7f8473f178a89b94163a11076629825d664964799bbbd5844da5 $(package)_dependencies=native_capnp define $(package)_config_cmds - $($(package)_cmake) + $($(package)_cmake) . endef define $(package)_build_cmds @@ -14,5 +14,5 @@ define $(package)_build_cmds endef define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install + $(MAKE) DESTDIR=$($(package)_staging_dir) install-bin endef diff --git a/depends/packages/native_libtapi.mk b/depends/packages/native_libtapi.mk index 1633213a423..052bb236893 100644 --- a/depends/packages/native_libtapi.mk +++ b/depends/packages/native_libtapi.mk @@ -13,7 +13,5 @@ define $(package)_build_cmds endef define $(package)_stage_cmds - ./install.sh && \ - mkdir -p $($(package)_staging_prefix_dir)/include/llvm-c && \ - cp src/llvm/include/llvm-c/lto.h $($(package)_staging_prefix_dir)/include/llvm-c + ./install.sh endef diff --git a/depends/packages/native_mac_alias.mk b/depends/packages/native_mac_alias.mk index 783f87ca7c0..ddd631186ed 100644 --- a/depends/packages/native_mac_alias.mk +++ b/depends/packages/native_mac_alias.mk @@ -1,6 +1,6 @@ package=native_mac_alias $(package)_version=2.2.0 -$(package)_download_path=https://github.com/al45tair/mac_alias/archive/ +$(package)_download_path=https://github.com/dmgbuild/mac_alias/archive/ $(package)_file_name=v$($(package)_version).tar.gz $(package)_sha256_hash=421e6d7586d1f155c7db3e7da01ca0dacc9649a509a253ad7077b70174426499 $(package)_install_libdir=$(build_prefix)/lib/python3/dist-packages diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 991db7f46e2..b3600b72d0b 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -1,4 +1,8 @@ -packages:=boost libevent +packages:= + +boost_packages = boost + +libevent_packages = libevent qrencode_linux_packages = qrencode qrencode_android_packages = qrencode @@ -26,7 +30,7 @@ usdt_linux_packages=systemtap darwin_native_packages = native_ds_store native_mac_alias ifneq ($(build_os),darwin) -darwin_native_packages += native_cctools native_libtapi native_libdmg-hfsplus +darwin_native_packages += native_cctools native_libtapi ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),) darwin_native_packages+= native_clang diff --git a/depends/packages/qrencode.mk b/depends/packages/qrencode.mk index d1687883bcd..2afd95d7c4f 100644 --- a/depends/packages/qrencode.mk +++ b/depends/packages/qrencode.mk @@ -1,15 +1,16 @@ package=qrencode -$(package)_version=3.4.4 +$(package)_version=4.1.1 $(package)_download_path=https://fukuchi.org/works/qrencode/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=efe5188b1ddbcbf98763b819b146be6a90481aac30cfc8d858ab78a19cde1fa5 +$(package)_sha256_hash=e455d9732f8041cf5b9c388e345a641fd15707860f928e94507b1961256a6923 define $(package)_set_vars -$(package)_config_opts=--disable-shared --without-tools --without-tests --disable-sdltest +$(package)_config_opts=--disable-shared --without-tools --without-tests --without-png $(package)_config_opts += --disable-gprof --disable-gcov --disable-mudflap $(package)_config_opts += --disable-dependency-tracking --enable-option-checking $(package)_config_opts_linux=--with-pic $(package)_config_opts_android=--with-pic +$(package)_cflags += -Wno-int-conversion -Wno-implicit-function-declaration endef define $(package)_preprocess_cmds diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index e5fb135da05..136ce325798 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -1,30 +1,39 @@ package=qt -$(package)_version=5.15.3 +$(package)_version=5.15.5 $(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules $(package)_suffix=everywhere-opensource-src-$($(package)_version).tar.xz $(package)_file_name=qtbase-$($(package)_suffix) -$(package)_sha256_hash=26394ec9375d52c1592bd7b689b1619c6b8dbe9b6f91fdd5c355589787f3a0b6 +$(package)_sha256_hash=0c42c799aa7c89e479a07c451bf5a301e291266ba789e81afc18f95049524edc $(package)_linux_dependencies=freetype fontconfig libxcb libxkbcommon libxcb_util libxcb_util_render libxcb_util_keysyms libxcb_util_image libxcb_util_wm $(package)_qt_libs=corelib network widgets gui plugins testlib $(package)_linguist_tools = lrelease lupdate lconvert -$(package)_patches = qt.pro qttools_src.pro -$(package)_patches += fix_qt_pkgconfig.patch mac-qmake.conf fix_no_printer.patch no-xlib.patch -$(package)_patches += dont_hardcode_x86_64.patch fix_montery_include.patch -$(package)_patches += fix_android_jni_static.patch dont_hardcode_pwd.patch -$(package)_patches += qtbase-moc-ignore-gcc-macro.patch fix_limits_header.patch +$(package)_patches = qt.pro +$(package)_patches += qttools_src.pro +$(package)_patches += mac-qmake.conf +$(package)_patches += fix_qt_pkgconfig.patch +$(package)_patches += no-xlib.patch +$(package)_patches += dont_hardcode_x86_64.patch +$(package)_patches += fix_montery_include.patch +$(package)_patches += fix_android_jni_static.patch +$(package)_patches += dont_hardcode_pwd.patch +$(package)_patches += qtbase-moc-ignore-gcc-macro.patch $(package)_patches += use_android_ndk23.patch $(package)_patches += rcc_hardcode_timestamp.patch +$(package)_patches += duplicate_lcqpafonts.patch +$(package)_patches += fast_fixed_dtoa_no_optimize.patch +$(package)_patches += guix_cross_lib_path.patch $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix) -$(package)_qttranslations_sha256_hash=5d7869f670a135ad0986e266813b9dd5bbae2b09577338f9cdf8904d4af52db0 +$(package)_qttranslations_sha256_hash=c92af4171397a0ed272330b4fa0669790fcac8d050b07c8b8cc565ebeba6735e $(package)_qttools_file_name=qttools-$($(package)_suffix) -$(package)_qttools_sha256_hash=463b2fe71a085e7ab4e39333ae360ab0ec857b966d7a08f752c427e5df55f90d +$(package)_qttools_sha256_hash=6d0778b71b2742cb527561791d1d3d255366163d54a10f78c683a398f09ffc6c $(package)_extra_sources = $($(package)_qttranslations_file_name) $(package)_extra_sources += $($(package)_qttools_file_name) define $(package)_set_vars +$(package)_config_env = QT_MAC_SDK_NO_VERSION_CHECK=1 $(package)_config_opts_release = -release $(package)_config_opts_release += -silent $(package)_config_opts_debug = -debug @@ -48,6 +57,7 @@ $(package)_config_opts += -no-linuxfb $(package)_config_opts += -no-libjpeg $(package)_config_opts += -no-libproxy $(package)_config_opts += -no-libudev +$(package)_config_opts += -no-mimetype-database $(package)_config_opts += -no-mtdev $(package)_config_opts += -no-openssl $(package)_config_opts += -no-openvg @@ -123,6 +133,9 @@ $(package)_config_opts_darwin += -no-feature-corewlan $(package)_config_opts_darwin += -no-freetype $(package)_config_opts_darwin += QMAKE_MACOSX_DEPLOYMENT_TARGET=$(OSX_MIN_VERSION) +# Optimizing using > -O1 causes non-determinism when building across arches. +$(package)_config_opts_aarch64_darwin += "QMAKE_CFLAGS_OPTIMIZE_FULL = -O1" + ifneq ($(build_os),darwin) $(package)_config_opts_darwin += -xplatform macx-clang-linux $(package)_config_opts_darwin += -device-option MAC_SDK_PATH=$(OSX_SDK) @@ -145,18 +158,19 @@ $(package)_config_opts_linux += -fontconfig $(package)_config_opts_linux += -no-opengl $(package)_config_opts_linux += -no-feature-vulkan $(package)_config_opts_linux += -dbus-runtime -$(package)_config_opts_arm_linux += -platform linux-g++ -xplatform bitcoin-linux-g++ -$(package)_config_opts_i686_linux = -xplatform linux-g++-32 -ifneq (,$(findstring -stdlib=libc++,$($(1)_cxx))) -$(package)_config_opts_x86_64_linux = -xplatform linux-clang-libc++ +ifneq ($(LTO),) +$(package)_config_opts_linux += -ltcg +endif + +ifneq (,$(findstring clang,$($(package)_cxx))) + ifneq (,$(findstring -stdlib=libc++,$($(package)_cxx))) + $(package)_config_opts_linux += -platform linux-clang-libc++ -xplatform linux-clang-libc++ + else + $(package)_config_opts_linux += -platform linux-clang -xplatform linux-clang + endif else -$(package)_config_opts_x86_64_linux = -xplatform linux-g++-64 + $(package)_config_opts_linux += -platform linux-g++ -xplatform bitcoin-linux-g++ endif -$(package)_config_opts_aarch64_linux = -xplatform linux-aarch64-gnu-g++ -$(package)_config_opts_powerpc64_linux = -platform linux-g++ -xplatform bitcoin-linux-g++ -$(package)_config_opts_powerpc64le_linux = -platform linux-g++ -xplatform bitcoin-linux-g++ -$(package)_config_opts_riscv64_linux = -platform linux-g++ -xplatform bitcoin-linux-g++ -$(package)_config_opts_s390x_linux = -platform linux-g++ -xplatform bitcoin-linux-g++ $(package)_config_opts_mingw32 = -no-opengl $(package)_config_opts_mingw32 += -no-dbus @@ -164,8 +178,9 @@ $(package)_config_opts_mingw32 += -no-freetype $(package)_config_opts_mingw32 += -xplatform win32-g++ $(package)_config_opts_mingw32 += "QMAKE_CFLAGS = '$($(package)_cflags) $($(package)_cppflags)'" $(package)_config_opts_mingw32 += "QMAKE_CXX = '$($(package)_cxx)'" -$(package)_config_opts_mingw32 += "QMAKE_CXXFLAGS = '$($(package)_cflags) $($(package)_cppflags)'" +$(package)_config_opts_mingw32 += "QMAKE_CXXFLAGS = '$($(package)_cxxflags) $($(package)_cppflags)'" $(package)_config_opts_mingw32 += "QMAKE_LFLAGS = '$($(package)_ldflags)'" +$(package)_config_opts_mingw32 += "QMAKE_LIB = '$($(package)_ar) rc'" $(package)_config_opts_mingw32 += -device-option CROSS_COMPILE="$(host)-" $(package)_config_opts_mingw32 += -pch @@ -215,49 +230,44 @@ endef # 2. Create a macOS-Clang-Linux mkspec using our mac-qmake.conf. # # 3. After making a copy of the mkspec for the linux-arm-gnueabi host, named -# bitcoin-linux-g++, replace instances of linux-arm-gnueabi with $(host). This -# way we can generically support hosts like riscv64-linux-gnu, which Qt doesn't -# ship a mkspec for. See it's usage in config_opts_* above. +# bitcoin-linux-g++, replace tool names with $($($(package)_type)_TOOL). # # 4. Put our C, CXX and LD FLAGS into gcc-base.conf. Only used for non-host builds. # -# 5. Do similar for the win32-g++ mkspec. -# -# 6. In clang.conf, swap out clang & clang++, for our compiler + flags. See #17466. -# -# 7. Adjust a regex in toolchain.prf, to accommodate Guix's usage of -# CROSS_LIBRARY_PATH. See #15277. +# 5. In clang.conf, swap out clang & clang++, for our compiler + flags. See #17466. define $(package)_preprocess_cmds cp $($(package)_patch_dir)/qt.pro qt.pro && \ cp $($(package)_patch_dir)/qttools_src.pro qttools/src/src.pro && \ patch -p1 -i $($(package)_patch_dir)/dont_hardcode_pwd.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_android_jni_static.patch && \ patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \ patch -p1 -i $($(package)_patch_dir)/dont_hardcode_x86_64.patch && \ patch -p1 -i $($(package)_patch_dir)/qtbase-moc-ignore-gcc-macro.patch && \ - patch -p1 -i $($(package)_patch_dir)/fix_limits_header.patch && \ patch -p1 -i $($(package)_patch_dir)/fix_montery_include.patch && \ patch -p1 -i $($(package)_patch_dir)/use_android_ndk23.patch && \ patch -p1 -i $($(package)_patch_dir)/rcc_hardcode_timestamp.patch && \ + patch -p1 -i $($(package)_patch_dir)/duplicate_lcqpafonts.patch && \ + patch -p1 -i $($(package)_patch_dir)/fast_fixed_dtoa_no_optimize.patch && \ + patch -p1 -i $($(package)_patch_dir)/guix_cross_lib_path.patch && \ mkdir -p qtbase/mkspecs/macx-clang-linux &&\ cp -f qtbase/mkspecs/macx-clang/qplatformdefs.h qtbase/mkspecs/macx-clang-linux/ &&\ cp -f $($(package)_patch_dir)/mac-qmake.conf qtbase/mkspecs/macx-clang-linux/qmake.conf && \ cp -r qtbase/mkspecs/linux-arm-gnueabi-g++ qtbase/mkspecs/bitcoin-linux-g++ && \ - sed -i.old "s/arm-linux-gnueabi-/$(host)-/g" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ + sed -i.old "s|arm-linux-gnueabi-gcc|$($($(package)_type)_CC)|" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ + sed -i.old "s|arm-linux-gnueabi-g++|$($($(package)_type)_CXX)|" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ + sed -i.old "s|arm-linux-gnueabi-ar|$($($(package)_type)_AR)|" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ + sed -i.old "s|arm-linux-gnueabi-objcopy|$($($(package)_type)_OBJCOPY)|" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ + sed -i.old "s|arm-linux-gnueabi-nm|$($($(package)_type)_NM)|" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ + sed -i.old "s|arm-linux-gnueabi-strip|$($($(package)_type)_STRIP)|" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \ echo "!host_build: QMAKE_CFLAGS += $($(package)_cflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_CXXFLAGS += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ echo "!host_build: QMAKE_LFLAGS += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf && \ sed -i.old "s|QMAKE_CC = \$$$$\$$$${CROSS_COMPILE}clang|QMAKE_CC = $($(package)_cc)|" qtbase/mkspecs/common/clang.conf && \ - sed -i.old "s|QMAKE_CXX = \$$$$\$$$${CROSS_COMPILE}clang++|QMAKE_CXX = $($(package)_cxx)|" qtbase/mkspecs/common/clang.conf && \ - sed -i.old "s/LIBRARY_PATH/(CROSS_)?\0/g" qtbase/mkspecs/features/toolchain.prf + sed -i.old "s|QMAKE_CXX = \$$$$\$$$${CROSS_COMPILE}clang++|QMAKE_CXX = $($(package)_cxx)|" qtbase/mkspecs/common/clang.conf endef define $(package)_config_cmds - export PKG_CONFIG_SYSROOT_DIR=/ && \ - export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \ - export QT_MAC_SDK_NO_VERSION_CHECK=1 && \ cd qtbase && \ ./configure -top-level $($(package)_config_opts) endef diff --git a/depends/packages/sqlite.mk b/depends/packages/sqlite.mk index 126781ceebb..a8ec89c6c64 100644 --- a/depends/packages/sqlite.mk +++ b/depends/packages/sqlite.mk @@ -1,15 +1,20 @@ package=sqlite -$(package)_version=3320100 -$(package)_download_path=https://sqlite.org/2020/ +$(package)_version=3380500 +$(package)_download_path=https://sqlite.org/2022/ $(package)_file_name=sqlite-autoconf-$($(package)_version).tar.gz -$(package)_sha256_hash=486748abfb16abd8af664e3a5f03b228e5f124682b0c942e157644bf6fff7d10 +$(package)_sha256_hash=5af07de982ba658fd91a03170c945f99c971f6955bc79df3266544373e39869c define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-readline --disable-dynamic-extensions --enable-option-checking +$(package)_config_opts+= --disable-rtree --disable-fts4 --disable-fts5 $(package)_config_opts_linux=--with-pic $(package)_config_opts_freebsd=--with-pic $(package)_config_opts_netbsd=--with-pic $(package)_config_opts_openbsd=--with-pic +$(package)_config_opts_debug=--enable-debug +$(package)_cflags+=-DSQLITE_DQS=0 -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_OMIT_DEPRECATED +$(package)_cflags+=-DSQLITE_OMIT_SHARED_CACHE -DSQLITE_OMIT_JSON -DSQLITE_LIKE_DOESNT_MATCH_BLOBS +$(package)_cflags+=-DSQLITE_OMIT_DECLTYPE -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_AUTOINIT endef define $(package)_preprocess_cmds diff --git a/depends/packages/systemtap.mk b/depends/packages/systemtap.mk index 833e75b9786..541ebeee014 100644 --- a/depends/packages/systemtap.mk +++ b/depends/packages/systemtap.mk @@ -1,12 +1,13 @@ package=systemtap -$(package)_version=4.5 -$(package)_download_path=https://sourceware.org/systemtap/ftp/releases/ +$(package)_version=4.8 +$(package)_download_path=https://sourceware.org/ftp/systemtap/releases/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=75078ed37e0dd2a769c9d1f9394170b2d9f4d7daa425f43ca80c13bad6cfc925 -$(package)_patches=remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch +$(package)_sha256_hash=cbd50a4eba5b261394dc454c12448ddec73e55e6742fda7f508f9fbc1331c223 +$(package)_patches=remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch fix_variadic_warning.patch define $(package)_preprocess_cmds patch -p1 < $($(package)_patch_dir)/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch && \ + patch -p1 < $($(package)_patch_dir)/fix_variadic_warning.patch && \ mkdir -p $($(package)_staging_prefix_dir)/include/sys && \ cp includes/sys/sdt.h $($(package)_staging_prefix_dir)/include/sys/sdt.h endef diff --git a/depends/packages/zeromq.mk b/depends/packages/zeromq.mk index c74ae15b313..d7152327934 100644 --- a/depends/packages/zeromq.mk +++ b/depends/packages/zeromq.mk @@ -16,17 +16,16 @@ define $(package)_set_vars $(package)_config_opts_netbsd=--with-pic $(package)_config_opts_openbsd=--with-pic $(package)_config_opts_android=--with-pic - $(package)_cxxflags+=-std=c++17 endef define $(package)_preprocess_cmds patch -p1 < $($(package)_patch_dir)/remove_libstd_link.patch && \ - patch -p1 < $($(package)_patch_dir)/netbsd_kevent_void.patch && \ - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config + patch -p1 < $($(package)_patch_dir)/netbsd_kevent_void.patch endef define $(package)_config_cmds ./autogen.sh && \ + cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub config && \ $($(package)_autoconf) endef diff --git a/depends/patches/libxcb/remove_pthread_stubs.patch b/depends/patches/libxcb/remove_pthread_stubs.patch new file mode 100644 index 00000000000..1f32dea527e --- /dev/null +++ b/depends/patches/libxcb/remove_pthread_stubs.patch @@ -0,0 +1,12 @@ +Remove uneeded pthread-stubs dependency +--- a/configure ++++ b/configure +@@ -19695,7 +19695,7 @@ fi + NEEDED="xau >= 0.99.2" + case $host_os in + linux*) ;; +- *) NEEDED="$NEEDED pthread-stubs" ;; ++ *) NEEDED="$NEEDED" ;; + esac + + pkg_failed=no diff --git a/depends/patches/miniupnpc/respect_mingw_cflags.patch b/depends/patches/miniupnpc/respect_mingw_cflags.patch new file mode 100644 index 00000000000..a44580ddab6 --- /dev/null +++ b/depends/patches/miniupnpc/respect_mingw_cflags.patch @@ -0,0 +1,23 @@ +commit fec515a7ac9991a0ee91068fda046b54b191155e +Author: fanquake +Date: Wed Jul 27 15:52:37 2022 +0100 + + build: respect CFLAGS in makefile.mingw + + Similar to the other Makefile. + + Cherry-pick of https://github.com/miniupnp/miniupnp/pull/619. + +diff --git a/Makefile.mingw b/Makefile.mingw +index 2bff7bd..88430d2 100644 +--- a/Makefile.mingw ++++ b/Makefile.mingw +@@ -19,7 +19,7 @@ else + RM = rm -f + endif + #CFLAGS = -Wall -g -DDEBUG -D_WIN32_WINNT=0X501 +-CFLAGS = -Wall -W -Wstrict-prototypes -Os -DNDEBUG -D_WIN32_WINNT=0X501 ++CFLAGS ?= -Wall -W -Wstrict-prototypes -Os -DNDEBUG -D_WIN32_WINNT=0X501 + LDLIBS = -lws2_32 -liphlpapi + # -lwsock32 + # -liphlpapi is needed for GetBestRoute() and GetIpAddrTable() diff --git a/depends/patches/native_libdmg-hfsplus/remove-libcrypto-dependency.patch b/depends/patches/native_libdmg-hfsplus/remove-libcrypto-dependency.patch deleted file mode 100644 index f346c8f2cff..00000000000 --- a/depends/patches/native_libdmg-hfsplus/remove-libcrypto-dependency.patch +++ /dev/null @@ -1,45 +0,0 @@ -From 3e5fd3fb56bc9ff03beb535979e33dcf83fe1f70 Mon Sep 17 00:00:00 2001 -From: Cory Fields -Date: Thu, 8 May 2014 12:39:42 -0400 -Subject: [PATCH] dmg: remove libcrypto dependency - ---- - dmg/CMakeLists.txt | 16 ---------------- - 1 file changed, 16 deletions(-) - -diff --git a/dmg/CMakeLists.txt b/dmg/CMakeLists.txt -index eec62d6..3969f64 100644 ---- a/dmg/CMakeLists.txt -+++ b/dmg/CMakeLists.txt -@@ -1,12 +1,5 @@ --INCLUDE(FindOpenSSL) - INCLUDE(FindZLIB) - --FIND_LIBRARY(CRYPTO_LIBRARIES crypto -- PATHS -- /usr/lib -- /usr/local/lib -- ) -- - IF(NOT ZLIB_FOUND) - message(FATAL_ERROR "zlib is required for dmg!") - ENDIF(NOT ZLIB_FOUND) -@@ -18,15 +11,6 @@ link_directories(${PROJECT_BINARY_DIR}/common ${PROJECT_BINARY_DIR}/hfs) - - add_library(dmg adc.c base64.c checksum.c dmgfile.c dmglib.c filevault.c io.c partition.c resources.c udif.c) - --IF(OPENSSL_FOUND) -- add_definitions(-DHAVE_CRYPT) -- include_directories(${OPENSSL_INCLUDE_DIR}) -- target_link_libraries(dmg ${CRYPTO_LIBRARIES}) -- IF(WIN32) -- TARGET_LINK_LIBRARIES(dmg gdi32) -- ENDIF(WIN32) --ENDIF(OPENSSL_FOUND) -- - target_link_libraries(dmg common hfs z) - - add_executable(dmg-bin dmg.c) --- -2.22.0 - diff --git a/depends/patches/qt/dont_hardcode_x86_64.patch b/depends/patches/qt/dont_hardcode_x86_64.patch index 5c1e030fa45..a66426877ad 100644 --- a/depends/patches/qt/dont_hardcode_x86_64.patch +++ b/depends/patches/qt/dont_hardcode_x86_64.patch @@ -73,7 +73,7 @@ diff --git a/mkspecs/features/mac/default_post.prf b/mkspecs/features/mac/defaul index 92a9112bca6..d888731ec8d 100644 --- old/qtbase/mkspecs/features/mac/default_post.prf +++ new/qtbase/mkspecs/features/mac/default_post.prf -@@ -90,6 +90,11 @@ app_extension_api_only { +@@ -95,6 +95,11 @@ app_extension_api_only { QMAKE_LFLAGS += $$QMAKE_CFLAGS_APPLICATION_EXTENSION } @@ -85,7 +85,7 @@ index 92a9112bca6..d888731ec8d 100644 macx-xcode { qmake_pkginfo_typeinfo.name = QMAKE_PKGINFO_TYPEINFO !isEmpty(QMAKE_PKGINFO_TYPEINFO): \ -@@ -145,9 +150,6 @@ macx-xcode { +@@ -150,9 +155,6 @@ macx-xcode { simulator: VALID_SIMULATOR_ARCHS = $$QMAKE_APPLE_SIMULATOR_ARCHS VALID_ARCHS = $$VALID_DEVICE_ARCHS $$VALID_SIMULATOR_ARCHS diff --git a/depends/patches/qt/duplicate_lcqpafonts.patch b/depends/patches/qt/duplicate_lcqpafonts.patch new file mode 100644 index 00000000000..c460b51dcff --- /dev/null +++ b/depends/patches/qt/duplicate_lcqpafonts.patch @@ -0,0 +1,104 @@ +QtGui: Fix duplication of logging category lcQpaFonts + +Move it to qplatformfontdatabase.h. + +Upstream commit: + - Qt 6.0: ab01885e48873fb2ad71841a3f1627fe4d9cd835 + +--- a/qtbase/src/gui/text/qplatformfontdatabase.cpp ++++ b/qtbase/src/gui/text/qplatformfontdatabase.cpp +@@ -52,6 +52,8 @@ + + QT_BEGIN_NAMESPACE + ++Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") ++ + void qt_registerFont(const QString &familyname, const QString &stylename, + const QString &foundryname, int weight, + QFont::Style style, int stretch, bool antialiased, + +--- a/qtbase/src/gui/text/qplatformfontdatabase.h ++++ b/qtbase/src/gui/text/qplatformfontdatabase.h +@@ -50,6 +50,7 @@ + // + + #include ++#include + #include + #include + #include +@@ -62,6 +63,7 @@ + + QT_BEGIN_NAMESPACE + ++Q_DECLARE_LOGGING_CATEGORY(lcQpaFonts) + + class QWritingSystemsPrivate; + + +--- a/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm ++++ b/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm +@@ -86,8 +86,6 @@ + + QT_BEGIN_NAMESPACE + +-Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") +- + static float SYNTHETIC_ITALIC_SKEW = std::tan(14.f * std::acos(0.f) / 90.f); + + bool QCoreTextFontEngine::ct_getSfntTable(void *user_data, uint tag, uchar *buffer, uint *length) + +--- a/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h ++++ b/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext_p.h +@@ -64,8 +64,6 @@ + + QT_BEGIN_NAMESPACE + +-Q_DECLARE_LOGGING_CATEGORY(lcQpaFonts) +- + class QCoreTextFontEngine : public QFontEngine + { + Q_GADGET + +--- a/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase.cpp ++++ b/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase.cpp +@@ -68,8 +68,6 @@ + + QT_BEGIN_NAMESPACE + +-Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") +- + #ifndef QT_NO_DIRECTWRITE + // ### fixme: Consider direct linking of dwrite.dll once Windows Vista pre SP2 is dropped (QTBUG-49711) + + +--- a/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase_p.h ++++ b/qtbase/src/platformsupport/fontdatabases/windows/qwindowsfontdatabase_p.h +@@ -63,8 +63,6 @@ + + QT_BEGIN_NAMESPACE + +-Q_DECLARE_LOGGING_CATEGORY(lcQpaFonts) +- + class QWindowsFontEngineData + { + Q_DISABLE_COPY_MOVE(QWindowsFontEngineData) + +--- a/qtbase/src/platformsupport/themes/genericunix/qgenericunixthemes.cpp ++++ b/qtbase/src/platformsupport/themes/genericunix/qgenericunixthemes.cpp +@@ -40,6 +40,7 @@ + #include "qgenericunixthemes_p.h" + + #include "qpa/qplatformtheme_p.h" ++#include "qpa/qplatformfontdatabase.h" + + #include + #include +@@ -76,7 +77,6 @@ + QT_BEGIN_NAMESPACE + + Q_DECLARE_LOGGING_CATEGORY(qLcTray) +-Q_LOGGING_CATEGORY(lcQpaFonts, "qt.qpa.fonts") + + ResourceHelper::ResourceHelper() + { diff --git a/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch b/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch new file mode 100644 index 00000000000..d4d6539f56d --- /dev/null +++ b/depends/patches/qt/fast_fixed_dtoa_no_optimize.patch @@ -0,0 +1,20 @@ +Modify the optimisation flags for FastFixedDtoa. +This fixes a non-determinism issue in the asm produced for +this function when cross-compiling on x86_64 and aarch64 for +the arm-linux-gnueabihf HOST. + +--- a/qtbase/src/3rdparty/double-conversion/fixed-dtoa.h ++++ b/qtbase/src/3rdparty/double-conversion/fixed-dtoa.h +@@ -48,9 +48,12 @@ namespace double_conversion { + // + // This method only works for some parameters. If it can't handle the input it + // returns false. The output is null-terminated when the function succeeds. ++#pragma GCC push_options ++#pragma GCC optimize ("-O1") + bool FastFixedDtoa(double v, int fractional_count, + Vector buffer, int* length, int* decimal_point); + ++#pragma GCC pop_options + } // namespace double_conversion + + #endif // DOUBLE_CONVERSION_FIXED_DTOA_H_ diff --git a/depends/patches/qt/fix_android_jni_static.patch b/depends/patches/qt/fix_android_jni_static.patch index 22a4d5ab0e1..3c7f66c2e53 100644 --- a/depends/patches/qt/fix_android_jni_static.patch +++ b/depends/patches/qt/fix_android_jni_static.patch @@ -1,6 +1,6 @@ --- old/qtbase/src/plugins/platforms/android/androidjnimain.cpp +++ new/qtbase/src/plugins/platforms/android/androidjnimain.cpp -@@ -934,6 +934,14 @@ Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void */*reserved*/) +@@ -943,6 +943,14 @@ Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void */*reserved*/) __android_log_print(ANDROID_LOG_FATAL, "Qt", "registerNatives failed"); return -1; } @@ -13,6 +13,5 @@ + } + QWindowSystemInterfacePrivate::TabletEvent::setPlatformSynthesizesMouse(false); - - m_javaVM = vm; + m_javaVM = vm; diff --git a/depends/patches/qt/fix_limits_header.patch b/depends/patches/qt/fix_limits_header.patch deleted file mode 100644 index 258128c0ca1..00000000000 --- a/depends/patches/qt/fix_limits_header.patch +++ /dev/null @@ -1,33 +0,0 @@ -Fix compiling with GCC 11 - -Upstream: - - bug report: https://bugreports.qt.io/browse/QTBUG-89977 - - fix in Qt 6.1: 813a928c7c3cf98670b6043149880ed5c955efb9 - ---- old/qtbase/src/corelib/text/qbytearraymatcher.h -+++ new/qtbase/src/corelib/text/qbytearraymatcher.h -@@ -42,6 +42,8 @@ - - #include - -+#include -+ - QT_BEGIN_NAMESPACE - - - -Upstream fix and backports: - - Qt 6.1: 3eab20ad382569cb2c9e6ccec2322c3d08c0f716 - - Qt 6.2: 380294a5971da85010a708dc23b0edec192cbf27 - - Qt 6.3: 2b2b3155d9f6ba1e4f859741468fbc47db09292b - ---- old/qtbase/src/corelib/tools/qoffsetstringarray_p.h -+++ new/qtbase/src/corelib/tools/qoffsetstringarray_p.h -@@ -55,6 +55,7 @@ - - #include - #include -+#include - - QT_BEGIN_NAMESPACE - diff --git a/depends/patches/qt/fix_no_printer.patch b/depends/patches/qt/fix_no_printer.patch deleted file mode 100644 index 13723561384..00000000000 --- a/depends/patches/qt/fix_no_printer.patch +++ /dev/null @@ -1,19 +0,0 @@ ---- x/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h -+++ y/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h -@@ -52,6 +52,7 @@ - // - - #include -+#include - - #ifndef QT_NO_PRINTER - ---- x/qtbase/src/plugins/plugins.pro -+++ y/qtbase/src/plugins/plugins.pro -@@ -9,6 +9,3 @@ qtHaveModule(gui) { - !android:qtConfig(library): SUBDIRS *= generic - } - qtHaveModule(widgets): SUBDIRS += styles -- --!winrt:qtHaveModule(printsupport): \ -- SUBDIRS += printsupport diff --git a/depends/patches/qt/guix_cross_lib_path.patch b/depends/patches/qt/guix_cross_lib_path.patch new file mode 100644 index 00000000000..7911dc21d7d --- /dev/null +++ b/depends/patches/qt/guix_cross_lib_path.patch @@ -0,0 +1,17 @@ +Facilitate guix building with CROSS_LIBRARY_PATH + +See discussion in https://github.com/bitcoin/bitcoin/pull/15277. + +--- a/qtbase/mkspecs/features/toolchain.prf ++++ b/qtbase/mkspecs/features/toolchain.prf +@@ -236,8 +236,8 @@ isEmpty($${target_prefix}.INCDIRS) { + add_libraries = false + for (line, output) { + line ~= s/^[ \\t]*// # remove leading spaces +- contains(line, "LIBRARY_PATH=.*") { +- line ~= s/^LIBRARY_PATH=// # remove leading LIBRARY_PATH= ++ contains(line, "(CROSS_)?LIBRARY_PATH=.*") { ++ line ~= s/^(CROSS_)?LIBRARY_PATH=// # remove leading (CROSS_)?LIBRARY_PATH= + equals(QMAKE_HOST.os, Windows): \ + paths = $$split(line, ;) + else: \ diff --git a/depends/patches/qt/mac-qmake.conf b/depends/patches/qt/mac-qmake.conf index e4bfaa1463e..cb94bf07b42 100644 --- a/depends/patches/qt/mac-qmake.conf +++ b/depends/patches/qt/mac-qmake.conf @@ -15,10 +15,8 @@ QMAKE_MAC_SDK.macosx.SDKVersion = $${MAC_SDK_VERSION} QMAKE_MAC_SDK.macosx.PlatformPath = /phony !host_build: QMAKE_CFLAGS += -target $${MAC_TARGET} !host_build: QMAKE_OBJECTIVE_CFLAGS += $$QMAKE_CFLAGS -!host_build: QMAKE_CXXFLAGS += $$QMAKE_CFLAGS +!host_build: QMAKE_CXXFLAGS += -target $${MAC_TARGET} !host_build: QMAKE_LFLAGS += -target $${MAC_TARGET} QMAKE_AR = $${CROSS_COMPILE}ar cq QMAKE_RANLIB=$${CROSS_COMPILE}ranlib -QMAKE_LIBTOOL=$${CROSS_COMPILE}libtool -QMAKE_INSTALL_NAME_TOOL=$${CROSS_COMPILE}install_name_tool load(qt_config) diff --git a/depends/patches/systemtap/fix_variadic_warning.patch b/depends/patches/systemtap/fix_variadic_warning.patch new file mode 100644 index 00000000000..93cc2d6081d --- /dev/null +++ b/depends/patches/systemtap/fix_variadic_warning.patch @@ -0,0 +1,16 @@ +Could be dropped after a migration to C++20. +See: https://github.com/bitcoin/bitcoin/issues/26916. + +diff --git a/includes/sys/sdt.h b/includes/sys/sdt.h +index 4075a5f..7c6138c 100644 +--- a/includes/sys/sdt.h ++++ b/includes/sys/sdt.h +@@ -276,7 +276,7 @@ __extension__ extern unsigned long long __sdt_unsp; + _SDT_ASM_1(.purgem _SDT_TYPE_) \ + _SDT_ASM_1(.purgem _SDT_TYPE) + +-#define _SDT_ASM_BODY(provider, name, pack_args, args, ...) \ ++#define _SDT_ASM_BODY(provider, name, pack_args, args) \ + _SDT_DEF_MACROS \ + _SDT_ASM_1(990: _SDT_NOP) \ + _SDT_ASM_3( .pushsection .note.stapsdt,_SDT_ASM_AUTOGROUP,"note") \ diff --git a/depends/patches/systemtap/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch b/depends/patches/systemtap/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch index eae0cf72d6c..c115bc43e81 100644 --- a/depends/patches/systemtap/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch +++ b/depends/patches/systemtap/remove_SDT_ASM_SECTION_AUTOGROUP_SUPPORT_check.patch @@ -1,19 +1,15 @@ -commit b92d4c121486f3c6e8a2cea537c53eb09894479a -Author: 0xb10c <0xb10c@gmail.com> -Date: Tue Dec 7 11:02:07 2021 +0100 +Remove _SDT_ASM_SECTION_AUTOGROUP_SUPPORT check - Remove _SDT_ASM_SECTION_AUTOGROUP_SUPPORT check - - We assume that the assembler supports "?" in .pushsection directives. - This enables us to skip configure and make. - - See https://github.com/bitcoin/bitcoin/issues/23297. +We assume that the assembler supports "?" in .pushsection directives. +This enables us to skip configure and make. + +See https://github.com/bitcoin/bitcoin/issues/23297. diff --git a/includes/sys/sdt.h b/includes/sys/sdt.h -index 97766e710..352b4ee25 100644 +index ca0162b..f96e0ee 100644 --- a/includes/sys/sdt.h +++ b/includes/sys/sdt.h -@@ -230,12 +230,10 @@ __extension__ extern unsigned long long __sdt_unsp; +@@ -241,12 +241,10 @@ __extension__ extern unsigned long long __sdt_unsp; nice with code in COMDAT sections, which comes up in C++ code. Without that assembler support, some combinations of probe placements in certain kinds of C++ code may produce link-time errors. */ @@ -27,5 +23,5 @@ index 97766e710..352b4ee25 100644 -# define _SDT_ASM_AUTOGROUP "" -#endif - #define _SDT_ASM_BODY(provider, name, pack_args, args) \ - _SDT_ASM_1(990: _SDT_NOP) \ + #define _SDT_DEF_MACROS \ + _SDT_ASM_1(.altmacro) \ diff --git a/doc/JSON-RPC-interface.md b/doc/JSON-RPC-interface.md index 12807bfb865..ab5db58cdd1 100644 --- a/doc/JSON-RPC-interface.md +++ b/doc/JSON-RPC-interface.md @@ -5,6 +5,28 @@ The headless daemon `bitcoind` has the JSON-RPC API enabled by default, the GUI option. In the GUI it is possible to execute RPC methods in the Debug Console Dialog. +## Parameter passing + +The JSON-RPC server supports both _by-position_ and _by-name_ [parameter +structures](https://www.jsonrpc.org/specification#parameter_structures) +described in the JSON-RPC specification. For extra convenience, to avoid the +need to name every parameter value, all RPC methods accept a named parameter +called `args`, which can be set to an array of initial positional values that +are combined with named values. + +Examples: + +```sh +# "params": ["mywallet", false, false, "", false, false, true] +bitcoin-cli createwallet mywallet false false "" false false true + +# "params": {"wallet_name": "mywallet", "load_on_startup": true} +bitcoin-cli -named createwallet wallet_name=mywallet load_on_startup=true + +# "params": {"args": ["mywallet"], "load_on_startup": true} +bitcoin-cli -named createwallet mywallet load_on_startup=true +``` + ## Versioning The RPC interface might change from one major version of Bitcoin Core to the diff --git a/doc/README.md b/doc/README.md index 897fdb59cf0..44e581fed16 100644 --- a/doc/README.md +++ b/doc/README.md @@ -53,7 +53,6 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [Developer Notes](developer-notes.md) - [Productivity Notes](productivity.md) -- [Release Notes](release-notes.md) - [Release Process](release-process.md) - [Source Code Documentation (External Link)](https://doxygen.bitcoincore.org/) - [Translation Process](translation_process.md) @@ -64,6 +63,7 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th - [BIPS](bips.md) - [Dnsseed Policy](dnsseed-policy.md) - [Benchmarking](benchmarking.md) +- [Internal Design Docs](design/) ### Resources * Discuss on the [BitcoinTalk](https://bitcointalk.org/) forums, in the [Development & Technical Discussion board](https://bitcointalk.org/index.php?board=6.0). @@ -71,8 +71,8 @@ The Bitcoin repo's [root README](/README.md) contains relevant information on th ### Miscellaneous - [Assets Attribution](assets-attribution.md) -- [Assumeutxo design](assumeutxo.md) - [bitcoin.conf Configuration File](bitcoin-conf.md) +- [CJDNS Support](cjdns.md) - [Files](files.md) - [Fuzz-testing](fuzzing.md) - [I2P Support](i2p.md) diff --git a/doc/REST-interface.md b/doc/REST-interface.md index 1f0a07a2840..2d7d0e37693 100644 --- a/doc/REST-interface.md +++ b/doc/REST-interface.md @@ -31,13 +31,14 @@ Supported API `GET /rest/tx/.` Given a transaction hash: returns a transaction in binary, hex-encoded binary, or JSON formats. +Responds with 404 if the transaction doesn't exist. By default, this endpoint will only search the mempool. To query for a confirmed transaction, enable the transaction index via "txindex=1" command line / configuration option. #### Blocks -`GET /rest/block/.` -`GET /rest/block/notxdetails/.` +- `GET /rest/block/.` +- `GET /rest/block/notxdetails/.` Given a block hash: returns a block, in binary, hex-encoded binary or JSON formats. Responds with 404 if the block doesn't exist. @@ -47,18 +48,24 @@ The HTTP request and response are both handled entirely in-memory. With the /notxdetails/ option JSON response will only contain the transaction hash instead of the complete transaction details. The option only affects the JSON response. #### Blockheaders -`GET /rest/headers//.` +`GET /rest/headers/.?count=` Given a block hash: returns amount of blockheaders in upward direction. Returns empty if the block doesn't exist or it isn't in the active chain. +*Deprecated (but not removed) since 24.0:* +`GET /rest/headers//.` + #### Blockfilter Headers -`GET /rest/blockfilterheaders///.` +`GET /rest/blockfilterheaders//.?count=` Given a block hash: returns amount of blockfilter headers in upward direction for the filter type . Returns empty if the block doesn't exist or it isn't in the active chain. +*Deprecated (but not removed) since 24.0:* +`GET /rest/blockfilterheaders///.` + #### Blockfilters `GET /rest/blockfilter//.` @@ -70,30 +77,32 @@ Responds with 404 if the block doesn't exist. `GET /rest/blockhashbyheight/.` Given a height: returns hash of block in best-block-chain at height provided. +Responds with 404 if block not found. #### Chaininfos `GET /rest/chaininfo.json` Returns various state info regarding block chain processing. Only supports JSON as output format. -* chain : (string) current network name (main, test, signet, regtest) -* blocks : (numeric) the current number of blocks processed in the server -* headers : (numeric) the current number of headers we have validated -* bestblockhash : (string) the hash of the currently best block -* difficulty : (numeric) the current difficulty -* mediantime : (numeric) the median time of the 11 blocks before the most recent block on the blockchain -* verificationprogress : (numeric) estimate of verification progress [0..1] -* chainwork : (string) total amount of work in active chain, in hexadecimal -* pruned : (boolean) if the blocks are subject to pruning -* pruneheight : (numeric) highest block available -* softforks : (array) status of softforks in progress +Refer to the `getblockchaininfo` RPC help for details. + +#### Deployment info +`GET /rest/deploymentinfo.json` +`GET /rest/deploymentinfo/.json` + +Returns an object containing various state info regarding deployments of +consensus changes at the current chain tip, or at if provided. +Only supports JSON as output format. +Refer to the `getdeploymentinfo` RPC help for details. #### Query UTXO set -`GET /rest/getutxos//-/-/.../-.` +- `GET /rest/getutxos/-/-/.../-.` +- `GET /rest/getutxos/checkmempool/-/-/.../-.` -The getutxo command allows querying of the UTXO set given a set of outpoints. -See BIP64 for input and output serialisation: -https://github.com/bitcoin/bips/blob/master/bip-0064.mediawiki +The getutxos endpoint allows querying the UTXO set, given a set of outpoints. +With the `/checkmempool/` option, the mempool is also taken into account. +See [BIP64](https://github.com/bitcoin/bips/blob/master/bip-0064.mediawiki) for +input and output serialization (relevant for `bin` and `hex` output formats). Example: ``` @@ -121,14 +130,19 @@ $ curl localhost:18332/rest/getutxos/checkmempool/b2cdfd7b89def827ff8af7cd9bff76 #### Memory pool `GET /rest/mempool/info.json` -Returns various information about the TX mempool. +Returns various information about the transaction mempool. Only supports JSON as output format. -Refer to the `getmempoolinfo` RPC for documentation of the fields. +Refer to the `getmempoolinfo` RPC help for details. -`GET /rest/mempool/contents.json` +`GET /rest/mempool/contents.json?verbose=&mempool_sequence=` -Returns transactions in the TX mempool. +Returns the transactions in the mempool. Only supports JSON as output format. +Refer to the `getrawmempool` RPC help for details. Defaults to setting +`verbose=true` and `mempool_sequence=false`. + +*Query parameters for `verbose` and `mempool_sequence` available in 25.0 and up.* + Risks ------------- diff --git a/doc/bips.md b/doc/bips.md index 0f3f61daf1f..1d5c91b8bdb 100644 --- a/doc/bips.md +++ b/doc/bips.md @@ -1,4 +1,4 @@ -BIPs that are implemented by Bitcoin Core (up-to-date up to **v23.0**): +BIPs that are implemented by Bitcoin Core (up-to-date up to **v24.0**): * [`BIP 9`](https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki): The changes allowing multiple soft-forks to be deployed in parallel have been implemented since **v0.12.1** ([PR #7575](https://github.com/bitcoin/bitcoin/pull/7575)) * [`BIP 11`](https://github.com/bitcoin/bips/blob/master/bip-0011.mediawiki): Multisig outputs are standard since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)). @@ -28,6 +28,7 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v23.0**): and it is disabled by default at build time since **v0.19.0** ([PR #15584](https://github.com/bitcoin/bitcoin/pull/15584)). It has been removed as of **v0.20.0** ([PR 17165](https://github.com/bitcoin/bitcoin/pull/17165)). * [`BIP 84`](https://github.com/bitcoin/bips/blob/master/bip-0084.mediawiki): The experimental descriptor wallets introduced in **v0.21.0** by default use the Hierarchical Deterministic Wallet derivation proposed by BIP 84. ([PR #16528](https://github.com/bitcoin/bitcoin/pull/16528)) +* [`BIP 86`](https://github.com/bitcoin/bips/blob/master/bip-0086.mediawiki): Descriptor wallets by default use the Hierarchical Deterministic Wallet derivation proposed by BIP 86 since **v23.0** ([PR #22364](https://github.com/bitcoin/bitcoin/pull/22364)). * [`BIP 90`](https://github.com/bitcoin/bips/blob/master/bip-0090.mediawiki): Trigger mechanism for activation of BIPs 34, 65, and 66 has been simplified to block height checks since **v0.14.0** ([PR #8391](https://github.com/bitcoin/bitcoin/pull/8391)). * [`BIP 111`](https://github.com/bitcoin/bips/blob/master/bip-0111.mediawiki): `NODE_BLOOM` service bit added, and enforced for all peer versions as of **v0.13.0** ([PR #6579](https://github.com/bitcoin/bitcoin/pull/6579) and [PR #6641](https://github.com/bitcoin/bitcoin/pull/6641)). * [`BIP 112`](https://github.com/bitcoin/bips/blob/master/bip-0112.mediawiki): The CHECKSEQUENCEVERIFY opcode has been implemented since **v0.12.1** ([PR #7524](https://github.com/bitcoin/bitcoin/pull/7524)), and has been *buried* since **v0.19.0** ([PR #16060](https://github.com/bitcoin/bitcoin/pull/16060)). @@ -58,6 +59,7 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v23.0**): with mainnet activation as of **v0.21.1** ([PR 21377](https://github.com/bitcoin/bitcoin/pull/21377), [PR 21686](https://github.com/bitcoin/bitcoin/pull/21686)). * [`BIP 350`](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki): Addresses for native v1+ segregated Witness outputs use Bech32m instead of Bech32 as of **v22.0** ([PR 20861](https://github.com/bitcoin/bitcoin/pull/20861)). +* [`BIP 371`](https://github.com/bitcoin/bips/blob/master/bip-0371.mediawiki): Taproot fields for PSBT as of **v24.0** ([PR 22558](https://github.com/bitcoin/bitcoin/pull/22558)). * [`BIP 380`](https://github.com/bitcoin/bips/blob/master/bip-0380.mediawiki) [`381`](https://github.com/bitcoin/bips/blob/master/bip-0381.mediawiki) [`382`](https://github.com/bitcoin/bips/blob/master/bip-0382.mediawiki) diff --git a/doc/bitcoin-conf.md b/doc/bitcoin-conf.md index 8c9035c45b6..1ebfb4c1de9 100644 --- a/doc/bitcoin-conf.md +++ b/doc/bitcoin-conf.md @@ -6,6 +6,8 @@ All command-line options (except for `-?`, `-help`, `-version` and `-conf`) may Changes to the configuration file while `bitcoind` or `bitcoin-qt` is running only take effect after restarting. +Users should never make any configuration changes which they do not understand. Furthermore, users should always be wary of accepting any configuration changes provided to them by another source (even if they believe that they do understand them). + ## Configuration File Format The configuration file is a plain text file and consists of `option=value` entries, one per line. Leading and trailing whitespaces are removed. @@ -61,4 +63,12 @@ Windows | `%APPDATA%\Bitcoin\` | `C:\Users\username\AppData\Roaming\Bitcoin\bitc Linux | `$HOME/.bitcoin/` | `/home/username/.bitcoin/bitcoin.conf` macOS | `$HOME/Library/Application Support/Bitcoin/` | `/Users/username/Library/Application Support/Bitcoin/bitcoin.conf` -You can find an example bitcoin.conf file in [share/examples/bitcoin.conf](../share/examples/bitcoin.conf). +An example configuration file can be generated by [contrib/devtools/gen-bitcoin-conf.sh](../contrib/devtools/gen-bitcoin-conf.sh). +Run this script after compiling to generate an up-to-date configuration file. +The output is placed under `share/examples/bitcoin.conf`. +To use the generated configuration file, copy the example file into your data directory and edit it there, like so: + +``` +# example copy command for linux user +cp share/examples/bitcoin.conf ~/.bitcoin +``` diff --git a/doc/build-freebsd.md b/doc/build-freebsd.md index da2ab61c2a7..aa10e4a8919 100644 --- a/doc/build-freebsd.md +++ b/doc/build-freebsd.md @@ -1,48 +1,21 @@ # FreeBSD Build Guide -**Updated for FreeBSD [12.2](https://www.freebsd.org/releases/12.2R/announce.html)** +**Updated for FreeBSD [12.3](https://www.freebsd.org/releases/12.3R/announce/)** This guide describes how to build bitcoind, command-line utilities, and GUI on FreeBSD. -## Dependencies - -The following dependencies are **required**: - - Library | Purpose | Description - ----------------------------------------------------------------------|------------|---------------------- - [autoconf](https://svnweb.freebsd.org/ports/head/devel/autoconf/) | Build | Automatically configure software source code - [automake](https://svnweb.freebsd.org/ports/head/devel/automake/) | Build | Generate makefile (requires autoconf) - [libtool](https://svnweb.freebsd.org/ports/head/devel/libtool/) | Build | Shared library support - [pkgconf](https://svnweb.freebsd.org/ports/head/devel/pkgconf/) | Build | Configure compiler and linker flags - [git](https://svnweb.freebsd.org/ports/head/devel/git/) | Clone | Version control system - [gmake](https://svnweb.freebsd.org/ports/head/devel/gmake/) | Compile | Generate executables - [boost-libs](https://svnweb.freebsd.org/ports/head/devel/boost-libs/) | Utility | Library for threading, data structures, etc - [libevent](https://svnweb.freebsd.org/ports/head/devel/libevent/) | Networking | OS independent asynchronous networking - - -The following dependencies are **optional**: - - Library | Purpose | Description - ---------------------------------------------------------------------------|------------------|---------------------- - [db5](https://svnweb.freebsd.org/ports/head/databases/db5/) | Berkeley DB | Wallet storage (only needed when wallet enabled) - [qt5](https://svnweb.freebsd.org/ports/head/devel/qt5/) | GUI | GUI toolkit (only needed when GUI enabled) - [libqrencode](https://svnweb.freebsd.org/ports/head/graphics/libqrencode/) | QR codes in GUI | Generating QR codes (only needed when GUI enabled) - [libzmq4](https://svnweb.freebsd.org/ports/head/net/libzmq4/) | ZMQ notification | Allows generating ZMQ notifications (requires ZMQ version >= 4.0.0) - [sqlite3](https://svnweb.freebsd.org/ports/head/databases/sqlite3/) | SQLite DB | Wallet storage (only needed when wallet enabled) - [python3](https://svnweb.freebsd.org/ports/head/lang/python3/) | Testing | Python Interpreter (only needed when running the test suite) - - See [dependencies.md](dependencies.md) for a complete overview. - ## Preparation ### 1. Install Required Dependencies -Install the required dependencies the usual way you [install software on FreeBSD](https://www.freebsd.org/doc/en/books/handbook/ports.html) - either with `pkg` or via the Ports collection. The example commands below use `pkg` which is usually run as `root` or via `sudo`. If you want to use `sudo`, and you haven't set it up: [use this guide](http://www.freebsdwiki.net/index.php/Sudo%2C_configuring) to setup `sudo` access on FreeBSD. +Run the following as root to install the base dependencies for building. ```bash pkg install autoconf automake boost-libs git gmake libevent libtool pkgconf ``` +See [dependencies.md](dependencies.md) for a complete overview. + ### 2. Clone Bitcoin Repo Now that `git` and all the required dependencies are installed, let's clone the Bitcoin Core repository to a directory. All build scripts and commands will run from this directory. ``` bash @@ -52,22 +25,41 @@ git clone https://github.com/bitcoin/bitcoin.git ### 3. Install Optional Dependencies #### Wallet Dependencies -It is not necessary to build wallet functionality to run bitcoind or the GUI. To enable legacy wallets, you must install `db5`. To enable [descriptor wallets](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md), `sqlite3` is required. Skip `db5` if you intend to *exclusively* use descriptor wallets +It is not necessary to build wallet functionality to run either `bitcoind` or `bitcoin-qt`. + +###### Descriptor Wallet Support + +`sqlite3` is required to support [descriptor wallets](descriptors.md). +Skip if you don't intend to use descriptor wallets. +``` bash +pkg install sqlite3 +``` ###### Legacy Wallet Support -`db5` is required to enable support for legacy wallets. Skip if you don't intend to use legacy wallets +BerkeleyDB is only required if legacy wallet support is required. -```bash -pkg install db5 +It is required to use Berkeley DB 4.8. You **cannot** use the BerkeleyDB library +from ports. However, you can build DB 4.8 yourself [using depends](/depends). + +``` +gmake -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_NATPMP=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 ``` -###### Descriptor Wallet Support +When the build is complete, the Berkeley DB installation location will be displayed: -`sqlite3` is required to enable support for descriptor wallets. Skip if you don't intend to use descriptor wallets. -``` bash -pkg install sqlite3 ``` ---- +to: /path/to/bitcoin/depends/x86_64-unknown-freebsd[release-number] +``` + +Finally, set `BDB_PREFIX` to this path according to your shell: + +``` +csh: setenv BDB_PREFIX [path displayed above] +``` + +``` +sh/bash: export BDB_PREFIX=[path displayed above] +``` #### GUI Dependencies ###### Qt5 @@ -84,12 +76,20 @@ pkg install libqrencode ``` --- +#### Notifications +###### ZeroMQ + +Bitcoin Core can provide notifications via ZeroMQ. If the package is installed, support will be compiled in. +```bash +pkg install libzmq4 +``` + #### Test Suite Dependencies There is an included test suite that is useful for testing code changes when developing. To run the test suite (recommended), you will need to have Python 3 installed: ```bash -pkg install python3 +pkg install python3 databases/py-sqlite3 ``` --- @@ -98,22 +98,25 @@ pkg install python3 ### 1. Configuration There are many ways to configure Bitcoin Core, here are a few common examples: -##### Wallet (BDB + SQlite) Support, No GUI: -This explicitly enables legacy wallet support and disables the GUI. If `sqlite3` is installed, then descriptor wallet support will be built. + +##### Descriptor Wallet and GUI: +This explicitly enables the GUI and disables legacy wallet support, assuming `sqlite` and `qt` are installed. ```bash ./autogen.sh -./configure --with-gui=no --with-incompatible-bdb \ - BDB_LIBS="-ldb_cxx-5" \ - BDB_CFLAGS="-I/usr/local/include/db5" \ - MAKE=gmake +./configure --without-bdb --with-gui=yes MAKE=gmake ``` -##### Wallet (only SQlite) and GUI Support: -This explicitly enables the GUI and disables legacy wallet support. If `qt5` is not installed, this will throw an error. If `sqlite3` is installed then descriptor wallet functionality will be built. If `sqlite3` is not installed, then wallet functionality will be disabled. +##### Descriptor & Legacy Wallet. No GUI: +This enables support for both wallet types and disables the GUI, assuming +`sqlite3` and `db4` are both installed. ```bash ./autogen.sh -./configure --without-bdb --with-gui=yes MAKE=gmake +./configure --with-gui=no \ + BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ + BDB_CFLAGS="-I${BDB_PREFIX}/include" \ + MAKE=gmake ``` + ##### No Wallet or GUI ``` bash ./autogen.sh diff --git a/doc/build-netbsd.md b/doc/build-netbsd.md index edabd716117..0f05cdcba78 100644 --- a/doc/build-netbsd.md +++ b/doc/build-netbsd.md @@ -1,81 +1,116 @@ -NetBSD build guide -====================== -(updated for NetBSD 8.0) +# NetBSD Build Guide -This guide describes how to build bitcoind and command-line utilities on NetBSD. +Updated for NetBSD [9.2](https://netbsd.org/releases/formal-9/NetBSD-9.2.html). -This guide does not contain instructions for building the GUI. +This guide describes how to build bitcoind, command-line utilities, and GUI on NetBSD. -Preparation -------------- +## Preparation -You will need the following modules, which can be installed via pkgsrc or pkgin: +### 1. Install Required Dependencies + +Install the required dependencies the usual way you [install software on NetBSD](https://www.netbsd.org/docs/guide/en/chap-boot.html#chap-boot-pkgsrc). +The example commands below use `pkgin`. + +```bash +pkgin install autoconf automake libtool pkg-config git gmake boost libevent ``` -autoconf -automake -boost -git -gmake -libevent -libtool -pkg-config -python37 -git clone https://github.com/bitcoin/bitcoin.git +NetBSD currently ships with an older version of `gcc` than is needed to build. You should upgrade your `gcc` and then pass this new version to the configure script. + +For example, grab `gcc9`: +``` +pkgin install gcc9 +``` + +Then, when configuring, pass the following: +```bash +./configure + ... + CC="/usr/pkg/gcc9/bin/gcc" \ + CXX="/usr/pkg/gcc9/bin/g++" \ + ... ``` See [dependencies.md](dependencies.md) for a complete overview. -### Building BerkeleyDB +### 2. Clone Bitcoin Repo + +Clone the Bitcoin Core repository to a directory. All build scripts and commands will run from this directory. -BerkeleyDB is only necessary for the wallet functionality. To skip this, pass -`--disable-wallet` to `./configure` and skip to the next section. +```bash +git clone https://github.com/bitcoin/bitcoin.git +``` + +### 3. Install Optional Dependencies + +#### Wallet Dependencies -It is recommended to use Berkeley DB 4.8. You cannot use the BerkeleyDB library -from ports, for the same reason as boost above (g++/libstd++ incompatibility). -If you have to build it yourself, you can use [the installation script included -in contrib/](/contrib/install_db4.sh) like so: +It is not necessary to build wallet functionality to run bitcoind or the GUI. + +###### Descriptor Wallet Support + +`sqlite3` is required to enable support for [descriptor wallets](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md). ```bash -./contrib/install_db4.sh `pwd` +pkgin install sqlite3 ``` -from the root of the repository. Then set `BDB_PREFIX` for the next section: +###### Legacy Wallet Support + +`db4` is required to enable support for legacy wallets. ```bash -export BDB_PREFIX="$PWD/db4" +pkgin install db4 ``` -### Building Bitcoin Core +#### GUI Dependencies -**Important**: Use `gmake` (the non-GNU `make` will exit with an error). +Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, we need to install `qt5`. -With wallet: ```bash -./autogen.sh -./configure --with-gui=no CPPFLAGS="-I/usr/pkg/include" \ - LDFLAGS="-L/usr/pkg/lib" \ - BOOST_CPPFLAGS="-I/usr/pkg/include" \ - BOOST_LDFLAGS="-L/usr/pkg/lib" \ - BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ - BDB_CFLAGS="-I${BDB_PREFIX}/include" \ - MAKE=gmake +pkgin install qt5 +``` + +The GUI can encode addresses in a QR Code. To build in QR support for the GUI, install `qrencode`. + +```bash +pkgin install qrencode ``` -Without wallet: +#### Test Suite Dependencies + +There is an included test suite that is useful for testing code changes when developing. +To run the test suite (recommended), you will need to have Python 3 installed: + +```bash +pkgin install python37 +``` + +### Building Bitcoin Core + +**Note**: Use `gmake` (the non-GNU `make` will exit with an error). + + +### 1. Configuration + +There are many ways to configure Bitcoin Core. Here is an example that +explicitly disables the wallet and GUI: + ```bash ./autogen.sh -./configure --with-gui=no --disable-wallet \ +./configure --without-wallet --with-gui=no \ CPPFLAGS="-I/usr/pkg/include" \ - LDFLAGS="-L/usr/pkg/lib" \ - BOOST_CPPFLAGS="-I/usr/pkg/include" \ - BOOST_LDFLAGS="-L/usr/pkg/lib" \ MAKE=gmake ``` +For a full list of configuration options, see the output of `./configure --help` + +### 2. Compile + Build and run the tests: + ```bash gmake # use "-j N" here for N parallel jobs -gmake check +gmake check # Run tests if Python 3 is available ``` diff --git a/doc/build-openbsd.md b/doc/build-openbsd.md index 275b7ce124b..96ee714341f 100644 --- a/doc/build-openbsd.md +++ b/doc/build-openbsd.md @@ -1,123 +1,128 @@ -OpenBSD build guide -====================== -(updated for OpenBSD 6.9) +# OpenBSD Build Guide -This guide describes how to build bitcoind, bitcoin-qt, and command-line utilities on OpenBSD. +**Updated for OpenBSD [7.3](https://www.openbsd.org/73.html)** -Preparation -------------- +This guide describes how to build bitcoind, command-line utilities, and GUI on OpenBSD. -Run the following as root to install the base dependencies for building: +## Preparation + +### 1. Install Required Dependencies +Run the following as root to install the base dependencies for building. ```bash -pkg_add git gmake libevent libtool boost -pkg_add qt5 # (optional for enabling the GUI) -pkg_add autoconf # (select highest version, e.g. 2.69) -pkg_add automake # (select highest version, e.g. 1.16) -pkg_add python # (select highest version, e.g. 3.8) -pkg_add bash +pkg_add bash git gmake libevent libtool boost +# Select the newest version of the following packages: +pkg_add autoconf automake python +``` + +See [dependencies.md](dependencies.md) for a complete overview. +### 2. Clone Bitcoin Repo +Clone the Bitcoin Core repository to a directory. All build scripts and commands will run from this directory. +``` bash git clone https://github.com/bitcoin/bitcoin.git ``` -See [dependencies.md](dependencies.md) for a complete overview. +### 3. Install Optional Dependencies + +#### Wallet Dependencies + +It is not necessary to build wallet functionality to run either `bitcoind` or `bitcoin-qt`. + +###### Descriptor Wallet Support -**Important**: From OpenBSD 6.2 onwards a C++11-supporting clang compiler is -part of the base image, and while building it is necessary to make sure that -this compiler is used and not ancient g++ 4.2.1. This is done by appending -`CC=cc CXX=c++` to configuration commands. Mixing different compilers within -the same executable will result in errors. +`sqlite3` is required to support [descriptor wallets](descriptors.md). -### Building BerkeleyDB +``` bash +pkg_add sqlite3 +``` -BerkeleyDB is only necessary for the wallet functionality. To skip this, pass -`--disable-wallet` to `./configure` and skip to the next section. +###### Legacy Wallet Support +BerkeleyDB is only required to support legacy wallets. It is recommended to use Berkeley DB 4.8. You cannot use the BerkeleyDB library -from ports, for the same reason as boost above (g++/libstd++ incompatibility). -If you have to build it yourself, you can use [the installation script included -in contrib/](/contrib/install_db4.sh) like so: +from ports. However you can build it yourself, [using depends](/depends). + +```bash +gmake -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_NATPMP=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 +... +to: /path/to/bitcoin/depends/x86_64-unknown-openbsd +``` + +Then set `BDB_PREFIX`: ```bash -./contrib/install_db4.sh `pwd` CC=cc CXX=c++ +export BDB_PREFIX="/path/to/bitcoin/depends/x86_64-unknown-openbsd" ``` -from the root of the repository. Then set `BDB_PREFIX` for the next section: +#### GUI Dependencies +###### Qt5 + +Bitcoin Core includes a GUI built with the cross-platform Qt Framework. To compile the GUI, Qt 5 is required. ```bash -export BDB_PREFIX="$PWD/db4" +pkg_add qt5 ``` -### Building Bitcoin Core +## Building Bitcoin Core **Important**: Use `gmake` (the non-GNU `make` will exit with an error). Preparation: ```bash -# Replace this with the autoconf version that you installed. Include only -# the major and minor parts of the version: use "2.69" for "autoconf-2.69p2". -export AUTOCONF_VERSION=2.69 - -# Replace this with the automake version that you installed. Include only -# the major and minor parts of the version: use "1.16" for "automake-1.16.1". +# Adapt the following for the version you installed (major.minor only): +export AUTOCONF_VERSION=2.71 export AUTOMAKE_VERSION=1.16 ./autogen.sh ``` -Make sure `BDB_PREFIX` is set to the appropriate path from the above steps. -Note that building with external signer support currently fails on OpenBSD, -hence you have to explicitly disable it by passing the parameter -`--disable-external-signer` to the configure script. -(Background: the feature requires the header-only library boost::process, which -is available on OpenBSD 6.9 via Boost 1.72.0, but contains certain system calls -and preprocessor defines like `waitid()` and `WEXITED` that are not available.) +### 1. Configuration -To configure with wallet: -```bash -./configure --with-gui=no --disable-external-signer CC=cc CXX=c++ \ - BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ - BDB_CFLAGS="-I${BDB_PREFIX}/include" \ - MAKE=gmake -``` +There are many ways to configure Bitcoin Core, here are a few common examples: + +##### Descriptor Wallet and GUI: +This enables the GUI and descriptor wallet support, assuming `sqlite` and `qt5` are installed. -To configure without wallet: ```bash -./configure --disable-wallet --with-gui=no --disable-external-signer CC=cc CXX=c++ MAKE=gmake +./configure MAKE=gmake ``` -To configure with GUI: +##### Descriptor & Legacy Wallet. No GUI: +This enables support for both wallet types and disables the GUI: + ```bash -./configure --with-gui=yes --disable-external-signer CC=cc CXX=c++ \ +./configure --with-gui=no \ BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ BDB_CFLAGS="-I${BDB_PREFIX}/include" \ MAKE=gmake ``` -Build and run the tests: +### 2. Compile +**Important**: Use `gmake` (the non-GNU `make` will exit with an error). + ```bash -gmake # use "-j N" here for N parallel jobs -gmake check +gmake # use "-j N" for N parallel jobs +gmake check # Run tests if Python 3 is available ``` -Resource limits -------------------- +## Resource limits If the build runs into out-of-memory errors, the instructions in this section might help. The standard ulimit restrictions in OpenBSD are very strict: - - data(kbytes) 1572864 +```bash +data(kbytes) 1572864 +``` This is, unfortunately, in some cases not enough to compile some `.cpp` files in the project, (see issue [#6658](https://github.com/bitcoin/bitcoin/issues/6658)). If your user is in the `staff` group the limit can be raised with: - - ulimit -d 3000000 - +```bash +ulimit -d 3000000 +``` The change will only affect the current shell and processes spawned by it. To make the change system-wide, change `datasize-cur` and `datasize-max` in `/etc/login.conf`, and reboot. - diff --git a/doc/build-osx.md b/doc/build-osx.md index bf20a0dd521..cde90f17c6b 100644 --- a/doc/build-osx.md +++ b/doc/build-osx.md @@ -4,42 +4,6 @@ This guide describes how to build elementsd, command-line utilities, and GUI on macOS -**Note:** The following is for Intel Macs only! - -## Dependencies - -The following dependencies are **required**: - -Library | Purpose | Description ------------------------------------------------------------|------------|---------------------- -[automake](https://formulae.brew.sh/formula/automake) | Build | Generate makefile -[libtool](https://formulae.brew.sh/formula/libtool) | Build | Shared library support -[pkg-config](https://formulae.brew.sh/formula/pkg-config) | Build | Configure compiler and linker flags -[boost](https://formulae.brew.sh/formula/boost) | Utility | Library for threading, data structures, etc -[libevent](https://formulae.brew.sh/formula/libevent) | Networking | OS independent asynchronous networking - -The following dependencies are **optional**: - -Library | Purpose | Description ---------------------------------------------------------------- |------------------|---------------------- -[berkeley-db@4](https://formulae.brew.sh/formula/berkeley-db@4) | Berkeley DB | Wallet storage (only needed when wallet enabled) -[qt@5](https://formulae.brew.sh/formula/qt@5) | GUI | GUI toolkit (only needed when GUI enabled) -[qrencode](https://formulae.brew.sh/formula/qrencode) | QR codes in GUI | Generating QR codes (only needed when GUI enabled) -[zeromq](https://formulae.brew.sh/formula/zeromq) | ZMQ notification | Allows generating ZMQ notifications (requires ZMQ version >= 4.0.0) -[sqlite](https://formulae.brew.sh/formula/sqlite) | SQLite DB | Wallet storage (only needed when wallet enabled) -[miniupnpc](https://formulae.brew.sh/formula/miniupnpc) | UPnP Support | Firewall-jumping support (needed for port mapping support) -[libnatpmp](https://formulae.brew.sh/formula/libnatpmp) | NAT-PMP Support | Firewall-jumping support (needed for port mapping support) -[python3](https://formulae.brew.sh/formula/python@3.9) | Testing | Python Interpreter (only needed when running the test suite) - -The following dependencies are **optional** packages required for deploying: - -Library | Purpose | Description -----------------------------------------------------|------------------|---------------------- -[ds_store](https://pypi.org/project/ds-store/) | Deploy Dependency| Examine and modify .DS_Store files -[mac_alias](https://pypi.org/project/mac-alias/) | Deploy Dependency| Generate/Read binary alias and bookmark records - -See [dependencies.md](dependencies.md) for a complete overview. - ## Preparation The commands in this guide should be executed in a Terminal application. @@ -78,6 +42,9 @@ Note: If you run into issues while installing Homebrew or pulling packages, refe The first step is to download the required dependencies. These dependencies represent the packages required to get a barebones installation up and running. + +See [dependencies.md](dependencies.md) for a complete overview. + To install, run the following from your terminal: ``` bash @@ -99,29 +66,21 @@ git clone https://github.com/ElementsProject/elements.git #### Wallet Dependencies It is not necessary to build wallet functionality to run `elementsd` or `elements-qt`. -To enable legacy wallets, you must install `berkeley-db@4`. -To enable [descriptor wallets](https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md), `sqlite` is required. -Skip `berkeley-db@4` if you intend to *exclusively* use descriptor wallets. - -###### Legacy Wallet Support -`berkeley-db@4` is required to enable support for legacy wallets. -Skip if you don't intend to use legacy wallets. +###### Descriptor Wallet Support -``` bash -brew install berkeley-db@4 -``` +`sqlite` is required to support for descriptor wallets. -###### Descriptor Wallet Support +macOS ships with a useable `sqlite` package, meaning you don't need to +install anything. -Note: Apple has included a useable `sqlite` package since macOS 10.14. -You may not need to install this package. +###### Legacy Wallet Support -`sqlite` is required to enable support for descriptor wallets. -Skip if you don't intend to use descriptor wallets. +`berkeley-db@4` is only required to support for legacy wallets. +Skip if you don't intend to use legacy wallets. ``` bash -brew install sqlite +brew install berkeley-db@4 ``` --- @@ -137,14 +96,6 @@ Skip if you don't intend to use the GUI. brew install qt@5 ``` -Ensure that the `qt@5` package is installed, not the `qt` package. -If 'qt' is installed, the build process will fail. -if installed, remove the `qt` package with the following command: - -``` bash -brew uninstall qt -``` - Note: Building with Qt binaries downloaded from the Qt website is not officially supported. See the notes in [#7714](https://github.com/bitcoin/bitcoin/issues/7714). diff --git a/doc/build-unix.md b/doc/build-unix.md index fe239cc372b..017268313d5 100644 --- a/doc/build-unix.md +++ b/doc/build-unix.md @@ -26,30 +26,7 @@ make install # optional This will build elements-qt as well if the dependencies are met. -Dependencies ---------------------- - -These dependencies are required: - - Library | Purpose | Description - ------------|------------------|---------------------- - libboost | Utility | Library for threading, data structures, etc - libevent | Networking | OS independent asynchronous networking - -Optional dependencies: - - Library | Purpose | Description - ------------|------------------|---------------------- - miniupnpc | UPnP Support | Firewall-jumping support - libnatpmp | NAT-PMP Support | Firewall-jumping support - libdb4.8 | Berkeley DB | Wallet storage (only needed when legacy wallet enabled) - qt | GUI | GUI toolkit (only needed when GUI enabled) - libqrencode | QR codes in GUI | QR code generation (only needed when GUI enabled) - libzmq3 | ZMQ notification | ZMQ notifications (requires ZMQ version >= 4.0.0) - sqlite3 | SQLite DB | Wallet storage (only needed when descriptor wallet enabled) - systemtap | Tracing (USDT) | Statically defined tracepoints - -For the versions used, see [dependencies.md](dependencies.md) +See [dependencies.md](dependencies.md) for a complete overview. Memory Requirements -------------------- @@ -95,7 +72,7 @@ executables, which are based on BerkeleyDB 4.8. If you do not care about wallet To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) -Optional port mapping libraries (see: `--with-miniupnpc`, `--enable-upnp-default`, and `--with-natpmp`, `--enable-natpmp-default`): +Optional port mapping libraries (see: `--with-miniupnpc` and `--with-natpmp`): sudo apt install libminiupnpc-dev libnatpmp-dev @@ -158,7 +135,7 @@ pass `--with-incompatible-bdb` to configure. Otherwise, you can build Berkeley D To build Bitcoin Core without wallet, see [*Disable-wallet mode*](#disable-wallet-mode) -Optional port mapping libraries (see: `--with-miniupnpc`, `--enable-upnp-default`, and `--with-natpmp`, `--enable-natpmp-default`): +Optional port mapping libraries (see: `--with-miniupnpc` and `--with-natpmp`): sudo dnf install miniupnpc-devel libnatpmp-devel @@ -201,40 +178,36 @@ miniupnpc [miniupnpc](https://miniupnp.tuxfamily.org) may be used for UPnP port mapping. It can be downloaded from [here]( https://miniupnp.tuxfamily.org/files/). UPnP support is compiled in and -turned off by default. See the configure options for UPnP behavior desired: - - --without-miniupnpc No UPnP support, miniupnp not required - --disable-upnp-default (the default) UPnP support turned off by default at runtime - --enable-upnp-default UPnP support turned on by default at runtime +turned off by default. libnatpmp --------- [libnatpmp](https://miniupnp.tuxfamily.org/libnatpmp.html) may be used for NAT-PMP port mapping. It can be downloaded from [here](https://miniupnp.tuxfamily.org/files/). NAT-PMP support is compiled in and -turned off by default. See the configure options for NAT-PMP behavior desired: - - --without-natpmp No NAT-PMP support, libnatpmp not required - --disable-natpmp-default (the default) NAT-PMP support turned off by default at runtime - --enable-natpmp-default NAT-PMP support turned on by default at runtime +turned off by default. Berkeley DB ----------- The legacy wallet uses Berkeley DB. To ensure backwards compatibility it is -recommended to use Berkeley DB 4.8. If you have to build it yourself, you can -use [the installation script included in contrib/](/contrib/install_db4.sh) -like so: - -```shell -./contrib/install_db4.sh `pwd` +recommended to use Berkeley DB 4.8. If you have to build it yourself, and don't +want to use any other libraries built in depends, you can do: +```bash +make -C depends NO_BOOST=1 NO_LIBEVENT=1 NO_QT=1 NO_SQLITE=1 NO_NATPMP=1 NO_UPNP=1 NO_ZMQ=1 NO_USDT=1 +... +to: /path/to/bitcoin/depends/x86_64-pc-linux-gnu ``` +and configure using the following: +```bash +export BDB_PREFIX="/path/to/bitcoin/depends/x86_64-pc-linux-gnu" -from the root of the repository. - -Otherwise, you can build Bitcoin Core from self-compiled [depends](/depends/README.md). +./configure \ + BDB_LIBS="-L${BDB_PREFIX}/lib -ldb_cxx-4.8" \ + BDB_CFLAGS="-I${BDB_PREFIX}/include" +``` -**Note**: You only need Berkeley DB if the wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). +**Note**: You only need Berkeley DB if the legacy wallet is enabled (see [*Disable-wallet mode*](#disable-wallet-mode)). Security -------- @@ -284,12 +257,12 @@ Hardening enables the following features: Disable-wallet mode -------------------- -When the intention is to run only a P2P node without a wallet, Bitcoin Core may be compiled in -disable-wallet mode with: +When the intention is to only run a P2P node, without a wallet, Bitcoin Core can +be compiled in disable-wallet mode with: ./configure --disable-wallet -In this case there is no dependency on Berkeley DB 4.8 and SQLite. +In this case there is no dependency on SQLite or Berkeley DB. Mining is also possible in disable-wallet mode using the `getblocktemplate` RPC call. @@ -302,42 +275,14 @@ A list of additional configure flags can be displayed with: Setup and Build Example: Arch Linux ----------------------------------- -This example lists the steps necessary to setup and build a command line only, non-wallet distribution of the latest changes on Arch Linux: +This example lists the steps necessary to setup and build a command line only distribution of the latest changes on Arch Linux: - pacman -S git base-devel boost libevent python + pacman --sync --needed autoconf automake boost gcc git libevent libtool make pkgconf python sqlite git clone https://github.com/bitcoin/bitcoin.git cd bitcoin/ ./autogen.sh - ./configure --disable-wallet --without-gui --without-miniupnpc + ./configure make check + ./src/bitcoind -Note: -Enabling wallet support requires either compiling against a Berkeley DB newer than 4.8 (package `db`) using `--with-incompatible-bdb`, -or building and depending on a local version of Berkeley DB 4.8. The readily available Arch Linux packages are currently built using -`--with-incompatible-bdb` according to the [PKGBUILD](https://github.com/archlinux/svntogit-community/blob/packages/bitcoin/trunk/PKGBUILD). -As mentioned above, when maintaining portability of the wallet between the standard Bitcoin Core distributions and independently built -node software is desired, Berkeley DB 4.8 must be used. - - -ARM Cross-compilation -------------------- -These steps can be performed on, for example, an Ubuntu VM. The depends system -will also work on other Linux distributions, however the commands for -installing the toolchain will be different. - -Make sure you install the build requirements mentioned above. -Then, install the toolchain and curl: - - sudo apt-get install g++-arm-linux-gnueabihf curl - -To build executables for ARM: - - cd depends - make HOST=arm-linux-gnueabihf NO_QT=1 - cd .. - ./autogen.sh - CONFIG_SITE=$PWD/depends/arm-linux-gnueabihf/share/config.site ./configure --enable-reduce-exports LDFLAGS=-static-libstdc++ - make - - -For further documentation on the depends system see [README.md](../depends/README.md) in the depends directory. +If you intend to work with legacy Berkeley DB wallets, see [Berkeley DB](#berkeley-db) section. diff --git a/doc/build-windows.md b/doc/build-windows.md index e35d3bcbd09..027e8f80f50 100644 --- a/doc/build-windows.md +++ b/doc/build-windows.md @@ -6,8 +6,8 @@ Below are some notes on how to build Bitcoin Core for Windows. The options known to work for building Bitcoin Core on Windows are: * On Linux, using the [Mingw-w64](https://www.mingw-w64.org/) cross compiler tool chain. -* On Windows, using [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/windows/wsl/about) and Mingw-w64. -* On Windows, using [Microsoft Visual Studio](https://www.visualstudio.com). See [README.md](/build_msvc/README.md). +* On Windows, using [Windows Subsystem for Linux (WSL)](https://learn.microsoft.com/en-us/windows/wsl/about) and Mingw-w64. +* On Windows, using [Microsoft Visual Studio](https://visualstudio.microsoft.com). See [README.md](/build_msvc/README.md). Other options which may work, but which have not been extensively tested are (please contribute instructions): @@ -16,7 +16,7 @@ Other options which may work, but which have not been extensively tested are (pl Installing Windows Subsystem for Linux --------------------------------------- -Follow the upstream installation instructions, available [here](https://docs.microsoft.com/windows/wsl/install-win10). +Follow the upstream installation instructions, available [here](https://learn.microsoft.com/en-us/windows/wsl/install). Cross-compilation for Ubuntu and Windows Subsystem for Linux ------------------------------------------------------------ @@ -66,7 +66,7 @@ Note that for WSL the Bitcoin Core source path MUST be somewhere in the default example /usr/src/bitcoin, AND not under /mnt/d/. If this is not the case the dependency autoconf scripts will fail. This means you cannot use a directory that is located directly on the host Windows file system to perform the build. -Additional WSL Note: WSL support for [launching Win32 applications](https://docs.microsoft.com/en-us/archive/blogs/wsl/windows-and-ubuntu-interoperability#launching-win32-applications-from-within-wsl) +Additional WSL Note: WSL support for [launching Win32 applications](https://learn.microsoft.com/en-us/archive/blogs/wsl/windows-and-ubuntu-interoperability#launching-win32-applications-from-within-wsl) results in `Autoconf` configure scripts being able to execute Windows Portable Executable files. This can cause unexpected behaviour during the build, such as Win32 error dialogs for missing libraries. The recommended approach is to temporarily disable WSL support for Win32 applications. diff --git a/doc/cjdns.md b/doc/cjdns.md new file mode 100644 index 00000000000..b69564729f8 --- /dev/null +++ b/doc/cjdns.md @@ -0,0 +1,116 @@ +# CJDNS support in Bitcoin Core + +It is possible to run Bitcoin Core over CJDNS, an encrypted IPv6 network that +uses public-key cryptography for address allocation and a distributed hash table +for routing. + +## What is CJDNS? + +CJDNS is like a distributed, shared VPN with multiple entry points where every +participant can reach any other participant. All participants use addresses from +the `fc00::/8` network (reserved IPv6 range). Installation and configuration is +done outside of Bitcoin Core, similarly to a VPN (either in the host/OS or on +the network router). See https://github.com/cjdelisle/cjdns#readme and +https://github.com/hyperboria/docs#hyperboriadocs for more information. + +Compared to IPv4/IPv6, CJDNS provides end-to-end encryption and protects nodes +from traffic analysis and filtering. + +Used with Tor and I2P, CJDNS is a complementary option that can enhance network +redundancy and robustness for both the Bitcoin network and individual nodes. + +Each network has different characteristics. For instance, Tor is widely used but +somewhat centralized. I2P connections have a source address and I2P is slow. +CJDNS is fast but does not hide the sender and the recipient from intermediate +routers. + +## Installing CJDNS and finding a peer to connect to the network + +To install and set up CJDNS, follow the instructions at +https://github.com/cjdelisle/cjdns#how-to-install-cjdns. + +You need to initiate an outbound connection to a peer on the CJDNS network +before it will work with your Bitcoin Core node. This is described in steps +["2. Find a friend"](https://github.com/cjdelisle/cjdns#2-find-a-friend) and +["3. Connect your node to your friend's +node"](https://github.com/cjdelisle/cjdns#3-connect-your-node-to-your-friends-node) +in the CJDNS documentation. + +One quick way to accomplish these two steps is to query for available public +peers on [Hyperboria](https://github.com/hyperboria) by running the following: + +``` +git clone https://github.com/hyperboria/peers hyperboria-peers +cd hyperboria-peers +./testAvailable.py +``` + +For each peer, the `./testAvailable.py` script prints the filename of the peer's +credentials followed by the ping result. + +Choose one or several peers, copy their credentials from their respective files, +paste them into the relevant IPv4 or IPv6 "connectTo" JSON object in the +`cjdroute.conf` file you created in step ["1. Generate a new configuration +file"](https://github.com/cjdelisle/cjdns#1-generate-a-new-configuration-file), +and save the file. + +## Launching CJDNS + +Typically, CJDNS might be launched from its directory with +`sudo ./cjdroute < cjdroute.conf` and it sheds permissions after setting up the +[TUN](https://en.wikipedia.org/wiki/TUN/TAP) interface. You may also [launch it as an +unprivileged user](https://github.com/cjdelisle/cjdns/blob/master/doc/non-root-user.md) +with some additional setup. + +The network connection can be checked by running `./tools/peerStats` from the +CJDNS directory. + +## Run Bitcoin Core with CJDNS + +Once you are connected to the CJDNS network, the following Bitcoin Core +configuration option makes CJDNS peers automatically reachable: + +``` +-cjdnsreachable +``` + +When enabled, this option tells Bitcoin Core that it is running in an +environment where a connection to an `fc00::/8` address will be to the CJDNS +network instead of to an [RFC4193](https://datatracker.ietf.org/doc/html/rfc4193) +IPv6 local network. This helps Bitcoin Core perform better address management: + - Your node can consider incoming `fc00::/8` connections to be from the CJDNS + network rather than from an IPv6 private one. + - If one of your node's local addresses is `fc00::/8`, then it can choose to + gossip that address to peers. + +## Additional configuration options related to CJDNS + +``` +-onlynet=cjdns +``` + +Make automatic outbound connections only to CJDNS addresses. Inbound and manual +connections are not affected by this option. It can be specified multiple times +to allow multiple networks, e.g. onlynet=cjdns, onlynet=i2p, onlynet=onion. + +CJDNS support was added to Bitcoin Core in version 23.0 and there may be fewer +CJDNS peers than Tor or IP ones. You can use `bitcoin-cli -addrinfo` to see the +number of CJDNS addresses known to your node. + +In general, a node can be run with both an onion service and CJDNS (or any/all +of IPv4/IPv6/onion/I2P/CJDNS), which can provide a potential fallback if one of +the networks has issues. There are a number of ways to configure this; see +[doc/tor.md](https://github.com/bitcoin/bitcoin/blob/master/doc/tor.md) for +details. + +## CJDNS-related information in Bitcoin Core + +There are several ways to see your CJDNS address in Bitcoin Core: +- in the "Local addresses" output of CLI `-netinfo` +- in the "localaddresses" output of RPC `getnetworkinfo` + +To see which CJDNS peers your node is connected to, use `bitcoin-cli -netinfo 4` +or the `getpeerinfo` RPC (i.e. `bitcoin-cli getpeerinfo`). + +To see which CJDNS addresses your node knows, use the `getnodeaddresses 0 cjdns` +RPC. diff --git a/doc/dependencies.md b/doc/dependencies.md index 57d2b994df6..182d026c581 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -1,48 +1,50 @@ -Dependencies -============ - -These are the dependencies currently used by Bitcoin Core. You can find instructions for installing them in the `build-*.md` file for your platform. - -| Dependency | Version used | Minimum required | CVEs | Shared | [Bundled Qt library](https://doc.qt.io/qt-5/configure-options.html#third-party-libraries) | -| --- | --- | --- | --- | --- | --- | -| Berkeley DB | [4.8.30](https://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) | 4.8.x | No | | | -| Boost | [1.77.0](https://www.boost.org/users/download/) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | | | -| Clang[ \* ](#note1) | | [7.0](https://releases.llvm.org/download.html) (C++17 & std::filesystem support) | | | | -| Fontconfig | [2.12.6](https://www.freedesktop.org/software/fontconfig/release/) | | No | Yes | | -| FreeType | [2.11.0](https://download.savannah.gnu.org/releases/freetype) | | No | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Android only) | -| GCC | | [8.1](https://gcc.gnu.org/) (C++17 & std::filesystem support) | | | | -| glibc | | [2.18](https://www.gnu.org/software/libc/) | | | | | -| HarfBuzz-NG | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| libevent | [2.1.12-stable](https://github.com/libevent/libevent/releases) | [2.0.21](https://github.com/bitcoin/bitcoin/pull/18676) | No | | | -| libnatpmp | git commit [4536032...](https://github.com/miniupnp/libnatpmp/tree/4536032ae32268a45c073a4d5e91bbab4534773a) | | No | | | -| libpng | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| MiniUPnPc | [2.2.2](https://miniupnp.tuxfamily.org/files) | | No | | | -| PCRE | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | -| Python (tests) | | [3.6](https://www.python.org/downloads) | | | | -| qrencode | [3.4.4](https://fukuchi.org/works/qrencode) | | No | | | -| Qt | [5.15.3](https://download.qt.io/official_releases/qt/) | [5.9.5](https://github.com/bitcoin/bitcoin/issues/20104) | No | | | -| SQLite | [3.32.1](https://sqlite.org/download.html) | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | | | | -| XCB | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | -| systemtap ([tracing](tracing.md))| [4.5](https://sourceware.org/systemtap/ftp/releases/) | | | | | -| xkbcommon | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) (Linux only) | -| ZeroMQ | [4.3.1](https://github.com/zeromq/libzmq/releases) | 4.0.0 | No | | | -| zlib | | | | | [Yes](https://github.com/bitcoin/bitcoin/blob/master/depends/packages/qt.mk) | - -Note \* : When compiling with `-stdlib=libc++`, the minimum supported libc++ version is 7.0. - -Controlling dependencies ------------------------- -Some dependencies are not needed in all configurations. The following are some factors that affect the dependency list. - -#### Options passed to `./configure` -* MiniUPnPc is not needed with `--without-miniupnpc`. -* libnatpmp is not needed with `--without-natpmp`. -* Berkeley DB is not needed with `--disable-wallet` or `--without-bdb`. -* SQLite is not needed with `--disable-wallet` or `--without-sqlite`. -* Qt is not needed with `--without-gui`. -* If the qrencode dependency is absent, QR support won't be added. To force an error when that happens, pass `--with-qrencode`. -* If the systemtap dependency is absent, USDT support won't compiled in. -* ZeroMQ is needed only with the `--with-zmq` option. - -#### Other -* Not-Qt-bundled zlib is required to build the [DMG tool](../contrib/macdeploy/README.md#deterministic-macos-dmg-notes) from the libdmg-hfsplus project. +# Dependencies + +These are the dependencies used by Bitcoin Core. +You can find installation instructions in the `build-*.md` file for your platform. +"Runtime" and "Version Used" are both in reference to the release binaries. + +| Dependency | Minimum required | +| --- | --- | +| [Autoconf](https://www.gnu.org/software/autoconf/) | [2.69](https://github.com/bitcoin/bitcoin/pull/17769) | +| [Automake](https://www.gnu.org/software/automake/) | [1.13](https://github.com/bitcoin/bitcoin/pull/18290) | +| [Clang](https://clang.llvm.org) | [8.0](https://github.com/bitcoin/bitcoin/pull/24164) | +| [GCC](https://gcc.gnu.org) | [8.1](https://github.com/bitcoin/bitcoin/pull/23060) | +| [Python](https://www.python.org) (tests) | [3.7](https://github.com/bitcoin/bitcoin/pull/26226) | +| [systemtap](https://sourceware.org/systemtap/) ([tracing](tracing.md))| N/A | + +## Required + +| Dependency | Releases | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | --- | +| [Boost](../depends/packages/boost.mk) | [link](https://www.boost.org/users/download/) | [1.81.0](https://github.com/bitcoin/bitcoin/pull/26557) | [1.64.0](https://github.com/bitcoin/bitcoin/pull/22320) | No | +| [libevent](../depends/packages/libevent.mk) | [link](https://github.com/libevent/libevent/releases) | [2.1.12-stable](https://github.com/bitcoin/bitcoin/pull/21991) | [2.1.8](https://github.com/bitcoin/bitcoin/pull/24681) | No | +| glibc | [link](https://www.gnu.org/software/libc/) | N/A | [2.27](https://github.com/bitcoin/bitcoin/pull/27029) | Yes | +| Linux Kernel | [link](https://www.kernel.org/) | N/A | 3.2.0 | Yes | + +## Optional + +### GUI +| Dependency | Releases | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | --- | +| [Fontconfig](../depends/packages/fontconfig.mk) | [link](https://www.freedesktop.org/wiki/Software/fontconfig/) | [2.12.6](https://github.com/bitcoin/bitcoin/pull/23495) | 2.6 | Yes | +| [FreeType](../depends/packages/freetype.mk) | [link](https://freetype.org) | [2.11.0](https://github.com/bitcoin/bitcoin/commit/01544dd78ccc0b0474571da854e27adef97137fb) | 2.3.0 | Yes | +| [qrencode](../depends/packages/qrencode.mk) | [link](https://fukuchi.org/works/qrencode/) | [4.1.1](https://github.com/bitcoin/bitcoin/pull/27312) | | No | +| [Qt](../depends/packages/qt.mk) | [link](https://download.qt.io/official_releases/qt/) | [5.15.5](https://github.com/bitcoin/bitcoin/pull/25719) | [5.11.3](https://github.com/bitcoin/bitcoin/pull/24132) | No | + +### Networking +| Dependency | Releases | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | --- | +| [libnatpmp](../depends/packages/libnatpmp.mk) | [link](https://github.com/miniupnp/libnatpmp/) | commit [07004b9...](https://github.com/bitcoin/bitcoin/pull/25917) | | No | +| [MiniUPnPc](../depends/packages/miniupnpc.mk) | [link](https://miniupnp.tuxfamily.org/) | [2.2.2](https://github.com/bitcoin/bitcoin/pull/20421) | 2.1 | No | + +### Notifications +| Dependency | Releases | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | --- | +| [ZeroMQ](../depends/packages/zeromq.mk) | [link](https://github.com/zeromq/libzmq/releases) | [4.3.4](https://github.com/bitcoin/bitcoin/pull/23956) | 4.0.0 | No | + +### Wallet +| Dependency | Releases | Version used | Minimum required | Runtime | +| --- | --- | --- | --- | --- | +| [Berkeley DB](../depends/packages/bdb.mk) (legacy wallet) | [link](https://www.oracle.com/technetwork/database/database-technologies/berkeleydb/downloads/index.html) | 4.8.30 | 4.8.x | No | +| [SQLite](../depends/packages/sqlite.mk) | [link](https://sqlite.org) | [3.38.5](https://github.com/bitcoin/bitcoin/pull/25378) | [3.7.17](https://github.com/bitcoin/bitcoin/pull/19077) | No | diff --git a/doc/descriptors.md b/doc/descriptors.md index 318d065fdbc..1baf652f308 100644 --- a/doc/descriptors.md +++ b/doc/descriptors.md @@ -11,13 +11,18 @@ Supporting RPCs are: addresses. - `listunspent` outputs a specialized descriptor for the reported unspent outputs. - `getaddressinfo` outputs a descriptor for solvable addresses (since v0.18). -- `importmulti` takes as input descriptors to import into the wallet +- `importmulti` takes as input descriptors to import into a legacy wallet (since v0.18). - `generatetodescriptor` takes as input a descriptor and generates coins to it (`regtest` only, since v0.19). - `utxoupdatepsbt` takes as input descriptors to add information to the psbt (since v0.19). -- `createmultisig` and `addmultisigaddress` return descriptors as well (since v0.20) +- `createmultisig` and `addmultisigaddress` return descriptors as well (since v0.20). +- `importdescriptors` takes as input descriptors to import into a descriptor wallet + (since v0.21). +- `listdescriptors` outputs descriptors imported into a descriptor wallet (since v22). +- `scanblocks` takes as input descriptors to scan for in blocks and returns the + relevant blockhashes (since v25). This document describes the language. For the specifics on usage, see the RPC documentation for the functions mentioned above. @@ -33,6 +38,7 @@ Output descriptors currently support: - Pay-to-taproot outputs (P2TR), through the `tr` function. - Multisig scripts, through the `multi` function. - Multisig scripts where the public keys are sorted lexicographically, through the `sortedmulti` function. +- Multisig scripts inside taproot script trees, through the `multi_a` (and `sortedmulti_a`) function. - Any type of supported address through the `addr` function. - Raw hex scripts through the `raw` function. - Public keys (compressed and uncompressed) in hex notation, or BIP32 extended pubkeys with derivation paths. @@ -56,6 +62,7 @@ Output descriptors currently support: - `wsh(multi(1,xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/1/0/*,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/0/0/*))` describes a set of *1-of-2* P2WSH multisig outputs where the first multisig key is the *1/0/`i`* child of the first specified xpub and the second multisig key is the *0/0/`i`* child of the second specified xpub, and `i` is any number in a configurable range (`0-1000` by default). - `wsh(sortedmulti(1,xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/1/0/*,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/0/0/*))` describes a set of *1-of-2* P2WSH multisig outputs where one multisig key is the *1/0/`i`* child of the first specified xpub and the other multisig key is the *0/0/`i`* child of the second specified xpub, and `i` is any number in a configurable range (`0-1000` by default). The order of public keys in the resulting witnessScripts is determined by the lexicographic order of the public keys at that index. - `tr(c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5,{pk(fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556),pk(e493dbf1c10d80f3581e4904930b1404cc6c13900ee0758474fa94abe8c4cd13)})` describes a P2TR output with the `c6...` x-only pubkey as internal key, and two script paths. +- `tr(c6047f9441ed7d6d3045406e95c07cd85c778e4b8cef3ca7abac09b95c709ee5,sortedmulti_a(2,2f8bde4d1a07209355b4a7250a5c5128e88b84bddc619ab7cba8d569b240efe4,5cbdf0646e5db4eaa398f365f2ea7a0e3d419b7e0330e39ce92bddedcac4f9bc))` describes a P2TR output with the `c6...` x-only pubkey as internal key, and a single `multi_a` script that needs 2 signatures with 2 specified x-only keys, which will be sorted lexicographically. ## Reference @@ -68,11 +75,14 @@ Descriptors consist of several types of expressions. The top level expression is - `pkh(KEY)` (not inside `tr`): P2PKH output for the given public key (use `addr` if you only know the pubkey hash). - `wpkh(KEY)` (top level or inside `sh` only): P2WPKH output for the given compressed pubkey. - `combo(KEY)` (top level only): an alias for the collection of `pk(KEY)` and `pkh(KEY)`. If the key is compressed, it also includes `wpkh(KEY)` and `sh(wpkh(KEY))`. -- `multi(k,KEY_1,KEY_2,...,KEY_n)` (not inside `tr`): k-of-n multisig script. +- `multi(k,KEY_1,KEY_2,...,KEY_n)` (not inside `tr`): k-of-n multisig script using OP_CHECKMULTISIG. - `sortedmulti(k,KEY_1,KEY_2,...,KEY_n)` (not inside `tr`): k-of-n multisig script with keys sorted lexicographically in the resulting script. +- `multi_a(k,KEY_1,KEY_2,...,KEY_N)` (only inside `tr`): k-of-n multisig script using OP_CHECKSIG, OP_CHECKSIGADD, and OP_NUMEQUAL. +- `sortedmulti_a(k,KEY_1,KEY_2,...,KEY_N)` (only inside `tr`): similar to `multi_a`, but the (x-only) public keys in it will be sorted lexicographically. - `tr(KEY)` or `tr(KEY,TREE)` (top level only): P2TR output with the specified key as internal key, and optionally a tree of script paths. - `addr(ADDR)` (top level only): the script which ADDR expands to. - `raw(HEX)` (top level only): the script whose hex encoding is HEX. +- `rawtr(KEY)` (top level only): P2TR output with the specified key as output key. NOTE: while it's possible to use this to construct wallets, it has several downsides, like being unable to prove no hidden script path exists. Use at your own risk. `KEY` expressions: - Optionally, key origin information, consisting of: @@ -83,7 +93,7 @@ Descriptors consist of several types of expressions. The top level expression is - Followed by the actual key, which is either: - Hex encoded public keys (either 66 characters starting with `02` or `03` for a compressed pubkey, or 130 characters starting with `04` for an uncompressed pubkey). - Inside `wpkh` and `wsh`, only compressed public keys are permitted. - - Inside `tr`, x-only pubkeys are also permitted (64 hex characters). + - Inside `tr` and `rawtr`, x-only pubkeys are also permitted (64 hex characters). - [WIF](https://en.bitcoin.it/wiki/Wallet_import_format) encoded private keys may be specified instead of the corresponding public key, with the same meaning. - `xpub` encoded extended public key or `xprv` encoded extended private key (as defined in [BIP 32](https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki)). - Followed by zero or more `/NUM` unhardened and `/NUM'` hardened BIP32 derivation steps. @@ -256,6 +266,6 @@ ones. For larger numbers of errors, or other types of errors, there is a roughly 1 in a trillion chance of not detecting the errors. All RPCs in Bitcoin Core will include the checksum in their output. Only -certain RPCs require checksums on input, including `deriveaddress` and +certain RPCs require checksums on input, including `deriveaddresses` and `importmulti`. The checksum for a descriptor without one can be computed using the `getdescriptorinfo` RPC. diff --git a/doc/assumeutxo.md b/doc/design/assumeutxo.md similarity index 69% rename from doc/assumeutxo.md rename to doc/design/assumeutxo.md index 2726cf779b2..469c5515360 100644 --- a/doc/assumeutxo.md +++ b/doc/design/assumeutxo.md @@ -3,9 +3,9 @@ Assumeutxo is a feature that allows fast bootstrapping of a validating bitcoind instance with a very similar security model to assumevalid. -The RPC commands `dumptxoutset` and `loadtxoutset` are used to respectively generate -and load UTXO snapshots. The utility script `./contrib/devtools/utxo_snapshot.sh` may -be of use. +The RPC commands `dumptxoutset` and `loadtxoutset` (yet to be merged) are used to +respectively generate and load UTXO snapshots. The utility script +`./contrib/devtools/utxo_snapshot.sh` may be of use. ## General background @@ -22,10 +22,6 @@ be of use. chainstate running asynchronously in the background. We also use this flag to control which index entries are added to setBlockIndexCandidates during LoadBlockIndex(). -- Indexing implementations via BaseIndex can no longer assume that indexation happens - sequentially, since background validation chainstates can submit BlockConnected - events out of order with the active chain. - - The concept of UTXO snapshots is treated as an implementation detail that lives behind the ChainstateManager interface. The external presentation of the changes required to facilitate the use of UTXO snapshots is the understanding that there are @@ -41,7 +37,7 @@ be of use. Chainstate within the system goes through a number of phases when UTXO snapshots are used, as managed by `ChainstateManager`. At various points there can be multiple -`CChainState` objects in existence to facilitate both maintaining the network tip and +`Chainstate` objects in existence to facilitate both maintaining the network tip and performing historical validation of the assumed-valid chain. It is worth noting that though there are multiple separate chainstates, those @@ -53,7 +49,7 @@ data. ### "Normal" operation via initial block download -`ChainstateManager` manages a single CChainState object, for which +`ChainstateManager` manages a single Chainstate object, for which `m_snapshot_blockhash` is null. This chainstate is (maybe obviously) considered active. This is the "traditional" mode of operation for bitcoind. @@ -76,8 +72,15 @@ original chainstate remains in use as active. Once the snapshot chainstate is loaded and validated, it is promoted to active chainstate and a sync to tip begins. A new chainstate directory is created in the -datadir for the snapshot chainstate called -`chainstate_[SHA256 blockhash of snapshot base block]`. +datadir for the snapshot chainstate called `chainstate_snapshot`. + +When this directory is present in the datadir, the snapshot chainstate will be detected +and loaded as active on node startup (via `DetectSnapshotChainstate()`). + +A special file is created within that directory, `base_blockhash`, which contains the +serialized `uint256` of the base block of the snapshot. This is used to reinitialize +the snapshot chainstate on subsequent inits. Otherwise, the directory is a normal +leveldb database. | | | | ---------- | ----------- | @@ -87,7 +90,7 @@ datadir for the snapshot chainstate called The snapshot begins to sync to tip from its base block, technically in parallel with the original chainstate, but it is given priority during block download and is allocated most of the cache (see `MaybeRebalanceCaches()` and usages) as our chief -consideration is getting to network tip. +goal is getting to network tip. **Failure consideration:** if shutdown happens at any point during this phase, both chainstates will be detected during the next init and the process will resume. @@ -106,33 +109,32 @@ sequentially. ### Background chainstate hits snapshot base block Once the tip of the background chainstate hits the base block of the snapshot -chainstate, we stop use of the background chainstate by setting `m_stop_use` (not yet -committed - see #15606), in `CompleteSnapshotValidation()`, which is checked in -`ActivateBestChain()`). We hash the background chainstate's UTXO set contents and -ensure it matches the compiled value in `CMainParams::m_assumeutxo_data`. - -The background chainstate data lingers on disk until shutdown, when in -`ChainstateManager::Reset()`, the background chainstate is cleaned up with -`ValidatedSnapshotShutdownCleanup()`, which renames the `chainstate_[hash]` datadir as -`chainstate`. +chainstate, we stop use of the background chainstate by setting `m_disabled`, in +`CompleteSnapshotValidation()`, which is checked in `ActivateBestChain()`). We hash the +background chainstate's UTXO set contents and ensure it matches the compiled value in +`CMainParams::m_assumeutxo_data`. | | | | ---------- | ----------- | -| number of chainstates | 2 (ibd has `m_stop_use=true`) | +| number of chainstates | 2 (ibd has `m_disabled=true`) | | active chainstate | snapshot | -**Failure consideration:** if bitcoind unexpectedly halts after `m_stop_use` is set on -the background chainstate but before `CompleteSnapshotValidation()` can finish, the -need to complete snapshot validation will be detected on subsequent init by -`ChainstateManager::CheckForUncleanShutdown()`. +The background chainstate data lingers on disk until the program is restarted. ### Bitcoind restarts sometime after snapshot validation has completed -When bitcoind initializes again, what began as the snapshot chainstate is now -indistinguishable from a chainstate that has been built from the traditional IBD -process, and will be initialized as such. +After a shutdown and subsequent restart, `LoadChainstate()` cleans up the background +chainstate with `ValidatedSnapshotCleanup()`, which renames the `chainstate_snapshot` +datadir as `chainstate` and removes the now unnecessary background chainstate data. | | | | ---------- | ----------- | | number of chainstates | 1 | -| active chainstate | ibd | +| active chainstate | ibd (was snapshot, but is now fully validated) | + +What began as the snapshot chainstate is now indistinguishable from a chainstate that +has been built from the traditional IBD process, and will be initialized as such. + +A file will be left in `chainstate/base_blockhash`, which indicates that the +chainstate, even though now fully validated, was originally started from a snapshot +with the corresponding base blockhash. diff --git a/doc/design/libraries.md b/doc/design/libraries.md new file mode 100644 index 00000000000..7cda64e713a --- /dev/null +++ b/doc/design/libraries.md @@ -0,0 +1,104 @@ +# Libraries + +| Name | Description | +|--------------------------|-------------| +| *libbitcoin_cli* | RPC client functionality used by *bitcoin-cli* executable | +| *libbitcoin_common* | Home for common functionality shared by different executables and libraries. Similar to *libbitcoin_util*, but higher-level (see [Dependencies](#dependencies)). | +| *libbitcoin_consensus* | Stable, backwards-compatible consensus functionality used by *libbitcoin_node* and *libbitcoin_wallet* and also exposed as a [shared library](../shared-libraries.md). | +| *libbitcoinconsensus* | Shared library build of static *libbitcoin_consensus* library | +| *libbitcoin_kernel* | Consensus engine and support library used for validation by *libbitcoin_node* and also exposed as a [shared library](../shared-libraries.md). | +| *libbitcoinqt* | GUI functionality used by *bitcoin-qt* and *bitcoin-gui* executables | +| *libbitcoin_ipc* | IPC functionality used by *bitcoin-node*, *bitcoin-wallet*, *bitcoin-gui* executables to communicate when [`--enable-multiprocess`](multiprocess.md) is used. | +| *libbitcoin_node* | P2P and RPC server functionality used by *bitcoind* and *bitcoin-qt* executables. | +| *libbitcoin_util* | Home for common functionality shared by different executables and libraries. Similar to *libbitcoin_common*, but lower-level (see [Dependencies](#dependencies)). | +| *libbitcoin_wallet* | Wallet functionality used by *bitcoind* and *bitcoin-wallet* executables. | +| *libbitcoin_wallet_tool* | Lower-level wallet functionality used by *bitcoin-wallet* executable. | +| *libbitcoin_zmq* | [ZeroMQ](../zmq.md) functionality used by *bitcoind* and *bitcoin-qt* executables. | + +## Conventions + +- Most libraries are internal libraries and have APIs which are completely unstable! There are few or no restrictions on backwards compatibility or rules about external dependencies. Exceptions are *libbitcoin_consensus* and *libbitcoin_kernel* which have external interfaces documented at [../shared-libraries.md](../shared-libraries.md). + +- Generally each library should have a corresponding source directory and namespace. Source code organization is a work in progress, so it is true that some namespaces are applied inconsistently, and if you look at [`libbitcoin_*_SOURCES`](../../src/Makefile.am) lists you can see that many libraries pull in files from outside their source directory. But when working with libraries, it is good to follow a consistent pattern like: + + - *libbitcoin_node* code lives in `src/node/` in the `node::` namespace + - *libbitcoin_wallet* code lives in `src/wallet/` in the `wallet::` namespace + - *libbitcoin_ipc* code lives in `src/ipc/` in the `ipc::` namespace + - *libbitcoin_util* code lives in `src/util/` in the `util::` namespace + - *libbitcoin_consensus* code lives in `src/consensus/` in the `Consensus::` namespace + +## Dependencies + +- Libraries should minimize what other libraries they depend on, and only reference symbols following the arrows shown in the dependency graph below: + +
+ +```mermaid + +%%{ init : { "flowchart" : { "curve" : "basis" }}}%% + +graph TD; + +bitcoin-cli[bitcoin-cli]-->libbitcoin_cli; + +bitcoind[bitcoind]-->libbitcoin_node; +bitcoind[bitcoind]-->libbitcoin_wallet; + +bitcoin-qt[bitcoin-qt]-->libbitcoin_node; +bitcoin-qt[bitcoin-qt]-->libbitcoinqt; +bitcoin-qt[bitcoin-qt]-->libbitcoin_wallet; + +bitcoin-wallet[bitcoin-wallet]-->libbitcoin_wallet; +bitcoin-wallet[bitcoin-wallet]-->libbitcoin_wallet_tool; + +libbitcoin_cli-->libbitcoin_util; +libbitcoin_cli-->libbitcoin_common; + +libbitcoin_common-->libbitcoin_consensus; +libbitcoin_common-->libbitcoin_util; + +libbitcoin_kernel-->libbitcoin_consensus; +libbitcoin_kernel-->libbitcoin_util; + +libbitcoin_node-->libbitcoin_consensus; +libbitcoin_node-->libbitcoin_kernel; +libbitcoin_node-->libbitcoin_common; +libbitcoin_node-->libbitcoin_util; + +libbitcoinqt-->libbitcoin_common; +libbitcoinqt-->libbitcoin_util; + +libbitcoin_wallet-->libbitcoin_common; +libbitcoin_wallet-->libbitcoin_util; + +libbitcoin_wallet_tool-->libbitcoin_wallet; +libbitcoin_wallet_tool-->libbitcoin_util; + +classDef bold stroke-width:2px, font-weight:bold, font-size: smaller; +class bitcoin-qt,bitcoind,bitcoin-cli,bitcoin-wallet bold +``` +
+ +**Dependency graph**. Arrows show linker symbol dependencies. *Consensus* lib depends on nothing. *Util* lib is depended on by everything. *Kernel* lib depends only on consensus and util. + +
+ +- The graph shows what _linker symbols_ (functions and variables) from each library other libraries can call and reference directly, but it is not a call graph. For example, there is no arrow connecting *libbitcoin_wallet* and *libbitcoin_node* libraries, because these libraries are intended to be modular and not depend on each other's internal implementation details. But wallet code is still able to call node code indirectly through the `interfaces::Chain` abstract class in [`interfaces/chain.h`](../../src/interfaces/chain.h) and node code calls wallet code through the `interfaces::ChainClient` and `interfaces::Chain::Notifications` abstract classes in the same file. In general, defining abstract classes in [`src/interfaces/`](../../src/interfaces/) can be a convenient way of avoiding unwanted direct dependencies or circular dependencies between libraries. + +- *libbitcoin_consensus* should be a standalone dependency that any library can depend on, and it should not depend on any other libraries itself. + +- *libbitcoin_util* should also be a standalone dependency that any library can depend on, and it should not depend on other internal libraries. + +- *libbitcoin_common* should serve a similar function as *libbitcoin_util* and be a place for miscellaneous code used by various daemon, GUI, and CLI applications and libraries to live. It should not depend on anything other than *libbitcoin_util* and *libbitcoin_consensus*. The boundary between _util_ and _common_ is a little fuzzy but historically _util_ has been used for more generic, lower-level things like parsing hex, and _common_ has been used for bitcoin-specific, higher-level things like parsing base58. The difference between util and common is mostly important because *libbitcoin_kernel* is not supposed to depend on *libbitcoin_common*, only *libbitcoin_util*. In general, if it is ever unclear whether it is better to add code to *util* or *common*, it is probably better to add it to *common* unless it is very generically useful or useful particularly to include in the kernel. + + +- *libbitcoin_kernel* should only depend on *libbitcoin_util* and *libbitcoin_consensus*. + +- The only thing that should depend on *libbitcoin_kernel* internally should be *libbitcoin_node*. GUI and wallet libraries *libbitcoinqt* and *libbitcoin_wallet* in particular should not depend on *libbitcoin_kernel* and the unneeded functionality it would pull in, like block validation. To the extent that GUI and wallet code need scripting and signing functionality, they should be get able it from *libbitcoin_consensus*, *libbitcoin_common*, and *libbitcoin_util*, instead of *libbitcoin_kernel*. + +- GUI, node, and wallet code internal implementations should all be independent of each other, and the *libbitcoinqt*, *libbitcoin_node*, *libbitcoin_wallet* libraries should never reference each other's symbols. They should only call each other through [`src/interfaces/`](`../../src/interfaces/`) abstract interfaces. + +## Work in progress + +- Validation code is moving from *libbitcoin_node* to *libbitcoin_kernel* as part of [The libbitcoinkernel Project #24303](https://github.com/bitcoin/bitcoin/issues/24303) +- Source code organization is discussed in general in [Library source code organization #15732](https://github.com/bitcoin/bitcoin/issues/15732) diff --git a/doc/multiprocess.md b/doc/design/multiprocess.md similarity index 100% rename from doc/multiprocess.md rename to doc/design/multiprocess.md diff --git a/doc/developer-notes.md b/doc/developer-notes.md index bfb64093e19..08dde2aa61a 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -17,6 +17,7 @@ Developer Notes - [`debug.log`](#debuglog) - [Signet, testnet, and regtest modes](#signet-testnet-and-regtest-modes) - [DEBUG_LOCKORDER](#debug_lockorder) + - [DEBUG_LOCKCONTENTION](#debug_lockcontention) - [Valgrind suppressions file](#valgrind-suppressions-file) - [Compiling for test coverage](#compiling-for-test-coverage) - [Performance profiling with perf](#performance-profiling-with-perf) @@ -31,6 +32,7 @@ Developer Notes - [C++ data structures](#c-data-structures) - [Strings and formatting](#strings-and-formatting) - [Shadowing](#shadowing) + - [Lifetimebound](#lifetimebound) - [Threads and synchronization](#threads-and-synchronization) - [Scripts](#scripts) - [Shebang](#shebang) @@ -95,7 +97,10 @@ code. Guidelines](https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Renum-caps), which recommend using `snake_case`. Please use what seems appropriate. - Class names, function names, and method names are UpperCamelCase - (PascalCase). Do not prefix class names with `C`. + (PascalCase). Do not prefix class names with `C`. See [Internal interface + naming style](#internal-interface-naming-style) for an exception to this + convention. + - Test suite naming convention: The Boost test suite in file `src/test/foo_tests.cpp` should be named `foo_tests`. Test suite names must be unique. @@ -104,6 +109,32 @@ code. - `++i` is preferred over `i++`. - `nullptr` is preferred over `NULL` or `(void*)0`. - `static_assert` is preferred over `assert` where possible. Generally; compile-time checking is preferred over run-time checking. + - Use a named cast or functional cast, not a C-Style cast. When casting + between integer types, use functional casts such as `int(x)` or `int{x}` + instead of `(int) x`. When casting between more complex types, use `static_cast`. + Use `reinterpret_cast` and `const_cast` as appropriate. + +For function calls a namespace should be specified explicitly, unless such functions have been declared within it. +Otherwise, [argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl), also known as ADL, could be +triggered that makes code harder to maintain and reason about: +```c++ +#include + +namespace fs { +class path : public std::filesystem::path +{ +}; +// The intention is to disallow this function. +bool exists(const fs::path& p) = delete; +} // namespace fs + +int main() +{ + //fs::path p; // error + std::filesystem::path p; // compiled + exists(p); // ADL being used for unqualified name lookup +} +``` Block style example: ```c++ @@ -137,11 +168,81 @@ public: } // namespace foo ``` +Coding Style (C++ functions and methods) +-------------------- + +- When ordering function parameters, place input parameters first, then any + in-out parameters, followed by any output parameters. + +- *Rationale*: API consistency. + +- Prefer returning values directly to using in-out or output parameters. Use + `std::optional` where helpful for returning values. + +- *Rationale*: Less error-prone (no need for assumptions about what the output + is initialized to on failure), easier to read, and often the same or better + performance. + +- Generally, use `std::optional` to represent optional by-value inputs (and + instead of a magic default value, if there is no real default). Non-optional + input parameters should usually be values or const references, while + non-optional in-out and output parameters should usually be references, as + they cannot be null. + +Coding Style (C++ named arguments) +------------------------------ + +When passing named arguments, use a format that clang-tidy understands. The +argument names can otherwise not be verified by clang-tidy. + +For example: + +```c++ +void function(Addrman& addrman, bool clear); + +int main() +{ + function(g_addrman, /*clear=*/false); +} +``` + +### Running clang-tidy + +To run clang-tidy on Ubuntu/Debian, install the dependencies: + +```sh +apt install clang-tidy bear clang +``` + +Then, pass clang as compiler to configure, and use bear to produce the `compile_commands.json`: + +```sh +./autogen.sh && ./configure CC=clang CXX=clang++ --enable-suppress-external-warnings +make clean && bear --config src/.bear-tidy-config -- make -j $(nproc) +``` + +The output is denoised of errors from external dependencies and includes with +`--enable-suppress-external-warnings` and `--config src/.bear-tidy-config`. Both +options may be omitted to view the full list of errors. + +To run clang-tidy on all source files: + +```sh +( cd ./src/ && run-clang-tidy -j $(nproc) ) +``` + +To run clang-tidy on the changed source lines: + +```sh +git diff | ( cd ./src/ && clang-tidy-diff -p2 -j $(nproc) ) +``` + Coding Style (Python) --------------------- Refer to [/test/functional/README.md#style-guidelines](/test/functional/README.md#style-guidelines). + Coding Style (Doxygen-compatible comments) ------------------------------------------ @@ -292,8 +393,10 @@ Run configure with the `--enable-gprof` option, then make. If the code is behaving strangely, take a look in the `debug.log` file in the data directory; error and debugging messages are written there. -The `-debug=...` command-line option controls debugging; running with just `-debug` or `-debug=1` will turn -on all categories (and give you a very large `debug.log` file). +Debug logging can be enabled on startup with the `-debug` and `-loglevel` +configuration options and toggled while bitcoind is running with the `logging` +RPC. For instance, launching bitcoind with `-debug` or `-debug=1` will turn on +all log categories and `-loglevel=trace` will turn on all log severity levels. The Qt code routes `qDebug()` output to `debug.log` under category "qt": run with `-debug=qt` to see it. @@ -316,6 +419,21 @@ configure option adds `-DDEBUG_LOCKORDER` to the compiler flags. This inserts run-time checks to keep track of which locks are held and adds warnings to the `debug.log` file if inconsistencies are detected. +### DEBUG_LOCKCONTENTION + +Defining `DEBUG_LOCKCONTENTION` adds a "lock" logging category to the logging +RPC that, when enabled, logs the location and duration of each lock contention +to the `debug.log` file. + +The `--enable-debug` configure option adds `-DDEBUG_LOCKCONTENTION` to the +compiler flags. You may also enable it manually for a non-debug build by running +configure with `-DDEBUG_LOCKCONTENTION` added to your CPPFLAGS, +i.e. `CPPFLAGS="-DDEBUG_LOCKCONTENTION"`, then build and run bitcoind. + +You can then use the `-debug=lock` configuration option at bitcoind startup or +`bitcoin-cli logging '["lock"]'` at runtime to turn on lock contention logging. +It can be toggled off again with `bitcoin-cli logging [] '["lock"]'`. + ### Assertions and Checks The util file `src/util/check.h` offers helpers to protect against coding and @@ -445,8 +563,19 @@ address sanitizer, libtsan for the thread sanitizer, and libubsan for the undefined sanitizer. If you are missing required libraries, the configure script will fail with a linker error when testing the sanitizer flags. -The test suite should pass cleanly with the `thread` and `undefined` sanitizers, -but there are a number of known problems when using the `address` sanitizer. The +The test suite should pass cleanly with the `thread` and `undefined` sanitizers. You +may need to use a suppressions file, see `test/sanitizer_suppressions`. They may be +used as follows: +```bash +export LSAN_OPTIONS="suppressions=$(pwd)/test/sanitizer_suppressions/lsan" +export TSAN_OPTIONS="suppressions=$(pwd)/test/sanitizer_suppressions/tsan:halt_on_error=1:second_deadlock_stack=1" +export UBSAN_OPTIONS="suppressions=$(pwd)/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" +``` + +See the CI config for more examples, and upstream documentation for more information +about any additional options. + +There are a number of known problems when using the `address` sanitizer. The address sanitizer is known to fail in [sha256_sse4::Transform](/src/crypto/sha256_sse4.cpp) which makes it unusable unless you also use `--disable-asm` when running configure. We would like to fix @@ -494,10 +623,10 @@ Threads : Started from `main()` in `bitcoind.cpp`. Responsible for starting up and shutting down the application. -- [ThreadImport (`b-loadblk`)](https://doxygen.bitcoincore.org/init_8cpp.html#ae9e290a0e829ec0198518de2eda579d1) +- [ThreadImport (`b-loadblk`)](https://doxygen.bitcoincore.org/namespacenode.html#ab4305679079866f0f420f7dbf278381d) : Loads blocks from `blk*.dat` files or `-loadblock=` on startup. -- [ThreadScriptCheck (`b-scriptch.x`)](https://doxygen.bitcoincore.org/validation_8cpp.html#a925a33e7952a157922b0bbb8dab29a20) +- [CCheckQueue::Loop (`b-scriptch.x`)](https://doxygen.bitcoincore.org/class_c_check_queue.html#a6e7fa51d3a25e7cb65446d4b50e6a987) : Parallel script validation threads for transactions in blocks. - [ThreadHTTP (`b-http`)](https://doxygen.bitcoincore.org/httpserver_8cpp.html#abb9f6ea8819672bd9a62d3695070709c) @@ -513,7 +642,7 @@ Threads : Does asynchronous background tasks like dumping wallet contents, dumping addrman and running asynchronous validationinterface callbacks. -- [TorControlThread (`b-torcontrol`)](https://doxygen.bitcoincore.org/torcontrol_8cpp.html#a4faed3692d57a0d7bdbecf3b37f72de0) +- [TorControlThread (`b-torcontrol`)](https://doxygen.bitcoincore.org/torcontrol_8cpp.html#a52a3efff23634500bb42c6474f306091) : Libevent thread for tor connections. - Net threads: @@ -525,7 +654,7 @@ Threads - [ThreadDNSAddressSeed (`b-dnsseed`)](https://doxygen.bitcoincore.org/class_c_connman.html#aa7c6970ed98a4a7bafbc071d24897d13) : Loads addresses of peers from the DNS. - - [ThreadMapPort (`b-upnp`)](https://doxygen.bitcoincore.org/net_8cpp.html#a63f82a71c4169290c2db1651a9bbe249) + - ThreadMapPort (`b-mapport`) : Universal plug-and-play startup/shutdown. - [ThreadSocketHandler (`b-net`)](https://doxygen.bitcoincore.org/class_c_connman.html#a765597cbfe99c083d8fa3d61bb464e34) @@ -537,6 +666,9 @@ Threads - [ThreadOpenConnections (`b-opencon`)](https://doxygen.bitcoincore.org/class_c_connman.html#a55e9feafc3bab78e5c9d408c207faa45) : Initiates new connections to peers. + - [ThreadI2PAcceptIncoming (`b-i2paccept`)](https://doxygen.bitcoincore.org/class_c_connman.html#a57787b4f9ac847d24065fbb0dd6e70f8) + : Listens for and accepts incoming I2P connections through the I2P SAM proxy. + Ignoring IDE/editor files -------------------------- @@ -597,10 +729,6 @@ Wallet - Make sure that no crashes happen with run-time option `-disablewallet`. -- Include `db_cxx.h` (BerkeleyDB header) only when `ENABLE_WALLET` is set. - - - *Rationale*: Otherwise compilation of the disable-wallet build will fail in environments without BerkeleyDB. - General C++ ------------- @@ -726,23 +854,18 @@ int GetInt(Tabs tab) Strings and formatting ------------------------ -- Be careful of `LogPrint` versus `LogPrintf`. `LogPrint` takes a `category` argument, `LogPrintf` does not. - - - *Rationale*: Confusion of these can result in runtime exceptions due to - formatting mismatch, and it is easy to get wrong because of subtly similar naming. - - Use `std::string`, avoid C string manipulation functions. - *Rationale*: C++ string handling is marginally safer, less scope for buffer overflows, and surprises with `\0` characters. Also, some C string manipulations tend to act differently depending on platform, or even the user locale. -- Use `ParseInt32`, `ParseInt64`, `ParseUInt32`, `ParseUInt64`, `ParseDouble` from `utilstrencodings.h` for number parsing. +- Use `ToIntegral` from [`strencodings.h`](/src/util/strencodings.h) for number parsing. In legacy code you might also find `ParseInt*` family of functions, `ParseDouble` or `LocaleIndependentAtoi`. - *Rationale*: These functions do overflow checking and avoid pesky locale issues. - Avoid using locale dependent functions if possible. You can use the provided - [`lint-locale-dependence.sh`](/test/lint/lint-locale-dependence.sh) + [`lint-locale-dependence.py`](/test/lint/lint-locale-dependence.py) to check for accidental use of locale dependent functions. - *Rationale*: Unnecessary locale dependence can cause bugs that are very tricky to isolate and fix. @@ -803,22 +926,37 @@ from using a different variable with the same name), please name variables so that their names do not shadow variables defined in the source code. When using nested cycles, do not name the inner cycle variable the same as in -the upper cycle, etc. +the outer cycle, etc. + +Lifetimebound +-------------- + +The [Clang `lifetimebound` +attribute](https://clang.llvm.org/docs/AttributeReference.html#lifetimebound) +can be used to tell the compiler that a lifetime is bound to an object and +potentially see a compile-time warning if the object has a shorter lifetime from +the invalid use of a temporary. You can use the attribute by adding a `LIFETIMEBOUND` +annotation defined in `src/attributes.h`; please grep the codebase for examples. Threads and synchronization ---------------------------- -- Prefer `Mutex` type to `RecursiveMutex` one +- Prefer `Mutex` type to `RecursiveMutex` one. - Consistently use [Clang Thread Safety Analysis](https://clang.llvm.org/docs/ThreadSafetyAnalysis.html) annotations to - get compile-time warnings about potential race conditions in code. Combine annotations in function declarations with - run-time asserts in function definitions: + get compile-time warnings about potential race conditions or deadlocks in code. - In functions that are declared separately from where they are defined, the thread safety annotations should be added exclusively to the function declaration. Annotations on the definition could lead to false positives (lack of compile failure) at call sites between the two. + - Prefer locks that are in a class rather than global, and that are + internal to a class (private or protected) rather than public. + + - Combine annotations in function declarations with run-time asserts in + function definitions: + ```C++ // txmempool.h class CTxMemPool @@ -842,21 +980,37 @@ void CTxMemPool::UpdateTransactionsFromBlock(...) ```C++ // validation.h -class ChainstateManager +class Chainstate { +protected: + ... + Mutex m_chainstate_mutex; + ... public: ... - bool ProcessNewBlock(...) LOCKS_EXCLUDED(::cs_main); + bool ActivateBestChain( + BlockValidationState& state, + std::shared_ptr pblock = nullptr) + EXCLUSIVE_LOCKS_REQUIRED(!m_chainstate_mutex) + LOCKS_EXCLUDED(::cs_main); + ... + bool PreciousBlock(BlockValidationState& state, CBlockIndex* pindex) + EXCLUSIVE_LOCKS_REQUIRED(!m_chainstate_mutex) + LOCKS_EXCLUDED(::cs_main); ... } // validation.cpp -bool ChainstateManager::ProcessNewBlock(...) +bool Chainstate::PreciousBlock(BlockValidationState& state, CBlockIndex* pindex) { + AssertLockNotHeld(m_chainstate_mutex); AssertLockNotHeld(::cs_main); - ... - LOCK(::cs_main); - ... + { + LOCK(cs_main); + ... + } + + return ActivateBestChain(state, std::shared_ptr()); } ``` @@ -887,6 +1041,8 @@ TRY_LOCK(cs_vNodes, lockNodes); Scripts -------------------------- +Write scripts in Python rather than bash, when possible. + ### Shebang - Use `#!/usr/bin/env bash` instead of obsolete `#!/bin/bash`. @@ -1025,10 +1181,6 @@ Current subtrees include: - src/crypto/ctaes - Upstream at https://github.com/bitcoin-core/ctaes ; maintained by Core contributors. -- src/univalue - - Subtree at https://github.com/bitcoin-core/univalue-subtree ; maintained by Core contributors. - - Deviates from upstream https://github.com/jgarzik/univalue. - - src/minisketch - Upstream at https://github.com/sipa/minisketch ; maintained by Core contributors. @@ -1160,7 +1312,10 @@ A few guidelines for introducing and reviewing new RPC interfaces: - *Rationale*: Consistency with the existing interface. -- Argument naming: use snake case `fee_delta` (and not, e.g. camel case `feeDelta`) +- Argument and field naming: please consider whether there is already a naming + style or spelling convention in the API for the type of object in question + (`blockhash`, for example), and if so, try to use that. If not, use snake case + `fee_delta` (and not, e.g. `feedelta` or camel case `feeDelta`). - *Rationale*: Consistency with the existing interface. @@ -1326,22 +1481,9 @@ communication: virtual boost::signals2::scoped_connection connectTipChanged(TipChangedFn fn) = 0; ``` -- For consistency and friendliness to code generation tools, interface method - input and inout parameters should be ordered first and output parameters - should come last. - - Example: - - ```c++ - // Good: error output param is last - virtual bool broadcastTransaction(const CTransactionRef& tx, CAmount max_fee, std::string& error) = 0; - - // Bad: error output param is between input params - virtual bool broadcastTransaction(const CTransactionRef& tx, std::string& error, CAmount max_fee) = 0; - ``` +- Interface methods should not be overloaded. -- For friendliness to code generation tools, interface methods should not be - overloaded: + *Rationale*: consistency and friendliness to code generation tools. Example: @@ -1355,10 +1497,13 @@ communication: virtual bool disconnect(NodeId id) = 0; ``` -- For consistency and friendliness to code generation tools, interface method - names should be `lowerCamelCase` and standalone function names should be +### Internal interface naming style + +- Interface method names should be `lowerCamelCase` and standalone function names should be `UpperCamelCase`. + *Rationale*: consistency and friendliness to code generation tools. + Examples: ```c++ diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 9abfbc9213e..84ebb0986d2 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -136,10 +136,10 @@ You may also need to take care of giving the correct path for `clang` and `clang++`, like `CC=/path/to/clang CXX=/path/to/clang++` if the non-systems `clang` does not come first in your path. -Full configure that was tested on macOS Catalina with `brew` installed `llvm`: +Full configure that was tested on macOS with `brew` installed `llvm`: ```sh -./configure --enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=/usr/local/opt/llvm/bin/clang CXX=/usr/local/opt/llvm/bin/clang++ --disable-asm +./configure --enable-fuzz --with-sanitizers=fuzzer,address,undefined --disable-asm CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ ``` Read the [libFuzzer documentation](https://llvm.org/docs/LibFuzzer.html) for more information. This [libFuzzer tutorial](https://github.com/google/fuzzing/blob/master/tutorial/libFuzzerTutorial.md) might also be of interest. @@ -283,8 +283,8 @@ $ sudo apt-get install libtool libtool-bin wget automake autoconf bison gdb ``` At this point, you must install the .NET core. The process differs, depending on your Linux distribution. -See [this link](https://docs.microsoft.com/en-us/dotnet/core/install/linux) for details. -On ubuntu 20.04, the following should work: +See [this link](https://learn.microsoft.com/en-us/dotnet/core/install/linux) for details. +On Ubuntu 20.04, the following should work: ```sh $ wget -q https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb diff --git a/doc/i2p.md b/doc/i2p.md index ee650f39998..0432136554d 100644 --- a/doc/i2p.md +++ b/doc/i2p.md @@ -16,8 +16,7 @@ enabled is required. Options include: Java - [i2pd (I2P Daemon)](https://github.com/PurpleI2P/i2pd) ([documentation](https://i2pd.readthedocs.io/en/latest)), a lighter - alternative in C++ (successfully tested with version 2.23 and up; version 2.36 - or later recommended) + alternative in C++ - [i2p-zero](https://github.com/i2p-zero/i2p-zero) - [other alternatives](https://en.wikipedia.org/wiki/I2P#Routers) @@ -33,12 +32,10 @@ Core configuration options: none) -i2pacceptincoming - If set and -i2psam is also set then incoming I2P connections are - accepted via the SAM proxy. If this is not set but -i2psam is set - then only outgoing connections will be made to the I2P network. - Ignored if -i2psam is not set. Listening for incoming I2P - connections is done through the SAM proxy, not by binding to a - local address and port (default: 1) + Whether to accept inbound I2P connections (default: 1). Ignored if + -i2psam is not set. Listening for inbound I2P connections is + done through the SAM proxy, not by binding to a local address and + port. ``` In a typical situation, this suffices: @@ -47,10 +44,6 @@ In a typical situation, this suffices: bitcoind -i2psam=127.0.0.1:7656 ``` -The first time Bitcoin Core connects to the I2P router, its I2P address (and -corresponding private key) will be automatically generated and saved in a file -named `i2p_private_key` in the Bitcoin Core data directory. - ## Additional configuration options related to I2P ``` @@ -65,9 +58,9 @@ logging` for more information. -onlynet=i2p ``` -Make outgoing connections only to I2P addresses. Incoming connections are not -affected by this option. It can be specified multiple times to allow multiple -network types, e.g. onlynet=onion, onlynet=i2p. +Make automatic outbound connections only to I2P addresses. Inbound and manual +connections are not affected by this option. It can be specified multiple times +to allow multiple networks, e.g. onlynet=onion, onlynet=i2p. I2P support was added to Bitcoin Core in version 22.0 and there may be fewer I2P peers than Tor or IP ones. Therefore, using I2P alone without other networks may @@ -80,15 +73,38 @@ phase when syncing up a new node can be very slow. This phase can be sped up by using other networks, for instance `onlynet=onion`, at the same time. In general, a node can be run with both onion and I2P hidden services (or -any/all of IPv4/IPv6/onion/I2P), which can provide a potential fallback if one -of the networks has issues. +any/all of IPv4/IPv6/onion/I2P/CJDNS), which can provide a potential fallback if +one of the networks has issues. + +## Persistent vs transient I2P addresses + +The first time Bitcoin Core connects to the I2P router, it automatically +generates a persistent I2P address and its corresponding private key by default +or if `-i2pacceptincoming=1` is set. The private key is saved in a file named +`i2p_private_key` in the Bitcoin Core data directory. The persistent I2P +address is used for making outbound connections and accepting inbound +connections. + +In the I2P network, the receiver of an inbound connection sees the address of +the initiator. This is unlike the Tor network, where the recipient does not +know who is connecting to it. + +If your node is configured by setting `-i2pacceptincoming=0` to not accept +inbound I2P connections, then it will use a random transient I2P address for +itself on each outbound connection to make it harder to discriminate, +fingerprint or analyze it based on its I2P address. -## I2P-related information in Bitcoin Core +I2P addresses are designed to be long-lived. Waiting for tunnels to be built +for every peer connection adds delay to connection setup time. Therefore, I2P +listening should only be turned off if really needed. -There are several ways to see your I2P address in Bitcoin Core: -- in the debug log (grep for `AddLocal`, the I2P address ends in `.b32.i2p`) -- in the output of the `getnetworkinfo` RPC in the "localaddresses" section -- in the output of `bitcoin-cli -netinfo` peer connections dashboard +## Fetching I2P-related information from Bitcoin Core + +There are several ways to see your I2P address in Bitcoin Core if accepting +incoming I2P connections (`-i2pacceptincoming`): +- in the "Local addresses" output of CLI `-netinfo` +- in the "localaddresses" output of RPC `getnetworkinfo` +- in the debug log (grep for `AddLocal`; the I2P address ends in `.b32.i2p`) To see which I2P peers your node is connected to, use `bitcoin-cli -netinfo 4` or the `getpeerinfo` RPC (e.g. `bitcoin-cli getpeerinfo`). @@ -115,3 +131,40 @@ listening port to 0 when listening for incoming I2P connections and advertises its own I2P address with port 0. Furthermore, it will not attempt to connect to I2P addresses with a non-zero port number because with SAM v3.1 the destination port (`TO_PORT`) is always set to 0 and is not in the control of Bitcoin Core. + +## Bandwidth + +By default, your node shares bandwidth and transit tunnels with the I2P network +in order to increase your anonymity with cover traffic, help the I2P router used +by your node integrate optimally with the network, and give back to the network. +It's important that the nodes of a popular application like Bitcoin contribute +as much to the I2P network as they consume. + +It is possible, though strongly discouraged, to change your I2P router +configuration to limit the amount of I2P traffic relayed by your node. + +With `i2pd`, this can be done by adjusting the `bandwidth`, `share` and +`transittunnels` options in your `i2pd.conf` file. For example, to limit total +I2P traffic to 256KB/s and share 50% of this limit for a maximum of 20 transit +tunnels: + +``` +bandwidth = 256 +share = 50 + +[limits] +transittunnels = 20 +``` + +Similar bandwidth configuration options for the Java I2P router can be found in +`http://127.0.0.1:7657/config` under the "Bandwidth" tab. + +Before doing this, please see the "Participating Traffic Considerations" section +in [Embedding I2P in your Application](https://geti2p.net/en/docs/applications/embedding). + +In most cases, the default router settings should work fine. + +## Bundling I2P in a Bitcoin application + +Please see the "General Guidance for Developers" section in https://geti2p.net/en/docs/api/samv3 +if you are developing a downstream application that may be bundling I2P with Bitcoin. diff --git a/doc/init.md b/doc/init.md index 399b819bf41..7f790277180 100644 --- a/doc/init.md +++ b/doc/init.md @@ -70,7 +70,7 @@ NOTE: When using the systemd .service file, the creation of the aforementioned directories and the setting of their permissions is automatically handled by systemd. Directories are given a permission of 710, giving the bitcoin group access to files under it _if_ the files themselves give permission to the -bitcoin group to do so (e.g. when `-sysperms` is specified). This does not allow +bitcoin group to do so. This does not allow for the listing of files under the directory. NOTE: It is not currently possible to override `datadir` in diff --git a/doc/managing-wallets.md b/doc/managing-wallets.md index 6c1e13c503c..22e006c9639 100644 --- a/doc/managing-wallets.md +++ b/doc/managing-wallets.md @@ -88,7 +88,7 @@ In the RPC, the destination parameter must include the name of the file. Otherwi $ bitcoin-cli -rpcwallet="wallet-01" backupwallet /home/node01/Backups/backup-01.dat ``` -In the GUI, the wallet is selected in the `Wallet` drop-down list in the upper right corner. If this list is not present, the wallet can be loaded in `File` ->`Open wallet` if necessary. Then, the backup can be done in `File` -> `Backup Wallet...`. +In the GUI, the wallet is selected in the `Wallet` drop-down list in the upper right corner. If this list is not present, the wallet can be loaded in `File` ->`Open Wallet` if necessary. Then, the backup can be done in `File` -> `Backup Wallet…`. This backup file can be stored on one or multiple offline devices, which must be reliable enough to work in an emergency and be malware free. Backup files can be regularly tested to avoid problems in the future. @@ -108,7 +108,7 @@ Wallets created before version 0.13 are not HD and must be backed up every 100 k ### 1.6 Restoring the Wallet From a Backup -To restore a wallet, the `restorewallet` RPC must be used. +To restore a wallet, the `restorewallet` RPC or the `Restore Wallet` GUI menu item (`File` -> `Restore Wallet…`) must be used. ``` $ bitcoin-cli restorewallet "restored-wallet" /home/node01/Backups/backup-01.dat @@ -120,4 +120,29 @@ After that, `getwalletinfo` can be used to check if the wallet has been fully re $ bitcoin-cli -rpcwallet="restored-wallet" getwalletinfo ``` -The restored wallet can also be loaded in the GUI via `File` ->`Open wallet`. \ No newline at end of file +The restored wallet can also be loaded in the GUI via `File` ->`Open wallet`. + +## Migrating Legacy Wallets to Descriptor Wallets + +Legacy wallets (traditional non-descriptor wallets) can be migrated to become Descriptor wallets +through the use of the `migratewallet` RPC. Migrated wallets will have all of their addresses and private keys added to +a newly created Descriptor wallet that has the same name as the original wallet. Because Descriptor +wallets do not support having private keys and watch-only scripts, there may be up to two +additional wallets created after migration. In addition to a descriptor wallet of the same name, +there may also be a wallet named `_watchonly` and `_solvables`. `_watchonly` +contains all of the watchonly scripts. `_solvables` contains any scripts which the wallet +knows but is not watching the corresponding P2(W)SH scripts. + +Migrated wallets will also generate new addresses differently. While the same BIP 32 seed will be +used, the BIP 44, 49, 84, and 86 standard derivation paths will be used. After migrating, a new +backup of the wallet(s) will need to be created. + +Given that there is an extremely large number of possible configurations for the scripts that +Legacy wallets can know about, be watching for, and be able to sign for, `migratewallet` only +makes a best effort attempt to capture all of these things into Descriptor wallets. There may be +unforeseen configurations which result in some scripts being excluded. If a migration fails +unexpectedly or otherwise misses any scripts, please create an issue on GitHub. A backup of the +original wallet can be found in the wallet directory with the name `-.legacy.bak`. + +The backup can be restored using the methods discussed in the +[Restoring the Wallet From a Backup](#16-restoring-the-wallet-from-a-backup) section. diff --git a/doc/multisig-tutorial.md b/doc/multisig-tutorial.md index 0793040418b..1d2b3244bca 100644 --- a/doc/multisig-tutorial.md +++ b/doc/multisig-tutorial.md @@ -29,7 +29,7 @@ These three wallets should not be used directly for privacy reasons (public key ```bash for ((n=1;n<=3;n++)) do - ./src/bitcoin-cli -signet -named createwallet wallet_name="participant_${n}" descriptors=true + ./src/bitcoin-cli -signet createwallet "participant_${n}" done ``` @@ -109,7 +109,7 @@ Then import the descriptors created in the previous step using the `importdescri After that, `getwalletinfo` can be used to check if the wallet was created successfully. ```bash -./src/bitcoin-cli -signet -named createwallet wallet_name="multisig_wallet_01" disable_private_keys=true blank=true descriptors=true +./src/bitcoin-cli -signet -named createwallet wallet_name="multisig_wallet_01" disable_private_keys=true blank=true ./src/bitcoin-cli -signet -rpcwallet="multisig_wallet_01" importdescriptors "$multisig_desc" @@ -238,4 +238,4 @@ psbt_2=$(./src/bitcoin-cli -signet -rpcwallet="participant_2" walletprocesspsbt finalized_psbt_hex=$(./src/bitcoin-cli -signet finalizepsbt $psbt_2 | jq -r '.hex') ./src/bitcoin-cli -signet sendrawtransaction $finalized_psbt_hex -``` \ No newline at end of file +``` diff --git a/doc/p2p-bad-ports.md b/doc/p2p-bad-ports.md index 0dd7d36cf46..4f717f97a29 100644 --- a/doc/p2p-bad-ports.md +++ b/doc/p2p-bad-ports.md @@ -1,6 +1,6 @@ -When Bitcoin Core automatically opens outgoing P2P connections it chooses +When Bitcoin Core automatically opens outgoing P2P connections, it chooses a peer (address and port) from its list of potential peers. This list is -populated with unchecked data, gossiped over the P2P network by other peers. +populated with unchecked data gossiped over the P2P network by other peers. A malicious actor may gossip an address:port where no Bitcoin node is listening, or one where a service is listening that is not related to the Bitcoin network. @@ -17,7 +17,7 @@ authentication are unlikely to be considered a malicious action, e.g. port 80 (http). Below is a list of "bad" ports which Bitcoin Core avoids when choosing a peer to -connect to. If a node is listening on such a port, it will likely receive less +connect to. If a node is listening on such a port, it will likely receive fewer incoming connections. 1: tcpmux diff --git a/doc/policy/README.md b/doc/policy/README.md index 6e8686365dc..27536407e7d 100644 --- a/doc/policy/README.md +++ b/doc/policy/README.md @@ -3,7 +3,7 @@ **Policy** (Mempool or Transaction Relay Policy) is the node's set of validation rules, in addition to consensus, enforced for unconfirmed transactions before submitting them to the mempool. These rules are local to the node and configurable (e.g. `-minrelaytxfee`, `-limitancestorsize`, -`-incrementalRelayFee`). Policy may include restrictions on the transaction itself, the transaction +`-incrementalrelayfee`). Policy may include restrictions on the transaction itself, the transaction in relation to the current chain tip, and the transaction in relation to the node's mempool contents. Policy is *not* applied to transactions in blocks. diff --git a/doc/policy/mempool-replacements.md b/doc/policy/mempool-replacements.md index 3e844f8d7b5..b3c4239b731 100644 --- a/doc/policy/mempool-replacements.md +++ b/doc/policy/mempool-replacements.md @@ -15,6 +15,8 @@ other consensus and policy rules, each of the following conditions are met: *Rationale*: See [BIP125 explanation](https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki#motivation). + Use the (`-mempoolfullrbf`) configuration option to allow transaction replacement without enforcement of the + opt-in signaling rule. 2. The replacement transaction only include an unconfirmed input if that input was included in one of the directly conflicting transactions. An unconfirmed input spends an output from a @@ -51,6 +53,13 @@ other consensus and policy rules, each of the following conditions are met: significant portions of the node's mempool using replacements with multiple directly conflicting transactions, each with large descendant sets. +6. The replacement transaction's feerate is greater than the feerates of all directly conflicting + transactions. + + *Rationale*: This rule was originally intended to ensure that the replacement transaction is + preferable for block-inclusion, compared to what would be removed from the mempool. This rule + predates ancestor feerate-based transaction selection. + This set of rules is similar but distinct from BIP125. ## History @@ -62,8 +71,11 @@ This set of rules is similar but distinct from BIP125. Bitcoin Core implementation. * The incremental relay feerate used to calculate the required additional fees is distinct from - `minRelayTxFee` and configurable using `-incrementalrelayfee` + `-minrelaytxfee` and configurable using `-incrementalrelayfee` ([PR #9380](https://github.com/bitcoin/bitcoin/pull/9380)). * RBF enabled by default in the wallet GUI as of **v0.18.1** ([PR #11605](https://github.com/bitcoin/bitcoin/pull/11605)). + +* Full replace-by-fee enabled as a configurable mempool policy as of **v24.0** ([PR + #25353](https://github.com/bitcoin/bitcoin/pull/25353)). diff --git a/doc/policy/packages.md b/doc/policy/packages.md index 7f7fbe18cdf..274854ddf93 100644 --- a/doc/policy/packages.md +++ b/doc/policy/packages.md @@ -33,7 +33,7 @@ The following rules are enforced for all packages: * Packages cannot have conflicting transactions, i.e. no two transactions in a package can spend the same inputs. Packages cannot have duplicate transactions. (#20833) -* No transaction in a package can conflict with a mempool transaction. BIP125 Replace By Fee is +* No transaction in a package can conflict with a mempool transaction. Replace By Fee is currently disabled for packages. (#20833) - Package RBF may be enabled in the future. @@ -72,3 +72,48 @@ test accepts): a competing package or transaction with a mutated witness, even though the two same-txid-different-witness transactions are conflicting and cannot replace each other, the honest package should still be considered for acceptance. + +### Package Fees and Feerate + +*Package Feerate* is the total modified fees (base fees + any fee delta from +`prioritisetransaction`) divided by the total virtual size of all transactions in the package. +If any transactions in the package are already in the mempool, they are not submitted again +("deduplicated") and are thus excluded from this calculation. + +To meet the two feerate requirements of a mempool, i.e., the pre-configured minimum relay feerate +(`-minrelaytxfee`) and the dynamic mempool minimum feerate, the total package feerate is used instead +of the individual feerate. The individual transactions are allowed to be below the feerate +requirements if the package meets the feerate requirements. For example, the parent(s) in the +package can pay no fees but be paid for by the child. + +*Rationale*: This can be thought of as "CPFP within a package," solving the issue of a parent not +meeting minimum fees on its own. This would allow contracting applications to adjust their fees at +broadcast time instead of overshooting or risking becoming stuck or pinned. + +*Rationale*: It would be incorrect to use the fees of transactions that are already in the mempool, as +we do not want a transaction's fees to be double-counted. + +Implementation Note: Transactions within a package are always validated individually first, and +package validation is used for the transactions that failed. Since package feerate is only +calculated using transactions that are not in the mempool, this implementation detail affects the +outcome of package validation. + +*Rationale*: Packages are intended for incentive-compatible fee-bumping: transaction B is a +"legitimate" fee-bump for transaction A only if B is a descendant of A and has a *higher* feerate +than A. We want to prevent "parents pay for children" behavior; fees of parents should not help +their children, since the parents can be mined without the child. More generally, if transaction A +is not needed in order for transaction B to be mined, A's fees cannot help B. In a +child-with-parents package, simply excluding any parent transactions that meet feerate requirements +individually is sufficient to ensure this. + +*Rationale*: We must not allow a low-feerate child to prevent its parent from being accepted; fees +of children should not negatively impact their parents, since they are not necessary for the parents +to be mined. More generally, if transaction B is not needed in order for transaction A to be mined, +B's fees cannot harm A. In a child-with-parents package, simply validating parents individually +first is sufficient to ensure this. + +*Rationale*: As a principle, we want to avoid accidentally restricting policy in order to be +backward-compatible for users and applications that rely on p2p transaction relay. Concretely, +package validation should not prevent the acceptance of a transaction that would otherwise be +policy-valid on its own. By always accepting a transaction that passes individual validation before +trying package validation, we prevent any unintentional restriction of policy. diff --git a/doc/productivity.md b/doc/productivity.md index a01c6f545db..e9b7bc270c6 100644 --- a/doc/productivity.md +++ b/doc/productivity.md @@ -9,6 +9,7 @@ Table of Contents * [Disable features with `./configure`](#disable-features-with-configure) * [Make use of your threads with `make -j`](#make-use-of-your-threads-with-make--j) * [Only build what you need](#only-build-what-you-need) + * [Compile on multiple machines](#compile-on-multiple-machines) * [Multiple working directories with `git worktrees`](#multiple-working-directories-with-git-worktrees) * [Interactive "dummy rebases" for fixups and execs with `git merge-base`](#interactive-dummy-rebases-for-fixups-and-execs-with-git-merge-base) * [Writing code](#writing-code) @@ -81,6 +82,10 @@ make -C src bitcoin_bench (You can and should combine this with `-j`, as above, for a parallel build.) +### Compile on multiple machines + +If you have more than one computer at your disposal, you can use [distcc](https://www.distcc.org) to speed up compilation. This is easiest when all computers run the same operating system and compiler version. + ### Multiple working directories with `git worktrees` If you work with multiple branches or multiple copies of the repository, you should try `git worktrees`. diff --git a/doc/reduce-memory.md b/doc/reduce-memory.md index 296b172bdea..25205258b84 100644 --- a/doc/reduce-memory.md +++ b/doc/reduce-memory.md @@ -16,11 +16,11 @@ The size of some in-memory caches can be reduced. As caches trade off memory usa - The minimum value for `-maxmempool` is 5. - A lower maximum mempool size means that transactions will be evicted sooner. This will affect any uses of `bitcoind` that process unconfirmed transactions. -- To completely disable mempool functionality there is the option `-blocksonly`. This will make the client opt out of receiving (and thus relaying) transactions completely, except as part of blocks. +- Since `0.14.0`, unused memory allocated to the mempool (default: 300MB) is shared with the UTXO cache, so when trying to reduce memory usage you should limit the mempool, with the `-maxmempool` command line argument. - - Do not use this when using the client to broadcast transactions as any transaction sent will stick out like a sore thumb, affecting privacy. When used with the wallet it should be combined with `-walletbroadcast=0` and `-spendzeroconfchange=0`. Another mechanism for broadcasting outgoing transactions (if any) should be used. +- To disable most of the mempool functionality there is the `-blocksonly` option. This will reduce the default memory usage to 5MB and make the client opt out of receiving (and thus relaying) transactions, except from peers who have the `relay` permission set (e.g. whitelisted peers), and as part of blocks. -- Since `0.14.0`, unused memory allocated to the mempool (default: 300MB) is shared with the UTXO cache, so when trying to reduce memory usage you should limit the mempool, with the `-maxmempool` command line argument. + - Do not use this when using the client to broadcast transactions as any transaction sent will stick out like a sore thumb, affecting privacy. When used with the wallet it should be combined with `-walletbroadcast=0` and `-spendzeroconfchange=0`. Another mechanism for broadcasting outgoing transactions (if any) should be used. ## Number of peers diff --git a/doc/release-notes.md b/doc/release-notes-empty-template.md similarity index 77% rename from doc/release-notes.md rename to doc/release-notes-empty-template.md index 2342342ae20..4cd2314308d 100644 --- a/doc/release-notes.md +++ b/doc/release-notes-empty-template.md @@ -1,17 +1,7 @@ -*After branching off for a major version release of Bitcoin Core, use this -template to create the initial release notes draft.* - *The release notes draft is a temporary file that can be added to by anyone. See [/doc/developer-notes.md#release-notes](/doc/developer-notes.md#release-notes) for the process.* -*Create the draft, named* "*version* Release Notes Draft" -*(e.g. "23.0 Release Notes Draft"), as a collaborative wiki in:* - -https://github.com/bitcoin-core/bitcoin-devwiki/wiki/ - -*Before the final release, move the notes back to this git repository.* - *version* Release Notes Draft =============================== @@ -35,7 +25,7 @@ How to Upgrade If you are running an older version, shut it down. Wait until it has completely shut down (which might take a few minutes in some cases), then run the -installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) or `bitcoind`/`bitcoin-qt` (on Linux). Upgrading directly from a version of Bitcoin Core that has reached its EOL is @@ -54,9 +44,51 @@ unsupported systems. Notable changes =============== -Example item +P2P and network changes +----------------------- + +Updated RPCs +------------ + + +Changes to wallet related RPCs can be found in the Wallet section below. + +New RPCs +-------- + +Build System +------------ + +Updated settings +---------------- + + +Changes to GUI or wallet related settings can be found in the GUI or Wallet section below. + +New settings ------------ +Tools and Utilities +------------------- + +Wallet +------ + +GUI changes +----------- + +Low-level changes +================= + +RPC +--- + +Tests +----- + +*version* change log +==================== + Credits ======= diff --git a/doc/release-notes/release-notes-0.20.2.md b/doc/release-notes/release-notes-0.20.2.md new file mode 100644 index 00000000000..ad001bc9c12 --- /dev/null +++ b/doc/release-notes/release-notes-0.20.2.md @@ -0,0 +1,165 @@ +0.20.2 Release Notes +==================== + +Bitcoin Core version 0.20.2 is now available from: + + + +This minor release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.12+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 0.20.0 onwards, macOS versions earlier than 10.12 are no +longer supported. Additionally, Bitcoin Core does not yet change appearance +when macOS "dark mode" is activated. + +Known Bugs +========== + +The process for generating the source code release ("tarball") has changed in an +effort to make it more complete, however, there are a few regressions in +this release: + +- The generated `configure` script is currently missing, and you will need to + install autotools and run `./autogen.sh` before you can run + `./configure`. This is the same as when checking out from git. + +- Instead of running `make` simply, you should instead run + `BITCOIN_GENBUILD_NO_GIT=1 make`. + +Notable changes +=============== + +Changes regarding misbehaving peers +----------------------------------- + +Peers that misbehave (e.g. send us invalid blocks) are now referred to as +discouraged nodes in log output, as they're not (and weren't) strictly banned: +incoming connections are still allowed from them, but they're preferred for +eviction. + +Furthermore, a few additional changes are introduced to how discouraged +addresses are treated: + +- Discouraging an address does not time out automatically after 24 hours + (or the `-bantime` setting). Depending on traffic from other peers, + discouragement may time out at an indeterminate time. + +- Discouragement is not persisted over restarts. + +- There is no method to list discouraged addresses. They are not returned by + the `listbanned` RPC. That RPC also no longer reports the `ban_reason` + field, as `"manually added"` is the only remaining option. + +- Discouragement cannot be removed with the `setban remove` RPC command. + If you need to remove a discouragement, you can remove all discouragements by + stop-starting your node. + +Notification changes +-------------------- + +`-walletnotify` notifications are now sent for wallet transactions that are +removed from the mempool because they conflict with a new block. These +notifications were sent previously before the v0.19 release, but had been +broken since that release (bug +[#18325](https://github.com/bitcoin/bitcoin/issues/18325)). + +PSBT changes +------------ + +PSBTs will contain both the non-witness utxo and the witness utxo for segwit +inputs in order to restore compatibility with wallet software that are now +requiring the full previous transaction for segwit inputs. The witness utxo +is still provided to maintain compatibility with software which relied on its +existence to determine whether an input was segwit. + +0.20.2 change log +================= + +### P2P protocol and network code + +- #19620 Add txids with non-standard inputs to reject filter (sdaftuar) +- #20146 Send post-verack handshake messages at most once (MarcoFalke) + +### Wallet + +- #19740 Simplify and fix CWallet::SignTransaction (achow101) + +### RPC and other APIs + +- #19836 Properly deserialize txs with witness before signing (MarcoFalke) +- #20731 Add missing description of vout in getrawtransaction help text (benthecarman) + +### Build system + +- #20142 build: set minimum required Boost to 1.48.0 (fanquake) +- #20298 use the new plistlib API (jonasschnelli) +- #20880 gitian: Use custom MacOS code signing tool (achow101) +- #22190 Use latest signapple commit (achow101) + +### Tests and QA + +- #19839 Set appveyor vm version to previous Visual Studio 2019 release. (sipsorcery) +- #19842 Update the vcpkg checkout commit ID in appveyor config. (sipsorcery) +- #20562 Test that a fully signed tx given to signrawtx is unchanged (achow101) + +### Miscellaneous + +- #19192 Extract net permissions doc (MarcoFalke) +- #19777 Correct description for getblockstats's txs field (shesek) +- #20080 Strip any trailing / in -datadir and -blocksdir paths (hebasto) +- #20082 fixes read buffer to use min rather than max (EthanHeilman) +- #20141 Avoid the use of abs64 in timedata (sipa) +- #20756 Add missing field (permissions) to the getpeerinfo help (amitiuttarwar) +- #20861 BIP 350: Implement Bech32m and use it for v1+ segwit addresses (sipa) +- #22124 Update translations after closing 0.20.x on Transifex (hebasto) +- #21471 fix bech32_encode calls in gen_key_io_test_vectors.py (sipa) +- #22837 mention bech32m/BIP350 in doc/descriptors.md (sipa) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Aaron Clauson +- Amiti Uttarwar +- Andrew Chow +- Ethan Heilman +- fanquake +- Hennadii Stepanov +- Jonas Schnelli +- MarcoFalke +- Nadav Ivgi +- Pieter Wuille +- Suhas Daftuar + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-0.21.2.md b/doc/release-notes/release-notes-0.21.2.md new file mode 100644 index 00000000000..3b33c48a269 --- /dev/null +++ b/doc/release-notes/release-notes-0.21.2.md @@ -0,0 +1,109 @@ +0.21.2 Release Notes +==================== + +Bitcoin Core version 0.21.2 is now available from: + + + +This minor release includes various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.12+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 0.20.0 onwards, macOS versions earlier than 10.12 are no +longer supported. Additionally, Bitcoin Core does not yet change appearance +when macOS "dark mode" is activated. + + +0.21.2 change log +================= + +### P2P protocol and network code + +- #21644 use NetPermissions::HasFlag() in CConnman::Bind() (jonatack) +- #22569 Rate limit the processing of rumoured addresses (sipa) + +### Wallet + +- #21907 Do not iterate a directory if having an error while accessing it (hebasto) + +### RPC + +- #19361 Reset scantxoutset progress before inferring descriptors (prusnak) + +### Build System + +- #21932 depends: update Qt 5.9 source url (kittywhiskers) +- #22017 Update Windows code signing certificate (achow101) +- #22191 Use custom MacOS code signing tool (achow101) +- #22713 Fix build with Boost 1.77.0 (sizeofvoid) + +### Tests and QA + +- #20182 Build with --enable-werror by default, and document exceptions (hebasto) +- #20535 Fix intermittent feature_taproot issue (MarcoFalke) +- #21663 Fix macOS brew install command (hebasto) +- #22279 add missing ECCVerifyHandle to base_encode_decode (apoelstra) +- #22730 Run fuzzer task for the master branch only (hebasto) + +### GUI + +- #277 Do not use QClipboard::Selection on Windows and macOS. (hebasto) +- #280 Remove user input from URI error message (prayank23) +- #365 Draw "eye" sign at the beginning of watch-only addresses (hebasto) + +### Miscellaneous + +- #22002 Fix crash when parsing command line with -noincludeconf=0 (MarcoFalke) +- #22137 util: Properly handle -noincludeconf on command line (take 2) (MarcoFalke) + + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Andrew Chow +- Andrew Poelstra +- fanquake +- Hennadii Stepanov +- Jon Atack +- Kittywhiskers Van Gogh +- Luke Dashjr +- MarcoFalke +- Pavol Rusnak +- Pieter Wuille +- prayank23 +- Rafael Sadowski +- W. J. van der Laan + + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-22.1.md b/doc/release-notes/release-notes-22.1.md new file mode 100644 index 00000000000..d304b7e57ae --- /dev/null +++ b/doc/release-notes/release-notes-22.1.md @@ -0,0 +1,128 @@ +22.1 Release Notes +================== + +Bitcoin Core version 22.1 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.14+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +From Bitcoin Core 22.0 onwards, macOS versions earlier than 10.14 are no longer supported. + +Notable changes +=============== + +Updated settings +---------------- + +- In previous releases, the meaning of the command line option + `-persistmempool` (without a value provided) incorrectly disabled mempool + persistence. `-persistmempool` is now treated like other boolean options to + mean `-persistmempool=1`. Passing `-persistmempool=0`, `-persistmempool=1` + and `-nopersistmempool` is unaffected. (#23061) + +### P2P + +### RPC and other APIs + +- #25237 rpc: Capture UniValue by ref for rpcdoccheck +- #25983 Prevent data race for pathHandlers +- #26275 Fix crash on deriveaddresses when index is 2147483647 (2^31-1) + +### Wallet + +- #22781 wallet: fix the behavior of IsHDEnabled +- #22949 fee: Round up fee calculation to avoid a lower than expected feerate +- #23333 wallet: fix segfault by avoiding invalid default-ctored external_spk_managers entry + +### Build system + +- #22820 build, qt: Fix typo in QtInputSupport check +- #23045 build: Restrict check for CRC32C intrinsic to aarch64 +- #23148 build: Fix guix linker-loader path and add check_ELF_interpreter +- #23314 build: explicitly disable libsecp256k1 openssl based tests +- #23580 build: patch qt to explicitly define previously implicit header include +- #24215 guix: ignore additional failing certvalidator test +- #24256 build: Bump depends packages (zmq, libXau) +- #25201 windeploy: Renewed windows code signing certificate +- #25985 Revert "build: Use Homebrew's sqlite package if it is available" +- #26633 depends: update qt 5.12 url to archive location + +### GUI + +- #gui631 Disallow encryption of watchonly wallets +- #gui680 Fixes MacOS 13 segfault by preventing certain notifications +- #24498 qt: Avoid crash on startup if int specified in settings.json + +### Tests + +- #23716 test: replace hashlib.ripemd160 with an own implementation +- #24239 test: fix ceildiv division by using integers + +### Utilities + +- #22390 system: skip trying to set the locale on NetBSD +- #22895 don't call GetBlockPos in ReadBlockFromDisk without cs_main lock +- #24104 fs: Make compatible with boost 1.78 + +### Miscellaneous + +- #23335 refactor: include a missing header in fs.cpp +- #23504 ci: Replace soon EOL hirsute with jammy +- #26321 Adjust .tx/config for new Transifex CLI + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Andrew Chow +- BlackcoinDev +- Carl Dong +- Hennadii Stepanov +- Joan Karadimov +- John Moffett +- Jon Atack +- Kittywhiskers Van Gogh +- Marco Falke +- Martin Zumsande +- Michael Ford +- muxator +- Pieter Wuille +- Ryan Ofsky +- Saibato +- Sebastian Falbesoner +- W. J. van der Laan + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-23.0.md b/doc/release-notes/release-notes-23.0.md new file mode 100644 index 00000000000..b1467a0f711 --- /dev/null +++ b/doc/release-notes/release-notes-23.0.md @@ -0,0 +1,373 @@ +23.0 Release Notes +================== + +Bitcoin Core version 23.0 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on Mac) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notable changes +=============== + +P2P and network changes +----------------------- + +- A bitcoind node will no longer rumour addresses to inbound peers by default. + They will become eligible for address gossip after sending an ADDR, ADDRV2, + or GETADDR message. (#21528) + +- Before this release, Bitcoin Core had a strong preference to try to connect only to peers that listen on port 8333. As a result of that, Bitcoin nodes listening on non-standard ports would likely not get any Bitcoin Core peers connecting to them. This preference has been removed. (#23542) + +- Full support has been added for the CJDNS network. See the new option `-cjdnsreachable` and [doc/cjdns.md](https://github.com/bitcoin/bitcoin/tree/23.x/doc/cjdns.md) (#23077) + +Fee estimation changes +---------------------- + +- Fee estimation now takes the feerate of replacement (RBF) transactions into + account. (#22539) + +Rescan startup parameter removed +-------------------------------- + +The `-rescan` startup parameter has been removed. Wallets which require +rescanning due to corruption will still be rescanned on startup. +Otherwise, please use the `rescanblockchain` RPC to trigger a rescan. (#23123) + +Tracepoints and Userspace, Statically Defined Tracing support +------------------------------------------------------------- + +Bitcoin Core release binaries for Linux now include experimental tracepoints which +act as an interface for process-internal events. These can be used for review, +debugging, monitoring, and more. The tracepoint API is semi-stable. While the API +is tested, process internals might change between releases requiring changes to the +tracepoints. Information about the existing tracepoints can be found under +[doc/tracing.md](https://github.com/bitcoin/bitcoin/blob/23.x/doc/tracing.md) and +usage examples are provided in [contrib/tracing/](https://github.com/bitcoin/bitcoin/tree/23.x/contrib/tracing). + +Updated RPCs +------------ + +- The `validateaddress` RPC now returns an `error_locations` array for invalid + addresses, with the indices of invalid character locations in the address (if + known). For example, this will attempt to locate up to two Bech32 errors, and + return their locations if successful. Success and correctness are only guaranteed + if fewer than two substitution errors have been made. + The error message returned in the `error` field now also returns more specific + errors when decoding fails. (#16807) + +- The `-deprecatedrpc=addresses` configuration option has been removed. RPCs + `gettxout`, `getrawtransaction`, `decoderawtransaction`, `decodescript`, + `gettransaction verbose=true` and REST endpoints `/rest/tx`, `/rest/getutxos`, + `/rest/block` no longer return the `addresses` and `reqSigs` fields, which + were previously deprecated in 22.0. (#22650) +- The `getblock` RPC command now supports verbosity level 3 containing transaction inputs' + `prevout` information. The existing `/rest/block/` REST endpoint is modified to contain + this information too. Every `vin` field will contain an additional `prevout` subfield + describing the spent output. `prevout` contains the following keys: + - `generated` - true if the spent coins was a coinbase. + - `height` + - `value` + - `scriptPubKey` + +- The top-level fee fields `fee`, `modifiedfee`, `ancestorfees` and `descendantfees` + returned by RPCs `getmempoolentry`,`getrawmempool(verbose=true)`, + `getmempoolancestors(verbose=true)` and `getmempooldescendants(verbose=true)` + are deprecated and will be removed in the next major version (use + `-deprecated=fees` if needed in this version). The same fee fields can be accessed + through the `fees` object in the result. WARNING: deprecated + fields `ancestorfees` and `descendantfees` are denominated in sats, whereas all + fields in the `fees` object are denominated in BTC. (#22689) + +- Both `createmultisig` and `addmultisigaddress` now include a `warnings` + field, which will show a warning if a non-legacy address type is requested + when using uncompressed public keys. (#23113) + +Changes to wallet related RPCs can be found in the Wallet section below. + +New RPCs +-------- + +- Information on soft fork status has been moved from `getblockchaininfo` + to the new `getdeploymentinfo` RPC which allows querying soft fork status at any + block, rather than just at the chain tip. Inclusion of soft fork + status in `getblockchaininfo` can currently be restored using the + configuration `-deprecatedrpc=softforks`, but this will be removed in + a future release. Note that in either case, the `status` field + now reflects the status of the current block rather than the next + block. (#23508) + +Files +----- + +* On startup, the list of banned hosts and networks (via `setban` RPC) in + `banlist.dat` is ignored and only `banlist.json` is considered. Bitcoin Core + version 22.x is the only version that can read `banlist.dat` and also write + it to `banlist.json`. If `banlist.json` already exists, version 22.x will not + try to translate the `banlist.dat` into json. After an upgrade, `listbanned` + can be used to double check the parsed entries. (#22570) + +Updated settings +---------------- + +- In previous releases, the meaning of the command line option + `-persistmempool` (without a value provided) incorrectly disabled mempool + persistence. `-persistmempool` is now treated like other boolean options to + mean `-persistmempool=1`. Passing `-persistmempool=0`, `-persistmempool=1` + and `-nopersistmempool` is unaffected. (#23061) + +- `-maxuploadtarget` now allows human readable byte units [k|K|m|M|g|G|t|T]. + E.g. `-maxuploadtarget=500g`. No whitespace, +- or fractions allowed. + Default is `M` if no suffix provided. (#23249) + +- If `-proxy=` is given together with `-noonion` then the provided proxy will + not be set as a proxy for reaching the Tor network. So it will not be + possible to open manual connections to the Tor network for example with the + `addnode` RPC. To mimic the old behavior use `-proxy=` together with + `-onlynet=` listing all relevant networks except `onion`. (#22834) + +Tools and Utilities +------------------- + +- Update `-getinfo` to return data in a user-friendly format that also reduces vertical space. (#21832) + +- CLI `-addrinfo` now returns a single field for the number of `onion` addresses + known to the node instead of separate `torv2` and `torv3` fields, as support + for Tor V2 addresses was removed from Bitcoin Core in 22.0. (#22544) + +Wallet +------ + +- Descriptor wallets are now the default wallet type. Newly created wallets + will use descriptors unless `descriptors=false` is set during `createwallet`, or + the `Descriptor wallet` checkbox is unchecked in the GUI. + + Note that wallet RPC commands like `importmulti` and `dumpprivkey` cannot be + used with descriptor wallets, so if your client code relies on these commands + without specifying `descriptors=false` during wallet creation, you will need + to update your code. + +- Newly created descriptor wallets will contain an automatically generated `tr()` + descriptor which allows for creating single key Taproot receiving addresses. + +- `upgradewallet` will now automatically flush the keypool if upgrading + from a non-HD wallet to an HD wallet, to immediately start using the + newly-generated HD keys. (#23093) + +- a new RPC `newkeypool` has been added, which will flush (entirely + clear and refill) the keypool. (#23093) + +- `listunspent` now includes `ancestorcount`, `ancestorsize`, and + `ancestorfees` for each transaction output that is still in the mempool. + (#12677) + +- `lockunspent` now optionally takes a third parameter, `persistent`, which + causes the lock to be written persistently to the wallet database. This + allows UTXOs to remain locked even after node restarts or crashes. (#23065) + +- `receivedby` RPCs now include coinbase transactions. Previously, the + following wallet RPCs excluded coinbase transactions: `getreceivedbyaddress`, + `getreceivedbylabel`, `listreceivedbyaddress`, `listreceivedbylabel`. This + release changes this behaviour and returns results accounting for received + coins from coinbase outputs. The previous behaviour can be restored using the + configuration `-deprecatedrpc=exclude_coinbase`, but may be removed in a + future release. (#14707) + +- A new option in the same `receivedby` RPCs, `include_immature_coinbase` + (default=`false`), determines whether to account for immature coinbase + transactions. Immature coinbase transactions are coinbase transactions that + have 100 or fewer confirmations, and are not spendable. (#14707) + +GUI changes +----------- + +- UTXOs which are locked via the GUI are now stored persistently in the + wallet database, so are not lost on node shutdown or crash. (#23065) + +- The Bech32 checkbox has been replaced with a dropdown for all address types, including the new Bech32m (BIP-350) standard for Taproot enabled wallets. + +Low-level changes +================= + +RPC +--- + +- `getblockchaininfo` now returns a new `time` field, that provides the chain tip time. (#22407) + +Tests +----- + +- For the `regtest` network the activation heights of several softforks were + set to block height 1. They can be changed by the runtime setting + `-testactivationheight=name@height`. (#22818) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- 0xb10c +- 0xree +- Aaron Clauson +- Adrian-Stefan Mares +- agroce +- aitorjs +- Alex Groce +- amadeuszpawlik +- Amiti Uttarwar +- Andrew Chow +- Andrew Poelstra +- Andrew Toth +- anouar kappitou +- Anthony Towns +- Antoine Poinsot +- Arnab Sen +- Ben Woosley +- benthecarman +- Bitcoin Hodler +- BitcoinTsunami +- brianddk +- Bruno Garcia +- CallMeMisterOwl +- Calvin Kim +- Carl Dong +- Cory Fields +- Cuong V. Nguyen +- Darius Parvin +- Dhruv Mehta +- Dimitri Deijs +- Dimitris Apostolou +- Dmitry Goncharov +- Douglas Chimento +- eugene +- Fabian Jahr +- fanquake +- Florian Baumgartl +- fyquah +- Gleb Naumenko +- glozow +- Gregory Sanders +- Heebs +- Hennadii Stepanov +- hg333 +- HiLivin +- Igor Cota +- Jadi +- James O'Beirne +- Jameson Lopp +- Jarol Rodriguez +- Jeremy Rand +- Jeremy Rubin +- Joan Karadimov +- John Newbery +- Jon Atack +- João Barbosa +- josibake +- junderw +- Karl-Johan Alm +- katesalazar +- Kennan Mell +- Kiminuo +- Kittywhiskers Van Gogh +- Klement Tan +- Kristaps Kaupe +- Kuro +- Larry Ruane +- lsilva01 +- lucash-dev +- Luke Dashjr +- MarcoFalke +- Martin Leitner-Ankerl +- Martin Zumsande +- Matt Corallo +- Matt Whitlock +- MeshCollider +- Michael Dietz +- Murch +- naiza +- Nathan Garabedian +- Nelson Galdeman +- NikhilBartwal +- Niklas Gögge +- node01 +- nthumann +- Pasta +- Patrick Kamin +- Pavel Safronov +- Pavol Rusnak +- Perlover +- Pieter Wuille +- practicalswift +- pradumnasaraf +- pranabp-bit +- Prateek Sancheti +- Prayank +- Rafael Sadowski +- rajarshimaitra +- randymcmillan +- ritickgoenka +- Rob Fielding +- Rojar Smith +- Russell Yanofsky +- S3RK +- Saibato +- Samuel Dobson +- sanket1729 +- seaona +- Sebastian Falbesoner +- sh15h4nk +- Shashwat +- Shorya +- ShubhamPalriwala +- Shubhankar Gambhir +- Sjors Provoost +- sogoagain +- sstone +- stratospher +- Suriyaa Rocky Sundararuban +- Taeik Lim +- TheCharlatan +- Tim Ruffing +- Tobin Harding +- Troy Giorshev +- Tyler Chambers +- Vasil Dimov +- W. J. van der Laan +- w0xlt +- willcl-ark +- William Casarin +- zealsham +- Zero-1729 + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-23.1.md b/doc/release-notes/release-notes-23.1.md new file mode 100644 index 00000000000..31d9b7f068d --- /dev/null +++ b/doc/release-notes/release-notes-23.1.md @@ -0,0 +1,90 @@ +23.1 Release Notes +================== + +Bitcoin Core version 23.1 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +### P2P + +- #25314 p2p: always set nTime for self-advertisements + +### RPC and other APIs + +- #25220 rpc: fix incorrect warning for address type p2sh-segwit in createmultisig +- #25237 rpc: Capture UniValue by ref for rpcdoccheck +- #25983 Prevent data race for pathHandlers +- #26275 Fix crash on deriveaddresses when index is 2147483647 (2^31-1) + +### Build system + +- #25201 windeploy: Renewed windows code signing certificate +- #25788 guix: patch NSIS to remove .reloc sections from installer stubs +- #25861 guix: use --build={arch}-guix-linux-gnu in cross toolchain +- #25985 Revert "build: Use Homebrew's sqlite package if it is available" + +### GUI + +- #24668 build, qt: bump Qt5 version to 5.15.3 +- gui#631 Disallow encryption of watchonly wallets +- gui#680 Fixes MacOS 13 segfault by preventing certain notifications + +### Tests + +- #24454 tests: Fix calculation of external input weights + +### Miscellaneous + +- #26321 Adjust .tx/config for new Transifex CLI + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- Andrew Chow +- brunoerg +- Hennadii Stepanov +- John Moffett +- MacroFake +- Martin Zumsande +- Michael Ford +- muxator +- Pavol Rusnak +- Sebastian Falbesoner +- W. J. van der Laan + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-24.0.1.md b/doc/release-notes/release-notes-24.0.1.md new file mode 100644 index 00000000000..24920ba4507 --- /dev/null +++ b/doc/release-notes/release-notes-24.0.1.md @@ -0,0 +1,391 @@ +24.0.1 Release Notes +==================== + +Due to last-minute issues (#26616), 24.0, although tagged, was never fully +announced or released. + +Bitcoin Core version 24.0.1 is now available from: + + + +This release includes new features, various bug fixes and performance +improvements, as well as updated translations. + +Please report bugs using the issue tracker at GitHub: + + + +To receive security and update notifications, please subscribe to: + + + +How to Upgrade +============== + +If you are running an older version, shut it down. Wait until it has completely +shut down (which might take a few minutes in some cases), then run the +installer (on Windows) or just copy over `/Applications/Bitcoin-Qt` (on macOS) +or `bitcoind`/`bitcoin-qt` (on Linux). + +Upgrading directly from a version of Bitcoin Core that has reached its EOL is +possible, but it might take some time if the data directory needs to be migrated. Old +wallet versions of Bitcoin Core are generally supported. + +Compatibility +============== + +Bitcoin Core is supported and extensively tested on operating systems +using the Linux kernel, macOS 10.15+, and Windows 7 and newer. Bitcoin +Core should also work on most other Unix-like systems but is not as +frequently tested on them. It is not recommended to use Bitcoin Core on +unsupported systems. + +Notice of new option for transaction replacement policies +========================================================= + +This version of Bitcoin Core adds a new `mempoolfullrbf` configuration +option which allows users to change the policy their individual node +will use for relaying and mining unconfirmed transactions. The option +defaults to the same policy that was used in previous releases and no +changes to node policy will occur if everyone uses the default. + +Some Bitcoin services today expect that the first version of an +unconfirmed transaction that they see will be the version of the +transaction that ultimately gets confirmed---a transaction acceptance +policy sometimes called "first-seen". + +The Bitcoin Protocol does not, and cannot, provide any assurance that +the first version of an unconfirmed transaction seen by a particular +node will be the version that gets confirmed. If there are multiple +versions of the same unconfirmed transaction available, only the miner +who includes one of those transactions in a block gets to decide which +version of the transaction gets confirmed. + +Despite this lack of assurance, multiple merchants and services today +still make this assumption. + +There are several benefits to users from removing this *first-seen* +simplification. One key benefit, the ability for the sender of a +transaction to replace it with an alternative version paying higher +fees, was realized in [Bitcoin Core 0.12.0][] (February 2016) with the +introduction of [BIP125][] opt-in Replace By Fee (RBF). + +Since then, there has been discussion about completely removing the +first-seen simplification and allowing users to replace any of their +older unconfirmed transactions with newer transactions, a feature called +*full-RBF*. This release includes a `mempoolfullrbf` configuration +option that allows enabling full-RBF, although it defaults to off +(allowing only opt-in RBF). + +Several alternative node implementations have already enabled full-RBF by +default for years, and several contributors to Bitcoin Core are +advocating for enabling full-RBF by default in a future version of +Bitcoin Core. + +As more nodes that participate in relay and mining begin enabling +full-RBF, replacement of unconfirmed transactions by ones offering higher +fees may rapidly become more reliable. + +Contributors to this project strongly recommend that merchants and services +not accept unconfirmed transactions as final, and if they insist on doing so, +to take the appropriate steps to ensure they have some recourse or plan for +when their assumptions do not hold. + +[Bitcoin Core 0.12.0]: https://bitcoincore.org/en/releases/0.12.0/#opt-in-replace-by-fee-transactions +[bip125]: https://github.com/bitcoin/bips/blob/master/bip-0125.mediawiki + +Notable changes +=============== + +P2P and network changes +----------------------- + +- To address a potential denial-of-service, the logic to download headers from peers + has been reworked. This is particularly relevant for nodes starting up for the + first time (or for nodes which are starting up after being offline for a long time). + + Whenever headers are received from a peer that have a total chainwork that is either + less than the node's `-minimumchainwork` value or is sufficiently below the work at + the node's tip, a "presync" phase will begin, in which the node will download the + peer's headers and verify the cumulative work on the peer's chain, prior to storing + those headers permanently. Once that cumulative work is verified to be sufficiently high, + the headers will be redownloaded from that peer and fully validated and stored. + + This may result in initial headers sync taking longer for new nodes starting up for + the first time, both because the headers will be downloaded twice, and because the effect + of a peer disconnecting during the presync phase (or while the node's best headers chain has less + than `-minimumchainwork`), will result in the node needing to use the headers presync mechanism + with the next peer as well (downloading the headers twice, again). (#25717) + +- With I2P connections, a new, transient address is used for each outbound + connection if `-i2pacceptincoming=0`. (#25355) + +Updated RPCs +------------ + +- The `-deprecatedrpc=softforks` configuration option has been removed. The + RPC `getblockchaininfo` no longer returns the `softforks` field, which was + previously deprecated in 23.0. (#23508) Information on soft fork status is + now only available via the `getdeploymentinfo` RPC. + +- The `deprecatedrpc=exclude_coinbase` configuration option has been removed. + The `receivedby` RPCs (`listreceivedbyaddress`, `listreceivedbylabel`, + `getreceivedbyaddress` and `getreceivedbylabel`) now always return results + accounting for received coins from coinbase outputs, without an option to + change that behaviour. Excluding coinbases was previously deprecated in 23.0. + (#25171) + +- The `deprecatedrpc=fees` configuration option has been removed. The top-level + fee fields `fee`, `modifiedfee`, `ancestorfees` and `descendantfees` are no + longer returned by RPCs `getmempoolentry`, `getrawmempool(verbose=true)`, + `getmempoolancestors(verbose=true)` and `getmempooldescendants(verbose=true)`. + The same fee fields can be accessed through the `fees` object in the result. + The top-level fee fields were previously deprecated in 23.0. (#25204) + +- The `getpeerinfo` RPC has been updated with a new `presynced_headers` field, + indicating the progress on the presync phase mentioned in the + "P2P and network changes" section above. + +Changes to wallet related RPCs can be found in the Wallet section below. + +New RPCs +-------- + +- The `sendall` RPC spends specific UTXOs to one or more recipients + without creating change. By default, the `sendall` RPC will spend + every UTXO in the wallet. `sendall` is useful to empty wallets or to + create a changeless payment from select UTXOs. When creating a payment + from a specific amount for which the recipient incurs the transaction + fee, continue to use the `subtractfeefromamount` option via the + `send`, `sendtoaddress`, or `sendmany` RPCs. (#24118) + +- A new `gettxspendingprevout` RPC has been added, which scans the mempool to find + transactions spending any of the given outpoints. (#24408) + +- The `simulaterawtransaction` RPC iterates over the inputs and outputs of the given + transactions, and tallies up the balance change for the given wallet. This can be + useful e.g. when verifying that a coin join like transaction doesn't contain unexpected + inputs that the wallet will then sign for unintentionally. (#22751) + +Updated REST APIs +----------------- + +- The `/headers/` and `/blockfilterheaders/` endpoints have been updated to use + a query parameter instead of path parameter to specify the result count. The + count parameter is now optional, and defaults to 5 for both endpoints. The old + endpoints are still functional, and have no documented behaviour change. + + For `/headers`, use + `GET /rest/headers/.?count=` + instead of + `GET /rest/headers//.` (deprecated) + + For `/blockfilterheaders/`, use + `GET /rest/blockfilterheaders//.?count=` + instead of + `GET /rest/blockfilterheaders///.` (deprecated) + + (#24098) + +Build System +------------ + +- Guix builds are now reproducible across architectures (x86_64 & aarch64). (#21194) + +New settings +------------ + +- A new `mempoolfullrbf` option has been added, which enables the mempool to + accept transaction replacement without enforcing BIP125 replaceability + signaling. (#25353) + +Wallet +------ + +- The `-walletrbf` startup option will now default to `true`. The + wallet will now default to opt-in RBF on transactions that it creates. (#25610) + +- The `replaceable` option for the `createrawtransaction` and + `createpsbt` RPCs will now default to `true`. Transactions created + with these RPCs will default to having opt-in RBF enabled. (#25610) + +- The `wsh()` output descriptor was extended with Miniscript support. You can import Miniscript + descriptors for P2WSH in a watchonly wallet to track coins, but you can't spend from them using + the Bitcoin Core wallet yet. + You can find more about Miniscript on the [reference website](https://bitcoin.sipa.be/miniscript/). (#24148) + +- The `tr()` output descriptor now supports multisig scripts through the `multi_a()` and + `sortedmulti_a()` functions. (#24043) + +- To help prevent fingerprinting transactions created by the Bitcoin Core wallet, change output + amounts are now randomized. (#24494) + +- The `listtransactions`, `gettransaction`, and `listsinceblock` + RPC methods now include a wtxid field (hash of serialized transaction, + including witness data) for each transaction. (#24198) + +- The `listsinceblock`, `listtransactions` and `gettransaction` output now contain a new + `parent_descs` field for every "receive" entry. (#25504) + +- A new optional `include_change` parameter was added to the `listsinceblock` command. + +- RPC `getreceivedbylabel` now returns an error, "Label not found + in wallet" (-4), if the label is not in the address book. (#25122) + +Migrating Legacy Wallets to Descriptor Wallets +--------------------------------------------- + +An experimental RPC `migratewallet` has been added to migrate Legacy (non-descriptor) wallets to +Descriptor wallets. More information about the migration process is available in the +[documentation](https://github.com/bitcoin/bitcoin/blob/master/doc/managing-wallets.md#migrating-legacy-wallets-to-descriptor-wallets). + +GUI changes +----------- + +- A new menu item to restore a wallet from a backup file has been added (gui#471). + +- Configuration changes made in the bitcoin GUI (such as the pruning setting, +proxy settings, UPNP preferences) are now saved to `/settings.json` +file rather than to the Qt settings backend (windows registry or unix desktop +config files), so these settings will now apply to bitcoind, instead of being +ignored. (#15936, gui#602) + +- Also, the interaction between GUI settings and `bitcoin.conf` settings is +simplified. Settings from `bitcoin.conf` are now displayed normally in the GUI +settings dialog, instead of in a separate warning message ("Options set in this +dialog are overridden by the configuration file: -setting=value"). And these +settings can now be edited because `settings.json` values take precedence over +`bitcoin.conf` values. (#15936) + +Low-level changes +================= + +RPC +--- + +- The `deriveaddresses`, `getdescriptorinfo`, `importdescriptors` and `scantxoutset` commands now + accept Miniscript expression within a `wsh()` descriptor. (#24148) + +- The `getaddressinfo`, `decodescript`, `listdescriptors` and `listunspent` commands may now output + a Miniscript descriptor inside a `wsh()` where a `wsh(raw())` descriptor was previously returned. (#24148) + +Credits +======= + +Thanks to everyone who directly contributed to this release: + +- /dev/fd0 +- 0xb10c +- Adam Jonas +- akankshakashyap +- Ali Sherief +- amadeuszpawlik +- Andreas Kouloumos +- Andrew Chow +- Anthony Towns +- Antoine Poinsot +- Antoine Riard +- Aurèle Oulès +- avirgovi +- Ayush Sharma +- Baas +- Ben Woosley +- BrokenProgrammer +- brunoerg +- brydinh +- Bushstar +- Calvin Kim +- CAnon +- Carl Dong +- chinggg +- Cory Fields +- Daniel Kraft +- Daniela Brozzoni +- darosior +- Dave Scotese +- David Bakin +- dergoegge +- dhruv +- Dimitri +- dontbyte +- Duncan Dean +- eugene +- Eunoia +- Fabian Jahr +- furszy +- Gleb Naumenko +- glozow +- Greg Weber +- Gregory Sanders +- gruve-p +- Hennadii Stepanov +- hiago +- Igor Bubelov +- ishaanam +- Jacob P. +- Jadi +- James O'Beirne +- Janna +- Jarol Rodriguez +- Jeremy Rand +- Jeremy Rubin +- jessebarton +- João Barbosa +- John Newbery +- Jon Atack +- Josiah Baker +- Karl-Johan Alm +- KevinMusgrave +- Kiminuo +- klementtan +- Kolby Moroz +- kouloumos +- Kristaps Kaupe +- Larry Ruane +- Luke Dashjr +- MarcoFalke +- Marnix +- Martin Leitner-Ankerl +- Martin Zumsande +- Michael Dietz +- Michael Folkson +- Michael Ford +- Murch +- mutatrum +- muxator +- Oskar Mendel +- Pablo Greco +- pasta +- Patrick Strateman +- Pavol Rusnak +- Peter Bushnell +- phyBrackets +- Pieter Wuille +- practicalswift +- randymcmillan +- Robert Spigler +- Russell Yanofsky +- S3RK +- Samer Afach +- Sebastian Falbesoner +- Seibart Nedor +- Shashwat +- Sjors Provoost +- Smlep +- sogoagain +- Stacie +- Stéphan Vuylsteke +- Suhail Saqan +- Suhas Daftuar +- t-bast +- TakeshiMusgrave +- Vasil Dimov +- W. J. van der Laan +- w0xlt +- whiteh0rse +- willcl-ark +- William Casarin +- Yancy Ribbens + +As well as to everyone that helped with translations on +[Transifex](https://www.transifex.com/bitcoin/bitcoin/). diff --git a/doc/release-notes/release-notes-24.0.md b/doc/release-notes/release-notes-24.0.md new file mode 100644 index 00000000000..a0227aa17fc --- /dev/null +++ b/doc/release-notes/release-notes-24.0.md @@ -0,0 +1,4 @@ +Due to last-minute issues (#26616), 24.0, although tagged, was never fully +announced or released. + +See the release notes for 24.0.1 instead. diff --git a/doc/release-process.md b/doc/release-process.md index 5a74f72b6e2..9c2e03d402b 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -8,6 +8,7 @@ Release Process * Update translations see [translation_process.md](https://github.com/bitcoin/bitcoin/blob/master/doc/translation_process.md#synchronising-translations). * Update release candidate version in `configure.ac` (`CLIENT_VERSION_RC`). * Update manpages (after rebuilding the binaries), see [gen-manpages.py](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-manpagespy). +* Update bitcoin.conf and commit, see [gen-bitcoin-conf.sh](https://github.com/bitcoin/bitcoin/blob/master/contrib/devtools/README.md#gen-bitcoin-confsh). ### Before every major and minor release @@ -28,32 +29,41 @@ Release Process #### Before branch-off * Update hardcoded [seeds](/contrib/seeds/README.md), see [this pull request](https://github.com/bitcoin/bitcoin/pull/7415) for an example. -* Update [`src/chainparams.cpp`](/src/chainparams.cpp) m_assumed_blockchain_size and m_assumed_chain_state_size with the current size plus some overhead (see [this](#how-to-calculate-assumed-blockchain-and-chain-state-size) for information on how to calculate them). -* Update [`src/chainparams.cpp`](/src/chainparams.cpp) chainTxData with statistics about the transaction count and rate. Use the output of the `getchaintxstats` RPC, see - [this pull request](https://github.com/bitcoin/bitcoin/pull/20263) for an example. Reviewers can verify the results by running `getchaintxstats ` with the `window_block_count` and `window_final_block_hash` from your output. -* Update `src/chainparams.cpp` nMinimumChainWork and defaultAssumeValid (and the block height comment) with information from the `getblockheader` (and `getblockhash`) RPCs. - - The selected value must not be orphaned so it may be useful to set the value two blocks back from the tip. - - Testnet should be set some tens of thousands back from the tip due to reorgs there. - - This update should be reviewed with a reindex-chainstate with assumevalid=0 to catch any defect - that causes rejection of blocks in the past history. +* Update the following variables in [`src/kernel/chainparams.cpp`](/src/kernel/chainparams.cpp) for mainnet, testnet, and signet: + - `m_assumed_blockchain_size` and `m_assumed_chain_state_size` with the current size plus some overhead (see + [this](#how-to-calculate-assumed-blockchain-and-chain-state-size) for information on how to calculate them). + - The following updates should be reviewed with `reindex-chainstate` and `assumevalid=0` to catch any defect + that causes rejection of blocks in the past history. + - `chainTxData` with statistics about the transaction count and rate. Use the output of the `getchaintxstats` RPC with an + `nBlocks` of 4096 (28 days) and a `bestblockhash` of RPC `getbestblockhash`; see + [this pull request](https://github.com/bitcoin/bitcoin/pull/20263) for an example. Reviewers can verify the results by running + `getchaintxstats ` with the `window_block_count` and `window_final_block_hash` from your output. + - `defaultAssumeValid` with the output of RPC `getblockhash` using the `height` of `window_final_block_height` above + (and update the block height comment with that height), taking into account the following: + - On mainnet, the selected value must not be orphaned, so it may be useful to set the height two blocks back from the tip. + - Testnet should be set with a height some tens of thousands back from the tip, due to reorgs there. + - `nMinimumChainWork` with the "chainwork" value of RPC `getblockheader` using the same height as that selected for the previous step. - Clear the release notes and move them to the wiki (see "Write the release notes" below). -- Translations on Transifex - - Create [a new resource](https://www.transifex.com/bitcoin/bitcoin/content/) named after the major version with the slug `[bitcoin.qt-translation-x]`, where `RRR` is the major branch number padded with zeros. Use `src/qt/locale/bitcoin_en.xlf` to create it. - - In the project workflow settings, ensure that [Translation Memory Fill-up](https://docs.transifex.com/translation-memory/enabling-autofill) is enabled and that [Translation Memory Context Matching](https://docs.transifex.com/translation-memory/translation-memory-with-context) is disabled. +- Translations on Transifex: + - Pull translations from Transifex into the master branch. + - Create [a new resource](https://www.transifex.com/bitcoin/bitcoin/content/) named after the major version with the slug `qt-translation-x`, where `RRR` is the major branch number padded with zeros. Use `src/qt/locale/bitcoin_en.xlf` to create it. + - In the project workflow settings, ensure that [Translation Memory Fill-up](https://help.transifex.com/en/articles/6224817-setting-up-translation-memory-fill-up) is enabled and that [Translation Memory Context Matching](https://help.transifex.com/en/articles/6224753-translation-memory-with-context) is disabled. - Update the Transifex slug in [`.tx/config`](/.tx/config) to the slug of the resource created in the first step. This identifies which resource the translations will be synchronized from. - - Make an announcement that translators can start translating for the new version. You can use one of the [previous announcements](https://www.transifex.com/bitcoin/bitcoin/announcements/) as a template. + - Make an announcement that translators can start translating for the new version. You can use one of the [previous announcements](https://www.transifex.com/bitcoin/communication/) as a template. - Change the auto-update URL for the resource to `master`, e.g. `https://raw.githubusercontent.com/bitcoin/bitcoin/master/src/qt/locale/bitcoin_en.xlf`. (Do this only after the previous steps, to prevent an auto-update from interfering.) #### After branch-off (on the major release branch) - Update the versions. +- Create the draft, named "*version* Release Notes Draft", as a [collaborative wiki](https://github.com/bitcoin-core/bitcoin-devwiki/wiki/_new). +- Clear the release notes: `cp doc/release-notes-empty-template.md doc/release-notes.md` - Create a pinned meta-issue for testing the release candidate (see [this issue](https://github.com/bitcoin/bitcoin/issues/17079) for an example) and provide a link to it in the release announcements where useful. - Translations on Transifex - Change the auto-update URL for the new major version's resource away from `master` and to the branch, e.g. `https://raw.githubusercontent.com/bitcoin/bitcoin//src/qt/locale/bitcoin_en.xlf`. Do not forget this or it will keep tracking the translations on master instead, drifting away from the specific major release. #### Before final release -- Merge the release notes from the wiki into the branch. +- Merge the release notes from [the wiki](https://github.com/bitcoin-core/bitcoin-devwiki/wiki/) into the branch. - Ensure the "Needs release note" label is removed from all relevant pull requests and issues. #### Tagging a release (candidate) @@ -88,7 +98,7 @@ Generate the change log. As this is a huge amount of work to do manually, there Generate list of authors: - git log --format='- %aN' v(current version, e.g. 0.20.0)..v(new version, e.g. 0.20.1) | sort -fiu + git log --format='- %aN' v(current version, e.g. 24.0)..v(new version, e.g. 24.1) | sort -fiu ### Setup and perform Guix builds @@ -97,7 +107,7 @@ Checkout the Bitcoin Core version you'd like to build: ```sh pushd ./bitcoin SIGNER='(your builder key, ie bluematt, sipa, etc)' -VERSION='(new version without v-prefix, e.g. 0.20.0)' +VERSION='(new version without v-prefix, e.g. 24.0)' git fetch origin "v${VERSION}" git checkout "v${VERSION}" popd @@ -110,28 +120,23 @@ against other `guix-attest` signatures. git -C ./guix.sigs pull ``` -### Create the macOS SDK tarball: (first time, or when SDK version changes) +### Create the macOS SDK tarball (first time, or when SDK version changes) Create the macOS SDK tarball, see the [macdeploy instructions](/contrib/macdeploy/README.md#deterministic-macos-dmg-notes) for details. -### Build and attest to build outputs: +### Build and attest to build outputs Follow the relevant Guix README.md sections: - [Building](/contrib/guix/README.md#building) - [Attesting to build outputs](/contrib/guix/README.md#attesting-to-build-outputs) -### Verify other builders' signatures to your own. (Optional) - -Add other builders keys to your gpg keyring, and/or refresh keys: See `../bitcoin/contrib/builder-keys/README.md`. +### Verify other builders' signatures to your own (optional) -Follow the relevant Guix README.md sections: - [Verifying build output attestations](/contrib/guix/README.md#verifying-build-output-attestations) -### Next steps: - -Commit your signature to guix.sigs: +### Commit your non codesigned signature to guix.sigs ```sh pushd ./guix.sigs @@ -141,29 +146,27 @@ git push # Assuming you can push to the guix.sigs tree popd ``` -Codesigner only: Create Windows/macOS detached signatures: -- Only one person handles codesigning. Everyone else should skip to the next step. -- Only once the Windows/macOS builds each have 3 matching signatures may they be signed with their respective release keys. +## Codesigning -Codesigner only: Sign the macOS binary: +### macOS codesigner only: Create detached macOS signatures (assuming [signapple](https://github.com/achow101/signapple/) is installed and up to date with master branch) - transfer bitcoin-osx-unsigned.tar.gz to macOS for signing tar xf bitcoin-osx-unsigned.tar.gz - ./detached-sig-create.sh -s "Key ID" + ./detached-sig-create.sh /path/to/codesign.p12 Enter the keychain password and authorize the signature - Move signature-osx.tar.gz back to the guix-build host + signature-osx.tar.gz will be created -Codesigner only: Sign the windows binaries: +### Windows codesigner only: Create detached Windows signatures tar xf bitcoin-win-unsigned.tar.gz ./detached-sig-create.sh -key /path/to/codesign.key Enter the passphrase for the key when prompted signature-win.tar.gz will be created -Code-signer only: It is advised to test that the code signature attaches properly prior to tagging by performing the `guix-codesign` step. +### Windows and macOS codesigners only: test code signatures +It is advised to test that the code signature attaches properly prior to tagging by performing the `guix-codesign` step. However if this is done, once the release has been tagged in the bitcoin-detached-sigs repo, the `guix-codesign` step must be performed again in order for the guix attestation to be valid when compared against the attestations of non-codesigner builds. -Codesigner only: Commit the detached codesign payloads: +### Windows and macOS codesigners only: Commit the detached codesign payloads ```sh pushd ./bitcoin-detached-sigs @@ -178,16 +181,20 @@ git push the current branch and new tag popd ``` -Non-codesigners: wait for Windows/macOS detached signatures: +### Non-codesigners: wait for Windows and macOS detached signatures -- Once the Windows/macOS builds each have 3 matching signatures, they will be signed with their respective release keys. +- Once the Windows and macOS builds each have 3 matching signatures, they will be signed with their respective release keys. - Detached signatures will then be committed to the [bitcoin-detached-sigs](https://github.com/bitcoin-core/bitcoin-detached-sigs) repository, which can be combined with the unsigned apps to create signed binaries. -Create (and optionally verify) the codesigned outputs: +### Create the codesigned build outputs -- [Codesigning](/contrib/guix/README.md#codesigning) +- [Codesigning build outputs](/contrib/guix/README.md#codesigning-build-outputs) -Commit your signature for the signed macOS/Windows binaries: +### Verify other builders' signatures to your own (optional) + +- [Verifying build output attestations](/contrib/guix/README.md#verifying-build-output-attestations) + +### Commit your codesigned signature to guix.sigs (for the signed macOS/Windows binaries) ```sh pushd ./guix.sigs @@ -197,7 +204,7 @@ git push # Assuming you can push to the guix.sigs tree popd ``` -### After 3 or more people have guix-built and their results match: +## After 3 or more people have guix-built and their results match Combine the `all.SHA256SUMS.asc` file from all signers into `SHA256SUMS.asc`: @@ -300,15 +307,16 @@ cat "$VERSION"/*/all.SHA256SUMS.asc > SHA256SUMS.asc Both variables are used as a guideline for how much space the user needs on their drive in total, not just strictly for the blockchain. Note that all values should be taken from a **fully synced** node and have an overhead of 5-10% added on top of its base value. -To calculate `m_assumed_blockchain_size`: -- For `mainnet` -> Take the size of the data directory, excluding `/regtest` and `/testnet3` directories. -- For `testnet` -> Take the size of the `/testnet3` directory. - +To calculate `m_assumed_blockchain_size`, take the size in GiB of these directories: +- For `mainnet` -> the data directory, excluding the `/testnet3`, `/signet`, and `/regtest` directories and any overly large files, e.g. a huge `debug.log` +- For `testnet` -> `/testnet3` +- For `signet` -> `/signet` -To calculate `m_assumed_chain_state_size`: -- For `mainnet` -> Take the size of the `/chainstate` directory. -- For `testnet` -> Take the size of the `/testnet3/chainstate` directory. +To calculate `m_assumed_chain_state_size`, take the size in GiB of these directories: +- For `mainnet` -> `/chainstate` +- For `testnet` -> `/testnet3/chainstate` +- For `signet` -> `/signet/chainstate` Notes: - When taking the size for `m_assumed_blockchain_size`, there's no need to exclude the `/chainstate` directory since it's a guideline value and an overhead will be added anyway. -- The expected overhead for growth may change over time, so it may not be the same value as last release; pay attention to that when changing the variables. +- The expected overhead for growth may change over time. Consider whether the percentage needs to be changed in response; if so, update it here in this section. diff --git a/doc/tor.md b/doc/tor.md index 086e6747bfd..581d124f7aa 100644 --- a/doc/tor.md +++ b/doc/tor.md @@ -16,9 +16,9 @@ configure Tor. ## How to see information about your Tor configuration via Bitcoin Core There are several ways to see your local onion address in Bitcoin Core: -- in the debug log (grep for "tor:" or "AddLocal") -- in the output of RPC `getnetworkinfo` in the "localaddresses" section -- in the output of the CLI `-netinfo` peer connections dashboard +- in the "Local addresses" output of CLI `-netinfo` +- in the "localaddresses" output of RPC `getnetworkinfo` +- in the debug log (grep for "AddLocal"; the Tor address ends in `.onion`) You may set the `-debug=tor` config logging option to have additional information in the debug log about your Tor configuration. @@ -27,6 +27,9 @@ CLI `-addrinfo` returns the number of addresses known to your node per network. This can be useful to see how many onion peers your node knows, e.g. for `-onlynet=onion`. +To fetch a number of onion addresses that your node knows, for example seven +addresses, use the `getnodeaddresses 7 onion` RPC. + ## 1. Run Bitcoin Core behind a Tor proxy The first step is running Bitcoin Core behind a Tor proxy. This will already anonymize all @@ -55,10 +58,10 @@ outgoing connections, but more is possible. -seednode=X SOCKS5. In Tor mode, such addresses can also be exchanged with other P2P nodes. - -onlynet=onion Make outgoing connections only to .onion addresses. Incoming - connections are not affected by this option. This option can be - specified multiple times to allow multiple network types, e.g. - onlynet=onion, onlynet=i2p. + -onlynet=onion Make automatic outbound connections only to .onion addresses. + Inbound and manual connections are not affected by this option. + It can be specified multiple times to allow multiple networks, + e.g. onlynet=onion, onlynet=i2p, onlynet=cjdns. In a typical situation, this suffices to run behind a Tor proxy: @@ -89,19 +92,13 @@ out by default (if not, add them): ControlPort 9051 CookieAuthentication 1 CookieAuthFileGroupReadable 1 +DataDirectoryGroupReadable 1 ``` Add or uncomment those, save, and restart Tor (usually `systemctl restart tor` or `sudo systemctl restart tor` on most systemd-based systems, including recent Debian and Ubuntu, or just restart the computer). -On some systems (such as Arch Linux), you may also need to add the following -line: - -``` -DataDirectoryGroupReadable 1 -``` - ### Authentication Connecting to Tor's control socket API requires one of two authentication diff --git a/doc/tracing.md b/doc/tracing.md index 6ec6a6c218a..d26cf52fc3b 100644 --- a/doc/tracing.md +++ b/doc/tracing.md @@ -76,7 +76,7 @@ the passed message. #### Tracepoint `net:outbound_message` -Is called when a message is send to a peer over the P2P network. Passes +Is called when a message is sent to a peer over the P2P network. Passes information about our peer, the connection and the message as arguments. Arguments passed: @@ -116,7 +116,7 @@ added to and removed (spent) from the cache when we connect a new block. (`chainstate.CoinsTip()`). For example, the RPCs `generateblock` and `getblocktemplate` call `TestBlockValidity()`, which applies the UTXO set changes to a temporary cache. Similarly, mempool consistency checks, which are -frequent on regtest, also apply the the UTXO set changes to a temporary cache. +frequent on regtest, also apply the UTXO set changes to a temporary cache. Changes to the _main_ UTXO cache and to temporary caches trigger the tracepoints. We can't tell if a temporary cache or the _main_ cache was changed. @@ -168,6 +168,101 @@ Arguments passed: 4. Value of the coin as `int64` 5. If the coin is a coinbase as `bool` +### Context `coin_selection` + +#### Tracepoint `coin_selection:selected_coins` + +Is called when `SelectCoins` completes. + +Arguments passed: +1. Wallet name as `pointer to C-style string` +2. Coin selection algorithm name as `pointer to C-style string` +3. Selection target value as `int64` +4. Calculated waste metric of the solution as `int64` +5. Total value of the selected inputs as `int64` + +#### Tracepoint `coin_selection:normal_create_tx_internal` + +Is called when the first `CreateTransactionInternal` completes. + +Arguments passed: +1. Wallet name as `pointer to C-style string` +2. Whether `CreateTransactionInternal` succeeded as `bool` +3. The expected transaction fee as an `int64` +4. The position of the change output as an `int32` + +#### Tracepoint `coin_selection:attempting_aps_create_tx` + +Is called when `CreateTransactionInternal` is called the second time for the optimistic +Avoid Partial Spends selection attempt. This is used to determine whether the next +tracepoints called are for the Avoid Partial Spends solution, or a different transaction. + +Arguments passed: +1. Wallet name as `pointer to C-style string` + +#### Tracepoint `coin_selection:aps_create_tx_internal` + +Is called when the second `CreateTransactionInternal` with Avoid Partial Spends enabled completes. + +Arguments passed: +1. Wallet name as `pointer to C-style string` +2. Whether the Avoid Partial Spends solution will be used as `bool` +3. Whether `CreateTransactionInternal` succeeded as` bool` +4. The expected transaction fee as an `int64` +5. The position of the change output as an `int32` + +### Context `mempool` + +#### Tracepoint `mempool:added` + +Is called when a transaction is added to the node's mempool. Passes information +about the transaction. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Transaction virtual size as `uint64` +3. Transaction fee as `int64` + +#### Tracepoint `mempool:removed` + +Is called when a transaction is removed from the node's mempool. Passes information +about the transaction. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Removal reason as `pointer to C-style String` (max. length 9 characters) +3. Transaction virtual size as `uint64` +4. Transaction fee as `int64` +5. Transaction mempool entry time (epoch) as `uint64` + +#### Tracepoint `mempool:replaced` + +Is called when a transaction in the node's mempool is getting replaced by another. +Passes information about the replaced and replacement transactions. + +Arguments passed: +1. Replaced transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Replaced transaction virtual size as `uint64` +3. Replaced transaction fee as `int64` +4. Replaced transaction mempool entry time (epoch) as `uint64` +5. Replacement transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +6. Replacement transaction virtual size as `uint64` +7. Replacement transaction fee as `int64` + +Note: In cases where a single replacement transaction replaces multiple +existing transactions in the mempool, the tracepoint is called once for each +replaced transaction, with data of the replacement transaction being the same +in each call. + +#### Tracepoint `mempool:rejected` + +Is called when a transaction is not permitted to enter the mempool. Passes +information about the rejected transaction. + +Arguments passed: +1. Transaction ID (hash) as `pointer to unsigned chars` (i.e. 32 bytes in little-endian) +2. Reject reason as `pointer to C-style String` (max. length 118 characters) + ## Adding tracepoints to Bitcoin Core To add a new tracepoint, `#include ` in the compilation unit where @@ -210,8 +305,8 @@ TRACE6(net, inbound_message, ### Guidelines and best practices -#### Clear motivation and use-case -Tracepoints need a clear motivation and use-case. The motivation should +#### Clear motivation and use case +Tracepoints need a clear motivation and use case. The motivation should outweigh the impact on, for example, code readability. There is no point in adding tracepoints that don't end up being used. diff --git a/doc/zmq.md b/doc/zmq.md index b832e717342..4055505d748 100644 --- a/doc/zmq.md +++ b/doc/zmq.md @@ -76,7 +76,7 @@ The option to set the PUB socket's outbound message high water mark -zmqpubhashblockhwm=n -zmqpubrawblockhwm=n -zmqpubrawtxhwm=n - -zmqpubsequencehwm=address + -zmqpubsequencehwm=n The high water mark value must be an integer greater than or equal to 0. diff --git a/share/examples/bitcoin.conf b/share/examples/bitcoin.conf index c5b79709c71..5bee4bf92e7 100644 --- a/share/examples/bitcoin.conf +++ b/share/examples/bitcoin.conf @@ -1,191 +1 @@ -## -## bitcoin.conf configuration file. Lines beginning with # are comments. -## - -# Network-related settings: - -# Note that if you use testnet, signet or regtest, particularly with the options -# addnode, connect, port, bind, rpcport, rpcbind or wallet, you will also -# want to read "[Sections]" further down. - -# Run on the testnet network -#testnet=0 - -# Run on a signet network -#signet=0 - -# Run a regression test network -#regtest=0 - -# Connect via a SOCKS5 proxy -#proxy=127.0.0.1:9050 - -# Bind to given address and always listen on it. Use [host]:port notation for IPv6 -#bind= - -# Bind to given address and add permission flags to peers connecting to it. Use [host]:port notation for IPv6 -#whitebind=perm@ - -############################################################## -## Quick Primer on addnode vs connect ## -## Let's say for instance you use addnode=4.2.2.4 ## -## addnode will connect you to and tell you about the ## -## nodes connected to 4.2.2.4. In addition it will tell ## -## the other nodes connected to it that you exist so ## -## they can connect to you. ## -## connect will not do the above when you 'connect' to it. ## -## It will *only* connect you to 4.2.2.4 and no one else.## -## ## -## So if you're behind a firewall, or have other problems ## -## finding nodes, add some using 'addnode'. ## -## ## -## If you want to stay private, use 'connect' to only ## -## connect to "trusted" nodes. ## -## ## -## If you run multiple nodes on a LAN, there's no need for ## -## all of them to open lots of connections. Instead ## -## 'connect' them all to one node that is port forwarded ## -## and has lots of connections. ## -## Thanks goes to [Noodle] on Freenode. ## -############################################################## - -# Use as many addnode= settings as you like to connect to specific peers -#addnode=69.164.218.197 -#addnode=10.0.0.2:8333 - -# Alternatively use as many connect= settings as you like to connect ONLY to specific peers -#connect=69.164.218.197 -#connect=10.0.0.1:8333 - -# Listening mode, enabled by default except when 'connect' is being used -#listen=1 - -# Port on which to listen for connections (default: 8333, testnet: 18333, signet: 38333, regtest: 18444) -#port= - -# Maximum number of inbound + outbound connections (default: 125). This option -# applies only if inbound connections are enabled; otherwise, the number of connections -# will not be more than 11: 8 full-relay connections, 2 block-relay-only ones, and -# occasionally 1 short-lived feeler or extra outbound block-relay-only connection. -# These limits do not apply to connections added manually with the -addnode -# configuration option or the addnode RPC, which have a separate limit of 8 connections. -#maxconnections= - -# Maximum upload bandwidth target in MiB per day (e.g. 'maxuploadtarget=1024' is 1 GiB per day). -# This limits the upload bandwidth for those with bandwidth limits. 0 = no limit (default: 0). -# -maxuploadtarget does not apply to peers with 'download' permission. -# For more information on reducing bandwidth utilization, see: doc/reduce-traffic.md. -#maxuploadtarget= - -# -# JSON-RPC options (for controlling a running Bitcoin/bitcoind process) -# - -# server=1 tells Bitcoin-Qt and bitcoind to accept JSON-RPC commands -#server=0 - -# Bind to given address to listen for JSON-RPC connections. -# Refer to the manpage or bitcoind -help for further details. -#rpcbind= - -# If no rpcpassword is set, rpc cookie auth is sought. The default `-rpccookiefile` name -# is .cookie and found in the `-datadir` being used for bitcoind. This option is typically used -# when the server and client are run as the same user. -# -# If not, you must set rpcuser and rpcpassword to secure the JSON-RPC API. -# -# The config option `rpcauth` can be added to server startup argument. It is set at initialization time -# using the output from the script in share/rpcauth/rpcauth.py after providing a username: -# -# ./share/rpcauth/rpcauth.py alice -# String to be appended to bitcoin.conf: -# rpcauth=alice:f7efda5c189b999524f151318c0c86$d5b51b3beffbc02b724e5d095828e0bc8b2456e9ac8757ae3211a5d9b16a22ae -# Your password: -# DONT_USE_THIS_YOU_WILL_GET_ROBBED_8ak1gI25KFTvjovL3gAM967mies3E= -# -# On client-side, you add the normal user/password pair to send commands: -#rpcuser=alice -#rpcpassword=DONT_USE_THIS_YOU_WILL_GET_ROBBED_8ak1gI25KFTvjovL3gAM967mies3E= -# -# You can even add multiple entries of these to the server conf file, and client can use any of them: -# rpcauth=bob:b2dd077cb54591a2f3139e69a897ac$4e71f08d48b4347cf8eff3815c0e25ae2e9a4340474079f55705f40574f4ec99 - -# How many seconds bitcoin will wait for a complete RPC HTTP request. -# after the HTTP connection is established. -#rpcclienttimeout=30 - -# By default, only RPC connections from localhost are allowed. -# Specify as many rpcallowip= settings as you like to allow connections from other hosts, -# either as a single IPv4/IPv6 or with a subnet specification. - -# NOTE: opening up the RPC port to hosts outside your local trusted network is NOT RECOMMENDED, -# because the rpcpassword is transmitted over the network unencrypted. - -# server=1 tells Bitcoin-Qt to accept JSON-RPC commands. -# it is also read by bitcoind to determine if RPC should be enabled -#rpcallowip=10.1.1.34/255.255.255.0 -#rpcallowip=1.2.3.4/24 -#rpcallowip=2001:db8:85a3:0:0:8a2e:370:7334/96 - -# Listen for RPC connections on this TCP port: -#rpcport=8332 - -# You can use Bitcoin or bitcoind to send commands to Bitcoin/bitcoind -# running on another host using this option: -#rpcconnect=127.0.0.1 - -# Wallet options - -# Specify where to find wallet, lockfile and logs. If not present, those files will be -# created as new. -#wallet= - -# Create transactions that have enough fees so they are likely to begin confirmation within n blocks (default: 6). -# This setting is over-ridden by the -paytxfee option. -#txconfirmtarget=n - -# Pay a transaction fee every time you send bitcoins. -#paytxfee=0.000x - -# Miscellaneous options - -# Pre-generate this many public/private key pairs, so wallet backups will be valid for -# both prior transactions and several dozen future transactions. -#keypool=100 - -# Maintain coinstats index used by the gettxoutsetinfo RPC (default: 0). -#coinstatsindex=1 - -# Enable pruning to reduce storage requirements by deleting old blocks. -# This mode is incompatible with -txindex and -coinstatsindex. -# 0 = default (no pruning). -# 1 = allows manual pruning via RPC. -# >=550 = target to stay under in MiB. -#prune=550 - -# User interface options - -# Start Bitcoin minimized -#min=1 - -# Minimize to the system tray -#minimizetotray=1 - -# [Sections] -# Most options apply to mainnet, testnet, signet and regtest. -# If you want to confine an option to just one network, you should add it in the -# relevant section below. -# EXCEPTIONS: The options addnode, connect, port, bind, rpcport, rpcbind and wallet -# only apply to mainnet unless they appear in the appropriate section below. - -# Options only for mainnet -[main] - -# Options only for testnet -[test] - -# Options only for signet -[signet] - -# Options only for regtest -[regtest] +# This is a placeholder file. Please follow the instructions in `contrib/devtools/README.md` to generate a bitcoin.conf file. diff --git a/share/rpcauth/rpcauth.py b/share/rpcauth/rpcauth.py index d441d5f21d4..cc7bba1f8b1 100755 --- a/share/rpcauth/rpcauth.py +++ b/share/rpcauth/rpcauth.py @@ -4,22 +4,20 @@ # file COPYING or http://www.opensource.org/licenses/mit-license.php. from argparse import ArgumentParser -from base64 import urlsafe_b64encode from getpass import getpass -from os import urandom - +from secrets import token_hex, token_urlsafe import hmac def generate_salt(size): """Create size byte hex salt""" - return urandom(size).hex() + return token_hex(size) def generate_password(): """Create 32 byte b64 password""" - return urlsafe_b64encode(urandom(32)).decode('utf-8') + return token_urlsafe(32) def password_to_hmac(salt, password): - m = hmac.new(bytearray(salt, 'utf-8'), bytearray(password, 'utf-8'), 'SHA256') + m = hmac.new(salt.encode('utf-8'), password.encode('utf-8'), 'SHA256') return m.hexdigest() def main(): @@ -38,8 +36,8 @@ def main(): password_hmac = password_to_hmac(salt, args.password) print('String to be appended to bitcoin.conf:') - print('rpcauth={0}:{1}${2}'.format(args.username, salt, password_hmac)) - print('Your password:\n{0}'.format(args.password)) + print(f'rpcauth={args.username}:{salt}${password_hmac}') + print(f'Your password:\n{args.password}') if __name__ == '__main__': main() diff --git a/share/setup.nsi.in b/share/setup.nsi.in index 196bb151400..cbce5030995 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -75,13 +75,15 @@ Section -Main SEC0000 File @abs_top_builddir@/release/@BITCOIN_GUI_NAME@@EXEEXT@ File /oname=COPYING.txt @abs_top_srcdir@/COPYING File /oname=readme.txt @abs_top_srcdir@/doc/README_windows.txt + File @abs_top_srcdir@/share/examples/bitcoin.conf + SetOutPath $INSTDIR\share\rpcauth + File @abs_top_srcdir@/share/rpcauth/*.* SetOutPath $INSTDIR\daemon File @abs_top_builddir@/release/@BITCOIN_DAEMON_NAME@@EXEEXT@ File @abs_top_builddir@/release/@BITCOIN_CLI_NAME@@EXEEXT@ File @abs_top_builddir@/release/@BITCOIN_TX_NAME@@EXEEXT@ File @abs_top_builddir@/release/@BITCOIN_WALLET_TOOL_NAME@@EXEEXT@ - SetOutPath $INSTDIR\doc - File /r /x Makefile* @abs_top_srcdir@/doc\*.* + File @abs_top_builddir@/release/@BITCOIN_TEST_NAME@@EXEEXT@ SetOutPath $INSTDIR WriteRegStr HKCU "${REGKEY}\Components" Main 1 SectionEnd @@ -128,8 +130,9 @@ Section /o -un.Main UNSEC0000 Delete /REBOOTOK $INSTDIR\@BITCOIN_GUI_NAME@@EXEEXT@ Delete /REBOOTOK $INSTDIR\COPYING.txt Delete /REBOOTOK $INSTDIR\readme.txt + Delete /REBOOTOK $INSTDIR\bitcoin.conf + RMDir /r /REBOOTOK $INSTDIR\share RMDir /r /REBOOTOK $INSTDIR\daemon - RMDir /r /REBOOTOK $INSTDIR\doc DeleteRegValue HKCU "${REGKEY}\Components" Main SectionEnd diff --git a/src/.bear-tidy-config b/src/.bear-tidy-config new file mode 100644 index 00000000000..434d9228f4a --- /dev/null +++ b/src/.bear-tidy-config @@ -0,0 +1,16 @@ +{ + "output": { + "content": { + "include_only_existing_source": true, + "paths_to_include": [], + "paths_to_exclude": [ + "src/leveldb", + "src/secp256k1" + ] + }, + "format": { + "command_as_array": true, + "drop_output_field": false + } + } +} diff --git a/src/.clang-tidy b/src/.clang-tidy index 27616ad072f..39566c0cb7f 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -1,2 +1,21 @@ -Checks: '-*,bugprone-argument-comment' -WarningsAsErrors: bugprone-argument-comment +Checks: ' +-*, +bugprone-argument-comment, +bugprone-use-after-move, +misc-unused-using-decls, +modernize-use-default-member-init, +modernize-use-nullptr, +performance-*, +-performance-inefficient-string-concatenation, +-performance-no-int-to-ptr, +-performance-noexcept-move-constructor, +-performance-unnecessary-value-param, +readability-const-return-type, +readability-redundant-declaration, +readability-redundant-string-init, +' +WarningsAsErrors: '*' +CheckOptions: + - key: performance-move-const-arg.CheckTriviallyCopyableMove + value: false +HeaderFilterRegex: '.' diff --git a/src/Makefile.am b/src/Makefile.am index 96c2a3be3d3..d7a5252636f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -8,21 +8,31 @@ print-%: FORCE DIST_SUBDIRS = secp256k1 -AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) $(LTO_LDFLAGS) -AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) $(LTO_CXXFLAGS) -AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) +AM_LDFLAGS = $(LIBTOOL_LDFLAGS) $(HARDENED_LDFLAGS) $(GPROF_LDFLAGS) $(SANITIZER_LDFLAGS) $(LTO_LDFLAGS) $(CORE_LDFLAGS) +AM_CXXFLAGS = $(DEBUG_CXXFLAGS) $(HARDENED_CXXFLAGS) $(WARN_CXXFLAGS) $(NOWARN_CXXFLAGS) $(ERROR_CXXFLAGS) $(GPROF_CXXFLAGS) $(SANITIZER_CXXFLAGS) $(LTO_CXXFLAGS) $(CORE_CXXFLAGS) +AM_CPPFLAGS = $(DEBUG_CPPFLAGS) $(HARDENED_CPPFLAGS) $(CORE_CPPFLAGS) AM_LIBTOOLFLAGS = --preserve-dup-deps PTHREAD_FLAGS = $(PTHREAD_CFLAGS) $(PTHREAD_LIBS) EXTRA_LIBRARIES = -BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) -I$(srcdir)/$(ELEMENTS_SIMPLICITY_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) +lib_LTLIBRARIES = +noinst_LTLIBRARIES = + +bin_PROGRAMS = +noinst_PROGRAMS = +check_PROGRAMS = +TESTS = +BENCHMARKS = + +BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(LEVELDB_CPPFLAGS) +BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) -I$(srcdir)/$(ELEMENTS_SIMPLICITY_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(LEVELDB_CPPFLAGS) LIBBITCOIN_NODE=libbitcoin_node.a LIBBITCOIN_COMMON=libbitcoin_common.a LIBBITCOIN_CONSENSUS=libelements_consensus.a LIBBITCOIN_CLI=libbitcoin_cli.a LIBBITCOIN_UTIL=libbitcoin_util.a -LIBBITCOIN_CRYPTO_BASE=crypto/libbitcoin_crypto_base.a +LIBBITCOIN_CRYPTO_BASE=crypto/libbitcoin_crypto_base.la LIBBITCOINQT=qt/libbitcoinqt.a LIBSECP256K1=secp256k1/libsecp256k1.la @@ -32,28 +42,32 @@ endif if BUILD_BITCOIN_LIBS LIBBITCOINCONSENSUS=libelementsconsensus.la endif +if BUILD_BITCOIN_KERNEL_LIB +LIBBITCOINKERNEL=libbitcoinkernel.la +endif if ENABLE_WALLET LIBBITCOIN_WALLET=libbitcoin_wallet.a LIBBITCOIN_WALLET_TOOL=libbitcoin_wallet_tool.a endif -LIBBITCOIN_CRYPTO= $(LIBBITCOIN_CRYPTO_BASE) +LIBBITCOIN_CRYPTO = $(LIBBITCOIN_CRYPTO_BASE) if ENABLE_SSE41 -LIBBITCOIN_CRYPTO_SSE41 = crypto/libbitcoin_crypto_sse41.a +LIBBITCOIN_CRYPTO_SSE41 = crypto/libbitcoin_crypto_sse41.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_SSE41) endif if ENABLE_AVX2 -LIBBITCOIN_CRYPTO_AVX2 = crypto/libbitcoin_crypto_avx2.a +LIBBITCOIN_CRYPTO_AVX2 = crypto/libbitcoin_crypto_avx2.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_AVX2) endif if ENABLE_X86_SHANI -LIBBITCOIN_CRYPTO_X86_SHANI = crypto/libbitcoin_crypto_x86_shani.a +LIBBITCOIN_CRYPTO_X86_SHANI = crypto/libbitcoin_crypto_x86_shani.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_X86_SHANI) endif if ENABLE_ARM_SHANI -LIBBITCOIN_CRYPTO_ARM_SHANI = crypto/libbitcoin_crypto_arm_shani.a +LIBBITCOIN_CRYPTO_ARM_SHANI = crypto/libbitcoin_crypto_arm_shani.la LIBBITCOIN_CRYPTO += $(LIBBITCOIN_CRYPTO_ARM_SHANI) endif +noinst_LTLIBRARIES += $(LIBBITCOIN_CRYPTO) $(LIBSECP256K1): $(wildcard secp256k1/src/*.h) $(wildcard secp256k1/src/*.c) $(wildcard secp256k1/include/*) $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C $(@D) $(@F) @@ -61,7 +75,6 @@ $(LIBSECP256K1): $(wildcard secp256k1/src/*.h) $(wildcard secp256k1/src/*.c) $(w # Make is not made aware of per-object dependencies to avoid limiting building parallelization # But to build the less dependent modules first, we manually select their order here: EXTRA_LIBRARIES += \ - $(LIBBITCOIN_CRYPTO) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_CONSENSUS) \ @@ -72,14 +85,6 @@ EXTRA_LIBRARIES += \ $(LIBBITCOIN_WALLET_TOOL) \ $(LIBBITCOIN_ZMQ) -lib_LTLIBRARIES = $(LIBBITCOINCONSENSUS) -noinst_LTLIBRARIES = - -bin_PROGRAMS = -noinst_PROGRAMS = -TESTS = -BENCHMARKS = - if BUILD_BITCOIND bin_PROGRAMS += elementsd endif @@ -136,12 +141,14 @@ BITCOIN_CORE_H = \ clientversion.h \ coins.h \ common/bloom.h \ - compat.h \ + common/init.h \ + common/run_command.h \ + common/url.h \ compat/assumptions.h \ compat/byteswap.h \ + compat/compat.h \ compat/cpuid.h \ compat/endian.h \ - compat/sanity.h \ compressor.h \ confidential_validation.h \ consensus/consensus.h \ @@ -156,7 +163,7 @@ BITCOIN_CORE_H = \ dynafed.h \ external_signer.h \ flatfile.h \ - fs.h \ + headerssync.h \ httprpc.h \ httpserver.h \ i2p.h \ @@ -176,6 +183,19 @@ BITCOIN_CORE_H = \ interfaces/node.h \ interfaces/wallet.h \ issuance.h \ + kernel/blockmanager_opts.h \ + kernel/chain.h \ + kernel/chainparams.h \ + kernel/chainstatemanager_opts.h \ + kernel/checks.h \ + kernel/coinstats.h \ + kernel/context.h \ + kernel/cs_main.h \ + kernel/mempool_entry.h \ + kernel/mempool_limits.h \ + kernel/mempool_options.h \ + kernel/mempool_persist.h \ + kernel/validation_cache_sizes.h \ key.h \ key_io.h \ logging.h \ @@ -190,25 +210,36 @@ BITCOIN_CORE_H = \ net_types.h \ netaddress.h \ netbase.h \ + netgroup.h \ netmessagemaker.h \ + node/blockmanager_args.h \ node/blockstorage.h \ node/caches.h \ node/chainstate.h \ + node/chainstatemanager_args.h \ node/coin.h \ - node/coinstats.h \ + node/coins_view_args.h \ + node/connection_types.h \ node/context.h \ + node/database_args.h \ + node/eviction.h \ + node/interface_ui.h \ + node/mempool_args.h \ + node/mempool_persist_args.h \ node/miner.h \ node/minisketchwrapper.h \ node/psbt.h \ node/transaction.h \ - node/ui_interface.h \ + node/txreconciliation.h \ node/utxo_snapshot.h \ + node/validation_cache_args.h \ noui.h \ outputtype.h \ pegins.h \ policy/discount.h \ policy/feerate.h \ policy/fees.h \ + policy/fees_args.h \ policy/packages.h \ policy/policy.h \ policy/rbf.h \ @@ -219,9 +250,11 @@ BITCOIN_CORE_H = \ psbt.h \ random.h \ randomenv.h \ + rest.h \ reverse_iterator.h \ rpc/blockchain.h \ rpc/client.h \ + rpc/mempool.h \ rpc/mining.h \ rpc/protocol.h \ rpc/rawtransaction_util.h \ @@ -235,6 +268,7 @@ BITCOIN_CORE_H = \ script/generic.hpp \ script/keyorigin.h \ script/pegins.h \ + script/miniscript.h \ script/sigcache.h \ script/sign.h \ script/signingprovider.h \ @@ -248,7 +282,6 @@ BITCOIN_CORE_H = \ support/events.h \ support/lockedpool.h \ sync.h \ - threadinterrupt.h \ threadsafety.h \ timedata.h \ torcontrol.h \ @@ -259,12 +292,16 @@ BITCOIN_CORE_H = \ undo.h \ util/asmap.h \ util/bip32.h \ + util/bitdeque.h \ util/bytevectorhash.h \ util/check.h \ util/epochguard.h \ util/error.h \ + util/exception.h \ util/fastrange.h \ util/fees.h \ + util/fs.h \ + util/fs_helpers.h \ util/getuniquepath.h \ util/golombrice.h \ util/hash_type.h \ @@ -276,14 +313,17 @@ BITCOIN_CORE_H = \ util/overloaded.h \ util/rbf.h \ util/readwritefile.h \ + util/result.h \ util/serfloat.h \ util/settings.h \ util/sock.h \ util/spanparsing.h \ util/string.h \ util/syscall_sandbox.h \ + util/syserror.h \ util/system.h \ util/thread.h \ + util/threadinterrupt.h \ util/threadnames.h \ util/time.h \ util/tokenpipe.h \ @@ -291,7 +331,6 @@ BITCOIN_CORE_H = \ util/translation.h \ util/types.h \ util/ui_change_type.h \ - util/url.h \ util/vector.h \ validation.h \ validationinterface.h \ @@ -306,7 +345,6 @@ BITCOIN_CORE_H = \ wallet/external_signer_scriptpubkeyman.h \ wallet/feebumper.h \ wallet/fees.h \ - wallet/ismine.h \ wallet/load.h \ wallet/receive.h \ wallet/rpc/util.h \ @@ -316,6 +354,7 @@ BITCOIN_CORE_H = \ wallet/spend.h \ wallet/sqlite.h \ wallet/transaction.h \ + wallet/types.h \ wallet/wallet.h \ wallet/walletdb.h \ wallet/wallettool.h \ @@ -335,13 +374,8 @@ obj/build.h: FORCE "$(abs_top_srcdir)" libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h -ipc/capnp/libbitcoin_ipc_a-ipc.$(OBJEXT): $(libbitcoin_ipc_mpgen_input:=.h) - -# server: shared between bitcoind and bitcoin-qt -# Contains code accessing mempool and chain state that is meant to be separated -# from wallet and gui code (see node/README.md). Shared code should go in -# libbitcoin_common or libbitcoin_util libraries, instead. -libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) +# node # +libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_node_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_node_a_SOURCES = \ addrdb.cpp \ @@ -357,6 +391,7 @@ libbitcoin_node_a_SOURCES = \ dbwrapper.cpp \ deploymentstatus.cpp \ flatfile.cpp \ + headerssync.cpp \ httprpc.cpp \ httpserver.cpp \ i2p.cpp \ @@ -365,25 +400,43 @@ libbitcoin_node_a_SOURCES = \ index/coinstatsindex.cpp \ index/txindex.cpp \ init.cpp \ + kernel/chain.cpp \ + kernel/checks.cpp \ + kernel/coinstats.cpp \ mainchainrpc.cpp \ + kernel/context.cpp \ + kernel/cs_main.cpp \ + kernel/mempool_persist.cpp \ mapport.cpp \ net.cpp \ net_processing.cpp \ + netgroup.cpp \ + node/blockmanager_args.cpp \ node/blockstorage.cpp \ node/caches.cpp \ node/chainstate.cpp \ + node/chainstatemanager_args.cpp \ node/coin.cpp \ - node/coinstats.cpp \ + node/coins_view_args.cpp \ + node/connection_types.cpp \ node/context.cpp \ + node/database_args.cpp \ + node/eviction.cpp \ + node/interface_ui.cpp \ node/interfaces.cpp \ + node/mempool_args.cpp \ + node/mempool_persist_args.cpp \ node/miner.cpp \ node/minisketchwrapper.cpp \ node/psbt.cpp \ node/transaction.cpp \ - node/ui_interface.cpp \ + node/txreconciliation.cpp \ + node/utxo_snapshot.cpp \ + node/validation_cache_args.cpp \ noui.cpp \ pegins.cpp \ policy/fees.cpp \ + policy/fees_args.cpp \ policy/packages.cpp \ policy/rbf.cpp \ policy/settings.cpp \ @@ -391,12 +444,17 @@ libbitcoin_node_a_SOURCES = \ primitives/pak.cpp \ rest.cpp \ rpc/blockchain.cpp \ + rpc/fees.cpp \ + rpc/mempool.cpp \ rpc/mining.cpp \ - rpc/misc.cpp \ rpc/net.cpp \ + rpc/node.cpp \ + rpc/output_script.cpp \ rpc/rawtransaction.cpp \ rpc/server.cpp \ rpc/server_util.cpp \ + rpc/signmessage.cpp \ + rpc/txoutproof.cpp \ script/sigcache.cpp \ shutdown.cpp \ signet.cpp \ @@ -413,11 +471,14 @@ libbitcoin_node_a_SOURCES = \ if ENABLE_WALLET libbitcoin_node_a_SOURCES += wallet/init.cpp +libbitcoin_node_a_CPPFLAGS += $(BDB_CPPFLAGS) endif if !ENABLE_WALLET libbitcoin_node_a_SOURCES += dummywallet.cpp endif +# +# zmq # if ENABLE_ZMQ libbitcoin_zmq_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(ZMQ_CFLAGS) libbitcoin_zmq_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) @@ -428,11 +489,10 @@ libbitcoin_zmq_a_SOURCES = \ zmq/zmqrpc.cpp \ zmq/zmqutil.cpp endif +# - -# wallet: shared between bitcoind and bitcoin-qt, but only linked -# when wallet enabled -libbitcoin_wallet_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(SQLITE_CFLAGS) +# wallet # +libbitcoin_wallet_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(BDB_CPPFLAGS) $(SQLITE_CFLAGS) libbitcoin_wallet_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_a_SOURCES = \ wallet/coincontrol.cpp \ @@ -475,17 +535,27 @@ endif if USE_BDB libbitcoin_wallet_a_SOURCES += wallet/bdb.cpp wallet/salvage.cpp endif +# -libbitcoin_wallet_tool_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +# wallet tool # +libbitcoin_wallet_tool_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libbitcoin_wallet_tool_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_tool_a_SOURCES = \ wallet/wallettool.cpp \ $(BITCOIN_CORE_H) +# + +# crypto # +crypto_libbitcoin_crypto_base_la_CPPFLAGS = $(AM_CPPFLAGS) + +# Specify -static in both CXXFLAGS and LDFLAGS so libtool will only build a +# static version of this library. We don't need a dynamic version, and a dynamic +# version can't be used on windows anyway because the library doesn't currently +# export DLL symbols. +crypto_libbitcoin_crypto_base_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_base_la_LDFLAGS = $(AM_LDFLAGS) -static -# crypto primitives library -crypto_libbitcoin_crypto_base_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_base_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_base_a_SOURCES = \ +crypto_libbitcoin_crypto_base_la_SOURCES = \ crypto/aes.cpp \ crypto/aes.h \ crypto/chacha_poly_aead.h \ @@ -517,34 +587,47 @@ crypto_libbitcoin_crypto_base_a_SOURCES = \ crypto/siphash.h if USE_ASM -crypto_libbitcoin_crypto_base_a_SOURCES += crypto/sha256_sse4.cpp +crypto_libbitcoin_crypto_base_la_SOURCES += crypto/sha256_sse4.cpp endif -crypto_libbitcoin_crypto_sse41_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_sse41_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_sse41_a_CXXFLAGS += $(SSE41_CXXFLAGS) -crypto_libbitcoin_crypto_sse41_a_CPPFLAGS += -DENABLE_SSE41 -crypto_libbitcoin_crypto_sse41_a_SOURCES = crypto/sha256_sse41.cpp - -crypto_libbitcoin_crypto_avx2_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_avx2_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_avx2_a_CXXFLAGS += $(AVX2_CXXFLAGS) -crypto_libbitcoin_crypto_avx2_a_CPPFLAGS += -DENABLE_AVX2 -crypto_libbitcoin_crypto_avx2_a_SOURCES = crypto/sha256_avx2.cpp - -crypto_libbitcoin_crypto_x86_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_x86_shani_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_x86_shani_a_CXXFLAGS += $(X86_SHANI_CXXFLAGS) -crypto_libbitcoin_crypto_x86_shani_a_CPPFLAGS += -DENABLE_X86_SHANI -crypto_libbitcoin_crypto_x86_shani_a_SOURCES = crypto/sha256_x86_shani.cpp - -crypto_libbitcoin_crypto_arm_shani_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -crypto_libbitcoin_crypto_arm_shani_a_CPPFLAGS = $(AM_CPPFLAGS) -crypto_libbitcoin_crypto_arm_shani_a_CXXFLAGS += $(ARM_SHANI_CXXFLAGS) -crypto_libbitcoin_crypto_arm_shani_a_CPPFLAGS += -DENABLE_ARM_SHANI -crypto_libbitcoin_crypto_arm_shani_a_SOURCES = crypto/sha256_arm_shani.cpp - -# consensus: shared between all executables that validate any consensus rules. +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_sse41_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_sse41_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_sse41_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_sse41_la_CXXFLAGS += $(SSE41_CXXFLAGS) +crypto_libbitcoin_crypto_sse41_la_CPPFLAGS += -DENABLE_SSE41 +crypto_libbitcoin_crypto_sse41_la_SOURCES = crypto/sha256_sse41.cpp + +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_avx2_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_avx2_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_avx2_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_avx2_la_CXXFLAGS += $(AVX2_CXXFLAGS) +crypto_libbitcoin_crypto_avx2_la_CPPFLAGS += -DENABLE_AVX2 +crypto_libbitcoin_crypto_avx2_la_SOURCES = crypto/sha256_avx2.cpp + +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_x86_shani_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_x86_shani_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_x86_shani_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_x86_shani_la_CXXFLAGS += $(X86_SHANI_CXXFLAGS) +crypto_libbitcoin_crypto_x86_shani_la_CPPFLAGS += -DENABLE_X86_SHANI +crypto_libbitcoin_crypto_x86_shani_la_SOURCES = crypto/sha256_x86_shani.cpp + +# See explanation for -static in crypto_libbitcoin_crypto_base_la's LDFLAGS and +# CXXFLAGS above +crypto_libbitcoin_crypto_arm_shani_la_LDFLAGS = $(AM_LDFLAGS) -static +crypto_libbitcoin_crypto_arm_shani_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crypto_libbitcoin_crypto_arm_shani_la_CPPFLAGS = $(AM_CPPFLAGS) +crypto_libbitcoin_crypto_arm_shani_la_CXXFLAGS += $(ARM_SHANI_CXXFLAGS) +crypto_libbitcoin_crypto_arm_shani_la_CPPFLAGS += -DENABLE_ARM_SHANI +crypto_libbitcoin_crypto_arm_shani_la_SOURCES = crypto/sha256_arm_shani.cpp +# + +# consensus # libelements_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libelements_consensus_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libelements_consensus_a_SOURCES = \ @@ -591,9 +674,10 @@ libelements_consensus_a_SOURCES = \ util/strencodings.cpp \ util/strencodings.h \ version.h +# -# common: shared between bitcoind, and bitcoin-qt and non-server tools -libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +# common # +libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ assetsdir.cpp \ @@ -606,6 +690,9 @@ libbitcoin_common_a_SOURCES = \ chainparams.cpp \ coins.cpp \ common/bloom.cpp \ + common/init.cpp \ + common/interfaces.cpp \ + common/run_command.cpp \ compressor.cpp \ core_read.cpp \ core_write.cpp \ @@ -613,6 +700,7 @@ libbitcoin_common_a_SOURCES = \ external_signer.cpp \ init/common.cpp \ issuance.cpp \ + kernel/chainparams.cpp \ key.cpp \ key_io.cpp \ merkleblock.cpp \ @@ -625,48 +713,51 @@ libbitcoin_common_a_SOURCES = \ policy/policy.cpp \ protocol.cpp \ psbt.cpp \ - rpc/rawtransaction_util.cpp \ rpc/external_signer.cpp \ + rpc/rawtransaction_util.cpp \ + rpc/request.cpp \ rpc/util.cpp \ scheduler.cpp \ script/descriptor.cpp \ script/pegins.cpp \ + script/miniscript.cpp \ script/sign.cpp \ script/signingprovider.cpp \ script/standard.cpp \ warnings.cpp \ $(BITCOIN_CORE_H) -# util: shared between all executables. -# This library *must* be included to make sure that the glibc -# sanity checks are linked. +if USE_LIBEVENT +libbitcoin_common_a_CPPFLAGS += $(EVENT_CFLAGS) +libbitcoin_common_a_SOURCES += common/url.cpp +endif +# + +# util # libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_util_a_SOURCES = \ support/lockedpool.cpp \ chainparamsbase.cpp \ clientversion.cpp \ - compat/glibcxx_sanity.cpp \ - compat/strnlen.cpp \ - fs.cpp \ - interfaces/echo.cpp \ - interfaces/handler.cpp \ - interfaces/init.cpp \ logging.cpp \ random.cpp \ randomenv.cpp \ - rpc/request.cpp \ support/cleanse.cpp \ sync.cpp \ - threadinterrupt.cpp \ util/asmap.cpp \ util/bip32.cpp \ util/bytevectorhash.cpp \ + util/check.cpp \ util/error.cpp \ + util/exception.cpp \ util/fees.cpp \ + util/fs.cpp \ + util/fs_helpers.cpp \ util/getuniquepath.cpp \ util/hasher.cpp \ util/sock.cpp \ + util/syserror.cpp \ util/system.cpp \ util/message.cpp \ util/moneystr.cpp \ @@ -674,6 +765,7 @@ libbitcoin_util_a_SOURCES = \ util/readwritefile.cpp \ util/settings.cpp \ util/thread.cpp \ + util/threadinterrupt.cpp \ util/threadnames.cpp \ util/serfloat.cpp \ util/spanparsing.cpp \ @@ -683,12 +775,9 @@ libbitcoin_util_a_SOURCES = \ util/time.cpp \ util/tokenpipe.cpp \ $(BITCOIN_CORE_H) +# -if USE_LIBEVENT -libbitcoin_util_a_SOURCES += util/url.cpp -endif - -# cli: shared between bitcoin-cli and bitcoin-qt +# cli # libbitcoin_cli_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_cli_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_cli_a_SOURCES = \ @@ -719,7 +808,6 @@ elements_bin_ldadd = \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) \ $(LIBELEMENTSSIMPLICITY) @@ -751,6 +839,7 @@ endif elements_cli_LDADD = \ $(LIBBITCOIN_CLI) \ $(LIBUNIVALUE) \ + $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CRYPTO) @@ -826,16 +915,75 @@ elements_util_LDADD = \ # # bitcoin-chainstate binary # -bitcoin_chainstate_SOURCES = \ - bitcoin-chainstate.cpp \ +bitcoin_chainstate_SOURCES = bitcoin-chainstate.cpp +bitcoin_chainstate_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) +bitcoin_chainstate_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) + +# ELEMENTS: additional sources for chainstate +bitcoin_chainstate_SOURCES += \ + asset.cpp \ + blind.cpp \ + blindpsbt.cpp \ + block_proof.cpp \ + common/bloom.cpp \ + confidential_validation.cpp \ + dynafed.cpp \ + issuance.cpp \ + mainchainrpc.cpp \ + merkleblock.cpp \ + pegins.cpp \ + primitives/bitcoin/block.cpp \ + primitives/bitcoin/merkleblock.cpp \ + primitives/bitcoin/transaction.cpp \ + primitives/confidential.cpp \ + primitives/pak.cpp \ + primitives/txwitness.cpp \ + psbt.cpp \ + rpc/request.cpp \ + script/pegins.cpp \ + script/sign.cpp \ + script/signingprovider.cpp \ + support/events.h + + +# $(LIBTOOL_APP_LDFLAGS) deliberately omitted here so that we can test linking +# bitcoin-chainstate against libbitcoinkernel as a shared or static library by +# setting --{en,dis}able-shared. +bitcoin_chainstate_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(PTHREAD_FLAGS) $(LIBTOOL_APP_LDFLAGS) -static +bitcoin_chainstate_LDADD = $(LIBBITCOINKERNEL) + +# libtool is unable to calculate this indirect dependency, presumably because it's a subproject. +# libsecp256k1 only needs to be linked in when libbitcoinkernel is static. +bitcoin_chainstate_LDADD += $(LIBSECP256K1) +# + +# bitcoinkernel library # +if BUILD_BITCOIN_KERNEL_LIB +lib_LTLIBRARIES += $(LIBBITCOINKERNEL) + +libbitcoinkernel_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) $(PTHREAD_FLAGS) +libbitcoinkernel_la_LIBADD = $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) $(LIBELEMENTSSIMPLICITY) +libbitcoinkernel_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) -I$(srcdir)/$(ELEMENTS_SIMPLICITY_INCLUDE_DIR_INT) + +# libbitcoinkernel requires default symbol visibility, explicitly specify that +# here so that things still work even when user configures with +# --enable-reduce-exports +# +# Note this is a quick hack that will be removed as we incrementally define what +# to export from the library. +libbitcoinkernel_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -fvisibility=default + +# TODO: libbitcoinkernel is a work in progress consensus engine library, as more +# and more modules are decoupled from the consensus engine, this list will +# shrink to only those which are absolutely necessary. +libbitcoinkernel_la_SOURCES = \ + kernel/bitcoinkernel.cpp \ arith_uint256.cpp \ - blockfilter.cpp \ chain.cpp \ chainparamsbase.cpp \ chainparams.cpp \ clientversion.cpp \ coins.cpp \ - compat/glibcxx_sanity.cpp \ compressor.cpp \ consensus/merkle.cpp \ consensus/tx_check.cpp \ @@ -845,19 +993,20 @@ bitcoin_chainstate_SOURCES = \ deploymentinfo.cpp \ deploymentstatus.cpp \ flatfile.cpp \ - fs.cpp \ hash.cpp \ - index/base.cpp \ - index/blockfilterindex.cpp \ - index/coinstatsindex.cpp \ - init/common.cpp \ + kernel/chain.cpp \ + kernel/checks.cpp \ + kernel/chainparams.cpp \ + kernel/coinstats.cpp \ + kernel/context.cpp \ + kernel/cs_main.cpp \ + kernel/mempool_persist.cpp \ key.cpp \ logging.cpp \ - netaddress.cpp \ node/blockstorage.cpp \ node/chainstate.cpp \ - node/coinstats.cpp \ - node/ui_interface.cpp \ + node/interface_ui.cpp \ + node/utxo_snapshot.cpp \ policy/feerate.cpp \ policy/fees.cpp \ policy/packages.cpp \ @@ -881,13 +1030,13 @@ bitcoin_chainstate_SOURCES = \ support/cleanse.cpp \ support/lockedpool.cpp \ sync.cpp \ - threadinterrupt.cpp \ - timedata.cpp \ txdb.cpp \ txmempool.cpp \ uint256.cpp \ - util/asmap.cpp \ - util/bytevectorhash.cpp \ + util/check.cpp \ + util/exception.cpp \ + util/fs.cpp \ + util/fs_helpers.cpp \ util/getuniquepath.cpp \ util/hasher.cpp \ util/moneystr.cpp \ @@ -895,7 +1044,9 @@ bitcoin_chainstate_SOURCES = \ util/serfloat.cpp \ util/settings.cpp \ util/strencodings.cpp \ + util/string.cpp \ util/syscall_sandbox.cpp \ + util/syserror.cpp \ util/system.cpp \ util/thread.cpp \ util/threadnames.cpp \ @@ -905,57 +1056,19 @@ bitcoin_chainstate_SOURCES = \ validationinterface.cpp \ versionbits.cpp \ warnings.cpp -# ELEMENTS: additional sources for chainstate -bitcoin_chainstate_SOURCES += \ - asset.cpp \ - blind.cpp \ - blindpsbt.cpp \ - block_proof.cpp \ - common/bloom.cpp \ - confidential_validation.cpp \ - dynafed.cpp \ - issuance.cpp \ - mainchainrpc.cpp \ - merkleblock.cpp \ - pegins.cpp \ - primitives/bitcoin/block.cpp \ - primitives/bitcoin/merkleblock.cpp \ - primitives/bitcoin/transaction.cpp \ - primitives/confidential.cpp \ - primitives/pak.cpp \ - primitives/txwitness.cpp \ - psbt.cpp \ - rpc/request.cpp \ - script/pegins.cpp \ - script/sign.cpp \ - script/signingprovider.cpp \ - support/events.h \ - util/url.cpp - -bitcoin_chainstate_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) -bitcoin_chainstate_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -bitcoin_chainstate_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) -bitcoin_chainstate_LDADD = \ - $(LIBBITCOIN_CRYPTO) \ - $(LIBUNIVALUE) \ - $(LIBSECP256K1) \ - $(LIBELEMENTSSIMPLICITY) \ - $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ - $(LIBMEMENV) -# ELEMENTS: -bitcoin_chainstate_LDADD += \ - $(EVENT_LIBS) # Required for obj/build.h to be generated first. # More details: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html -bitcoin_chainstate-clientversion.$(OBJEXT): obj/build.h +libbitcoinkernel_la-clientversion.l$(OBJEXT): obj/build.h +endif # BUILD_BITCOIN_KERNEL_LIB # # bitcoinconsensus library # if BUILD_BITCOIN_LIBS +lib_LTLIBRARIES += $(LIBBITCOINCONSENSUS) + include_HEADERS = script/bitcoinconsensus.h -libelementsconsensus_la_SOURCES = support/cleanse.cpp $(crypto_libbitcoin_crypto_base_a_SOURCES) $(libelements_consensus_a_SOURCES) +libelementsconsensus_la_SOURCES = support/cleanse.cpp $(crypto_libbitcoin_crypto_base_la_SOURCES) $(libelements_consensus_a_SOURCES) libelementsconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) libelementsconsensus_la_LIBADD = $(LIBSECP256K1) $(LIBELEMENTSSIMPLICITY) @@ -1030,6 +1143,10 @@ libbitcoin_ipc_mpgen_input = \ EXTRA_DIST += $(libbitcoin_ipc_mpgen_input) %.capnp: +# Explicitly list dependencies on generated headers as described in +# https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html#Recording-Dependencies-manually +ipc/capnp/libbitcoin_ipc_a-protocol.$(OBJEXT): $(libbitcoin_ipc_mpgen_input:=.h) + if BUILD_MULTIPROCESS LIBBITCOIN_IPC=libbitcoin_ipc.a libbitcoin_ipc_a_SOURCES = \ @@ -1059,6 +1176,15 @@ nodist_libbitcoin_ipc_a_SOURCES = $(libbitcoin_ipc_mpgen_output) CLEANFILES += $(libbitcoin_ipc_mpgen_output) endif +%.raw.h: %.raw + @$(MKDIR_P) $(@D) + @{ \ + echo "static unsigned const char $(*F)_raw[] = {" && \ + $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ + echo "};"; \ + } > "$@.new" && mv -f "$@.new" "$@" + @echo "Generated $@" + include Makefile.minisketch.include include Makefile.crc32c.include @@ -1067,9 +1193,7 @@ include Makefile.leveldb.include include Makefile.test_util.include include Makefile.test_fuzz.include -if ENABLE_TESTS include Makefile.test.include -endif if ENABLE_BENCH include Makefile.bench.include diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 49efbac3d79..64cf7a9b899 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -13,59 +13,65 @@ GENERATED_BENCH_FILES = $(RAW_BENCH_FILES:.raw=.raw.h) bench_bench_bitcoin_SOURCES = \ $(RAW_BENCH_FILES) \ bench/addrman.cpp \ - bench/bench_bitcoin.cpp \ + bench/base58.cpp \ + bench/bech32.cpp \ bench/bench.cpp \ bench/bench.h \ + bench/bench_bitcoin.cpp \ bench/block_assemble.cpp \ + bench/ccoins_caching.cpp \ + bench/chacha20.cpp \ + bench/chacha_poly_aead.cpp \ bench/checkblock.cpp \ bench/checkqueue.cpp \ - bench/data.h \ + bench/crypto_hash.cpp \ bench/data.cpp \ + bench/data.h \ + bench/descriptors.cpp \ bench/duplicate_inputs.cpp \ bench/examples.cpp \ - bench/rollingbloom.cpp \ - bench/chacha20.cpp \ - bench/chacha_poly_aead.cpp \ - bench/crypto_hash.cpp \ - bench/ccoins_caching.cpp \ bench/gcs_filter.cpp \ bench/hashpadding.cpp \ - bench/merkle_root.cpp \ + bench/load_external.cpp \ + bench/lockedpool.cpp \ + bench/logging.cpp \ bench/mempool_eviction.cpp \ bench/mempool_stress.cpp \ - bench/nanobench.h \ + bench/merkle_root.cpp \ bench/nanobench.cpp \ + bench/nanobench.h \ bench/peer_eviction.cpp \ + bench/poly1305.cpp \ + bench/prevector.cpp \ + bench/rollingbloom.cpp \ bench/rpc_blockchain.cpp \ bench/rpc_mempool.cpp \ + bench/strencodings.cpp \ bench/util_time.cpp \ - bench/verify_script.cpp \ - bench/base58.cpp \ - bench/bech32.cpp \ - bench/lockedpool.cpp \ - bench/poly1305.cpp \ - bench/prevector.cpp + bench/verify_script.cpp nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_BENCH_FILES) -bench_bench_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ +bench_bench_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/ bench_bench_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) bench_bench_bitcoin_LDADD = \ + $(LIBTEST_UTIL) \ $(LIBBITCOIN_NODE) \ $(LIBBITCOIN_WALLET) \ $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CONSENSUS) \ $(LIBBITCOIN_CRYPTO) \ - $(LIBTEST_UTIL) \ $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) \ $(LIBELEMENTSSIMPLICITY) \ $(LIBUNIVALUE) \ $(EVENT_PTHREADS_LIBS) \ - $(EVENT_LIBS) + $(EVENT_LIBS) \ + $(MINIUPNPC_LIBS) \ + $(NATPMP_LIBS) if ENABLE_ZMQ bench_bench_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) @@ -74,11 +80,11 @@ endif if ENABLE_WALLET bench_bench_bitcoin_SOURCES += bench/coin_selection.cpp bench_bench_bitcoin_SOURCES += bench/wallet_balance.cpp +bench_bench_bitcoin_SOURCES += bench/wallet_loading.cpp +bench_bench_bitcoin_SOURCES += bench/wallet_create_tx.cpp +bench_bench_bitcoin_LDADD += $(BDB_LIBS) $(SQLITE_LIBS) endif -bench_bench_bitcoin_LDADD += $(BDB_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) -bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) - CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno $(GENERATED_BENCH_FILES) CLEANFILES += $(CLEAN_BITCOIN_BENCH) @@ -92,12 +98,3 @@ bench: $(BENCH_BINARY) FORCE bitcoin_bench_clean : FORCE rm -f $(CLEAN_BITCOIN_BENCH) $(bench_bench_bitcoin_OBJECTS) $(BENCH_BINARY) - -%.raw.h: %.raw - @$(MKDIR_P) $(@D) - @{ \ - echo "static unsigned const char $(*F)_raw[] = {" && \ - $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ - echo "};"; \ - } > "$@.new" && mv -f "$@.new" "$@" - @echo "Generated $@" diff --git a/src/Makefile.crc32c.include b/src/Makefile.crc32c.include index 3cbe71792ca..c4dd84991d2 100644 --- a/src/Makefile.crc32c.include +++ b/src/Makefile.crc32c.include @@ -2,10 +2,9 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -LIBCRC32C_INT = crc32c/libcrc32c.a -LIBLEVELDB_SSE42_INT = leveldb/libleveldb_sse42.a +LIBCRC32C_INT = crc32c/libcrc32c.la -EXTRA_LIBRARIES += $(LIBCRC32C_INT) +noinst_LTLIBRARIES += $(LIBCRC32C_INT) LIBCRC32C = $(LIBCRC32C_INT) @@ -34,41 +33,49 @@ else CRC32C_CPPFLAGS_INT += -DBYTE_ORDER_BIG_ENDIAN=0 endif -crc32c_libcrc32c_a_CPPFLAGS = $(AM_CPPFLAGS) $(CRC32C_CPPFLAGS_INT) $(CRC32C_CPPFLAGS) -crc32c_libcrc32c_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) - -crc32c_libcrc32c_a_SOURCES = -crc32c_libcrc32c_a_SOURCES += crc32c/include/crc32c/crc32c.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_arm64.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_arm64_check.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_internal.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_prefetch.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_read_le.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_round_up.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_sse42_check.h -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_sse42.h - -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c.cc -crc32c_libcrc32c_a_SOURCES += crc32c/src/crc32c_portable.cc +crc32c_libcrc32c_la_CPPFLAGS = $(AM_CPPFLAGS) $(CRC32C_CPPFLAGS_INT) $(CRC32C_CPPFLAGS) + +# Specify -static in both CXXFLAGS and LDFLAGS so libtool will only build a +# static version of this library. We don't need a dynamic version, and a dynamic +# version can't be used on windows anyway because the library doesn't currently +# export DLL symbols. +crc32c_libcrc32c_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -static +crc32c_libcrc32c_la_LDFLAGS = $(AM_LDFLAGS) -static + +crc32c_libcrc32c_la_SOURCES = +crc32c_libcrc32c_la_SOURCES += crc32c/include/crc32c/crc32c.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_arm64.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_arm64_check.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_internal.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_prefetch.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_read_le.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_round_up.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_sse42_check.h +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_sse42.h + +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c.cc +crc32c_libcrc32c_la_SOURCES += crc32c/src/crc32c_portable.cc if ENABLE_SSE42 -LIBCRC32C_SSE42_INT = crc32c/libcrc32c_sse42.a -EXTRA_LIBRARIES += $(LIBCRC32C_SSE42_INT) +LIBCRC32C_SSE42_INT = crc32c/libcrc32c_sse42.la +noinst_LTLIBRARIES += $(LIBCRC32C_SSE42_INT) LIBCRC32C += $(LIBCRC32C_SSE42_INT) -crc32c_libcrc32c_sse42_a_CPPFLAGS = $(crc32c_libcrc32c_a_CPPFLAGS) -crc32c_libcrc32c_sse42_a_CXXFLAGS = $(crc32c_libcrc32c_a_CXXFLAGS) $(SSE42_CXXFLAGS) +crc32c_libcrc32c_sse42_la_CPPFLAGS = $(crc32c_libcrc32c_la_CPPFLAGS) +crc32c_libcrc32c_sse42_la_CXXFLAGS = $(crc32c_libcrc32c_la_CXXFLAGS) $(SSE42_CXXFLAGS) +crc32c_libcrc32c_sse42_la_LDFLAGS = $(crc32c_libcrc32c_la_LDFLAGS) -crc32c_libcrc32c_sse42_a_SOURCES = crc32c/src/crc32c_sse42.cc +crc32c_libcrc32c_sse42_la_SOURCES = crc32c/src/crc32c_sse42.cc endif if ENABLE_ARM_CRC -LIBCRC32C_ARM_CRC_INT = crc32c/libcrc32c_arm_crc.a -EXTRA_LIBRARIES += $(LIBCRC32C_ARM_CRC_INT) +LIBCRC32C_ARM_CRC_INT = crc32c/libcrc32c_arm_crc.la +noinst_LTLIBRARIES += $(LIBCRC32C_ARM_CRC_INT) LIBCRC32C += $(LIBCRC32C_ARM_CRC_INT) -crc32c_libcrc32c_arm_crc_a_CPPFLAGS = $(crc32c_libcrc32c_a_CPPFLAGS) -crc32c_libcrc32c_arm_crc_a_CXXFLAGS = $(crc32c_libcrc32c_a_CXXFLAGS) $(ARM_CRC_CXXFLAGS) +crc32c_libcrc32c_arm_crc_la_CPPFLAGS = $(crc32c_libcrc32c_la_CPPFLAGS) +crc32c_libcrc32c_arm_crc_la_CXXFLAGS = $(crc32c_libcrc32c_la_CXXFLAGS) $(ARM_CRC_CXXFLAGS) +crc32c_libcrc32c_arm_crc_la_LDFLAGS = $(crc32c_libcrc32c_la_LDFLAGS) -crc32c_libcrc32c_arm_crc_a_SOURCES = crc32c/src/crc32c_arm64.cc +crc32c_libcrc32c_arm_crc_la_SOURCES = crc32c/src/crc32c_arm64.cc endif diff --git a/src/Makefile.leveldb.include b/src/Makefile.leveldb.include index 3bec92482a2..bf14fe206b9 100644 --- a/src/Makefile.leveldb.include +++ b/src/Makefile.leveldb.include @@ -2,18 +2,17 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -LIBLEVELDB_INT = leveldb/libleveldb.a -LIBMEMENV_INT = leveldb/libmemenv.a +LIBLEVELDB_INT = leveldb/libleveldb.la +LIBMEMENV_INT = leveldb/libmemenv.la -EXTRA_LIBRARIES += $(LIBLEVELDB_INT) -EXTRA_LIBRARIES += $(LIBMEMENV_INT) +noinst_LTLIBRARIES += $(LIBLEVELDB_INT) +noinst_LTLIBRARIES += $(LIBMEMENV_INT) LIBLEVELDB = $(LIBLEVELDB_INT) $(LIBCRC32C) LIBMEMENV = $(LIBMEMENV_INT) LEVELDB_CPPFLAGS = LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/include -LEVELDB_CPPFLAGS += -I$(srcdir)/leveldb/helpers/memenv LEVELDB_CPPFLAGS_INT = LEVELDB_CPPFLAGS_INT += -I$(srcdir)/leveldb @@ -37,111 +36,118 @@ else LEVELDB_CPPFLAGS_INT += -DLEVELDB_PLATFORM_POSIX endif -leveldb_libleveldb_a_CPPFLAGS = $(AM_CPPFLAGS) $(LEVELDB_CPPFLAGS_INT) $(LEVELDB_CPPFLAGS) -leveldb_libleveldb_a_CXXFLAGS = $(filter-out -Wconditional-uninitialized -Werror=conditional-uninitialized -Wsuggest-override -Werror=suggest-override, $(AM_CXXFLAGS)) $(PIE_FLAGS) +leveldb_libleveldb_la_CPPFLAGS = $(AM_CPPFLAGS) $(LEVELDB_CPPFLAGS_INT) $(LEVELDB_CPPFLAGS) -leveldb_libleveldb_a_SOURCES= -leveldb_libleveldb_a_SOURCES += leveldb/port/port_stdcxx.h -leveldb_libleveldb_a_SOURCES += leveldb/port/port.h -leveldb_libleveldb_a_SOURCES += leveldb/port/thread_annotations.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/db.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/options.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/comparator.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/filter_policy.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/slice.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table_builder.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/env.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/export.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/c.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/iterator.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/cache.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/dumpfile.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/table.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/write_batch.h -leveldb_libleveldb_a_SOURCES += leveldb/include/leveldb/status.h -leveldb_libleveldb_a_SOURCES += leveldb/db/log_format.h -leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.h -leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.h -leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch_internal.h -leveldb_libleveldb_a_SOURCES += leveldb/db/filename.h -leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.h -leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.h -leveldb_libleveldb_a_SOURCES += leveldb/db/builder.h -leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.h -leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.h -leveldb_libleveldb_a_SOURCES += leveldb/db/skiplist.h -leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.h -leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.h -leveldb_libleveldb_a_SOURCES += leveldb/db/snapshot.h -leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.h -leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.h -leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.h -leveldb_libleveldb_a_SOURCES += leveldb/table/block.h -leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.h -leveldb_libleveldb_a_SOURCES += leveldb/table/merger.h -leveldb_libleveldb_a_SOURCES += leveldb/table/format.h -leveldb_libleveldb_a_SOURCES += leveldb/table/iterator_wrapper.h -leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.h -leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix_test_helper.h -leveldb_libleveldb_a_SOURCES += leveldb/util/env_windows_test_helper.h -leveldb_libleveldb_a_SOURCES += leveldb/util/arena.h -leveldb_libleveldb_a_SOURCES += leveldb/util/random.h -leveldb_libleveldb_a_SOURCES += leveldb/util/posix_logger.h -leveldb_libleveldb_a_SOURCES += leveldb/util/hash.h -leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.h -leveldb_libleveldb_a_SOURCES += leveldb/util/coding.h -leveldb_libleveldb_a_SOURCES += leveldb/util/testutil.h -leveldb_libleveldb_a_SOURCES += leveldb/util/mutexlock.h -leveldb_libleveldb_a_SOURCES += leveldb/util/logging.h -leveldb_libleveldb_a_SOURCES += leveldb/util/no_destructor.h -leveldb_libleveldb_a_SOURCES += leveldb/util/testharness.h -leveldb_libleveldb_a_SOURCES += leveldb/util/windows_logger.h +# Specify -static in both CXXFLAGS and LDFLAGS so libtool will only build a +# static version of this library. We don't need a dynamic version, and a dynamic +# version can't be used on windows anyway because the library doesn't currently +# export DLL symbols. +leveldb_libleveldb_la_CXXFLAGS = $(filter-out -Wconditional-uninitialized -Werror=conditional-uninitialized -Wsuggest-override -Werror=suggest-override, $(AM_CXXFLAGS)) $(PIE_FLAGS) -static +leveldb_libleveldb_la_LDFLAGS = $(AM_LDFLAGS) -static -leveldb_libleveldb_a_SOURCES += leveldb/db/builder.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/c.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/dbformat.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/db_impl.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/db_iter.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/dumpfile.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/filename.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/log_reader.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/log_writer.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/memtable.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/repair.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/table_cache.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/version_edit.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/version_set.cc -leveldb_libleveldb_a_SOURCES += leveldb/db/write_batch.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/block_builder.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/block.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/filter_block.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/format.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/iterator.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/merger.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/table_builder.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/table.cc -leveldb_libleveldb_a_SOURCES += leveldb/table/two_level_iterator.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/arena.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/bloom.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/cache.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/coding.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/comparator.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/crc32c.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/env.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/filter_policy.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/hash.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/histogram.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/logging.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/options.cc -leveldb_libleveldb_a_SOURCES += leveldb/util/status.cc +leveldb_libleveldb_la_SOURCES= +leveldb_libleveldb_la_SOURCES += leveldb/port/port_stdcxx.h +leveldb_libleveldb_la_SOURCES += leveldb/port/port.h +leveldb_libleveldb_la_SOURCES += leveldb/port/thread_annotations.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/db.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/options.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/comparator.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/filter_policy.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/slice.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/table_builder.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/env.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/export.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/c.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/iterator.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/cache.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/dumpfile.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/table.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/write_batch.h +leveldb_libleveldb_la_SOURCES += leveldb/include/leveldb/status.h +leveldb_libleveldb_la_SOURCES += leveldb/db/log_format.h +leveldb_libleveldb_la_SOURCES += leveldb/db/memtable.h +leveldb_libleveldb_la_SOURCES += leveldb/db/version_set.h +leveldb_libleveldb_la_SOURCES += leveldb/db/write_batch_internal.h +leveldb_libleveldb_la_SOURCES += leveldb/db/filename.h +leveldb_libleveldb_la_SOURCES += leveldb/db/version_edit.h +leveldb_libleveldb_la_SOURCES += leveldb/db/dbformat.h +leveldb_libleveldb_la_SOURCES += leveldb/db/builder.h +leveldb_libleveldb_la_SOURCES += leveldb/db/log_writer.h +leveldb_libleveldb_la_SOURCES += leveldb/db/db_iter.h +leveldb_libleveldb_la_SOURCES += leveldb/db/skiplist.h +leveldb_libleveldb_la_SOURCES += leveldb/db/db_impl.h +leveldb_libleveldb_la_SOURCES += leveldb/db/table_cache.h +leveldb_libleveldb_la_SOURCES += leveldb/db/snapshot.h +leveldb_libleveldb_la_SOURCES += leveldb/db/log_reader.h +leveldb_libleveldb_la_SOURCES += leveldb/table/filter_block.h +leveldb_libleveldb_la_SOURCES += leveldb/table/block_builder.h +leveldb_libleveldb_la_SOURCES += leveldb/table/block.h +leveldb_libleveldb_la_SOURCES += leveldb/table/two_level_iterator.h +leveldb_libleveldb_la_SOURCES += leveldb/table/merger.h +leveldb_libleveldb_la_SOURCES += leveldb/table/format.h +leveldb_libleveldb_la_SOURCES += leveldb/table/iterator_wrapper.h +leveldb_libleveldb_la_SOURCES += leveldb/util/crc32c.h +leveldb_libleveldb_la_SOURCES += leveldb/util/env_posix_test_helper.h +leveldb_libleveldb_la_SOURCES += leveldb/util/env_windows_test_helper.h +leveldb_libleveldb_la_SOURCES += leveldb/util/arena.h +leveldb_libleveldb_la_SOURCES += leveldb/util/random.h +leveldb_libleveldb_la_SOURCES += leveldb/util/posix_logger.h +leveldb_libleveldb_la_SOURCES += leveldb/util/hash.h +leveldb_libleveldb_la_SOURCES += leveldb/util/histogram.h +leveldb_libleveldb_la_SOURCES += leveldb/util/coding.h +leveldb_libleveldb_la_SOURCES += leveldb/util/testutil.h +leveldb_libleveldb_la_SOURCES += leveldb/util/mutexlock.h +leveldb_libleveldb_la_SOURCES += leveldb/util/logging.h +leveldb_libleveldb_la_SOURCES += leveldb/util/no_destructor.h +leveldb_libleveldb_la_SOURCES += leveldb/util/testharness.h +leveldb_libleveldb_la_SOURCES += leveldb/util/windows_logger.h + +leveldb_libleveldb_la_SOURCES += leveldb/db/builder.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/c.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/dbformat.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/db_impl.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/db_iter.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/dumpfile.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/filename.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/log_reader.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/log_writer.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/memtable.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/repair.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/table_cache.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/version_edit.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/version_set.cc +leveldb_libleveldb_la_SOURCES += leveldb/db/write_batch.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/block_builder.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/block.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/filter_block.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/format.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/iterator.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/merger.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/table_builder.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/table.cc +leveldb_libleveldb_la_SOURCES += leveldb/table/two_level_iterator.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/arena.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/bloom.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/cache.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/coding.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/comparator.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/crc32c.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/env.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/filter_policy.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/hash.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/histogram.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/logging.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/options.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/status.cc if TARGET_WINDOWS -leveldb_libleveldb_a_SOURCES += leveldb/util/env_windows.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/env_windows.cc else -leveldb_libleveldb_a_SOURCES += leveldb/util/env_posix.cc +leveldb_libleveldb_la_SOURCES += leveldb/util/env_posix.cc endif -leveldb_libmemenv_a_CPPFLAGS = $(leveldb_libleveldb_a_CPPFLAGS) -leveldb_libmemenv_a_CXXFLAGS = $(leveldb_libleveldb_a_CXXFLAGS) -leveldb_libmemenv_a_SOURCES = leveldb/helpers/memenv/memenv.cc -leveldb_libmemenv_a_SOURCES += leveldb/helpers/memenv/memenv.h +leveldb_libmemenv_la_CPPFLAGS = $(leveldb_libleveldb_la_CPPFLAGS) +leveldb_libmemenv_la_CXXFLAGS = $(leveldb_libleveldb_la_CXXFLAGS) +leveldb_libmemenv_la_LDFLAGS = $(leveldb_libleveldb_la_LDFLAGS) +leveldb_libmemenv_la_SOURCES = leveldb/helpers/memenv/memenv.cc +leveldb_libmemenv_la_SOURCES += leveldb/helpers/memenv/memenv.h diff --git a/src/Makefile.minisketch.include b/src/Makefile.minisketch.include index b337f483498..1363bec34ea 100644 --- a/src/Makefile.minisketch.include +++ b/src/Makefile.minisketch.include @@ -31,7 +31,7 @@ if ENABLE_TESTS if !ENABLE_FUZZ MINISKETCH_TEST = minisketch/test TESTS += $(MINISKETCH_TEST) -noinst_PROGRAMS += $(MINISKETCH_TEST) +check_PROGRAMS += $(MINISKETCH_TEST) minisketch_test_SOURCES = $(MINISKETCH_TEST_SOURCES_INT) minisketch_test_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS) diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 962d82b71a2..fa4db87c002 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -268,6 +268,7 @@ BITCOIN_QT_WALLET_CPP = \ qt/transactiondesc.cpp \ qt/transactiondescdialog.cpp \ qt/transactionfilterproxy.cpp \ + qt/transactionoverviewwidget.cpp \ qt/transactionrecord.cpp \ qt/transactiontablemodel.cpp \ qt/transactionview.cpp \ @@ -292,7 +293,7 @@ BITCOIN_QT_RC = qt/res/bitcoin-qt-res.rc BITCOIN_QT_INCLUDES = -DQT_NO_KEYWORDS -DQT_USE_QSTRINGBUILDER qt_libbitcoinqt_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ - $(QT_INCLUDES) $(QT_DBUS_INCLUDES) $(QR_CFLAGS) + $(QT_INCLUDES) $(QT_DBUS_INCLUDES) $(QR_CFLAGS) $(BOOST_CPPFLAGS) qt_libbitcoinqt_a_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) qt_libbitcoinqt_a_OBJCXXFLAGS = $(AM_OBJCXXFLAGS) $(QT_PIE_FLAGS) @@ -328,7 +329,7 @@ endif if ENABLE_ZMQ elements_qt_ldadd += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -elements_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ +elements_qt_ldadd += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \ $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(LIBSECP256K1) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(SQLITE_LIBS) $(LIBELEMENTSSIMPLICITY) elements_qt_ldflags = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) diff --git a/src/Makefile.qt_locale.include b/src/Makefile.qt_locale.include index 8638bd024cd..c44cf61a1a8 100644 --- a/src/Makefile.qt_locale.include +++ b/src/Makefile.qt_locale.include @@ -31,6 +31,7 @@ QT_TS = \ qt/locale/bitcoin_gl.ts \ qt/locale/bitcoin_gl_ES.ts \ qt/locale/bitcoin_gu.ts \ + qt/locale/bitcoin_ha.ts \ qt/locale/bitcoin_he.ts \ qt/locale/bitcoin_hr.ts \ qt/locale/bitcoin_hu.ts \ @@ -43,6 +44,7 @@ QT_TS = \ qt/locale/bitcoin_kl.ts \ qt/locale/bitcoin_km.ts \ qt/locale/bitcoin_ko.ts \ + qt/locale/bitcoin_ku.ts \ qt/locale/bitcoin_ku_IQ.ts \ qt/locale/bitcoin_ky.ts \ qt/locale/bitcoin_la.ts \ @@ -58,6 +60,7 @@ QT_TS = \ qt/locale/bitcoin_ne.ts \ qt/locale/bitcoin_nl.ts \ qt/locale/bitcoin_no.ts \ + qt/locale/bitcoin_pa.ts \ qt/locale/bitcoin_pam.ts \ qt/locale/bitcoin_pl.ts \ qt/locale/bitcoin_pt.ts \ @@ -78,10 +81,13 @@ QT_TS = \ qt/locale/bitcoin_ta.ts \ qt/locale/bitcoin_te.ts \ qt/locale/bitcoin_th.ts \ + qt/locale/bitcoin_tk.ts \ + qt/locale/bitcoin_tl.ts \ qt/locale/bitcoin_tr.ts \ qt/locale/bitcoin_ug.ts \ qt/locale/bitcoin_uk.ts \ qt/locale/bitcoin_ur.ts \ + qt/locale/bitcoin_uz.ts \ qt/locale/bitcoin_uz@Cyrl.ts \ qt/locale/bitcoin_uz@Latn.ts \ qt/locale/bitcoin_vi.ts \ diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index 3c67d099cd9..1c75f96cec0 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -27,7 +27,7 @@ TEST_QT_H = \ qt/test/wallettests.h qt_test_test_elements_qt_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BITCOIN_QT_INCLUDES) \ - $(QT_INCLUDES) $(QT_TEST_INCLUDES) + $(QT_INCLUDES) $(QT_TEST_INCLUDES) $(BOOST_CPPFLAGS) qt_test_test_elements_qt_SOURCES = \ init/bitcoin-qt.cpp \ @@ -55,7 +55,7 @@ if ENABLE_ZMQ qt_test_test_elements_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif qt_test_test_elements_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(QT_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) \ + $(LIBMEMENV) $(QT_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) \ $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(LIBSECP256K1) $(LIBELEMENTSSIMPLICITY) \ $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(SQLITE_LIBS) qt_test_test_elements_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 24631f8450f..5fdc666d3be 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -6,7 +6,7 @@ if ENABLE_FUZZ_BINARY noinst_PROGRAMS += test/fuzz/fuzz endif -if !ENABLE_FUZZ +if ENABLE_TESTS bin_PROGRAMS += test/test_bitcoin endif @@ -47,7 +47,6 @@ FUZZ_SUITE_LD_COMMON = \ $(LIBBITCOIN_CLI) \ $(LIBUNIVALUE) \ $(LIBLEVELDB) \ - $(LIBLEVELDB_SSE42) \ $(LIBMEMENV) \ $(LIBSECP256K1) \ $(LIBELEMENTSSIMPLICITY) \ @@ -69,6 +68,7 @@ BITCOIN_TESTS =\ test/addrman_tests.cpp \ test/allocator_tests.cpp \ test/amount_tests.cpp \ + test/argsman_tests.cpp \ test/arith_uint256_tests.cpp \ test/banman_tests.cpp \ test/base32_tests.cpp \ @@ -81,6 +81,7 @@ BITCOIN_TESTS =\ test/blockencodings_tests.cpp \ test/blockfilter_index_tests.cpp \ test/blockfilter_tests.cpp \ + test/blockmanager_tests.cpp \ test/bloom_tests.cpp \ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ @@ -97,6 +98,8 @@ BITCOIN_TESTS =\ test/fs_tests.cpp \ test/getarg_tests.cpp \ test/hash_tests.cpp \ + test/headers_sync_chainwork_tests.cpp \ + test/httpserver_tests.cpp \ test/i2p_tests.cpp \ test/interfaces_tests.cpp \ test/key_io_tests.cpp \ @@ -106,11 +109,13 @@ BITCOIN_TESTS =\ test/merkle_tests.cpp \ test/merkleblock_tests.cpp \ test/miner_tests.cpp \ + test/miniscript_tests.cpp \ test/minisketch_tests.cpp \ test/multisig_tests.cpp \ test/net_peer_eviction_tests.cpp \ test/net_tests.cpp \ test/netbase_tests.cpp \ + test/orphanage_tests.cpp \ test/pmt_tests.cpp \ test/policy_fee_tests.cpp \ test/policyestimator_tests.cpp \ @@ -118,12 +123,16 @@ BITCOIN_TESTS =\ test/prevector_tests.cpp \ test/raii_event_tests.cpp \ test/random_tests.cpp \ + test/rbf_tests.cpp \ + test/rest_tests.cpp \ + test/result_tests.cpp \ test/reverselock_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ test/scheduler_tests.cpp \ test/script_p2sh_tests.cpp \ test/script_parse_tests.cpp \ + test/script_segwit_tests.cpp \ test/script_standard_tests.cpp \ test/script_tests.cpp \ test/scriptnum10.h \ @@ -141,8 +150,10 @@ BITCOIN_TESTS =\ test/timedata_tests.cpp \ test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ + test/translation_tests.cpp \ test/txindex_tests.cpp \ test/txpackage_tests.cpp \ + test/txreconciliation_tests.cpp \ test/txrequest_tests.cpp \ test/txvalidation_tests.cpp \ test/txvalidationcache_tests.cpp \ @@ -156,6 +167,7 @@ BITCOIN_TESTS =\ test/validation_tests.cpp \ test/validationinterface_tests.cpp \ test/versionbits_tests.cpp \ + test/xoroshiro128plusplus_tests.cpp \ test/pegin_spent_tests.cpp \ test/dynafed_tests.cpp \ test/blind_tests.cpp @@ -163,6 +175,7 @@ BITCOIN_TESTS =\ if ENABLE_WALLET BITCOIN_TESTS += \ + wallet/test/feebumper_tests.cpp \ wallet/test/spend_tests.cpp \ wallet/test/wallet_tests.cpp \ wallet/test/walletdb_tests.cpp \ @@ -171,7 +184,10 @@ BITCOIN_TESTS += \ wallet/test/coinselector_tests.cpp \ wallet/test/init_tests.cpp \ wallet/test/ismine_tests.cpp \ - wallet/test/scriptpubkeyman_tests.cpp + wallet/test/rpc_util_tests.cpp \ + wallet/test/scriptpubkeyman_tests.cpp \ + wallet/test/walletload_tests.cpp \ + wallet/test/group_outputs_tests.cpp # ELEMENTS: disable PSBT test # wallet/test/psbt_wallet_tests.cpp @@ -183,14 +199,16 @@ if USE_BDB BITCOIN_TESTS += wallet/test/db_tests.cpp endif -if USE_SQLITE FUZZ_WALLET_SRC = \ + wallet/test/fuzz/coinselection.cpp \ + wallet/test/fuzz/parse_iso8601.cpp + +if USE_SQLITE +FUZZ_WALLET_SRC += \ wallet/test/fuzz/notifications.cpp endif # USE_SQLITE BITCOIN_TEST_SUITE += \ - wallet/test/util.cpp \ - wallet/test/util.h \ wallet/test/wallet_test_fixture.cpp \ wallet/test/wallet_test_fixture.h \ wallet/test/init_test_fixture.cpp \ @@ -198,14 +216,15 @@ BITCOIN_TEST_SUITE += \ endif # ENABLE_WALLET test_test_bitcoin_SOURCES = $(BITCOIN_TEST_SUITE) $(BITCOIN_TESTS) $(JSON_TEST_FILES) $(RAW_TEST_FILES) -test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(TESTDEFS) $(EVENT_CFLAGS) +test_test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(TESTDEFS) $(BOOST_CPPFLAGS) $(EVENT_CFLAGS) test_test_bitcoin_LDADD = $(LIBTEST_UTIL) if ENABLE_WALLET test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) +test_test_bitcoin_CPPFLAGS += $(BDB_CPPFLAGS) endif test_test_bitcoin_LDADD += $(LIBBITCOIN_NODE) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \ - $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(LIBSECP256K1) $(LIBELEMENTSSIMPLICITY) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS) + $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) $(LIBELEMENTSSIMPLICITY) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS) test_test_bitcoin_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_test_bitcoin_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(NATPMP_LIBS) $(SQLITE_LIBS) @@ -217,7 +236,7 @@ FUZZ_SUITE_LD_COMMON += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif if ENABLE_FUZZ_BINARY -test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_fuzz_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) test_fuzz_fuzz_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_fuzz_CFLAGS = $(AM_CFLAGS) $(PIE_FLAGS) test_fuzz_fuzz_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -232,6 +251,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/banman.cpp \ test/fuzz/base_encode_decode.cpp \ test/fuzz/bech32.cpp \ + test/fuzz/bitdeque.cpp \ test/fuzz/block.cpp \ test/fuzz/block_header.cpp \ test/fuzz/blockfilter.cpp \ @@ -240,6 +260,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/chain.cpp \ test/fuzz/checkqueue.cpp \ test/fuzz/coins_view.cpp \ + test/fuzz/coinscache_sim.cpp \ test/fuzz/connman.cpp \ test/fuzz/crypto.cpp \ test/fuzz/crypto_aes256.cpp \ @@ -260,6 +281,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/flatfile.cpp \ test/fuzz/float.cpp \ test/fuzz/golomb_rice.cpp \ + test/fuzz/headerssync.cpp \ test/fuzz/hex.cpp \ test/fuzz/http_request.cpp \ test/fuzz/i2p.cpp \ @@ -271,6 +293,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/locale.cpp \ test/fuzz/merkleblock.cpp \ test/fuzz/message.cpp \ + test/fuzz/miniscript.cpp \ test/fuzz/minisketch.cpp \ test/fuzz/muhash.cpp \ test/fuzz/multiplication_overflow.cpp \ @@ -281,10 +304,10 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/node_eviction.cpp \ test/fuzz/p2p_transport_serialization.cpp \ test/fuzz/parse_hd_keypath.cpp \ - test/fuzz/parse_iso8601.cpp \ test/fuzz/parse_numbers.cpp \ test/fuzz/parse_script.cpp \ test/fuzz/parse_univalue.cpp \ + test/fuzz/partially_downloaded_block.cpp \ test/fuzz/policy_estimator.cpp \ test/fuzz/policy_estimator_io.cpp \ test/fuzz/pow.cpp \ @@ -328,6 +351,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/tx_in.cpp \ test/fuzz/tx_out.cpp \ test/fuzz/tx_pool.cpp \ + test/fuzz/txorphan.cpp \ test/fuzz/txrequest.cpp \ test/fuzz/utxo_snapshot.cpp \ test/fuzz/validation_load_mempool.cpp \ @@ -339,7 +363,7 @@ nodist_test_test_bitcoin_SOURCES = $(GENERATED_TEST_FILES) $(BITCOIN_TESTS): $(GENERATED_TEST_FILES) -CLEAN_BITCOIN_TEST = test/*.gcda test/*.gcno test/fuzz/*.gcda test/fuzz/*.gcno test/util/*.gcda test/util/*.gcno $(GENERATED_TEST_FILES) $(BITCOIN_TESTS:=.log) +CLEAN_BITCOIN_TEST = test/*.gcda test/*.gcno test/fuzz/*.gcda test/fuzz/*.gcno test/util/*.gcda test/util/*.gcno $(GENERATED_TEST_FILES) $(addsuffix .log,$(basename $(BITCOIN_TESTS))) CLEANFILES += $(CLEAN_BITCOIN_TEST) @@ -369,14 +393,14 @@ endif if TARGET_WINDOWS else if ENABLE_BENCH - @echo "Running bench/bench_bitcoin ..." - $(BENCH_BINARY) > /dev/null + @echo "Running bench/bench_bitcoin (one iteration sanity check, only high priority)..." + $(BENCH_BINARY) -sanity-check -priority-level=high endif endif $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check -if !ENABLE_FUZZ -UNIVALUE_TESTS = univalue/test/object univalue/test/unitester univalue/test/no_nul +if ENABLE_TESTS +UNIVALUE_TESTS = univalue/test/object univalue/test/unitester noinst_PROGRAMS += $(UNIVALUE_TESTS) TESTS += $(UNIVALUE_TESTS) @@ -385,11 +409,6 @@ univalue_test_unitester_LDADD = $(LIBUNIVALUE) univalue_test_unitester_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) -DJSON_TEST_SRC=\"$(srcdir)/$(UNIVALUE_TEST_DATA_DIR_INT)\" univalue_test_unitester_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) -univalue_test_no_nul_SOURCES = $(UNIVALUE_TEST_NO_NUL_INT) -univalue_test_no_nul_LDADD = $(LIBUNIVALUE) -univalue_test_no_nul_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) -univalue_test_no_nul_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) - univalue_test_object_SOURCES = $(UNIVALUE_TEST_OBJECT_INT) univalue_test_object_LDADD = $(LIBUNIVALUE) univalue_test_object_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) @@ -397,8 +416,19 @@ univalue_test_object_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) endif %.cpp.test: %.cpp - @echo Running tests: `cat $< | grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1` from $< - $(AM_V_at)$(TEST_BINARY) --catch_system_errors=no -l test_suite -t "`cat $< | grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1`" -- DEBUG_LOG_OUT > $<.log 2>&1 || (cat $<.log && false) + @echo Running tests: $$(\ + cat $< | \ + grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | \ + cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1\ + ) from $< + $(AM_V_at)export TEST_LOGFILE=$(abs_builddir)/$$(\ + echo $< | grep -E -o "(wallet/test/.*\.cpp|test/.*\.cpp)" | $(SED) -e s/\.cpp/.log/ \ + ) && \ + $(TEST_BINARY) --catch_system_errors=no -l test_suite -t "$$(\ + cat $< | \ + grep -E "(BOOST_FIXTURE_TEST_SUITE\\(|BOOST_AUTO_TEST_SUITE\\()" | \ + cut -d '(' -f 2 | cut -d ',' -f 1 | cut -d ')' -f 1\ + )" -- DEBUG_LOG_OUT > "$$TEST_LOGFILE" 2>&1 || (cat "$$TEST_LOGFILE" && false) %.json.h: %.json @$(MKDIR_P) $(@D) @@ -409,12 +439,3 @@ endif echo "};};"; \ } > "$@.new" && mv -f "$@.new" "$@" @echo "Generated $@" - -%.raw.h: %.raw - @$(MKDIR_P) $(@D) - @{ \ - echo "static unsigned const char $(*F)_raw[] = {" && \ - $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ - echo "};"; \ - } > "$@.new" && mv -f "$@.new" "$@" - @echo "Generated $@" diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index 9574454fd27..aa9c0527509 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -10,16 +10,15 @@ EXTRA_LIBRARIES += \ TEST_FUZZ_H = \ test/fuzz/fuzz.h \ test/fuzz/FuzzedDataProvider.h \ - test/fuzz/util.h + test/fuzz/util.h \ + test/fuzz/util/mempool.h \ + test/fuzz/util/net.h -libtest_fuzz_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) +libtest_fuzz_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libtest_fuzz_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libtest_fuzz_a_SOURCES = \ test/fuzz/fuzz.cpp \ test/fuzz/util.cpp \ + test/fuzz/util/mempool.cpp \ + test/fuzz/util/net.cpp \ $(TEST_FUZZ_H) - -LIBTEST_FUZZ += $(LIBBITCOIN_NODE) -LIBTEST_FUZZ += $(LIBBITCOIN_COMMON) -LIBTEST_FUZZ += $(LIBBITCOIN_UTIL) -LIBTEST_FUZZ += $(LIBBITCOIN_CRYPTO_BASE) diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 92cb8a5ce6f..aefefe789a7 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -5,25 +5,35 @@ LIBTEST_UTIL=libtest_util.a EXTRA_LIBRARIES += \ - $(LIBTEST_UTIL) + $(LIBTEST_UTIL) TEST_UTIL_H = \ - test/util/blockfilter.h \ - test/util/chainstate.h \ - test/util/logging.h \ - test/util/mining.h \ - test/util/net.h \ - test/util/script.h \ - test/util/setup_common.h \ - test/util/str.h \ - test/util/transaction_utils.h \ - test/util/validation.h \ - test/util/wallet.h + test/util/blockfilter.h \ + test/util/chainstate.h \ + test/util/coins.h \ + test/util/json.h \ + test/util/logging.h \ + test/util/mining.h \ + test/util/net.h \ + test/util/random.h \ + test/util/script.h \ + test/util/setup_common.h \ + test/util/str.h \ + test/util/transaction_utils.h \ + test/util/txmempool.h \ + test/util/validation.h \ + test/util/xoroshiro128plusplus.h -libtest_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) +if ENABLE_WALLET +TEST_UTIL_H += wallet/test/util.h +endif # ENABLE_WALLET + +libtest_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libtest_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libtest_util_a_SOURCES = \ test/util/blockfilter.cpp \ + test/util/coins.cpp \ + test/util/json.cpp \ test/util/logging.cpp \ test/util/mining.cpp \ test/util/net.cpp \ @@ -31,11 +41,11 @@ libtest_util_a_SOURCES = \ test/util/setup_common.cpp \ test/util/str.cpp \ test/util/transaction_utils.cpp \ - test/util/validation.cpp \ - test/util/wallet.cpp \ - $(TEST_UTIL_H) + test/util/txmempool.cpp \ + test/util/validation.cpp + +if ENABLE_WALLET +libtest_util_a_SOURCES += wallet/test/util.cpp +endif # ENABLE_WALLET -LIBTEST_UTIL += $(LIBBITCOIN_NODE) -LIBTEST_UTIL += $(LIBBITCOIN_COMMON) -LIBTEST_UTIL += $(LIBBITCOIN_UTIL) -LIBTEST_UTIL += $(LIBBITCOIN_CRYPTO_BASE) +libtest_util_a_SOURCES += $(TEST_UTIL_H) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 0fa8f3c3da9..8ecccd4d22e 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2021 The Bitcoin Core developers +// Copyright (c) 2009-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,14 +9,17 @@ #include #include #include -#include #include +#include #include #include +#include #include #include #include #include +#include +#include #include #include #include @@ -33,10 +36,9 @@ bool SerializeDB(Stream& stream, const Data& data) { // Write and commit header, data try { - CHashWriter hasher(stream.GetType(), stream.GetVersion()); - stream << Params().MessageStart() << data; - hasher << Params().MessageStart() << data; - stream << hasher.GetHash(); + HashedSourceWriter hashwriter{stream}; + hashwriter << Params().MessageStart() << data; + stream << hashwriter.GetHash(); } catch (const std::exception& e) { return error("%s: Serialize or I/O error - %s", __func__, e.what()); } @@ -48,12 +50,11 @@ template bool SerializeFileDB(const std::string& prefix, const fs::path& path, const Data& data, int version) { // Generate random temporary filename - uint16_t randv = 0; - GetRandBytes((unsigned char*)&randv, sizeof(randv)); + const uint16_t randv{GetRand()}; std::string tmpfn = strprintf("%s.%04x", prefix, randv); // open temp output file, and associate with CAutoFile - fs::path pathTmp = gArgs.GetDataDirNet() / tmpfn; + fs::path pathTmp = gArgs.GetDataDirNet() / fs::u8path(tmpfn); FILE *file = fsbridge::fopen(pathTmp, "wb"); CAutoFile fileout(file, SER_DISK, version); if (fileout.IsNull()) { @@ -182,19 +183,19 @@ void ReadFromStream(AddrMan& addr, CDataStream& ssPeers) DeserializeDB(ssPeers, addr, false); } -std::optional LoadAddrman(const std::vector& asmap, const ArgsManager& args, std::unique_ptr& addrman) +std::optional LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr& addrman) { auto check_addrman = std::clamp(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); - addrman = std::make_unique(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + addrman = std::make_unique(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); - int64_t nStart = GetTimeMillis(); + const auto start{SteadyClock::now()}; const auto path_addr{args.GetDataDirNet() / "peers.dat"}; try { DeserializeFileDB(path_addr, *addrman, CLIENT_VERSION); - LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->size(), GetTimeMillis() - nStart); + LogPrintf("Loaded %i addresses from peers.dat %dms\n", addrman->Size(), Ticks(SteadyClock::now() - start)); } catch (const DbNotFoundError&) { // Addrman can be in an inconsistent state after failure, reset it - addrman = std::make_unique(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + addrman = std::make_unique(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating peers.dat because the file was not found (%s)\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const InvalidAddrManVersionError&) { @@ -203,7 +204,7 @@ std::optional LoadAddrman(const std::vector& asmap, const A return strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again.")); } // Addrman can be in an inconsistent state after failure, reset it - addrman = std::make_unique(asmap, /* deterministic */ false, /* consistency_check_ratio */ check_addrman); + addrman = std::make_unique(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating new peers.dat because the file version was not compatible (%s). Original backed up to peers.dat.bak\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const std::exception& e) { diff --git a/src/addrdb.h b/src/addrdb.h index f568cb05abb..d77ff0f1a33 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -6,9 +6,9 @@ #ifndef BITCOIN_ADDRDB_H #define BITCOIN_ADDRDB_H -#include #include // For banmap_t #include +#include #include #include @@ -18,6 +18,7 @@ class ArgsManager; class AddrMan; class CAddress; class CDataStream; +class NetGroupManager; struct bilingual_str; bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr); @@ -49,7 +50,7 @@ class CBanDB }; /** Returns an error string on failure */ -std::optional LoadAddrman(const std::vector& asmap, const ArgsManager& args, std::unique_ptr& addrman); +std::optional LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr& addrman); /** * Dump the anchor IP address database (anchors.dat) diff --git a/src/addrman.cpp b/src/addrman.cpp index f91a9799346..f5ca9a5c343 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2021 The Bitcoin Core developers +// Copyright (c) 2012-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,10 +14,10 @@ #include #include #include -#include #include #include #include +#include #include #include @@ -29,31 +29,31 @@ static constexpr uint32_t ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP{64}; /** Maximum number of times an address can occur in the new table */ static constexpr int32_t ADDRMAN_NEW_BUCKETS_PER_ADDRESS{8}; /** How old addresses can maximally be */ -static constexpr int64_t ADDRMAN_HORIZON_DAYS{30}; +static constexpr auto ADDRMAN_HORIZON{30 * 24h}; /** After how many failed attempts we give up on a new node */ static constexpr int32_t ADDRMAN_RETRIES{3}; /** How many successive failures are allowed ... */ static constexpr int32_t ADDRMAN_MAX_FAILURES{10}; -/** ... in at least this many days */ -static constexpr int64_t ADDRMAN_MIN_FAIL_DAYS{7}; +/** ... in at least this duration */ +static constexpr auto ADDRMAN_MIN_FAIL{7 * 24h}; /** How recent a successful connection should be before we allow an address to be evicted from tried */ -static constexpr int64_t ADDRMAN_REPLACEMENT_HOURS{4}; +static constexpr auto ADDRMAN_REPLACEMENT{4h}; /** The maximum number of tried addr collisions to store */ static constexpr size_t ADDRMAN_SET_TRIED_COLLISION_SIZE{10}; -/** The maximum time we'll spend trying to resolve a tried table collision, in seconds */ -static constexpr int64_t ADDRMAN_TEST_WINDOW{40*60}; // 40 minutes +/** The maximum time we'll spend trying to resolve a tried table collision */ +static constexpr auto ADDRMAN_TEST_WINDOW{40min}; -int AddrInfo::GetTriedBucket(const uint256& nKey, const std::vector& asmap) const +int AddrInfo::GetTriedBucket(const uint256& nKey, const NetGroupManager& netgroupman) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetCheapHash(); - uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); + uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << netgroupman.GetGroup(*this) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetCheapHash(); return hash2 % ADDRMAN_TRIED_BUCKET_COUNT; } -int AddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector& asmap) const +int AddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const NetGroupManager& netgroupman) const { - std::vector vchSourceGroupKey = src.GetGroup(asmap); - uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << vchSourceGroupKey).GetCheapHash(); + std::vector vchSourceGroupKey = netgroupman.GetGroup(src); + uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << netgroupman.GetGroup(*this) << vchSourceGroupKey).GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetCheapHash(); return hash2 % ADDRMAN_NEW_BUCKET_COUNT; } @@ -64,34 +64,39 @@ int AddrInfo::GetBucketPosition(const uint256& nKey, bool fNew, int nBucket) con return hash1 % ADDRMAN_BUCKET_SIZE; } -bool AddrInfo::IsTerrible(int64_t nNow) const +bool AddrInfo::IsTerrible(NodeSeconds now) const { - if (nLastTry && nLastTry >= nNow - 60) // never remove things tried in the last minute + if (now - m_last_try <= 1min) { // never remove things tried in the last minute return false; + } - if (nTime > nNow + 10 * 60) // came in a flying DeLorean + if (nTime > now + 10min) { // came in a flying DeLorean return true; + } - if (nTime == 0 || nNow - nTime > ADDRMAN_HORIZON_DAYS * 24 * 60 * 60) // not seen in recent history + if (now - nTime > ADDRMAN_HORIZON) { // not seen in recent history return true; + } - if (nLastSuccess == 0 && nAttempts >= ADDRMAN_RETRIES) // tried N times and never a success + if (TicksSinceEpoch(m_last_success) == 0 && nAttempts >= ADDRMAN_RETRIES) { // tried N times and never a success return true; + } - if (nNow - nLastSuccess > ADDRMAN_MIN_FAIL_DAYS * 24 * 60 * 60 && nAttempts >= ADDRMAN_MAX_FAILURES) // N successive failures in the last week + if (now - m_last_success > ADDRMAN_MIN_FAIL && nAttempts >= ADDRMAN_MAX_FAILURES) { // N successive failures in the last week return true; + } return false; } -double AddrInfo::GetChance(int64_t nNow) const +double AddrInfo::GetChance(NodeSeconds now) const { double fChance = 1.0; - int64_t nSinceLastTry = std::max(nNow - nLastTry, 0); // deprioritize very recent attempts away - if (nSinceLastTry < 60 * 10) + if (now - m_last_try < 10min) { fChance *= 0.01; + } // deprioritize 66% after each failed attempt, but at most 1/28th to avoid the search taking forever or overly penalizing outages. fChance *= pow(0.66, std::min(nAttempts, 8)); @@ -99,11 +104,11 @@ double AddrInfo::GetChance(int64_t nNow) const return fChance; } -AddrManImpl::AddrManImpl(std::vector&& asmap, bool deterministic, int32_t consistency_check_ratio) +AddrManImpl::AddrManImpl(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio) : insecure_rand{deterministic} , nKey{deterministic ? uint256{1} : insecure_rand.rand256()} , m_consistency_check_ratio{consistency_check_ratio} - , m_asmap{std::move(asmap)} + , m_netgroupman{netgroupman} { for (auto& bucket : vvNew) { for (auto& entry : bucket) { @@ -218,11 +223,7 @@ void AddrManImpl::Serialize(Stream& s_) const } // Store asmap checksum after bucket entries so that it // can be ignored by older clients for backward compatibility. - uint256 asmap_checksum; - if (m_asmap.size() != 0) { - asmap_checksum = SerializeHash(m_asmap); - } - s << asmap_checksum; + s << m_netgroupman.GetAsmapChecksum(); } template @@ -246,12 +247,18 @@ void AddrManImpl::Unserialize(Stream& s_) uint8_t compat; s >> compat; + if (compat < INCOMPATIBILITY_BASE) { + throw std::ios_base::failure(strprintf( + "Corrupted addrman database: The compat value (%u) " + "is lower than the expected minimum value %u.", + compat, INCOMPATIBILITY_BASE)); + } const uint8_t lowest_compatible = compat - INCOMPATIBILITY_BASE; if (lowest_compatible > FILE_FORMAT) { throw InvalidAddrManVersionError(strprintf( "Unsupported format of addrman database: %u. It is compatible with formats >=%u, " "but the maximum supported by this version of %s is %u.", - uint8_t{format}, uint8_t{lowest_compatible}, PACKAGE_NAME, uint8_t{FILE_FORMAT})); + uint8_t{format}, lowest_compatible, PACKAGE_NAME, uint8_t{FILE_FORMAT})); } s >> nKey; @@ -284,6 +291,7 @@ void AddrManImpl::Unserialize(Stream& s_) mapAddr[info] = n; info.nRandomPos = vRandom.size(); vRandom.push_back(n); + m_network_counts[info.GetNetwork()].n_new++; } nIdCount = nNew; @@ -292,7 +300,7 @@ void AddrManImpl::Unserialize(Stream& s_) for (int n = 0; n < nTried; n++) { AddrInfo info; s >> info; - int nKBucket = info.GetTriedBucket(nKey, m_asmap); + int nKBucket = info.GetTriedBucket(nKey, m_netgroupman); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); if (info.IsValid() && vvTried[nKBucket][nKBucketPos] == -1) { @@ -303,6 +311,7 @@ void AddrManImpl::Unserialize(Stream& s_) mapAddr[info] = nIdCount; vvTried[nKBucket][nKBucketPos] = nIdCount; nIdCount++; + m_network_counts[info.GetNetwork()].n_tried++; } else { nLost++; } @@ -329,10 +338,7 @@ void AddrManImpl::Unserialize(Stream& s_) // If the bucket count and asmap checksum haven't changed, then attempt // to restore the entries to the buckets/positions they were in before // serialization. - uint256 supplied_asmap_checksum; - if (m_asmap.size() != 0) { - supplied_asmap_checksum = SerializeHash(m_asmap); - } + uint256 supplied_asmap_checksum{m_netgroupman.GetAsmapChecksum()}; uint256 serialized_asmap_checksum; if (format >= Format::V2_ASMAP) { s >> serialized_asmap_checksum; @@ -365,7 +371,7 @@ void AddrManImpl::Unserialize(Stream& s_) } else { // In case the new table data cannot be used (bucket count wrong or new asmap), // try to give them a reference based on their primary source address. - bucket = info.GetNewBucket(nKey, m_asmap); + bucket = info.GetNewBucket(nKey, m_netgroupman); bucket_position = info.GetBucketPosition(nKey, true, bucket); if (vvNew[bucket][bucket_position] == -1) { vvNew[bucket][bucket_position] = entry_index; @@ -421,6 +427,8 @@ AddrInfo* AddrManImpl::Create(const CAddress& addr, const CNetAddr& addrSource, mapAddr[addr] = nId; mapInfo[nId].nRandomPos = vRandom.size(); vRandom.push_back(nId); + nNew++; + m_network_counts[addr.GetNetwork()].n_new++; if (pnId) *pnId = nId; return &mapInfo[nId]; @@ -460,6 +468,7 @@ void AddrManImpl::Delete(int nId) assert(info.nRefCount == 0); SwapRandom(info.nRandomPos, vRandom.size() - 1); + m_network_counts[info.GetNetwork()].n_new--; vRandom.pop_back(); mapAddr.erase(info); mapInfo.erase(nId); @@ -477,7 +486,7 @@ void AddrManImpl::ClearNew(int nUBucket, int nUBucketPos) assert(infoDelete.nRefCount > 0); infoDelete.nRefCount--; vvNew[nUBucket][nUBucketPos] = -1; - LogPrint(BCLog::ADDRMAN, "Removed %s from new[%i][%i]\n", infoDelete.ToString(), nUBucket, nUBucketPos); + LogPrint(BCLog::ADDRMAN, "Removed %s from new[%i][%i]\n", infoDelete.ToStringAddrPort(), nUBucket, nUBucketPos); if (infoDelete.nRefCount == 0) { Delete(nIdDelete); } @@ -489,7 +498,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) AssertLockHeld(cs); // remove the entry from all new buckets - const int start_bucket{info.GetNewBucket(nKey, m_asmap)}; + const int start_bucket{info.GetNewBucket(nKey, m_netgroupman)}; for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; ++n) { const int bucket{(start_bucket + n) % ADDRMAN_NEW_BUCKET_COUNT}; const int pos{info.GetBucketPosition(nKey, true, bucket)}; @@ -500,11 +509,12 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) } } nNew--; + m_network_counts[info.GetNetwork()].n_new--; assert(info.nRefCount == 0); // which tried bucket to move the entry to - int nKBucket = info.GetTriedBucket(nKey, m_asmap); + int nKBucket = info.GetTriedBucket(nKey, m_netgroupman); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); // first make space to add it (the existing tried entry there is moved to new, deleting whatever is there). @@ -518,9 +528,10 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) infoOld.fInTried = false; vvTried[nKBucket][nKBucketPos] = -1; nTried--; + m_network_counts[infoOld.GetNetwork()].n_tried--; // find which new bucket it belongs to - int nUBucket = infoOld.GetNewBucket(nKey, m_asmap); + int nUBucket = infoOld.GetNewBucket(nKey, m_netgroupman); int nUBucketPos = infoOld.GetBucketPosition(nKey, true, nUBucket); ClearNew(nUBucket, nUBucketPos); assert(vvNew[nUBucket][nUBucketPos] == -1); @@ -529,17 +540,19 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId) infoOld.nRefCount = 1; vvNew[nUBucket][nUBucketPos] = nIdEvict; nNew++; + m_network_counts[infoOld.GetNetwork()].n_new++; LogPrint(BCLog::ADDRMAN, "Moved %s from tried[%i][%i] to new[%i][%i] to make space\n", - infoOld.ToString(), nKBucket, nKBucketPos, nUBucket, nUBucketPos); + infoOld.ToStringAddrPort(), nKBucket, nKBucketPos, nUBucket, nUBucketPos); } assert(vvTried[nKBucket][nKBucketPos] == -1); vvTried[nKBucket][nKBucketPos] = nId; nTried++; info.fInTried = true; + m_network_counts[info.GetNetwork()].n_tried++; } -bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) +bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, std::chrono::seconds time_penalty) { AssertLockHeld(cs); @@ -551,22 +564,24 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_ // Do not set a penalty for a source's self-announcement if (addr == source) { - nTimePenalty = 0; + time_penalty = 0s; } if (pinfo) { // periodically update nTime - bool fCurrentlyOnline = (GetAdjustedTime() - addr.nTime < 24 * 60 * 60); - int64_t nUpdateInterval = (fCurrentlyOnline ? 60 * 60 : 24 * 60 * 60); - if (addr.nTime && (!pinfo->nTime || pinfo->nTime < addr.nTime - nUpdateInterval - nTimePenalty)) - pinfo->nTime = std::max((int64_t)0, addr.nTime - nTimePenalty); + const bool currently_online{NodeClock::now() - addr.nTime < 24h}; + const auto update_interval{currently_online ? 1h : 24h}; + if (pinfo->nTime < addr.nTime - update_interval - time_penalty) { + pinfo->nTime = std::max(NodeSeconds{0s}, addr.nTime - time_penalty); + } // add services pinfo->nServices = ServiceFlags(pinfo->nServices | addr.nServices); // do not update if no new information is present - if (!addr.nTime || (pinfo->nTime && addr.nTime <= pinfo->nTime)) + if (addr.nTime <= pinfo->nTime) { return false; + } // do not update if the entry was already in the "tried" table if (pinfo->fInTried) @@ -584,11 +599,10 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_ return false; } else { pinfo = Create(addr, source, &nId); - pinfo->nTime = std::max((int64_t)0, (int64_t)pinfo->nTime - nTimePenalty); - nNew++; + pinfo->nTime = std::max(NodeSeconds{0s}, pinfo->nTime - time_penalty); } - int nUBucket = pinfo->GetNewBucket(nKey, source, m_asmap); + int nUBucket = pinfo->GetNewBucket(nKey, source, m_netgroupman); int nUBucketPos = pinfo->GetBucketPosition(nKey, true, nUBucket); bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; if (vvNew[nUBucket][nUBucketPos] != nId) { @@ -604,7 +618,7 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_ pinfo->nRefCount++; vvNew[nUBucket][nUBucketPos] = nId; LogPrint(BCLog::ADDRMAN, "Added %s mapped to AS%i to new[%i][%i]\n", - addr.ToString(), addr.GetMappedAS(m_asmap), nUBucket, nUBucketPos); + addr.ToStringAddrPort(), m_netgroupman.GetMappedAS(addr), nUBucket, nUBucketPos); } else { if (pinfo->nRefCount == 0) { Delete(nId); @@ -614,13 +628,13 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_ return fInsert; } -bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nTime) +bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, NodeSeconds time) { AssertLockHeld(cs); int nId; - nLastGood = nTime; + m_last_good = time; AddrInfo* pinfo = Find(addr, &nId); @@ -630,8 +644,8 @@ bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nT AddrInfo& info = *pinfo; // update info - info.nLastSuccess = nTime; - info.nLastTry = nTime; + info.m_last_success = time; + info.m_last_try = time; info.nAttempts = 0; // nTime is not updated here, to avoid leaking information about // currently-connected peers. @@ -644,7 +658,7 @@ bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nT // which tried bucket to move the entry to - int tried_bucket = info.GetTriedBucket(nKey, m_asmap); + int tried_bucket = info.GetTriedBucket(nKey, m_netgroupman); int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket); // Will moving this address into tried evict another entry? @@ -655,32 +669,32 @@ bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nT // Output the entry we'd be colliding with, for debugging purposes auto colliding_entry = mapInfo.find(vvTried[tried_bucket][tried_bucket_pos]); LogPrint(BCLog::ADDRMAN, "Collision with %s while attempting to move %s to tried table. Collisions=%d\n", - colliding_entry != mapInfo.end() ? colliding_entry->second.ToString() : "", - addr.ToString(), + colliding_entry != mapInfo.end() ? colliding_entry->second.ToStringAddrPort() : "", + addr.ToStringAddrPort(), m_tried_collisions.size()); return false; } else { // move nId to the tried tables MakeTried(info, nId); LogPrint(BCLog::ADDRMAN, "Moved %s mapped to AS%i to tried[%i][%i]\n", - addr.ToString(), addr.GetMappedAS(m_asmap), tried_bucket, tried_bucket_pos); + addr.ToStringAddrPort(), m_netgroupman.GetMappedAS(addr), tried_bucket, tried_bucket_pos); return true; } } -bool AddrManImpl::Add_(const std::vector &vAddr, const CNetAddr& source, int64_t nTimePenalty) +bool AddrManImpl::Add_(const std::vector& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) { int added{0}; for (std::vector::const_iterator it = vAddr.begin(); it != vAddr.end(); it++) { - added += AddSingle(*it, source, nTimePenalty) ? 1 : 0; + added += AddSingle(*it, source, time_penalty) ? 1 : 0; } if (added > 0) { - LogPrint(BCLog::ADDRMAN, "Added %i addresses (of %i) from %s: %i tried, %i new\n", added, vAddr.size(), source.ToString(), nTried, nNew); + LogPrint(BCLog::ADDRMAN, "Added %i addresses (of %i) from %s: %i tried, %i new\n", added, vAddr.size(), source.ToStringAddr(), nTried, nNew); } return added > 0; } -void AddrManImpl::Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) +void AddrManImpl::Attempt_(const CService& addr, bool fCountFailure, NodeSeconds time) { AssertLockHeld(cs); @@ -693,14 +707,14 @@ void AddrManImpl::Attempt_(const CService& addr, bool fCountFailure, int64_t nTi AddrInfo& info = *pinfo; // update info - info.nLastTry = nTime; - if (fCountFailure && info.nLastCountAttempt < nLastGood) { - info.nLastCountAttempt = nTime; + info.m_last_try = time; + if (fCountFailure && info.m_last_count_attempt < m_last_good) { + info.m_last_count_attempt = time; info.nAttempts++; } } -std::pair AddrManImpl::Select_(bool newOnly) const +std::pair AddrManImpl::Select_(bool newOnly) const { AssertLockHeld(cs); @@ -732,8 +746,8 @@ std::pair AddrManImpl::Select_(bool newOnly) const const AddrInfo& info{it_found->second}; // With probability GetChance() * fChanceFactor, return the entry. if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { - LogPrint(BCLog::ADDRMAN, "Selected %s from tried\n", info.ToString()); - return {info, info.nLastTry}; + LogPrint(BCLog::ADDRMAN, "Selected %s from tried\n", info.ToStringAddrPort()); + return {info, info.m_last_try}; } // Otherwise start over with a (likely) different bucket, and increased chance factor. fChanceFactor *= 1.2; @@ -760,8 +774,8 @@ std::pair AddrManImpl::Select_(bool newOnly) const const AddrInfo& info{it_found->second}; // With probability GetChance() * fChanceFactor, return the entry. if (insecure_rand.randbits(30) < fChanceFactor * info.GetChance() * (1 << 30)) { - LogPrint(BCLog::ADDRMAN, "Selected %s from new\n", info.ToString()); - return {info, info.nLastTry}; + LogPrint(BCLog::ADDRMAN, "Selected %s from new\n", info.ToStringAddrPort()); + return {info, info.m_last_try}; } // Otherwise start over with a (likely) different bucket, and increased chance factor. fChanceFactor *= 1.2; @@ -782,7 +796,7 @@ std::vector AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct } // gather a list of random nodes, skipping those of low quality - const int64_t now{GetAdjustedTime()}; + const auto now{Now()}; std::vector addresses; for (unsigned int n = 0; n < vRandom.size(); n++) { if (addresses.size() >= nNodes) @@ -807,7 +821,7 @@ std::vector AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct return addresses; } -void AddrManImpl::Connected_(const CService& addr, int64_t nTime) +void AddrManImpl::Connected_(const CService& addr, NodeSeconds time) { AssertLockHeld(cs); @@ -820,9 +834,10 @@ void AddrManImpl::Connected_(const CService& addr, int64_t nTime) AddrInfo& info = *pinfo; // update info - int64_t nUpdateInterval = 20 * 60; - if (nTime - info.nTime > nUpdateInterval) - info.nTime = nTime; + const auto update_interval{20min}; + if (time - info.nTime > update_interval) { + info.nTime = time; + } } void AddrManImpl::SetServices_(const CService& addr, ServiceFlags nServices) @@ -857,7 +872,7 @@ void AddrManImpl::ResolveCollisions_() AddrInfo& info_new = mapInfo[id_new]; // Which tried bucket to move the entry to. - int tried_bucket = info_new.GetTriedBucket(nKey, m_asmap); + int tried_bucket = info_new.GetTriedBucket(nKey, m_netgroupman); int tried_bucket_pos = info_new.GetBucketPosition(nKey, false, tried_bucket); if (!info_new.IsValid()) { // id_new may no longer map to a valid address erase_collision = true; @@ -867,29 +882,31 @@ void AddrManImpl::ResolveCollisions_() int id_old = vvTried[tried_bucket][tried_bucket_pos]; AddrInfo& info_old = mapInfo[id_old]; + const auto current_time{Now()}; + // Has successfully connected in last X hours - if (GetAdjustedTime() - info_old.nLastSuccess < ADDRMAN_REPLACEMENT_HOURS*(60*60)) { + if (current_time - info_old.m_last_success < ADDRMAN_REPLACEMENT) { erase_collision = true; - } else if (GetAdjustedTime() - info_old.nLastTry < ADDRMAN_REPLACEMENT_HOURS*(60*60)) { // attempted to connect and failed in last X hours + } else if (current_time - info_old.m_last_try < ADDRMAN_REPLACEMENT) { // attempted to connect and failed in last X hours // Give address at least 60 seconds to successfully connect - if (GetAdjustedTime() - info_old.nLastTry > 60) { - LogPrint(BCLog::ADDRMAN, "Replacing %s with %s in tried table\n", info_old.ToString(), info_new.ToString()); + if (current_time - info_old.m_last_try > 60s) { + LogPrint(BCLog::ADDRMAN, "Replacing %s with %s in tried table\n", info_old.ToStringAddrPort(), info_new.ToStringAddrPort()); // Replaces an existing address already in the tried table with the new address - Good_(info_new, false, GetAdjustedTime()); + Good_(info_new, false, current_time); erase_collision = true; } - } else if (GetAdjustedTime() - info_new.nLastSuccess > ADDRMAN_TEST_WINDOW) { + } else if (current_time - info_new.m_last_success > ADDRMAN_TEST_WINDOW) { // If the collision hasn't resolved in some reasonable amount of time, // just evict the old entry -- we must not be able to // connect to it for some reason. - LogPrint(BCLog::ADDRMAN, "Unable to test; replacing %s with %s in tried table anyway\n", info_old.ToString(), info_new.ToString()); - Good_(info_new, false, GetAdjustedTime()); + LogPrint(BCLog::ADDRMAN, "Unable to test; replacing %s with %s in tried table anyway\n", info_old.ToStringAddrPort(), info_new.ToStringAddrPort()); + Good_(info_new, false, current_time); erase_collision = true; } } else { // Collision is not actually a collision anymore - Good_(info_new, false, GetAdjustedTime()); + Good_(info_new, false, Now()); erase_collision = true; } } @@ -902,7 +919,7 @@ void AddrManImpl::ResolveCollisions_() } } -std::pair AddrManImpl::SelectTriedCollision_() +std::pair AddrManImpl::SelectTriedCollision_() { AssertLockHeld(cs); @@ -923,11 +940,11 @@ std::pair AddrManImpl::SelectTriedCollision_() const AddrInfo& newInfo = mapInfo[id_new]; // which tried bucket to move the entry to - int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap); + int tried_bucket = newInfo.GetTriedBucket(nKey, m_netgroupman); int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket); const AddrInfo& info_old = mapInfo[vvTried[tried_bucket][tried_bucket_pos]]; - return {info_old, info_old.nLastTry}; + return {info_old, info_old.m_last_try}; } std::optional AddrManImpl::FindAddressEntry_(const CAddress& addr) @@ -939,18 +956,40 @@ std::optional AddrManImpl::FindAddressEntry_(const CAddress& ad if (!addr_info) return std::nullopt; if(addr_info->fInTried) { - int bucket{addr_info->GetTriedBucket(nKey, m_asmap)}; - return AddressPosition(/*tried=*/true, - /*multiplicity=*/1, - /*bucket=*/bucket, - /*position=*/addr_info->GetBucketPosition(nKey, false, bucket)); + int bucket{addr_info->GetTriedBucket(nKey, m_netgroupman)}; + return AddressPosition(/*tried_in=*/true, + /*multiplicity_in=*/1, + /*bucket_in=*/bucket, + /*position_in=*/addr_info->GetBucketPosition(nKey, false, bucket)); } else { - int bucket{addr_info->GetNewBucket(nKey, m_asmap)}; - return AddressPosition(/*tried=*/false, - /*multiplicity=*/addr_info->nRefCount, - /*bucket=*/bucket, - /*position=*/addr_info->GetBucketPosition(nKey, true, bucket)); + int bucket{addr_info->GetNewBucket(nKey, m_netgroupman)}; + return AddressPosition(/*tried_in=*/false, + /*multiplicity_in=*/addr_info->nRefCount, + /*bucket_in=*/bucket, + /*position_in=*/addr_info->GetBucketPosition(nKey, true, bucket)); + } +} + +size_t AddrManImpl::Size_(std::optional net, std::optional in_new) const +{ + AssertLockHeld(cs); + + if (!net.has_value()) { + if (in_new.has_value()) { + return *in_new ? nNew : nTried; + } else { + return vRandom.size(); + } + } + if (auto it = m_network_counts.find(*net); it != m_network_counts.end()) { + auto net_count = it->second; + if (in_new.has_value()) { + return *in_new ? net_count.n_new : net_count.n_tried; + } else { + return net_count.n_new + net_count.n_tried; + } } + return 0; } void AddrManImpl::Check() const @@ -977,6 +1016,7 @@ int AddrManImpl::CheckAddrman() const std::unordered_set setTried; std::unordered_map mapNew; + std::unordered_map local_counts; if (vRandom.size() != (size_t)(nTried + nNew)) return -7; @@ -985,17 +1025,20 @@ int AddrManImpl::CheckAddrman() const int n = entry.first; const AddrInfo& info = entry.second; if (info.fInTried) { - if (!info.nLastSuccess) + if (!TicksSinceEpoch(info.m_last_success)) { return -1; + } if (info.nRefCount) return -2; setTried.insert(n); + local_counts[info.GetNetwork()].n_tried++; } else { if (info.nRefCount < 0 || info.nRefCount > ADDRMAN_NEW_BUCKETS_PER_ADDRESS) return -3; if (!info.nRefCount) return -4; mapNew[n] = info.nRefCount; + local_counts[info.GetNetwork()].n_new++; } const auto it{mapAddr.find(info)}; if (it == mapAddr.end() || it->second != n) { @@ -1003,10 +1046,12 @@ int AddrManImpl::CheckAddrman() const } if (info.nRandomPos < 0 || (size_t)info.nRandomPos >= vRandom.size() || vRandom[info.nRandomPos] != n) return -14; - if (info.nLastTry < 0) + if (info.m_last_try < NodeSeconds{0s}) { return -6; - if (info.nLastSuccess < 0) + } + if (info.m_last_success < NodeSeconds{0s}) { return -8; + } } if (setTried.size() != (size_t)nTried) @@ -1020,7 +1065,7 @@ int AddrManImpl::CheckAddrman() const if (!setTried.count(vvTried[n][i])) return -11; const auto it{mapInfo.find(vvTried[n][i])}; - if (it == mapInfo.end() || it->second.GetTriedBucket(nKey, m_asmap) != n) { + if (it == mapInfo.end() || it->second.GetTriedBucket(nKey, m_netgroupman) != n) { return -17; } if (it->second.GetBucketPosition(nKey, false, n) != i) { @@ -1053,38 +1098,52 @@ int AddrManImpl::CheckAddrman() const if (nKey.IsNull()) return -16; + // It's possible that m_network_counts may have all-zero entries that local_counts + // doesn't have if addrs from a network were being added and then removed again in the past. + if (m_network_counts.size() < local_counts.size()) { + return -20; + } + for (const auto& [net, count] : m_network_counts) { + if (local_counts[net].n_new != count.n_new || local_counts[net].n_tried != count.n_tried) { + return -21; + } + } + return 0; } -size_t AddrManImpl::size() const +size_t AddrManImpl::Size(std::optional net, std::optional in_new) const { - LOCK(cs); // TODO: Cache this in an atomic to avoid this overhead - return vRandom.size(); + LOCK(cs); + Check(); + auto ret = Size_(net, in_new); + Check(); + return ret; } -bool AddrManImpl::Add(const std::vector& vAddr, const CNetAddr& source, int64_t nTimePenalty) +bool AddrManImpl::Add(const std::vector& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) { LOCK(cs); Check(); - auto ret = Add_(vAddr, source, nTimePenalty); + auto ret = Add_(vAddr, source, time_penalty); Check(); return ret; } -bool AddrManImpl::Good(const CService& addr, int64_t nTime) +bool AddrManImpl::Good(const CService& addr, NodeSeconds time) { LOCK(cs); Check(); - auto ret = Good_(addr, /*test_before_evict=*/true, nTime); + auto ret = Good_(addr, /*test_before_evict=*/true, time); Check(); return ret; } -void AddrManImpl::Attempt(const CService& addr, bool fCountFailure, int64_t nTime) +void AddrManImpl::Attempt(const CService& addr, bool fCountFailure, NodeSeconds time) { LOCK(cs); Check(); - Attempt_(addr, fCountFailure, nTime); + Attempt_(addr, fCountFailure, time); Check(); } @@ -1096,20 +1155,20 @@ void AddrManImpl::ResolveCollisions() Check(); } -std::pair AddrManImpl::SelectTriedCollision() +std::pair AddrManImpl::SelectTriedCollision() { LOCK(cs); Check(); - const auto ret = SelectTriedCollision_(); + auto ret = SelectTriedCollision_(); Check(); return ret; } -std::pair AddrManImpl::Select(bool newOnly) const +std::pair AddrManImpl::Select(bool newOnly) const { LOCK(cs); Check(); - const auto addrRet = Select_(newOnly); + auto addrRet = Select_(newOnly); Check(); return addrRet; } @@ -1118,16 +1177,16 @@ std::vector AddrManImpl::GetAddr(size_t max_addresses, size_t max_pct, { LOCK(cs); Check(); - const auto addresses = GetAddr_(max_addresses, max_pct, network); + auto addresses = GetAddr_(max_addresses, max_pct, network); Check(); return addresses; } -void AddrManImpl::Connected(const CService& addr, int64_t nTime) +void AddrManImpl::Connected(const CService& addr, NodeSeconds time) { LOCK(cs); Check(); - Connected_(addr, nTime); + Connected_(addr, time); Check(); } @@ -1148,13 +1207,8 @@ std::optional AddrManImpl::FindAddressEntry(const CAddress& add return entry; } -const std::vector& AddrManImpl::GetAsmap() const -{ - return m_asmap; -} - -AddrMan::AddrMan(std::vector asmap, bool deterministic, int32_t consistency_check_ratio) - : m_impl(std::make_unique(std::move(asmap), deterministic, consistency_check_ratio)) {} +AddrMan::AddrMan(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio) + : m_impl(std::make_unique(netgroupman, deterministic, consistency_check_ratio)) {} AddrMan::~AddrMan() = default; @@ -1171,32 +1225,31 @@ void AddrMan::Unserialize(Stream& s_) } // explicit instantiation -template void AddrMan::Serialize(CHashWriter& s) const; -template void AddrMan::Serialize(CAutoFile& s) const; +template void AddrMan::Serialize(HashedSourceWriter& s) const; template void AddrMan::Serialize(CDataStream& s) const; template void AddrMan::Unserialize(CAutoFile& s); template void AddrMan::Unserialize(CHashVerifier& s); template void AddrMan::Unserialize(CDataStream& s); template void AddrMan::Unserialize(CHashVerifier& s); -size_t AddrMan::size() const +size_t AddrMan::Size(std::optional net, std::optional in_new) const { - return m_impl->size(); + return m_impl->Size(net, in_new); } -bool AddrMan::Add(const std::vector& vAddr, const CNetAddr& source, int64_t nTimePenalty) +bool AddrMan::Add(const std::vector& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) { - return m_impl->Add(vAddr, source, nTimePenalty); + return m_impl->Add(vAddr, source, time_penalty); } -bool AddrMan::Good(const CService& addr, int64_t nTime) +bool AddrMan::Good(const CService& addr, NodeSeconds time) { - return m_impl->Good(addr, nTime); + return m_impl->Good(addr, time); } -void AddrMan::Attempt(const CService& addr, bool fCountFailure, int64_t nTime) +void AddrMan::Attempt(const CService& addr, bool fCountFailure, NodeSeconds time) { - m_impl->Attempt(addr, fCountFailure, nTime); + m_impl->Attempt(addr, fCountFailure, time); } void AddrMan::ResolveCollisions() @@ -1204,12 +1257,12 @@ void AddrMan::ResolveCollisions() m_impl->ResolveCollisions(); } -std::pair AddrMan::SelectTriedCollision() +std::pair AddrMan::SelectTriedCollision() { return m_impl->SelectTriedCollision(); } -std::pair AddrMan::Select(bool newOnly) const +std::pair AddrMan::Select(bool newOnly) const { return m_impl->Select(newOnly); } @@ -1219,9 +1272,9 @@ std::vector AddrMan::GetAddr(size_t max_addresses, size_t max_pct, std return m_impl->GetAddr(max_addresses, max_pct, network); } -void AddrMan::Connected(const CService& addr, int64_t nTime) +void AddrMan::Connected(const CService& addr, NodeSeconds time) { - m_impl->Connected(addr, nTime); + m_impl->Connected(addr, time); } void AddrMan::SetServices(const CService& addr, ServiceFlags nServices) @@ -1229,11 +1282,6 @@ void AddrMan::SetServices(const CService& addr, ServiceFlags nServices) m_impl->SetServices(addr, nServices); } -const std::vector& AddrMan::GetAsmap() const -{ - return m_impl->GetAsmap(); -} - std::optional AddrMan::FindAddressEntry(const CAddress& addr) { return m_impl->FindAddressEntry(addr); diff --git a/src/addrman.h b/src/addrman.h index 472282833b0..4985fc764cf 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -1,5 +1,5 @@ // Copyright (c) 2012 Pieter Wuille -// Copyright (c) 2012-2021 The Bitcoin Core developers +// Copyright (c) 2012-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -7,9 +7,10 @@ #define BITCOIN_ADDRMAN_H #include +#include #include #include -#include +#include #include #include @@ -88,7 +89,7 @@ class AddrMan const std::unique_ptr m_impl; public: - explicit AddrMan(std::vector asmap, bool deterministic, int32_t consistency_check_ratio); + explicit AddrMan(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio); ~AddrMan(); @@ -98,31 +99,37 @@ class AddrMan template void Unserialize(Stream& s_); - //! Return the number of (unique) addresses in all tables. - size_t size() const; + /** + * Return size information about addrman. + * + * @param[in] net Select addresses only from specified network (nullopt = all) + * @param[in] in_new Select addresses only from one table (true = new, false = tried, nullopt = both) + * @return Number of unique addresses that match specified options. + */ + size_t Size(std::optional net = std::nullopt, std::optional in_new = std::nullopt) const; /** * Attempt to add one or more addresses to addrman's new table. * * @param[in] vAddr Address records to attempt to add. * @param[in] source The address of the node that sent us these addr records. - * @param[in] nTimePenalty A "time penalty" to apply to the address record's nTime. If a peer + * @param[in] time_penalty A "time penalty" to apply to the address record's nTime. If a peer * sends us an address record with nTime=n, then we'll add it to our - * addrman with nTime=(n - nTimePenalty). + * addrman with nTime=(n - time_penalty). * @return true if at least one address is successfully added. */ - bool Add(const std::vector& vAddr, const CNetAddr& source, int64_t nTimePenalty = 0); + bool Add(const std::vector& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty = 0s); /** * Mark an address record as accessible and attempt to move it to addrman's tried table. * * @param[in] addr Address record to attempt to move to tried table. - * @param[in] nTime The time that we were last connected to this peer. + * @param[in] time The time that we were last connected to this peer. * @return true if the address is successfully moved from the new table to the tried table. */ - bool Good(const CService& addr, int64_t nTime = GetAdjustedTime()); + bool Good(const CService& addr, NodeSeconds time = Now()); //! Mark an entry as connection attempted to. - void Attempt(const CService& addr, bool fCountFailure, int64_t nTime = GetAdjustedTime()); + void Attempt(const CService& addr, bool fCountFailure, NodeSeconds time = Now()); //! See if any to-be-evicted tried table entries have been tested and if so resolve the collisions. void ResolveCollisions(); @@ -132,18 +139,18 @@ class AddrMan * attempting to evict. * * @return CAddress The record for the selected tried peer. - * int64_t The last time we attempted to connect to that peer. + * seconds The last time we attempted to connect to that peer. */ - std::pair SelectTriedCollision(); + std::pair SelectTriedCollision(); /** * Choose an address to connect to. * * @param[in] newOnly Whether to only select addresses from the new table. * @return CAddress The record for the selected peer. - * int64_t The last time we attempted to connect to that peer. + * seconds The last time we attempted to connect to that peer. */ - std::pair Select(bool newOnly = false) const; + std::pair Select(bool newOnly = false) const; /** * Return all or many randomly selected addresses, optionally by network. @@ -165,15 +172,13 @@ class AddrMan * not leak information about currently connected peers. * * @param[in] addr The address of the peer we were connected to - * @param[in] nTime The time that we were last connected to this peer + * @param[in] time The time that we were last connected to this peer */ - void Connected(const CService& addr, int64_t nTime = GetAdjustedTime()); + void Connected(const CService& addr, NodeSeconds time = Now()); //! Update an entry's service bits. void SetServices(const CService& addr, ServiceFlags nServices); - const std::vector& GetAsmap() const; - /** Test-only function * Find the address record in AddrMan and return information about its * position. diff --git a/src/addrman_impl.h b/src/addrman_impl.h index 5e76f723429..94fe81aca9b 100644 --- a/src/addrman_impl.h +++ b/src/addrman_impl.h @@ -1,4 +1,4 @@ -// Copyright (c) 2021 The Bitcoin Core developers +// Copyright (c) 2021-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,7 +11,9 @@ #include #include #include +#include #include +#include #include #include @@ -38,16 +40,16 @@ class AddrInfo : public CAddress { public: //! last try whatsoever by us (memory only) - int64_t nLastTry{0}; + NodeSeconds m_last_try{0s}; //! last counted attempt (memory only) - int64_t nLastCountAttempt{0}; + NodeSeconds m_last_count_attempt{0s}; //! where knowledge about this address first came from CNetAddr source; //! last successful connection by us - int64_t nLastSuccess{0}; + NodeSeconds m_last_success{0s}; //! connection attempts since last successful attempt int nAttempts{0}; @@ -64,7 +66,7 @@ class AddrInfo : public CAddress SERIALIZE_METHODS(AddrInfo, obj) { READWRITEAS(CAddress, obj); - READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts); + READWRITE(obj.source, Using>(obj.m_last_success), obj.nAttempts); } AddrInfo(const CAddress &addrIn, const CNetAddr &addrSource) : CAddress(addrIn), source(addrSource) @@ -76,31 +78,31 @@ class AddrInfo : public CAddress } //! Calculate in which "tried" bucket this entry belongs - int GetTriedBucket(const uint256 &nKey, const std::vector &asmap) const; + int GetTriedBucket(const uint256& nKey, const NetGroupManager& netgroupman) const; //! Calculate in which "new" bucket this entry belongs, given a certain source - int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector &asmap) const; + int GetNewBucket(const uint256& nKey, const CNetAddr& src, const NetGroupManager& netgroupman) const; //! Calculate in which "new" bucket this entry belongs, using its default source - int GetNewBucket(const uint256 &nKey, const std::vector &asmap) const + int GetNewBucket(const uint256& nKey, const NetGroupManager& netgroupman) const { - return GetNewBucket(nKey, source, asmap); + return GetNewBucket(nKey, source, netgroupman); } //! Calculate in which position of a bucket to store this entry. int GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const; //! Determine whether the statistics about this entry are bad enough so that it can just be deleted - bool IsTerrible(int64_t nNow = GetAdjustedTime()) const; + bool IsTerrible(NodeSeconds now = Now()) const; //! Calculate the relative chance this entry should be given when selecting nodes to connect to - double GetChance(int64_t nNow = GetAdjustedTime()) const; + double GetChance(NodeSeconds now = Now()) const; }; class AddrManImpl { public: - AddrManImpl(std::vector&& asmap, bool deterministic, int32_t consistency_check_ratio); + AddrManImpl(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio); ~AddrManImpl(); @@ -110,28 +112,28 @@ class AddrManImpl template void Unserialize(Stream& s_) EXCLUSIVE_LOCKS_REQUIRED(!cs); - size_t size() const EXCLUSIVE_LOCKS_REQUIRED(!cs); + size_t Size(std::optional net, std::optional in_new) const EXCLUSIVE_LOCKS_REQUIRED(!cs); - bool Add(const std::vector& vAddr, const CNetAddr& source, int64_t nTimePenalty) + bool Add(const std::vector& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) EXCLUSIVE_LOCKS_REQUIRED(!cs); - bool Good(const CService& addr, int64_t nTime) + bool Good(const CService& addr, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(!cs); - void Attempt(const CService& addr, bool fCountFailure, int64_t nTime) + void Attempt(const CService& addr, bool fCountFailure, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(!cs); void ResolveCollisions() EXCLUSIVE_LOCKS_REQUIRED(!cs); - std::pair SelectTriedCollision() EXCLUSIVE_LOCKS_REQUIRED(!cs); + std::pair SelectTriedCollision() EXCLUSIVE_LOCKS_REQUIRED(!cs); - std::pair Select(bool newOnly) const + std::pair Select(bool newOnly) const EXCLUSIVE_LOCKS_REQUIRED(!cs); std::vector GetAddr(size_t max_addresses, size_t max_pct, std::optional network) const EXCLUSIVE_LOCKS_REQUIRED(!cs); - void Connected(const CService& addr, int64_t nTime) + void Connected(const CService& addr, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(!cs); void SetServices(const CService& addr, ServiceFlags nServices) @@ -140,8 +142,6 @@ class AddrManImpl std::optional FindAddressEntry(const CAddress& addr) EXCLUSIVE_LOCKS_REQUIRED(!cs); - const std::vector& GetAsmap() const; - friend class AddrManDeterministic; private: @@ -204,7 +204,7 @@ class AddrManImpl int vvNew[ADDRMAN_NEW_BUCKET_COUNT][ADDRMAN_BUCKET_SIZE] GUARDED_BY(cs); //! last time Good was called (memory only). Initially set to 1 so that "never" is strictly worse. - int64_t nLastGood GUARDED_BY(cs){1}; + NodeSeconds m_last_good GUARDED_BY(cs){1s}; //! Holds addrs inserted into tried table that collide with existing entries. Test-before-evict discipline used to resolve these collisions. std::set m_tried_collisions; @@ -212,21 +212,16 @@ class AddrManImpl /** Perform consistency checks every m_consistency_check_ratio operations (if non-zero). */ const int32_t m_consistency_check_ratio; - // Compressed IP->ASN mapping, loaded from a file when a node starts. - // Should be always empty if no file was provided. - // This mapping is then used for bucketing nodes in Addrman. - // - // If asmap is provided, nodes will be bucketed by - // AS they belong to, in order to make impossible for a node - // to connect to several nodes hosted in a single AS. - // This is done in response to Erebus attack, but also to generally - // diversify the connections every node creates, - // especially useful when a large fraction of nodes - // operate under a couple of cloud providers. - // - // If a new asmap was provided, the existing records - // would be re-bucketed accordingly. - const std::vector m_asmap; + /** Reference to the netgroup manager. netgroupman must be constructed before addrman and destructed after. */ + const NetGroupManager& m_netgroupman; + + struct NewTriedCount { + size_t n_new; + size_t n_tried; + }; + + /** Number of entries in addrman per network and new/tried table. */ + std::unordered_map m_network_counts GUARDED_BY(cs); //! Find an entry. AddrInfo* Find(const CService& addr, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs); @@ -248,28 +243,30 @@ class AddrManImpl /** Attempt to add a single address to addrman's new table. * @see AddrMan::Add() for parameters. */ - bool AddSingle(const CAddress& addr, const CNetAddr& source, int64_t nTimePenalty) EXCLUSIVE_LOCKS_REQUIRED(cs); + bool AddSingle(const CAddress& addr, const CNetAddr& source, std::chrono::seconds time_penalty) EXCLUSIVE_LOCKS_REQUIRED(cs); - bool Good_(const CService& addr, bool test_before_evict, int64_t time) EXCLUSIVE_LOCKS_REQUIRED(cs); + bool Good_(const CService& addr, bool test_before_evict, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(cs); - bool Add_(const std::vector &vAddr, const CNetAddr& source, int64_t nTimePenalty) EXCLUSIVE_LOCKS_REQUIRED(cs); + bool Add_(const std::vector& vAddr, const CNetAddr& source, std::chrono::seconds time_penalty) EXCLUSIVE_LOCKS_REQUIRED(cs); - void Attempt_(const CService& addr, bool fCountFailure, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + void Attempt_(const CService& addr, bool fCountFailure, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(cs); - std::pair Select_(bool newOnly) const EXCLUSIVE_LOCKS_REQUIRED(cs); + std::pair Select_(bool newOnly) const EXCLUSIVE_LOCKS_REQUIRED(cs); std::vector GetAddr_(size_t max_addresses, size_t max_pct, std::optional network) const EXCLUSIVE_LOCKS_REQUIRED(cs); - void Connected_(const CService& addr, int64_t nTime) EXCLUSIVE_LOCKS_REQUIRED(cs); + void Connected_(const CService& addr, NodeSeconds time) EXCLUSIVE_LOCKS_REQUIRED(cs); void SetServices_(const CService& addr, ServiceFlags nServices) EXCLUSIVE_LOCKS_REQUIRED(cs); void ResolveCollisions_() EXCLUSIVE_LOCKS_REQUIRED(cs); - std::pair SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); + std::pair SelectTriedCollision_() EXCLUSIVE_LOCKS_REQUIRED(cs); std::optional FindAddressEntry_(const CAddress& addr) EXCLUSIVE_LOCKS_REQUIRED(cs); + size_t Size_(std::optional net, std::optional in_new) const EXCLUSIVE_LOCKS_REQUIRED(cs); + //! Consistency check, taking into account m_consistency_check_ratio. //! Will std::abort if an inconsistency is detected. void Check() const EXCLUSIVE_LOCKS_REQUIRED(cs); diff --git a/src/arith_uint256.cpp b/src/arith_uint256.cpp index f7f62dfc680..3776cfb6def 100644 --- a/src/arith_uint256.cpp +++ b/src/arith_uint256.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -146,13 +146,21 @@ double base_uint::getdouble() const template std::string base_uint::GetHex() const { - return ArithToUint256(*this).GetHex(); + base_blob b; + for (int x = 0; x < this->WIDTH; ++x) { + WriteLE32(b.begin() + x*4, this->pn[x]); + } + return b.GetHex(); } template void base_uint::SetHex(const char* psz) { - *this = UintToArith256(uint256S(psz)); + base_blob b; + b.SetHex(psz); + for (int x = 0; x < this->WIDTH; ++x) { + this->pn[x] = ReadLE32(b.begin() + x*4); + } } template @@ -164,7 +172,7 @@ void base_uint::SetHex(const std::string& str) template std::string base_uint::ToString() const { - return (GetHex()); + return GetHex(); } template @@ -183,20 +191,7 @@ unsigned int base_uint::bits() const } // Explicit instantiations for base_uint<256> -template base_uint<256>::base_uint(const std::string&); -template base_uint<256>& base_uint<256>::operator<<=(unsigned int); -template base_uint<256>& base_uint<256>::operator>>=(unsigned int); -template base_uint<256>& base_uint<256>::operator*=(uint32_t b32); -template base_uint<256>& base_uint<256>::operator*=(const base_uint<256>& b); -template base_uint<256>& base_uint<256>::operator/=(const base_uint<256>& b); -template int base_uint<256>::CompareTo(const base_uint<256>&) const; -template bool base_uint<256>::EqualTo(uint64_t) const; -template double base_uint<256>::getdouble() const; -template std::string base_uint<256>::GetHex() const; -template std::string base_uint<256>::ToString() const; -template void base_uint<256>::SetHex(const char*); -template void base_uint<256>::SetHex(const std::string&); -template unsigned int base_uint<256>::bits() const; +template class base_uint<256>; // This implementation directly uses shifts instead of going // through an intermediate MPI representation. diff --git a/src/arith_uint256.h b/src/arith_uint256.h index a0a0429c2a3..c710fe94712 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2019 The Bitcoin Core developers +// Copyright (c) 2009-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -24,22 +24,19 @@ template class base_uint { protected: + static_assert(BITS / 32 > 0 && BITS % 32 == 0, "Template parameter BITS must be a positive multiple of 32."); static constexpr int WIDTH = BITS / 32; uint32_t pn[WIDTH]; public: base_uint() { - static_assert(BITS/32 > 0 && BITS%32 == 0, "Template parameter BITS must be a positive multiple of 32."); - for (int i = 0; i < WIDTH; i++) pn[i] = 0; } base_uint(const base_uint& b) { - static_assert(BITS/32 > 0 && BITS%32 == 0, "Template parameter BITS must be a positive multiple of 32."); - for (int i = 0; i < WIDTH; i++) pn[i] = b.pn[i]; } @@ -53,8 +50,6 @@ class base_uint base_uint(uint64_t b) { - static_assert(BITS/32 > 0 && BITS%32 == 0, "Template parameter BITS must be a positive multiple of 32."); - pn[0] = (unsigned int)b; pn[1] = (unsigned int)(b >> 32); for (int i = 2; i < WIDTH; i++) @@ -63,7 +58,7 @@ class base_uint explicit base_uint(const std::string& str); - const base_uint operator~() const + base_uint operator~() const { base_uint ret; for (int i = 0; i < WIDTH; i++) @@ -71,7 +66,7 @@ class base_uint return ret; } - const base_uint operator-() const + base_uint operator-() const { base_uint ret; for (int i = 0; i < WIDTH; i++) @@ -176,7 +171,7 @@ class base_uint return *this; } - const base_uint operator++(int) + base_uint operator++(int) { // postfix operator const base_uint ret = *this; @@ -193,7 +188,7 @@ class base_uint return *this; } - const base_uint operator--(int) + base_uint operator--(int) { // postfix operator const base_uint ret = *this; @@ -204,16 +199,16 @@ class base_uint int CompareTo(const base_uint& b) const; bool EqualTo(uint64_t b) const; - friend inline const base_uint operator+(const base_uint& a, const base_uint& b) { return base_uint(a) += b; } - friend inline const base_uint operator-(const base_uint& a, const base_uint& b) { return base_uint(a) -= b; } - friend inline const base_uint operator*(const base_uint& a, const base_uint& b) { return base_uint(a) *= b; } - friend inline const base_uint operator/(const base_uint& a, const base_uint& b) { return base_uint(a) /= b; } - friend inline const base_uint operator|(const base_uint& a, const base_uint& b) { return base_uint(a) |= b; } - friend inline const base_uint operator&(const base_uint& a, const base_uint& b) { return base_uint(a) &= b; } - friend inline const base_uint operator^(const base_uint& a, const base_uint& b) { return base_uint(a) ^= b; } - friend inline const base_uint operator>>(const base_uint& a, int shift) { return base_uint(a) >>= shift; } - friend inline const base_uint operator<<(const base_uint& a, int shift) { return base_uint(a) <<= shift; } - friend inline const base_uint operator*(const base_uint& a, uint32_t b) { return base_uint(a) *= b; } + friend inline base_uint operator+(const base_uint& a, const base_uint& b) { return base_uint(a) += b; } + friend inline base_uint operator-(const base_uint& a, const base_uint& b) { return base_uint(a) -= b; } + friend inline base_uint operator*(const base_uint& a, const base_uint& b) { return base_uint(a) *= b; } + friend inline base_uint operator/(const base_uint& a, const base_uint& b) { return base_uint(a) /= b; } + friend inline base_uint operator|(const base_uint& a, const base_uint& b) { return base_uint(a) |= b; } + friend inline base_uint operator&(const base_uint& a, const base_uint& b) { return base_uint(a) &= b; } + friend inline base_uint operator^(const base_uint& a, const base_uint& b) { return base_uint(a) ^= b; } + friend inline base_uint operator>>(const base_uint& a, int shift) { return base_uint(a) >>= shift; } + friend inline base_uint operator<<(const base_uint& a, int shift) { return base_uint(a) <<= shift; } + friend inline base_uint operator*(const base_uint& a, uint32_t b) { return base_uint(a) *= b; } friend inline bool operator==(const base_uint& a, const base_uint& b) { return memcmp(a.pn, b.pn, sizeof(a.pn)) == 0; } friend inline bool operator!=(const base_uint& a, const base_uint& b) { return memcmp(a.pn, b.pn, sizeof(a.pn)) != 0; } friend inline bool operator>(const base_uint& a, const base_uint& b) { return a.CompareTo(b) > 0; } @@ -284,4 +279,6 @@ class arith_uint256 : public base_uint<256> { uint256 ArithToUint256(const arith_uint256 &); arith_uint256 UintToArith256(const uint256 &); +extern template class base_uint<256>; + #endif // BITCOIN_ARITH_UINT256_H diff --git a/src/assetsdir.cpp b/src/assetsdir.cpp index 8d3194d9c9b..4c3711135d0 100644 --- a/src/assetsdir.cpp +++ b/src/assetsdir.cpp @@ -29,7 +29,7 @@ void CAssetsDir::SetHex(const std::string& assetHex, const std::string& label) throw std::runtime_error("The asset must be hex string of length 64"); const std::vector protectedLabels = {"", "*", "bitcoin", "Bitcoin", "btc"}; - for (std::string proLabel : protectedLabels) { + for (const std::string& proLabel : protectedLabels) { if (label == proLabel) { throw std::runtime_error(strprintf("'%s' label is protected", proLabel)); } @@ -74,7 +74,7 @@ std::string CAssetsDir::GetLabel(const CAsset& asset) const std::string CAssetsDir::GetIdentifier(const CAsset& asset) const { - const std::string label = GetMetadata(asset).GetLabel(); + std::string label = GetMetadata(asset).GetLabel(); if (!label.empty()) return label; return asset.GetHex(); } @@ -110,4 +110,3 @@ void ClearGlobalAssetDir() { _gAssetsDir = CAssetsDir(); } - diff --git a/src/assetsdir.h b/src/assetsdir.h index b551b8b71e1..1e74cfe095c 100644 --- a/src/assetsdir.h +++ b/src/assetsdir.h @@ -10,7 +10,7 @@ class AssetMetadata { std::string label; public: - AssetMetadata() : label("") {}; + AssetMetadata() : label() {}; AssetMetadata(std::string _label) : label(_label) {}; const std::string& GetLabel() const diff --git a/src/banman.cpp b/src/banman.cpp index b28e3f7f7cc..ece949d9976 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -1,12 +1,12 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2021 The Bitcoin Core developers +// Copyright (c) 2009-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include -#include +#include #include #include #include @@ -16,26 +16,32 @@ BanMan::BanMan(fs::path ban_file, CClientUIInterface* client_interface, int64_t default_ban_time) : m_client_interface(client_interface), m_ban_db(std::move(ban_file)), m_default_ban_time(default_ban_time) { + LoadBanlist(); + DumpBanlist(); +} + +BanMan::~BanMan() +{ + DumpBanlist(); +} + +void BanMan::LoadBanlist() +{ + LOCK(m_cs_banned); + if (m_client_interface) m_client_interface->InitMessage(_("Loading banlist…").translated); - int64_t n_start = GetTimeMillis(); + const auto start{SteadyClock::now()}; if (m_ban_db.Read(m_banned)) { SweepBanned(); // sweep out unused entries LogPrint(BCLog::NET, "Loaded %d banned node addresses/subnets %dms\n", m_banned.size(), - GetTimeMillis() - n_start); + Ticks(SteadyClock::now() - start)); } else { LogPrintf("Recreating the banlist database\n"); m_banned = {}; m_is_dirty = true; } - - DumpBanlist(); -} - -BanMan::~BanMan() -{ - DumpBanlist(); } void BanMan::DumpBanlist() @@ -52,13 +58,13 @@ void BanMan::DumpBanlist() SetBannedSetDirty(false); } - int64_t n_start = GetTimeMillis(); + const auto start{SteadyClock::now()}; if (!m_ban_db.Write(banmap)) { SetBannedSetDirty(true); } LogPrint(BCLog::NET, "Flushed %d banned node addresses/subnets to disk %dms\n", banmap.size(), - GetTimeMillis() - n_start); + Ticks(SteadyClock::now() - start)); } void BanMan::ClearBanned() @@ -173,23 +179,24 @@ void BanMan::GetBanned(banmap_t& banmap) void BanMan::SweepBanned() { + AssertLockHeld(m_cs_banned); + int64_t now = GetTime(); bool notify_ui = false; - { - LOCK(m_cs_banned); - banmap_t::iterator it = m_banned.begin(); - while (it != m_banned.end()) { - CSubNet sub_net = (*it).first; - CBanEntry ban_entry = (*it).second; - if (!sub_net.IsValid() || now > ban_entry.nBanUntil) { - m_banned.erase(it++); - m_is_dirty = true; - notify_ui = true; - LogPrint(BCLog::NET, "Removed banned node address/subnet: %s\n", sub_net.ToString()); - } else - ++it; + banmap_t::iterator it = m_banned.begin(); + while (it != m_banned.end()) { + CSubNet sub_net = (*it).first; + CBanEntry ban_entry = (*it).second; + if (!sub_net.IsValid() || now > ban_entry.nBanUntil) { + m_banned.erase(it++); + m_is_dirty = true; + notify_ui = true; + LogPrint(BCLog::NET, "Removed banned node address/subnet: %s\n", sub_net.ToString()); + } else { + ++it; } } + // update UI if (notify_ui && m_client_interface) { m_client_interface->BannedListChanged(); diff --git a/src/banman.h b/src/banman.h index f268fffa5a7..5a5f5677b00 100644 --- a/src/banman.h +++ b/src/banman.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2021 The Bitcoin Core developers +// Copyright (c) 2009-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_BANMAN_H @@ -7,9 +7,9 @@ #include #include -#include #include // For banmap_t #include +#include #include #include @@ -80,11 +80,12 @@ class BanMan void DumpBanlist(); private: + void LoadBanlist() EXCLUSIVE_LOCKS_REQUIRED(!m_cs_banned); bool BannedSetIsDirty(); //!set the "dirty" flag for the banlist void SetBannedSetDirty(bool dirty = true); //!clean unused entries (if bantime has expired) - void SweepBanned(); + void SweepBanned() EXCLUSIVE_LOCKS_REQUIRED(m_cs_banned); RecursiveMutex m_cs_banned; banmap_t m_banned GUARDED_BY(m_cs_banned); diff --git a/src/base58.cpp b/src/base58.cpp index dfa2e8db554..cf5d62f164f 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2021 The Bitcoin Core developers +// Copyright (c) 2014-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -126,7 +126,7 @@ std::string EncodeBase58(Span input) bool DecodeBase58(const std::string& str, std::vector& vchRet, int max_ret_len) { - if (!ValidAsCString(str)) { + if (!ContainsNoNUL(str)) { return false; } return DecodeBase58(str.c_str(), vchRet, max_ret_len); @@ -160,7 +160,7 @@ std::string EncodeBase58Check(Span input) bool DecodeBase58Check(const std::string& str, std::vector& vchRet, int max_ret) { - if (!ValidAsCString(str)) { + if (!ContainsNoNUL(str)) { return false; } return DecodeBase58Check(str.c_str(), vchRet, max_ret); diff --git a/src/base58.h b/src/base58.h index 9ba5af73e05..2f4d0b74b1b 100644 --- a/src/base58.h +++ b/src/base58.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2020 The Bitcoin Core developers +// Copyright (c) 2009-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,7 +14,6 @@ #ifndef BITCOIN_BASE58_H #define BITCOIN_BASE58_H -#include #include #include diff --git a/src/bech32.cpp b/src/bech32.cpp index 70b736621b2..4c7ff78a406 100644 --- a/src/bech32.cpp +++ b/src/bech32.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2017, 2021 Pieter Wuille -// Copyright (c) 2021 The Bitcoin Core developers +// Copyright (c) 2021-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -238,7 +238,7 @@ constexpr std::array GenerateSyndromeConstants() { std::array SYNDROME_CONSTS{}; for (int k = 1; k < 6; ++k) { for (int shift = 0; shift < 5; ++shift) { - int16_t b = GF1024_LOG.at(1 << shift); + int16_t b = GF1024_LOG.at(size_t{1} << shift); int16_t c0 = GF1024_EXP.at((997*k + b) % 1023); int16_t c1 = GF1024_EXP.at((998*k + b) % 1023); int16_t c2 = GF1024_EXP.at((999*k + b) % 1023); diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index 3ca58b923e9..d6b52eb587a 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -1,9 +1,10 @@ -// Copyright (c) 2020-2021 The Bitcoin Core developers +// Copyright (c) 2020-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include +#include #include #include #include @@ -16,7 +17,7 @@ static constexpr size_t NUM_SOURCES = 64; static constexpr size_t NUM_ADDRESSES_PER_SOURCE = 256; -static const std::vector EMPTY_ASMAP; +static NetGroupManager EMPTY_NETGROUPMAN{std::vector()}; static constexpr uint32_t ADDRMAN_CONSISTENCY_CHECK_RATIO{0}; static std::vector g_sources; @@ -42,7 +43,7 @@ static void CreateAddresses() CAddress ret(CService(addr, port), NODE_NETWORK); - ret.nTime = GetAdjustedTime(); + ret.nTime = Now(); return ret; }; @@ -77,14 +78,14 @@ static void AddrManAdd(benchmark::Bench& bench) CreateAddresses(); bench.run([&] { - AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; + AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; AddAddressesToAddrMan(addrman); }); } static void AddrManSelect(benchmark::Bench& bench) { - AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; + AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; FillAddrMan(addrman); @@ -96,12 +97,12 @@ static void AddrManSelect(benchmark::Bench& bench) static void AddrManGetAddr(benchmark::Bench& bench) { - AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; + AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; FillAddrMan(addrman); bench.run([&] { - const auto& addresses = addrman.GetAddr(/* max_addresses */ 2500, /* max_pct */ 23, /* network */ std::nullopt); + const auto& addresses = addrman.GetAddr(/*max_addresses=*/2500, /*max_pct=*/23, /*network=*/std::nullopt); assert(addresses.size() > 0); }); } @@ -125,14 +126,14 @@ static void AddrManAddThenGood(benchmark::Bench& bench) // // This has some overhead (exactly the result of AddrManAdd benchmark), but that overhead is constant so improvements in // AddrMan::Good() will still be noticeable. - AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; + AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; AddAddressesToAddrMan(addrman); markSomeAsGood(addrman); }); } -BENCHMARK(AddrManAdd); -BENCHMARK(AddrManSelect); -BENCHMARK(AddrManGetAddr); -BENCHMARK(AddrManAddThenGood); +BENCHMARK(AddrManAdd, benchmark::PriorityLevel::HIGH); +BENCHMARK(AddrManSelect, benchmark::PriorityLevel::HIGH); +BENCHMARK(AddrManGetAddr, benchmark::PriorityLevel::HIGH); +BENCHMARK(AddrManAddThenGood, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index 6f6b4e3bfa9..78748bc5bde 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2020 The Bitcoin Core developers +// Copyright (c) 2016-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -50,6 +50,6 @@ static void Base58Decode(benchmark::Bench& bench) } -BENCHMARK(Base58Encode); -BENCHMARK(Base58CheckEncode); -BENCHMARK(Base58Decode); +BENCHMARK(Base58Encode, benchmark::PriorityLevel::HIGH); +BENCHMARK(Base58CheckEncode, benchmark::PriorityLevel::HIGH); +BENCHMARK(Base58Decode, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/bech32.cpp b/src/bench/bech32.cpp index 0f89a8e2b54..9922653766f 100644 --- a/src/bench/bech32.cpp +++ b/src/bench/bech32.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2021 The Bitcoin Core developers +// Copyright (c) 2018-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -32,5 +32,5 @@ static void Bech32Decode(benchmark::Bench& bench) } -BENCHMARK(Bech32Encode); -BENCHMARK(Bech32Decode); +BENCHMARK(Bech32Encode, benchmark::PriorityLevel::HIGH); +BENCHMARK(Bech32Decode, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 033d3197502..84b66bc4b22 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -1,11 +1,12 @@ -// Copyright (c) 2015-2021 The Bitcoin Core developers +// Copyright (c) 2015-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include -#include #include +#include +#include #include #include @@ -41,35 +42,73 @@ void GenerateTemplateResults(const std::vector& bench } // namespace -benchmark::BenchRunner::BenchmarkMap& benchmark::BenchRunner::benchmarks() +namespace benchmark { + +// map a label to one or multiple priority levels +std::map map_label_priority = { + {"high", PriorityLevel::HIGH}, + {"low", PriorityLevel::LOW}, + {"all", 0xff} +}; + +std::string ListPriorities() +{ + using item_t = std::pair; + auto sort_by_priority = [](item_t a, item_t b){ return a.second < b.second; }; + std::set sorted_priorities(map_label_priority.begin(), map_label_priority.end(), sort_by_priority); + return Join(sorted_priorities, ',', [](const auto& entry){ return entry.first; }); +} + +uint8_t StringToPriority(const std::string& str) +{ + auto it = map_label_priority.find(str); + if (it == map_label_priority.end()) throw std::runtime_error(strprintf("Unknown priority level %s", str)); + return it->second; +} + +BenchRunner::BenchmarkMap& BenchRunner::benchmarks() { - static std::map benchmarks_map; + static BenchmarkMap benchmarks_map; return benchmarks_map; } -benchmark::BenchRunner::BenchRunner(std::string name, benchmark::BenchFunction func) +BenchRunner::BenchRunner(std::string name, BenchFunction func, PriorityLevel level) { - benchmarks().insert(std::make_pair(name, func)); + benchmarks().insert(std::make_pair(name, std::make_pair(func, level))); } -void benchmark::BenchRunner::RunAll(const Args& args) +void BenchRunner::RunAll(const Args& args) { std::regex reFilter(args.regex_filter); std::smatch baseMatch; + if (args.sanity_check) { + std::cout << "Running with -sanity-check option, output is being suppressed as benchmark results will be useless." << std::endl; + } + std::vector benchmarkResults; - for (const auto& p : benchmarks()) { - if (!std::regex_match(p.first, baseMatch, reFilter)) { + for (const auto& [name, bench_func] : benchmarks()) { + const auto& [func, priority_level] = bench_func; + + if (!(priority_level & args.priority)) { + continue; + } + + if (!std::regex_match(name, baseMatch, reFilter)) { continue; } if (args.is_list_only) { - std::cout << p.first << std::endl; + std::cout << name << std::endl; continue; } Bench bench; - bench.name(p.first); + if (args.sanity_check) { + bench.epochs(1).epochIterations(1); + bench.output(nullptr); + } + bench.name(name); if (args.min_time > 0ms) { // convert to nanos before dividing to reduce rounding errors std::chrono::nanoseconds min_time_ns = args.min_time; @@ -77,11 +116,11 @@ void benchmark::BenchRunner::RunAll(const Args& args) } if (args.asymptote.empty()) { - p.second(bench); + func(bench); } else { for (auto n : args.asymptote) { bench.complexityN(n); - p.second(bench); + func(bench); } std::cout << bench.complexityBigO() << std::endl; } @@ -96,3 +135,5 @@ void benchmark::BenchRunner::RunAll(const Args& args) "{{/result}}"); GenerateTemplateResults(benchmarkResults, args.output_json, ankerl::nanobench::templates::json()); } + +} // namespace benchmark diff --git a/src/bench/bench.h b/src/bench/bench.h index 6634138beb0..78196134e78 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -1,11 +1,11 @@ -// Copyright (c) 2015-2021 The Bitcoin Core developers +// Copyright (c) 2015-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_BENCH_BENCH_H #define BITCOIN_BENCH_BENCH_H -#include +#include #include #include @@ -41,29 +41,42 @@ using ankerl::nanobench::Bench; typedef std::function BenchFunction; +enum PriorityLevel : uint8_t +{ + LOW = 1 << 0, + HIGH = 1 << 2, +}; + +// List priority labels, comma-separated and sorted by increasing priority +std::string ListPriorities(); +uint8_t StringToPriority(const std::string& str); + struct Args { bool is_list_only; + bool sanity_check; std::chrono::milliseconds min_time; std::vector asymptote; fs::path output_csv; fs::path output_json; std::string regex_filter; + uint8_t priority; }; class BenchRunner { - typedef std::map BenchmarkMap; + // maps from "name" -> (function, priority_level) + typedef std::map> BenchmarkMap; static BenchmarkMap& benchmarks(); public: - BenchRunner(std::string name, BenchFunction func); + BenchRunner(std::string name, BenchFunction func, PriorityLevel level); static void RunAll(const Args& args); }; } // namespace benchmark -// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo); -#define BENCHMARK(n) \ - benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n); +// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo, priority_level); +#define BENCHMARK(n, priority_level) \ + benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n, priority_level); #endif // BITCOIN_BENCH_BENCH_H diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 3f8bff4bcf5..7faaa1fb14e 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2021 The Bitcoin Core developers +// Copyright (c) 2015-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include @@ -18,6 +18,8 @@ static const char* DEFAULT_BENCH_FILTER = ".*"; static constexpr int64_t DEFAULT_MIN_TIME_MS{10}; +/** Priority level default value, run "all" priority levels */ +static const std::string DEFAULT_PRIORITY{"all"}; static void SetupBenchArgs(ArgsManager& argsman) { @@ -26,9 +28,12 @@ static void SetupBenchArgs(ArgsManager& argsman) argsman.AddArg("-asymptote=", "Test asymptotic growth of the runtime of an algorithm, if supported by the benchmark", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-filter=", strprintf("Regular expression filter to select benchmark by name (default: %s)", DEFAULT_BENCH_FILTER), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-list", "List benchmarks without executing them", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-min_time=", strprintf("Minimum runtime per benchmark, in milliseconds (default: %d)", DEFAULT_MIN_TIME_MS), ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS); - argsman.AddArg("-output_csv=", "Generate CSV file with the most important benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-output_json=", "Generate JSON file with all benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-min-time=", strprintf("Minimum runtime per benchmark, in milliseconds (default: %d)", DEFAULT_MIN_TIME_MS), ArgsManager::ALLOW_ANY | ArgsManager::DISALLOW_NEGATION, OptionsCategory::OPTIONS); + argsman.AddArg("-output-csv=", "Generate CSV file with the most important benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-output-json=", "Generate JSON file with all benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-sanity-check", "Run benchmarks for only one iteration with no output", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-priority-level=", strprintf("Run benchmarks of one or multiple priority level(s) (%s), default: '%s'", + benchmark::ListPriorities(), DEFAULT_PRIORITY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); } // parses a comma separated list like "10,20,30,50" @@ -44,6 +49,14 @@ static std::vector parseAsymptote(const std::string& str) { return numbers; } +static uint8_t parsePriorityLevel(const std::string& str) { + uint8_t levels{0}; + for (const auto& level: SplitString(str, ',')) { + levels |= benchmark::StringToPriority(level); + } + return levels; +} + int main(int argc, char** argv) { ArgsManager argsman; @@ -73,7 +86,7 @@ int main(int argc, char** argv) " sure each run has exactly the same preconditions.\n" "\n" " * If results are still not reliable, increase runtime with e.g.\n" - " -min_time=5000 to let a benchmark run for at least 5 seconds.\n" + " -min-time=5000 to let a benchmark run for at least 5 seconds.\n" "\n" " * bench_bitcoin uses nanobench [3] for which there is extensive\n" " documentation available online.\n" @@ -105,15 +118,22 @@ int main(int argc, char** argv) return EXIT_SUCCESS; } - benchmark::Args args; - args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", "")); - args.is_list_only = argsman.GetBoolArg("-list", false); - args.min_time = std::chrono::milliseconds(argsman.GetIntArg("-min_time", DEFAULT_MIN_TIME_MS)); - args.output_csv = fs::PathFromString(argsman.GetArg("-output_csv", "")); - args.output_json = fs::PathFromString(argsman.GetArg("-output_json", "")); - args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); + try { + benchmark::Args args; + args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", "")); + args.is_list_only = argsman.GetBoolArg("-list", false); + args.min_time = std::chrono::milliseconds(argsman.GetIntArg("-min-time", DEFAULT_MIN_TIME_MS)); + args.output_csv = argsman.GetPathArg("-output-csv"); + args.output_json = argsman.GetPathArg("-output-json"); + args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); + args.sanity_check = argsman.GetBoolArg("-sanity-check", false); + args.priority = parsePriorityLevel(argsman.GetArg("-priority-level", DEFAULT_PRIORITY)); - benchmark::BenchRunner::RunAll(args); + benchmark::BenchRunner::RunAll(args); - return EXIT_SUCCESS; + return EXIT_SUCCESS; + } catch (const std::exception& e) { + tfm::format(std::cerr, "Error: %s\n", e.what()); + return EXIT_FAILURE; + } } diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 4e1565b89f7..02cb01263e9 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -1,14 +1,14 @@ -// Copyright (c) 2011-2021 The Bitcoin Core developers +// Copyright (c) 2011-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include +#include #include #include #include -#include #include #include @@ -47,5 +47,18 @@ static void AssembleBlock(benchmark::Bench& bench) PrepareBlock(test_setup->m_node, P2WSH_OP_TRUE); }); } +static void BlockAssemblerAddPackageTxns(benchmark::Bench& bench) +{ + FastRandomContext det_rand{true}; + auto testing_setup{MakeNoLogFileContext()}; + testing_setup->PopulateMempool(det_rand, /*num_transactions=*/1000, /*submit=*/true); + node::BlockAssembler::Options assembler_options; + assembler_options.test_block_validity = false; + + bench.run([&] { + PrepareBlock(testing_setup->m_node, P2WSH_OP_TRUE, assembler_options); + }); +} -BENCHMARK(AssembleBlock); +BENCHMARK(AssembleBlock, benchmark::PriorityLevel::HIGH); +BENCHMARK(BlockAssemblerAddPackageTxns, benchmark::PriorityLevel::LOW); diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index 22dfb7aa5bb..4a3ec67c2b8 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2021 The Bitcoin Core developers +// Copyright (c) 2016-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,7 +18,6 @@ // (https://github.com/bitcoin/bitcoin/issues/7883#issuecomment-224807484) static void CCoinsCaching(benchmark::Bench& bench) { - const ECCVerifyHandle verify_handle; ECC_Start(); FillableSigningProvider keystore; @@ -51,4 +50,4 @@ static void CCoinsCaching(benchmark::Bench& bench) ECC_Stop(); } -BENCHMARK(CCoinsCaching); +BENCHMARK(CCoinsCaching, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/chacha20.cpp b/src/bench/chacha20.cpp index a6f4eec4ca9..115cd064bd9 100644 --- a/src/bench/chacha20.cpp +++ b/src/bench/chacha20.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2020 The Bitcoin Core developers +// Copyright (c) 2019-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,9 +14,9 @@ static const uint64_t BUFFER_SIZE_LARGE = 1024*1024; static void CHACHA20(benchmark::Bench& bench, size_t buffersize) { std::vector key(32,0); - ChaCha20 ctx(key.data(), key.size()); + ChaCha20 ctx(key.data()); ctx.SetIV(0); - ctx.Seek(0); + ctx.Seek64(0); std::vector in(buffersize,0); std::vector out(buffersize,0); bench.batch(in.size()).unit("byte").run([&] { @@ -39,6 +39,6 @@ static void CHACHA20_1MB(benchmark::Bench& bench) CHACHA20(bench, BUFFER_SIZE_LARGE); } -BENCHMARK(CHACHA20_64BYTES); -BENCHMARK(CHACHA20_256BYTES); -BENCHMARK(CHACHA20_1MB); +BENCHMARK(CHACHA20_64BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_256BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_1MB, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/chacha_poly_aead.cpp b/src/bench/chacha_poly_aead.cpp index e994279a4d8..db88841c327 100644 --- a/src/bench/chacha_poly_aead.cpp +++ b/src/bench/chacha_poly_aead.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2020 The Bitcoin Core developers +// Copyright (c) 2019-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -115,12 +115,12 @@ static void HASH_1MB(benchmark::Bench& bench) HASH(bench, BUFFER_SIZE_LARGE); } -BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ONLY_ENCRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ONLY_ENCRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ONLY_ENCRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ENCRYPT_DECRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ENCRYPT_DECRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ENCRYPT_DECRYPT); -BENCHMARK(HASH_64BYTES); -BENCHMARK(HASH_256BYTES); -BENCHMARK(HASH_1MB); +BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(HASH_64BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(HASH_256BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(HASH_1MB, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 52e5cb743fb..ee76f7b7674 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2020 The Bitcoin Core developers +// Copyright (c) 2016-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,6 +8,7 @@ #include #include #include +#include #include // These are the two major time-sinks which happen after we have fully received @@ -49,5 +50,5 @@ static void DeserializeAndCheckBlockTest(benchmark::Bench& bench) }); } -BENCHMARK(DeserializeBlockTest); -BENCHMARK(DeserializeAndCheckBlockTest); +BENCHMARK(DeserializeBlockTest, benchmark::PriorityLevel::HIGH); +BENCHMARK(DeserializeAndCheckBlockTest, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index c4ba85e5456..9d5f542fb2f 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2020 The Bitcoin Core developers +// Copyright (c) 2015-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -25,14 +25,10 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) // We shouldn't ever be running with the checkqueue on a single core machine. if (GetNumCores() <= 1) return; - const ECCVerifyHandle verify_handle; ECC_Start(); struct PrevectorJob { prevector p; - // ELEMENTS: fix unused member function warnings - // PrevectorJob(){ - // } explicit PrevectorJob(FastRandomContext& insecure_rand){ p.resize(insecure_rand.randrange(PREVECTOR_SIZE*2)); } @@ -40,7 +36,6 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) { return true; } - // void swap(PrevectorJob& x){p.swap(x.p);}; }; CCheckQueue queue {QUEUE_BATCH_SIZE}; // The main thread should be counted to prevent thread oversubscription, and @@ -61,7 +56,7 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) // Make insecure_rand here so that each iteration is identical. CCheckQueueControl control(&queue); for (auto vChecks : vBatches) { - control.Add(vChecks); + control.Add(std::move(vChecks)); } // control waits for completion by RAII, but // it is done explicitly here for clarity @@ -76,4 +71,4 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) ECC_Stop(); } -BENCHMARK(CCheckQueueSpeedPrevectorJob); +BENCHMARK(CCheckQueueSpeedPrevectorJob, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 9ebee47ab94..7b5277301d7 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2012-2021 The Bitcoin Core developers +// Copyright (c) 2012-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,7 @@ using node::NodeContext; using wallet::AttemptSelection; -using wallet::CInputCoin; +using wallet::CHANGE_LOWER; using wallet::COutput; using wallet::CWallet; using wallet::CWalletTx; @@ -47,7 +47,7 @@ static void CoinSelection(benchmark::Bench& bench) { NodeContext node; auto chain = interfaces::MakeChain(node); - CWallet wallet(chain.get(), "", gArgs, CreateDummyWalletDatabase()); + CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); std::vector> wtxs; LOCK(wallet.cs_wallet); @@ -58,20 +58,30 @@ static void CoinSelection(benchmark::Bench& bench) addCoin(3 * COIN, wallet, wtxs); // Create coins - std::vector coins; + wallet::CoinsResult available_coins; for (const auto& wtx : wtxs) { - coins.emplace_back(wallet, *wtx, 0 /* iIn */, 6 * 24 /* nDepthIn */, true /* spendable */, true /* solvable */, true /* safe */); + const auto txout = wtx->tx->vout.at(0); + available_coins.coins[OutputType::BECH32].emplace_back(COutPoint(wtx->GetHash(), 0), txout, /*depth=*/6 * 24, CalculateMaximumSignedInputSize(txout, &wallet, /*coin_control=*/nullptr), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true, /*fees=*/ 0); } const CoinEligibilityFilter filter_standard(1, 6, 0); - const CoinSelectionParams coin_selection_params(/* change_output_size= */ 34, - /* change_spend_size= */ 148, /* effective_feerate= */ CFeeRate(0), - /* long_term_feerate= */ CFeeRate(0), /* discard_feerate= */ CFeeRate(0), - /* tx_noinputs_size= */ 0, /* avoid_partial= */ false); + FastRandomContext rand{}; + const CoinSelectionParams coin_selection_params{ + rand, + /*change_output_size=*/ 34, + /*change_spend_size=*/ 148, + /*min_change_target=*/ CHANGE_LOWER, + /*effective_feerate=*/ CFeeRate(0), + /*long_term_feerate=*/ CFeeRate(0), + /*discard_feerate=*/ CFeeRate(0), + /*tx_noinputs_size=*/ 0, + /*avoid_partial=*/ false, + }; + auto group = wallet::GroupOutputs(wallet, available_coins, coin_selection_params, {{filter_standard}})[filter_standard]; bench.run([&] { CAmountMap mapValue; mapValue[::policyAsset] = 1003 * COIN; - auto result = AttemptSelection(wallet, mapValue, filter_standard, coins, coin_selection_params); + auto result = AttemptSelection(mapValue, group, coin_selection_params, /*allow_mixed_output_types=*/true); assert(result); assert(result->GetSelectedValue() == mapValue); assert(result->GetInputSet().size() == 2); @@ -87,10 +97,9 @@ static void CoinSelection(benchmark::Bench& bench) }); } -typedef std::set CoinSet; static NodeContext testNode; static auto testChain = interfaces::MakeChain(testNode); -static CWallet testWallet(testChain.get(), "", gArgs, CreateDummyWalletDatabase()); +static CWallet testWallet(testChain.get(), "", CreateDummyWalletDatabase()); // Copied from src/wallet/test/coinselector_tests.cpp static void add_coin(const CAmount& nValue, int nInput, std::vector& set) @@ -99,9 +108,9 @@ static void add_coin(const CAmount& nValue, int nInput, std::vector tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; CWalletTx wtx(MakeTransactionRef(tx), TxStateInactive{}); - CInputCoin coin(testWallet, &wtx, nInput); + COutput output(COutPoint(wtx.GetHash(), nInput), wtx.tx->vout.at(nInput), /*depth=*/ 0, /*input_bytes=*/ -1, /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, /*time=*/ 0, /*from_me=*/ true, /*fees=*/ 0); set.emplace_back(); - set.back().Insert(coin, 0, true, 0, 0, false); + set.back().Insert(std::make_shared(output), /*ancestors=*/ 0, /*descendants=*/ 0); } // Copied from src/wallet/test/coinselector_tests.cpp static CAmount make_hard_case(int utxos, std::vector& utxo_pool) @@ -109,9 +118,9 @@ static CAmount make_hard_case(int utxos, std::vector& utxo_pool) utxo_pool.clear(); CAmount target = 0; for (int i = 0; i < utxos; ++i) { - target += (CAmount)1 << (utxos+i); - add_coin((CAmount)1 << (utxos+i), 2*i, utxo_pool); - add_coin(((CAmount)1 << (utxos+i)) + ((CAmount)1 << (utxos-1-i)), 2*i + 1, utxo_pool); + target += CAmount{1} << (utxos+i); + add_coin(CAmount{1} << (utxos+i), 2*i, utxo_pool); + add_coin((CAmount{1} << (utxos+i)) + (CAmount{1} << (utxos-1-i)), 2*i + 1, utxo_pool); } return target; } @@ -131,5 +140,5 @@ static void BnBExhaustion(benchmark::Bench& bench) }); } -BENCHMARK(CoinSelection); -BENCHMARK(BnBExhaustion); +BENCHMARK(CoinSelection, benchmark::PriorityLevel::HIGH); +BENCHMARK(BnBExhaustion, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index d17ec503e7e..cf8d807d7b1 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2021 The Bitcoin Core developers +// Copyright (c) 2016-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,7 +18,7 @@ /* Number of bytes to hash per iteration */ static const uint64_t BUFFER_SIZE = 1000*1000; -static void RIPEMD160(benchmark::Bench& bench) +static void BenchRIPEMD160(benchmark::Bench& bench) { uint8_t hash[CRIPEMD160::OUTPUT_SIZE]; std::vector in(BUFFER_SIZE,0); @@ -150,19 +150,19 @@ static void MuHashPrecompute(benchmark::Bench& bench) }); } -BENCHMARK(RIPEMD160); -BENCHMARK(SHA1); -BENCHMARK(SHA256); -BENCHMARK(SHA512); -BENCHMARK(SHA3_256_1M); - -BENCHMARK(SHA256_32b); -BENCHMARK(SipHash_32b); -BENCHMARK(SHA256D64_1024); -BENCHMARK(FastRandom_32bit); -BENCHMARK(FastRandom_1bit); - -BENCHMARK(MuHash); -BENCHMARK(MuHashMul); -BENCHMARK(MuHashDiv); -BENCHMARK(MuHashPrecompute); +BENCHMARK(BenchRIPEMD160, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA1, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA256, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA512, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA3_256_1M, benchmark::PriorityLevel::HIGH); + +BENCHMARK(SHA256_32b, benchmark::PriorityLevel::HIGH); +BENCHMARK(SipHash_32b, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA256D64_1024, benchmark::PriorityLevel::HIGH); +BENCHMARK(FastRandom_32bit, benchmark::PriorityLevel::HIGH); +BENCHMARK(FastRandom_1bit, benchmark::PriorityLevel::HIGH); + +BENCHMARK(MuHash, benchmark::PriorityLevel::HIGH); +BENCHMARK(MuHashMul, benchmark::PriorityLevel::HIGH); +BENCHMARK(MuHashDiv, benchmark::PriorityLevel::HIGH); +BENCHMARK(MuHashPrecompute, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/descriptors.cpp b/src/bench/descriptors.cpp new file mode 100644 index 00000000000..5d28d269092 --- /dev/null +++ b/src/bench/descriptors.cpp @@ -0,0 +1,35 @@ +// Copyright (c) 2019-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include