diff --git a/configs/components/curl.rb b/configs/components/curl.rb index e07dd1810..b7100d54e 100644 --- a/configs/components/curl.rb +++ b/configs/components/curl.rb @@ -1,6 +1,17 @@ component 'curl' do |pkg, settings, platform| - pkg.version '7.88.1' - pkg.sha256sum 'cdb38b72e36bc5d33d5b8810f8018ece1baa29a8f215b4495e495ded82bbf3c7' + # Projects may define a :curl_version setting + version = settings[:curl_version] || '7.88.1' + pkg.version version + + case version + when '7.88.1' + pkg.sha256sum 'cdb38b72e36bc5d33d5b8810f8018ece1baa29a8f215b4495e495ded82bbf3c7' + when '8.7.1' + pkg.sha256sum 'f91249c87f68ea00cf27c44fdfa5a78423e41e71b7d408e5901a9896d905c495' + else + raise "curl version #{version} has not been configured; Cannot continue." + end + pkg.url "https://curl.se/download/curl-#{pkg.get_version}.tar.gz" pkg.mirror "#{settings[:buildsources_url]}/curl-#{pkg.get_version}.tar.gz" @@ -26,15 +37,17 @@ pkg.environment "PATH", "/opt/pl-build-tools/bin:$(PATH):#{settings[:bindir]}" end - # Following 3 lines should we removed once we upgrade CURL to 8.x.x - pkg.apply_patch 'resources/patches/curl/CVE-2023-27535.patch' - pkg.apply_patch 'resources/patches/curl/CVE-2023-28319.patch' - pkg.apply_patch 'resources/patches/curl/CVE-2023-32001.patch' - pkg.apply_patch 'resources/patches/curl/CVE-2023-38545.patch' - pkg.apply_patch 'resources/patches/curl/CVE-2023-38546.patch' - pkg.apply_patch 'resources/patches/curl/CVE-2023-46218.patch' - pkg.apply_patch 'resources/patches/curl/CVE-2024-2004.patch' - pkg.apply_patch 'resources/patches/curl/CVE-2024-2398.patch' + # Following lines should we removed once we drop curl 7 + if version.start_with?('7') + pkg.apply_patch 'resources/patches/curl/CVE-2023-27535.patch' + pkg.apply_patch 'resources/patches/curl/CVE-2023-28319.patch' + pkg.apply_patch 'resources/patches/curl/CVE-2023-32001.patch' + pkg.apply_patch 'resources/patches/curl/CVE-2023-38545.patch' + pkg.apply_patch 'resources/patches/curl/CVE-2023-38546.patch' + pkg.apply_patch 'resources/patches/curl/CVE-2023-46218.patch' + pkg.apply_patch 'resources/patches/curl/CVE-2024-2004.patch' + pkg.apply_patch 'resources/patches/curl/CVE-2024-2398.patch' + end configure_options = [] configure_options << "--with-ssl=#{settings[:prefix]}" diff --git a/configs/components/runtime-agent.rb b/configs/components/runtime-agent.rb index 7f9e96214..4bf916115 100644 --- a/configs/components/runtime-agent.rb +++ b/configs/components/runtime-agent.rb @@ -13,6 +13,14 @@ "zypper install -y pl-gcc8" end end + elsif platform.is_macos? && platform.is_cross_compiled? + if settings[:ruby_version] =~ /^3\./ + pkg.install do + # These are dependencies of ruby@3.x, remove symlinks from /usr/local + # so our build doesn't use the wrong headers + "cd /etc/homebrew && su test -c '#{platform.brew} unlink openssl libyaml'" + end + end end if platform.is_cross_compiled? diff --git a/configs/platforms/osx-11-arm64.rb b/configs/platforms/osx-11-arm64.rb index 2d762df6b..47ddef4a1 100644 --- a/configs/platforms/osx-11-arm64.rb +++ b/configs/platforms/osx-11-arm64.rb @@ -1,7 +1,7 @@ platform 'osx-11-arm64' do |plat| plat.inherit_from_default - packages = %w[cmake pkg-config yaml-cpp] + packages = %w[cmake pkg-config] plat.provision_with "su test -c '/usr/local/bin/brew install #{packages.join(' ')}'" plat.output_dir File.join('apple', '11', 'PC1', 'arm64') diff --git a/configs/platforms/osx-11-x86_64.rb b/configs/platforms/osx-11-x86_64.rb index 28dd84273..e06c83328 100644 --- a/configs/platforms/osx-11-x86_64.rb +++ b/configs/platforms/osx-11-x86_64.rb @@ -1,7 +1,7 @@ platform 'osx-11-x86_64' do |plat| plat.inherit_from_default - packages = %w[cmake pkg-config yaml-cpp] + packages = %w[cmake pkg-config] plat.provision_with "su test -c '/usr/local/bin/brew install #{packages.join(' ')}'" plat.output_dir File.join('apple', '11', 'PC1', 'x86_64') diff --git a/configs/platforms/osx-12-arm64.rb b/configs/platforms/osx-12-arm64.rb index 1c1fbdc0a..2a83b3fa0 100644 --- a/configs/platforms/osx-12-arm64.rb +++ b/configs/platforms/osx-12-arm64.rb @@ -1,7 +1,7 @@ platform 'osx-12-arm64' do |plat| plat.inherit_from_default - packages = %w[automake cmake pkg-config yaml-cpp] + packages = %w[automake cmake pkg-config] plat.provision_with "su test -c '/usr/local/bin/brew install #{packages.join(' ')}'" plat.output_dir File.join('apple', '12', 'PC1', 'arm64') diff --git a/configs/platforms/osx-12-x86_64.rb b/configs/platforms/osx-12-x86_64.rb index 2b583907e..242e2fc5d 100644 --- a/configs/platforms/osx-12-x86_64.rb +++ b/configs/platforms/osx-12-x86_64.rb @@ -1,7 +1,7 @@ platform 'osx-12-x86_64' do |plat| plat.inherit_from_default - packages = %w[cmake pkg-config yaml-cpp] + packages = %w[cmake pkg-config] plat.provision_with "su test -c '/usr/local/bin/brew install #{packages.join(' ')}'" plat.output_dir File.join('apple', '12', 'PC1', 'x86_64') diff --git a/configs/platforms/osx-13-arm64.rb b/configs/platforms/osx-13-arm64.rb index c4b7f5a67..cb68172e9 100644 --- a/configs/platforms/osx-13-arm64.rb +++ b/configs/platforms/osx-13-arm64.rb @@ -1,7 +1,7 @@ platform 'osx-13-arm64' do |plat| plat.inherit_from_default - packages = %w[automake cmake pkg-config yaml-cpp] + packages = %w[automake cmake pkg-config] plat.provision_with "su test -c '/opt/homebrew/bin/brew install #{packages.join(' ')}'" plat.output_dir File.join('apple', '13', 'PC1', 'arm64') diff --git a/configs/platforms/osx-13-x86_64.rb b/configs/platforms/osx-13-x86_64.rb index 758509891..74e0aee11 100644 --- a/configs/platforms/osx-13-x86_64.rb +++ b/configs/platforms/osx-13-x86_64.rb @@ -1,7 +1,7 @@ platform 'osx-13-x86_64' do |plat| plat.inherit_from_default - packages = %w[cmake pkg-config yaml-cpp] + packages = %w[cmake pkg-config] plat.provision_with "su test -c '/usr/local/bin/brew install #{packages.join(' ')}'" plat.output_dir File.join('apple', '13', 'PC1', 'x86_64') diff --git a/configs/platforms/osx-14-arm64.rb b/configs/platforms/osx-14-arm64.rb index 5706d641d..3fb0bac3d 100644 --- a/configs/platforms/osx-14-arm64.rb +++ b/configs/platforms/osx-14-arm64.rb @@ -1,7 +1,7 @@ platform 'osx-14-arm64' do |plat| plat.inherit_from_default - packages = %w[automake cmake pkg-config yaml-cpp] + packages = %w[automake cmake pkg-config] plat.provision_with "su test -c '/opt/homebrew/bin/brew install #{packages.join(' ')}'" plat.output_dir File.join('apple', '14', 'PC1', 'arm64') diff --git a/configs/platforms/osx-14-x86_64.rb b/configs/platforms/osx-14-x86_64.rb index 2dd1bef9f..00d772dae 100644 --- a/configs/platforms/osx-14-x86_64.rb +++ b/configs/platforms/osx-14-x86_64.rb @@ -1,7 +1,7 @@ platform 'osx-14-x86_64' do |plat| plat.inherit_from_default - packages = %w[cmake pkg-config yaml-cpp] + packages = %w[cmake pkg-config] plat.provision_with "su test -c '/usr/local/bin/brew install #{packages.join(' ')}'" plat.output_dir File.join('apple', '14', 'PC1', 'x86_64') diff --git a/configs/projects/agent-runtime-main.rb b/configs/projects/agent-runtime-main.rb index 355235cf6..26f20c4ca 100644 --- a/configs/projects/agent-runtime-main.rb +++ b/configs/projects/agent-runtime-main.rb @@ -14,6 +14,8 @@ proj.setting :augeas_version, '1.14.1' end + proj.setting :curl_version, '8.7.1' + ######## # Load shared agent settings ######## diff --git a/notes/README.md b/notes/README.md index 985c02a7f..b3454dbd6 100644 --- a/notes/README.md +++ b/notes/README.md @@ -13,6 +13,7 @@ The notes directory is sorted by platform. ## macOS +* [General](macos/README.md) * [Boost](macos/boost.md) ## Solaris diff --git a/notes/macos/README.md b/notes/macos/README.md new file mode 100644 index 000000000..4af6b9b88 --- /dev/null +++ b/notes/macos/README.md @@ -0,0 +1,121 @@ +# macOS Builds + +## Xcode + +Xcode contains the necessary tools to build packages on macOS such as `make`, `cc`, etc. Xcode is preinstalled on our macOS images. If it's missing or out of date, then the image will need to be updated. + +The following command will show you where xcode is installed: + +``` +# xcode-select -p +/Library/Developer/CommandLineTools +``` + +## Build Tools + +Xcode ships a `gcc` binary, but it's actually clang(!) + +``` +# which gcc +/usr/bin/gcc +# gcc --version +Configured with: --prefix=/Library/Developer/CommandLineTools/usr --with-gxx-include-dir=/Library/Developer/CommandLineTools/SDKs/MacOSX12.1.sdk/usr/include/c++/4.2.1 +Apple clang version 13.0.0 (clang-1300.0.27.3) +Target: x86_64-apple-darwin21.3.0 +Thread model: posix +InstalledDir: /Library/Developer/CommandLineTools/usr/bin +``` + +## Homebrew + +We use homebrew to install build dependencies like `automake`. It's a good idea to become [familiar with these Homebrew terms](https://docs.brew.sh/Manpage#terminology). + +### Installation Directory + +Homebrew installs itself into `/usr/local` on macOS Intel and [`/opt/homebrew` on macOS ARM](https://docs.brew.sh/FAQ#why-is-the-default-installation-prefix-opthomebrew-on-apple-silicon). To account for these differences you can use the [`brew` method in vanagon](https://github.com/puppetlabs/vanagon/commit/02134c79ce917fe82e6b201b9efd6faf73b1b116). + +### Permissions + +Homebrew does not allow itself to be run as root: + +``` +# brew list +Error: Running Homebrew as root is extremely dangerous and no longer supported. +As Homebrew does not drop privileges on installation you would be giving all +build scripts full access to your system +``` + +So we create a `test` user and execute all `brew` commands as that user. Importantly, brew must be executed from a current working directory that the `test` user has access to, so you'll see things like: + +``` +# cd /etc/homebrew +# su test -c '/usr/local/bin/brew install cmake' +``` + +### Symlinks + +The `brew install ` command will install a formula ("package definition from upstream sources") into the keg ("installation directory of a given formula version"). For example, if you `brew install cmake`, it will install into a versioned path such as: + +``` +/usr/local/Cellar/cmake/3.29.2/bin/cmake +``` + +Homebrew also creates symlinks in `/usr/local/bin`: + +``` +# ls -la /usr/local/bin/cmake +lrwxr-xr-x 1 test admin 32 May 6 20:42 /usr/local/bin/cmake -> ../Cellar/cmake/3.29.2/bin/cmake +``` + +The symlinks are useful when installing a tool that needs to be executed during the build, such as cmake, autoconf, perl, etc. + +However, you have to be careful when installing a homebrew formula (or one of its transitive dependencies) is also a vanagon component, such as `openssl`. For example, the `ruby@3.2` formula depends on `openssl@3` (currently 3.3.0). However, the agent's openssl component is currently `3.0.x`. By default, clang will prefer the headers that homebrew symlinked and compile against those. But at runtime, only the libraries that we built will be present. + +To avoid conflicts, you should run `brew unlink ` for any formula that is also a vanagon component. + +One exception is if the formula is keg-only, which just means homebrew won't create symlinks. For example, [`readline` is keg-only](https://github.com/Homebrew/homebrew-core/blob/c0218d50084e300cba26da84028acfd4917ce623/Formula/r/readline.rb#L77) + +## Troubleshooting + +To view library dependencies, use `otool -L` (instead of `ldd`): + +``` +# otool -L /opt/puppetlabs/puppet/lib/libcurl.dylib +/opt/puppetlabs/puppet/lib/libcurl.dylib: + /opt/puppetlabs/puppet/lib/libcurl.4.dylib (compatibility version 13.0.0, current version 13.0.0) + /opt/puppetlabs/puppet/lib/libssl.3.dylib (compatibility version 3.0.0, current version 3.0.0) + /opt/puppetlabs/puppet/lib/libcrypto.3.dylib (compatibility version 3.0.0, current version 3.0.0) + /usr/lib/libz.1.dylib (compatibility version 1.0.0, current version 1.2.11) + /System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation (compatibility version 150.0.0, current version 1856.105.0) + /System/Library/Frameworks/CoreServices.framework/Versions/A/CoreServices (compatibility version 1.0.0, current version 1141.1.0) + /System/Library/Frameworks/SystemConfiguration.framework/Versions/A/SystemConfiguration (compatibility version 1.0.0, current version 1163.60.3) + /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1311.0.0) +``` + +To view undefined symbols (which are expected to be defined in some other library). For example, `libcurl.dylib` expects the `_TLS_client_method` function to be defined in `libssl.dylib`: + +``` +# nm -m /opt/puppetlabs/puppet/lib/libcurl.dylib | grep TLS_ + (undefined) external _TLS_client_method (from libssl) +``` + +To view symbols defined in a library: + +``` +# nm -gU /opt/puppetlabs/puppet/lib/libcurl.dylib | grep _curl_easy_init +0000000000017fac T _curl_easy_init +``` + +To trace how the dynamic loader (dyld) resolves library dependencies: + +``` +# export DYLD_PRINT_LIBRARIES=1 +# /usr/local/Cellar/openssl\@3/3.3.0/bin/openssl version +dyld[15123]: /usr/local/Cellar/openssl@3/3.3.0/bin/openssl +dyld[15123]: /usr/local/Cellar/openssl@3/3.3.0/lib/libssl.3.dylib +dyld[15123]: <71192998-23D0-3BAD-AAC9-DC90966A8177> /usr/local/Cellar/openssl@3/3.3.0/lib/libcrypto.3.dylib +dyld[15123]: <155C5726-E0E6-3FAF-9CD5-CD8E043487D5> /usr/lib/libSystem.B.dylib +dyld[15123]: <952A7572-D3ED-388C-8190-DD17DDCC6522> /usr/lib/system/libcache.dylib +dyld[15123]: <9E46E39C-0DBB-333A-9597-23FA11E5B96C> /usr/lib/system/libcommonCrypto.dylib +... +``` diff --git a/resources/patches/curl/curl-7.55.1-aix-poll.patch b/resources/patches/curl/curl-7.55.1-aix-poll.patch deleted file mode 100644 index 4c7436265..000000000 --- a/resources/patches/curl/curl-7.55.1-aix-poll.patch +++ /dev/null @@ -1,40 +0,0 @@ -From 44b69e23fb7522e1e0d6052c4d379715f9aa6447 Mon Sep 17 00:00:00 2001 -From: Michael Smith -Date: Thu, 24 Aug 2017 16:43:01 -0700 -Subject: [PATCH] Compilation fails on AIX with 7.55.1 (#1828) - -Compilation fails on multi.c. The root of it is that on AIX, when -_ALL_SOURCE is set to 1 (as CURL does), sys/poll.h uses #define for -event and revent. Because multi.c includes sys/poll.h but multi.h -doesn't, we end up with a mismatch between names for the curl_waitfd -struct. - -Previously this appears to have built with curl 7.51.0 because multi.h -also included sys/poll.h, so they were equally wrong. Based on -https://curl.haxx.se/mail/lib-2013-02/0330.html, this appears to have -been an issue off-and-on for years. - -My workaround has been to undef _ALL_SOURCE before including select.h in -multi.c, and redefine it after. This probably isn't a good general -solution. ---- - lib/multi.c | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/lib/multi.c b/lib/multi.c -index d5bc532ea..ecc00d3af 100644 ---- a/lib/multi.c -+++ b/lib/multi.c -@@ -35,7 +35,9 @@ - #include "sendf.h" - #include "timeval.h" - #include "http.h" -+#undef _ALL_SOURCE - #include "select.h" -+#define _ALL_SOURCE 1 - #include "warnless.h" - #include "speedcheck.h" - #include "conncache.h" --- -2.14.1 -