From fc83ef834b343917bb3702273b9e9a2f00710db1 Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Wed, 8 May 2024 01:09:29 -0700 Subject: [PATCH 1/5] (PA-6422) Only install ffi gem once Previously, we were installing the ffi gem twice, but with different arguments, once in the base-rubygem component and again in the rubygem-ffi component. For example: /opt/puppetlabs/puppet/bin/gem install --no-document --local ffi-1.15.5.gem && \ /opt/puppetlabs/puppet/bin/gem install --no-document --local ffi-1.15.5.gem -- --enable-system-ffi Building native extensions. This could take a while... Successfully installed ffi-1.15.5 1 gem installed Building native extensions with: '--enable-system-ffi' This could take a while... Successfully installed ffi-1.15.5 1 gem installed Now we only execute the gem install command once. On Windows, we install ffi gems with precompiled binaries (so we don't need to specify enable/disable flags). On non-Windows, we continue to pass enable/disable based on the ruby version. --- configs/components/rubygem-ffi.rb | 47 +++++++++++++++---------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/configs/components/rubygem-ffi.rb b/configs/components/rubygem-ffi.rb index e86bdd585..09a5aeb7e 100644 --- a/configs/components/rubygem-ffi.rb +++ b/configs/components/rubygem-ffi.rb @@ -15,38 +15,16 @@ pkg.sha256sum '6f2ed2fa68047962d6072b964420cba91d82ce6fa8ee251950c17fca6af3c2a0' end - instance_eval File.read('configs/components/_base-rubygem.rb') - rb_major_minor_version = settings[:ruby_version].to_f - # Prior to ruby 3.2, both ruby and the ffi gem vendored a version of libffi. - # If libffi happened to be installed in /usr/lib, then the ffi gem preferred - # that instead of building libffi itself. To ensure consistency, we use - # --disable-system-ffi so that the ffi gem *always* builds libffi, then - # builds the ffi_c native extension and links it against libffi.so. - # - # In ruby 3.2 and up, libffi is no longer vendored. So we created a separate - # libffi vanagon component which is built before ruby. The ffi gem still - # vendors libffi, so we use the --enable-system-ffi option to ensure the ffi - # gem *always* uses the libffi.so we already built. Note the term "system" is - # misleading, because we override PKG_CONFIG_PATH below so that our libffi.so - # is preferred, not the one in /usr/lib. - if rb_major_minor_version > 2.7 - pkg.install do - "#{settings[:gem_install]} ffi-#{pkg.get_version}.gem -- --enable-system-ffi" - end - else - pkg.install do - "#{settings[:gem_install]} ffi-#{pkg.get_version}.gem -- --disable-system-ffi" - end - end - # Windows versions of the FFI gem have custom filenames, so we overwite the # defaults that _base-rubygem provides here, just for Windows for Ruby < 3.2 if platform.is_windows? && rb_major_minor_version < 3.2 # Pin this if lower than Ruby 2.7 pkg.version '1.9.25' if rb_major_minor_version < 2.7 + instance_eval File.read('configs/components/_base-rubygem.rb') + # Vanagon's `pkg.mirror` is additive, and the _base_rubygem sets the # non-Windows gem as the first mirror, which is incorrect. We need to unset # the list of mirrors before adding the Windows-appropriate ones here: @@ -81,6 +59,25 @@ pkg.install do "#{settings[:gem_install]} ffi-#{pkg.get_version}-#{platform.architecture}-mingw32.gem" end + else + # Prior to ruby 3.2, both ruby and the ffi gem vendored a version of libffi. + # If libffi happened to be installed in /usr/lib, then the ffi gem preferred + # that instead of building libffi itself. To ensure consistency, we use + # --disable-system-ffi so that the ffi gem *always* builds libffi, then + # builds the ffi_c native extension and links it against libffi.so. + # + # In ruby 3.2 and up, libffi is no longer vendored. So we created a separate + # libffi vanagon component which is built before ruby. The ffi gem still + # vendors libffi, so we use the --enable-system-ffi option to ensure the ffi + # gem *always* uses the libffi.so we already built. Note the term "system" is + # misleading, because we override PKG_CONFIG_PATH below so that our libffi.so + # is preferred, not the one in /usr/lib. + settings[:gem_install_options] = if rb_major_minor_version > 2.7 + "-- --enable-system-ffi" + else + "-- --disable-system-ffi" + end + instance_eval File.read('configs/components/_base-rubygem.rb') end # due to contrib/make_sunver.pl missing on solaris 11 we cannot compile libffi, so we provide the opencsw library @@ -157,4 +154,4 @@ %(#{platform[:sed]} -i '0,/ensure_required_ruby_version_met/b; /ensure_required_ruby_version_met/d' #{base_ruby}/rubygems/installer.rb) end end -end \ No newline at end of file +end From 63080c2c2d9bcfece7e77ec644677bb6115a293e Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Wed, 8 May 2024 01:19:04 -0700 Subject: [PATCH 2/5] (PA-6422) Enable/disable libffi library not ffi gem The ffi gem depends on the libffi native library. When installing the ffi gem, we need to specify the name of the library in the enable/disable-system-libffi option. --- configs/components/rubygem-ffi.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/components/rubygem-ffi.rb b/configs/components/rubygem-ffi.rb index 09a5aeb7e..5af3e8864 100644 --- a/configs/components/rubygem-ffi.rb +++ b/configs/components/rubygem-ffi.rb @@ -63,19 +63,19 @@ # Prior to ruby 3.2, both ruby and the ffi gem vendored a version of libffi. # If libffi happened to be installed in /usr/lib, then the ffi gem preferred # that instead of building libffi itself. To ensure consistency, we use - # --disable-system-ffi so that the ffi gem *always* builds libffi, then + # --disable-system-libffi so that the ffi gem *always* builds libffi, then # builds the ffi_c native extension and links it against libffi.so. # # In ruby 3.2 and up, libffi is no longer vendored. So we created a separate # libffi vanagon component which is built before ruby. The ffi gem still - # vendors libffi, so we use the --enable-system-ffi option to ensure the ffi + # vendors libffi, so we use the --enable-system-libffi option to ensure the ffi # gem *always* uses the libffi.so we already built. Note the term "system" is # misleading, because we override PKG_CONFIG_PATH below so that our libffi.so # is preferred, not the one in /usr/lib. settings[:gem_install_options] = if rb_major_minor_version > 2.7 - "-- --enable-system-ffi" + "-- --enable-system-libffi" else - "-- --disable-system-ffi" + "-- --disable-system-libffi" end instance_eval File.read('configs/components/_base-rubygem.rb') end From 58560cc96b67efe3a937b608b38ac35310b78f9c Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Wed, 8 May 2024 01:21:32 -0700 Subject: [PATCH 3/5] (PA-6422) Update rubygems patch for ruby 3.2.4 Previously, our rubygems patch wasn't being applied when cross compiling mac 12 ARM, because homebrew updated to ruby 3.2.4 and builder.rb was modified: - cmd = Gem.ruby.shellsplit + cmd = Shellwords.split(Gem.ruby) The code for patching is very fragile and has broken multiple times without any indication that the patch failed to apply, causing the resulting native extensions to be compiled for the wrong architecture. The code was also conflating the `settings[:ruby_version]` in our ruby component with the host's ruby version (the one we're actually trying to patch). Also the version that matters is the rubygems version (Gem::VERSION), not RUBY_VERSION. For example, if `gem update --system` was executed during provisioning, then our patch would fail to apply. This commit creates a script that runs on the remote system and so can introspect the host's ruby and rubygems versions. This way if brew updates to a newer ruby/rubygems, we can fail the build. --- configs/components/pl-ruby-patch.rb | 71 +++--------- resources/files/ruby/patch-hostruby.rb | 145 +++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 54 deletions(-) create mode 100644 resources/files/ruby/patch-hostruby.rb diff --git a/configs/components/pl-ruby-patch.rb b/configs/components/pl-ruby-patch.rb index 94e3aa592..11bfc075e 100644 --- a/configs/components/pl-ruby-patch.rb +++ b/configs/components/pl-ruby-patch.rb @@ -8,70 +8,33 @@ # This component should also be present in the puppet-agent project component "pl-ruby-patch" do |pkg, settings, platform| if platform.is_cross_compiled? - if platform.is_macos? - pkg.build_requires 'gnu-sed' - pkg.environment "PATH", "/usr/local/opt/gnu-sed/libexec/gnubin:$(PATH)" - end - ruby_api_version = settings[:ruby_version].gsub(/\.\d*$/, '.0') ruby_version_y = settings[:ruby_version].gsub(/(\d+)\.(\d+)\.(\d+)/, '\1.\2') - base_ruby = case platform.name - when /solaris-10/ - "/opt/csw/lib/ruby/2.0.0" - when /osx/ - "/usr/local/opt/ruby@#{ruby_version_y}/lib/ruby/#{ruby_api_version}" - else - "/opt/pl-build-tools/lib/ruby/2.1.0" - end + pkg.add_source("file://resources/files/ruby/patch-hostruby.rb") # The `target_triple` determines which directory native extensions are stored in the # compiled ruby and must match ruby's naming convention. - # - # solaris 10 uses ruby 2.0 which doesn't install native extensions based on architecture - unless platform.name =~ /solaris-10/ - # weird architecture naming conventions... - target_triple = if platform.architecture =~ /ppc64el|ppc64le/ - "powerpc64le-linux" - elsif platform.name == 'solaris-11-sparc' - "sparc-solaris-2.11" - elsif platform.is_macos? - if ruby_version_y.start_with?('2') - "aarch64-darwin" - else - "arm64-darwin" - end + # weird architecture naming conventions... + target_triple = if platform.architecture =~ /ppc64el|ppc64le/ + "powerpc64le-linux" + elsif platform.name == 'solaris-11-sparc' + "sparc-solaris-2.11" + elsif platform.name =~ /solaris-10/ + "sparc-solaris" + elsif platform.is_macos? + if ruby_version_y.start_with?('2') + "aarch64-darwin" else - "#{platform.architecture}-linux" + "arm64-darwin" end + else + "#{platform.architecture}-linux" + end - pkg.build do - [ - %(#{platform[:sed]} -i 's/Gem::Platform.local.to_s/"#{target_triple}"/' #{base_ruby}/rubygems/basic_specification.rb), - %(#{platform[:sed]} -i 's/Gem.extension_api_version/"#{ruby_api_version}"/' #{base_ruby}/rubygems/basic_specification.rb) - ] - end - end - - # make rubygems use our target rbconfig when installing gems - case File.basename(base_ruby) - when '2.0.0', '2.1.0' - sed_command = %(s|Gem.ruby|&, '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{settings[:ruby_version]}-orig.rb'|) - else - sed_command = %(s|Gem.ruby.shellsplit|& << '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{settings[:ruby_version]}-orig.rb'|) - end - - # rubygems switched which file has the command we need to patch starting in rubygems 3.4.10, which we install in our formula - # for ruby in homebrew-puppet - if Gem::Version.new(settings[:ruby_version]) >= Gem::Version.new('3.2.2') || platform.is_macos? && ruby_version_y.start_with?('2') - filename = 'builder.rb' - else - filename = 'ext_conf_builder.rb' - end - - pkg.build do + pkg.install do [ - %(#{platform[:sed]} -i "#{sed_command}" #{base_ruby}/rubygems/ext/#{filename}) + "#{settings[:host_ruby]} patch-hostruby.rb #{settings[:ruby_version]} #{target_triple}" ] end end diff --git a/resources/files/ruby/patch-hostruby.rb b/resources/files/ruby/patch-hostruby.rb new file mode 100644 index 000000000..03194bf95 --- /dev/null +++ b/resources/files/ruby/patch-hostruby.rb @@ -0,0 +1,145 @@ +# When cross compiling we need to run gem install using the host ruby, but +# force ruby to use our overridden rbconfig.rb. To do that, we insert a +# require statement between the ruby executable and it's first argument, +# thereby hooking the ruby process. +# +# In the future we could use the --target-rbconfig= option to point +# to our rbconfig.rb. But that option is only available in newer ruby versions. +require 'rbconfig' +require 'tempfile' + +if ARGV.length < 2 + warn < + +example: patch-hostruby.rb 3.2.2 arm64-darwin +USAGE + exit(1) +end + +# target ruby versions (what we're trying to build) +target_ruby_version = ARGV[0] +target_triple = ARGV[1] +target_api_version = target_ruby_version.gsub(/\.\d*$/, '.0') + +# host ruby (the ruby we execute to build the target) +host_rubylibdir = RbConfig::CONFIG['rubylibdir'] +GEM_VERSION = Gem::Version.new(Gem::VERSION) + +# Rewrite the file in-place securely, yielding each line to the caller +def rewrite(file) + # create temp file in the same directory as the file we're patching, + # so rename doesn't cross filesystems + tmpfile = Tempfile.new(File.basename(file), File.dirname(file)) + begin + File.open("#{file}.orig", "w") do |orig| + File.open(file, 'r').readlines.each do |line| + orig.write(line) + yield line + tmpfile.write(line) + end + end + ensure + tmpfile.close + File.unlink(file) + File.rename(tmpfile.path, file) + tmpfile.unlink + end +end + +# Based on the RUBYGEMS version of the host ruby, the line and file that needs patching is different +# Note the RUBY version doesn't matter (for either the host or target ruby). +# +# Here we define different intervals. For each interval, we specify the regexp to match, what to +# replace it with, and which file to edit in-place. Note `\&` is a placeholder for whatever the regexp +# was, that way we can easily append to it. And since it's in a double quoted string, it's escaped +# as `\\&` +# +if GEM_VERSION <= Gem::Version.new('2.0.0') + # $ git show v2.0.0:lib/rubygems/ext/ext_conf_builder.rb + # cmd = "#{Gem.ruby} #{File.basename extension}" + regexp = /{Gem\.ruby}/ + replace = "\\& -r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb" + builder = 'rubygems/ext/ext_conf_builder.rb' +elsif GEM_VERSION < Gem::Version.new('3.0.0') # there weren't any tags between >= 2.7.11 and < 3.0.0 + # $ git show v2.0.1:lib/rubygems/ext/ext_conf_builder.rb + # cmd = [Gem.ruby, File.basename(extension), *args].join ' ' + # + # $ git show v2.7.11:lib/rubygems/ext/ext_conf_builder.rb + # cmd = [Gem.ruby, "-r", get_relative_path(siteconf.path), File.basename(extension), *args].join ' ' + regexp = /Gem\.ruby/ + replace = "\\&, '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb'" + builder = 'rubygems/ext/ext_conf_builder.rb' +elsif GEM_VERSION <= Gem::Version.new('3.4.8') + # $ git show v3.0.0:lib/rubygems/ext/ext_conf_builder.rb + # cmd = Gem.ruby.shellsplit << "-I" << File.expand_path("../../..", __FILE__) << + # + # $ git show v3.4.8:lib/rubygems/ext/ext_conf_builder.rb + # cmd = Gem.ruby.shellsplit << "-I" << File.expand_path("../..", __dir__) << File.basename(extension) + regexp = /Gem\.ruby\.shellsplit/ + replace = "\\& << '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb'" + builder = 'rubygems/ext/ext_conf_builder.rb' +elsif GEM_VERSION <= Gem::Version.new('3.4.14') + # NOTE: rubygems 3.4.9 moved the code to builder.rb + # + # $ git show v3.4.9:lib/rubygems/ext/builder.rb + # cmd = Gem.ruby.shellsplit + # + # $ git show v3.4.14:lib/rubygems/ext/builder.rb + # cmd = Gem.ruby.shellsplit + regexp = /Gem\.ruby\.shellsplit/ + replace = "\\& << '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb'" + builder = 'rubygems/ext/builder.rb' +elsif GEM_VERSION <= Gem::Version.new('3.5.10') + # $ git show v3.4.9:lib/rubygems/ext/builder.rb + # cmd = Shellwords.split(Gem.ruby) + # + # $ git show v3.5.10:lib/rubygems/ext/builder.rb + # cmd = Shellwords.split(Gem.ruby) + regexp = /Shellwords\.split\(Gem\.ruby\)/ + replace = "\\& << '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb'" + builder = 'rubygems/ext/builder.rb' +else + raise "We don't know how to patch rubygems #{GEM_VERSION}" +end + +# path to the builder file on the HOST ruby +builder = File.join(host_rubylibdir, builder) + +raise "We can't patch #{builder} because it doesn't exist" unless File.exist?(builder) + +# hook rubygems builder so it loads our rbconfig when building native gems +patched = false +rewrite(builder) do |line| + if line.gsub!(regexp, replace) + patched = true + end +end + +raise "Failed to patch rubygems hook, because we couldn't match #{regexp} in #{builder}" unless patched + +puts "Patched '#{regexp.inspect}' in #{builder}" + +# solaris 10 uses ruby 2.0 which doesn't install native extensions based on architecture +if RUBY_PLATFORM !~ /solaris2\.10$/ || RUBY_VERSION != '2.0.0' + # ensure native extensions are written to a directory that matches the + # architecture of the target ruby we're building for. To do that we + # patch the host ruby to pretend to be the target architecture. + triple_patched = false + api_version_patched = false + spec_file = "#{host_rubylibdir}/rubygems/basic_specification.rb" + rewrite(spec_file) do |line| + if line.gsub!(/Gem::Platform\.local\.to_s/, "'#{target_triple}'") + triple_patched = true + end + if line.gsub!(/Gem\.extension_api_version/, "'#{target_api_version}'") + api_version_patched = true + end + end + + raise "Failed to patch '#{target_triple}' in #{spec_file}" unless triple_patched + puts "Patched '#{target_triple}' in #{spec_file}" + + raise "Failed to patch '#{target_api_version}' in #{spec_file}" unless api_version_patched + puts "Patched '#{target_api_version}' in #{spec_file}" +end From 5788a37ce71fac15a62db7b59001a9023d6f2a2c Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Wed, 8 May 2024 09:22:01 -0700 Subject: [PATCH 4/5] (PA-6422) Use platform=ruby to remove nokogiri workaround for macOS 12 ARM Now that we're patching homebrew's ruby 3.2.4 correctly, we can remove our workaround and use platform=ruby as suggested in https://nokogiri.org/tutorials/installing_nokogiri.html#installing-using-standard-system-libraries Note --platform is an option to `gem install` so it comes before double dashes. --- configs/components/rubygem-nokogiri.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/configs/components/rubygem-nokogiri.rb b/configs/components/rubygem-nokogiri.rb index ae64f0906..eca304201 100644 --- a/configs/components/rubygem-nokogiri.rb +++ b/configs/components/rubygem-nokogiri.rb @@ -1,14 +1,13 @@ component 'rubygem-nokogiri' do |pkg, _settings, _platform| pkg.version '1.14.2' pkg.sha256sum 'c765a74aac6cf430a710bb0b6038b8ee11f177393cd6ae8dadc7a44a6e2658b6' - # On macOS when we are not cross compiling we need to use runtime's libxml2 and libxslt - if platform.is_macos? && !platform.is_cross_compiled? - settings[:gem_install_options] = "-- --use-system-libraries \ + + settings[:gem_install_options] = "--platform=ruby -- \ + --use-system-libraries \ --with-xml2-lib=#{settings[:libdir]} \ --with-xml2-include=#{settings[:includedir]}/libxml2 \ --with-xslt-lib=#{settings[:libdir]} \ --with-xslt-include=#{settings[:includedir]}" - end instance_eval File.read('configs/components/_base-rubygem.rb') pkg.build_requires 'rubygem-mini_portile2' gem_home = settings[:gem_home] From 9c19360894d2b776aea2bea9c524dbf848be706b Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Wed, 8 May 2024 12:03:34 -0700 Subject: [PATCH 5/5] (PA-6422) Namespace gem_install_options Settings are shared across all components, so `settings[:gem_install_options]` in the ffi gem caused its options to be passed to gems that follow like gssapi in the bolt-runtime: /opt/puppetlabs/bolt/bin/gem install --no-document --local --bindir=/opt/puppetlabs/bolt/bin gssapi-1.3.1.gem -- --disable-system-libffi This commit namespaces the options so they only apply when installing that specific gem. The namespace is based on the full gem name like `rubygems-ffi`, not the short name `ffi`. --- configs/components/_base-rubygem.rb | 5 +++-- configs/components/rubygem-ffi.rb | 11 ++++++----- configs/components/rubygem-nokogiri.rb | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/configs/components/_base-rubygem.rb b/configs/components/_base-rubygem.rb index 90f399555..a5614b81b 100644 --- a/configs/components/_base-rubygem.rb +++ b/configs/components/_base-rubygem.rb @@ -39,13 +39,14 @@ # If a gem needs more command line options to install set the :gem_install_options # in its component file rubygem-, before the instance_eval of this file. -if settings[:gem_install_options].nil? +gem_install_options = settings["#{pkg.get_name}_gem_install_options".to_sym] +if gem_install_options.nil? pkg.install do "#{settings[:gem_install]} #{name}-#{version}.gem" end else pkg.install do - "#{settings[:gem_install]} #{name}-#{version}.gem #{settings[:gem_install_options]}" + "#{settings[:gem_install]} #{name}-#{version}.gem #{gem_install_options}" end end diff --git a/configs/components/rubygem-ffi.rb b/configs/components/rubygem-ffi.rb index 5af3e8864..f4c73444e 100644 --- a/configs/components/rubygem-ffi.rb +++ b/configs/components/rubygem-ffi.rb @@ -72,11 +72,12 @@ # gem *always* uses the libffi.so we already built. Note the term "system" is # misleading, because we override PKG_CONFIG_PATH below so that our libffi.so # is preferred, not the one in /usr/lib. - settings[:gem_install_options] = if rb_major_minor_version > 2.7 - "-- --enable-system-libffi" - else - "-- --disable-system-libffi" - end + settings["#{pkg.get_name}_gem_install_options".to_sym] = + if rb_major_minor_version > 2.7 + "-- --enable-system-libffi" + else + "-- --disable-system-libffi" + end instance_eval File.read('configs/components/_base-rubygem.rb') end diff --git a/configs/components/rubygem-nokogiri.rb b/configs/components/rubygem-nokogiri.rb index eca304201..79fd58038 100644 --- a/configs/components/rubygem-nokogiri.rb +++ b/configs/components/rubygem-nokogiri.rb @@ -1,8 +1,8 @@ -component 'rubygem-nokogiri' do |pkg, _settings, _platform| +component 'rubygem-nokogiri' do |pkg, settings, _platform| pkg.version '1.14.2' pkg.sha256sum 'c765a74aac6cf430a710bb0b6038b8ee11f177393cd6ae8dadc7a44a6e2658b6' - settings[:gem_install_options] = "--platform=ruby -- \ + settings["#{pkg.get_name}_gem_install_options".to_sym] = "--platform=ruby -- \ --use-system-libraries \ --with-xml2-lib=#{settings[:libdir]} \ --with-xml2-include=#{settings[:includedir]}/libxml2 \