From 58560cc96b67efe3a937b608b38ac35310b78f9c Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Wed, 8 May 2024 01:21:32 -0700 Subject: [PATCH] (PA-6422) Update rubygems patch for ruby 3.2.4 Previously, our rubygems patch wasn't being applied when cross compiling mac 12 ARM, because homebrew updated to ruby 3.2.4 and builder.rb was modified: - cmd = Gem.ruby.shellsplit + cmd = Shellwords.split(Gem.ruby) The code for patching is very fragile and has broken multiple times without any indication that the patch failed to apply, causing the resulting native extensions to be compiled for the wrong architecture. The code was also conflating the `settings[:ruby_version]` in our ruby component with the host's ruby version (the one we're actually trying to patch). Also the version that matters is the rubygems version (Gem::VERSION), not RUBY_VERSION. For example, if `gem update --system` was executed during provisioning, then our patch would fail to apply. This commit creates a script that runs on the remote system and so can introspect the host's ruby and rubygems versions. This way if brew updates to a newer ruby/rubygems, we can fail the build. --- configs/components/pl-ruby-patch.rb | 71 +++--------- resources/files/ruby/patch-hostruby.rb | 145 +++++++++++++++++++++++++ 2 files changed, 162 insertions(+), 54 deletions(-) create mode 100644 resources/files/ruby/patch-hostruby.rb diff --git a/configs/components/pl-ruby-patch.rb b/configs/components/pl-ruby-patch.rb index 94e3aa592..11bfc075e 100644 --- a/configs/components/pl-ruby-patch.rb +++ b/configs/components/pl-ruby-patch.rb @@ -8,70 +8,33 @@ # This component should also be present in the puppet-agent project component "pl-ruby-patch" do |pkg, settings, platform| if platform.is_cross_compiled? - if platform.is_macos? - pkg.build_requires 'gnu-sed' - pkg.environment "PATH", "/usr/local/opt/gnu-sed/libexec/gnubin:$(PATH)" - end - ruby_api_version = settings[:ruby_version].gsub(/\.\d*$/, '.0') ruby_version_y = settings[:ruby_version].gsub(/(\d+)\.(\d+)\.(\d+)/, '\1.\2') - base_ruby = case platform.name - when /solaris-10/ - "/opt/csw/lib/ruby/2.0.0" - when /osx/ - "/usr/local/opt/ruby@#{ruby_version_y}/lib/ruby/#{ruby_api_version}" - else - "/opt/pl-build-tools/lib/ruby/2.1.0" - end + pkg.add_source("file://resources/files/ruby/patch-hostruby.rb") # The `target_triple` determines which directory native extensions are stored in the # compiled ruby and must match ruby's naming convention. - # - # solaris 10 uses ruby 2.0 which doesn't install native extensions based on architecture - unless platform.name =~ /solaris-10/ - # weird architecture naming conventions... - target_triple = if platform.architecture =~ /ppc64el|ppc64le/ - "powerpc64le-linux" - elsif platform.name == 'solaris-11-sparc' - "sparc-solaris-2.11" - elsif platform.is_macos? - if ruby_version_y.start_with?('2') - "aarch64-darwin" - else - "arm64-darwin" - end + # weird architecture naming conventions... + target_triple = if platform.architecture =~ /ppc64el|ppc64le/ + "powerpc64le-linux" + elsif platform.name == 'solaris-11-sparc' + "sparc-solaris-2.11" + elsif platform.name =~ /solaris-10/ + "sparc-solaris" + elsif platform.is_macos? + if ruby_version_y.start_with?('2') + "aarch64-darwin" else - "#{platform.architecture}-linux" + "arm64-darwin" end + else + "#{platform.architecture}-linux" + end - pkg.build do - [ - %(#{platform[:sed]} -i 's/Gem::Platform.local.to_s/"#{target_triple}"/' #{base_ruby}/rubygems/basic_specification.rb), - %(#{platform[:sed]} -i 's/Gem.extension_api_version/"#{ruby_api_version}"/' #{base_ruby}/rubygems/basic_specification.rb) - ] - end - end - - # make rubygems use our target rbconfig when installing gems - case File.basename(base_ruby) - when '2.0.0', '2.1.0' - sed_command = %(s|Gem.ruby|&, '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{settings[:ruby_version]}-orig.rb'|) - else - sed_command = %(s|Gem.ruby.shellsplit|& << '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{settings[:ruby_version]}-orig.rb'|) - end - - # rubygems switched which file has the command we need to patch starting in rubygems 3.4.10, which we install in our formula - # for ruby in homebrew-puppet - if Gem::Version.new(settings[:ruby_version]) >= Gem::Version.new('3.2.2') || platform.is_macos? && ruby_version_y.start_with?('2') - filename = 'builder.rb' - else - filename = 'ext_conf_builder.rb' - end - - pkg.build do + pkg.install do [ - %(#{platform[:sed]} -i "#{sed_command}" #{base_ruby}/rubygems/ext/#{filename}) + "#{settings[:host_ruby]} patch-hostruby.rb #{settings[:ruby_version]} #{target_triple}" ] end end diff --git a/resources/files/ruby/patch-hostruby.rb b/resources/files/ruby/patch-hostruby.rb new file mode 100644 index 000000000..03194bf95 --- /dev/null +++ b/resources/files/ruby/patch-hostruby.rb @@ -0,0 +1,145 @@ +# When cross compiling we need to run gem install using the host ruby, but +# force ruby to use our overridden rbconfig.rb. To do that, we insert a +# require statement between the ruby executable and it's first argument, +# thereby hooking the ruby process. +# +# In the future we could use the --target-rbconfig= option to point +# to our rbconfig.rb. But that option is only available in newer ruby versions. +require 'rbconfig' +require 'tempfile' + +if ARGV.length < 2 + warn < + +example: patch-hostruby.rb 3.2.2 arm64-darwin +USAGE + exit(1) +end + +# target ruby versions (what we're trying to build) +target_ruby_version = ARGV[0] +target_triple = ARGV[1] +target_api_version = target_ruby_version.gsub(/\.\d*$/, '.0') + +# host ruby (the ruby we execute to build the target) +host_rubylibdir = RbConfig::CONFIG['rubylibdir'] +GEM_VERSION = Gem::Version.new(Gem::VERSION) + +# Rewrite the file in-place securely, yielding each line to the caller +def rewrite(file) + # create temp file in the same directory as the file we're patching, + # so rename doesn't cross filesystems + tmpfile = Tempfile.new(File.basename(file), File.dirname(file)) + begin + File.open("#{file}.orig", "w") do |orig| + File.open(file, 'r').readlines.each do |line| + orig.write(line) + yield line + tmpfile.write(line) + end + end + ensure + tmpfile.close + File.unlink(file) + File.rename(tmpfile.path, file) + tmpfile.unlink + end +end + +# Based on the RUBYGEMS version of the host ruby, the line and file that needs patching is different +# Note the RUBY version doesn't matter (for either the host or target ruby). +# +# Here we define different intervals. For each interval, we specify the regexp to match, what to +# replace it with, and which file to edit in-place. Note `\&` is a placeholder for whatever the regexp +# was, that way we can easily append to it. And since it's in a double quoted string, it's escaped +# as `\\&` +# +if GEM_VERSION <= Gem::Version.new('2.0.0') + # $ git show v2.0.0:lib/rubygems/ext/ext_conf_builder.rb + # cmd = "#{Gem.ruby} #{File.basename extension}" + regexp = /{Gem\.ruby}/ + replace = "\\& -r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb" + builder = 'rubygems/ext/ext_conf_builder.rb' +elsif GEM_VERSION < Gem::Version.new('3.0.0') # there weren't any tags between >= 2.7.11 and < 3.0.0 + # $ git show v2.0.1:lib/rubygems/ext/ext_conf_builder.rb + # cmd = [Gem.ruby, File.basename(extension), *args].join ' ' + # + # $ git show v2.7.11:lib/rubygems/ext/ext_conf_builder.rb + # cmd = [Gem.ruby, "-r", get_relative_path(siteconf.path), File.basename(extension), *args].join ' ' + regexp = /Gem\.ruby/ + replace = "\\&, '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb'" + builder = 'rubygems/ext/ext_conf_builder.rb' +elsif GEM_VERSION <= Gem::Version.new('3.4.8') + # $ git show v3.0.0:lib/rubygems/ext/ext_conf_builder.rb + # cmd = Gem.ruby.shellsplit << "-I" << File.expand_path("../../..", __FILE__) << + # + # $ git show v3.4.8:lib/rubygems/ext/ext_conf_builder.rb + # cmd = Gem.ruby.shellsplit << "-I" << File.expand_path("../..", __dir__) << File.basename(extension) + regexp = /Gem\.ruby\.shellsplit/ + replace = "\\& << '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb'" + builder = 'rubygems/ext/ext_conf_builder.rb' +elsif GEM_VERSION <= Gem::Version.new('3.4.14') + # NOTE: rubygems 3.4.9 moved the code to builder.rb + # + # $ git show v3.4.9:lib/rubygems/ext/builder.rb + # cmd = Gem.ruby.shellsplit + # + # $ git show v3.4.14:lib/rubygems/ext/builder.rb + # cmd = Gem.ruby.shellsplit + regexp = /Gem\.ruby\.shellsplit/ + replace = "\\& << '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb'" + builder = 'rubygems/ext/builder.rb' +elsif GEM_VERSION <= Gem::Version.new('3.5.10') + # $ git show v3.4.9:lib/rubygems/ext/builder.rb + # cmd = Shellwords.split(Gem.ruby) + # + # $ git show v3.5.10:lib/rubygems/ext/builder.rb + # cmd = Shellwords.split(Gem.ruby) + regexp = /Shellwords\.split\(Gem\.ruby\)/ + replace = "\\& << '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb'" + builder = 'rubygems/ext/builder.rb' +else + raise "We don't know how to patch rubygems #{GEM_VERSION}" +end + +# path to the builder file on the HOST ruby +builder = File.join(host_rubylibdir, builder) + +raise "We can't patch #{builder} because it doesn't exist" unless File.exist?(builder) + +# hook rubygems builder so it loads our rbconfig when building native gems +patched = false +rewrite(builder) do |line| + if line.gsub!(regexp, replace) + patched = true + end +end + +raise "Failed to patch rubygems hook, because we couldn't match #{regexp} in #{builder}" unless patched + +puts "Patched '#{regexp.inspect}' in #{builder}" + +# solaris 10 uses ruby 2.0 which doesn't install native extensions based on architecture +if RUBY_PLATFORM !~ /solaris2\.10$/ || RUBY_VERSION != '2.0.0' + # ensure native extensions are written to a directory that matches the + # architecture of the target ruby we're building for. To do that we + # patch the host ruby to pretend to be the target architecture. + triple_patched = false + api_version_patched = false + spec_file = "#{host_rubylibdir}/rubygems/basic_specification.rb" + rewrite(spec_file) do |line| + if line.gsub!(/Gem::Platform\.local\.to_s/, "'#{target_triple}'") + triple_patched = true + end + if line.gsub!(/Gem\.extension_api_version/, "'#{target_api_version}'") + api_version_patched = true + end + end + + raise "Failed to patch '#{target_triple}' in #{spec_file}" unless triple_patched + puts "Patched '#{target_triple}' in #{spec_file}" + + raise "Failed to patch '#{target_api_version}' in #{spec_file}" unless api_version_patched + puts "Patched '#{target_api_version}' in #{spec_file}" +end