diff --git a/configs/components/_base-rubygem.rb b/configs/components/_base-rubygem.rb index 90f399555..a5614b81b 100644 --- a/configs/components/_base-rubygem.rb +++ b/configs/components/_base-rubygem.rb @@ -39,13 +39,14 @@ # If a gem needs more command line options to install set the :gem_install_options # in its component file rubygem-, before the instance_eval of this file. -if settings[:gem_install_options].nil? +gem_install_options = settings["#{pkg.get_name}_gem_install_options".to_sym] +if gem_install_options.nil? pkg.install do "#{settings[:gem_install]} #{name}-#{version}.gem" end else pkg.install do - "#{settings[:gem_install]} #{name}-#{version}.gem #{settings[:gem_install_options]}" + "#{settings[:gem_install]} #{name}-#{version}.gem #{gem_install_options}" end end diff --git a/configs/components/pl-ruby-patch.rb b/configs/components/pl-ruby-patch.rb index 94e3aa592..11bfc075e 100644 --- a/configs/components/pl-ruby-patch.rb +++ b/configs/components/pl-ruby-patch.rb @@ -8,70 +8,33 @@ # This component should also be present in the puppet-agent project component "pl-ruby-patch" do |pkg, settings, platform| if platform.is_cross_compiled? - if platform.is_macos? - pkg.build_requires 'gnu-sed' - pkg.environment "PATH", "/usr/local/opt/gnu-sed/libexec/gnubin:$(PATH)" - end - ruby_api_version = settings[:ruby_version].gsub(/\.\d*$/, '.0') ruby_version_y = settings[:ruby_version].gsub(/(\d+)\.(\d+)\.(\d+)/, '\1.\2') - base_ruby = case platform.name - when /solaris-10/ - "/opt/csw/lib/ruby/2.0.0" - when /osx/ - "/usr/local/opt/ruby@#{ruby_version_y}/lib/ruby/#{ruby_api_version}" - else - "/opt/pl-build-tools/lib/ruby/2.1.0" - end + pkg.add_source("file://resources/files/ruby/patch-hostruby.rb") # The `target_triple` determines which directory native extensions are stored in the # compiled ruby and must match ruby's naming convention. - # - # solaris 10 uses ruby 2.0 which doesn't install native extensions based on architecture - unless platform.name =~ /solaris-10/ - # weird architecture naming conventions... - target_triple = if platform.architecture =~ /ppc64el|ppc64le/ - "powerpc64le-linux" - elsif platform.name == 'solaris-11-sparc' - "sparc-solaris-2.11" - elsif platform.is_macos? - if ruby_version_y.start_with?('2') - "aarch64-darwin" - else - "arm64-darwin" - end + # weird architecture naming conventions... + target_triple = if platform.architecture =~ /ppc64el|ppc64le/ + "powerpc64le-linux" + elsif platform.name == 'solaris-11-sparc' + "sparc-solaris-2.11" + elsif platform.name =~ /solaris-10/ + "sparc-solaris" + elsif platform.is_macos? + if ruby_version_y.start_with?('2') + "aarch64-darwin" else - "#{platform.architecture}-linux" + "arm64-darwin" end + else + "#{platform.architecture}-linux" + end - pkg.build do - [ - %(#{platform[:sed]} -i 's/Gem::Platform.local.to_s/"#{target_triple}"/' #{base_ruby}/rubygems/basic_specification.rb), - %(#{platform[:sed]} -i 's/Gem.extension_api_version/"#{ruby_api_version}"/' #{base_ruby}/rubygems/basic_specification.rb) - ] - end - end - - # make rubygems use our target rbconfig when installing gems - case File.basename(base_ruby) - when '2.0.0', '2.1.0' - sed_command = %(s|Gem.ruby|&, '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{settings[:ruby_version]}-orig.rb'|) - else - sed_command = %(s|Gem.ruby.shellsplit|& << '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{settings[:ruby_version]}-orig.rb'|) - end - - # rubygems switched which file has the command we need to patch starting in rubygems 3.4.10, which we install in our formula - # for ruby in homebrew-puppet - if Gem::Version.new(settings[:ruby_version]) >= Gem::Version.new('3.2.2') || platform.is_macos? && ruby_version_y.start_with?('2') - filename = 'builder.rb' - else - filename = 'ext_conf_builder.rb' - end - - pkg.build do + pkg.install do [ - %(#{platform[:sed]} -i "#{sed_command}" #{base_ruby}/rubygems/ext/#{filename}) + "#{settings[:host_ruby]} patch-hostruby.rb #{settings[:ruby_version]} #{target_triple}" ] end end diff --git a/configs/components/rubygem-ffi.rb b/configs/components/rubygem-ffi.rb index e86bdd585..f4c73444e 100644 --- a/configs/components/rubygem-ffi.rb +++ b/configs/components/rubygem-ffi.rb @@ -15,38 +15,16 @@ pkg.sha256sum '6f2ed2fa68047962d6072b964420cba91d82ce6fa8ee251950c17fca6af3c2a0' end - instance_eval File.read('configs/components/_base-rubygem.rb') - rb_major_minor_version = settings[:ruby_version].to_f - # Prior to ruby 3.2, both ruby and the ffi gem vendored a version of libffi. - # If libffi happened to be installed in /usr/lib, then the ffi gem preferred - # that instead of building libffi itself. To ensure consistency, we use - # --disable-system-ffi so that the ffi gem *always* builds libffi, then - # builds the ffi_c native extension and links it against libffi.so. - # - # In ruby 3.2 and up, libffi is no longer vendored. So we created a separate - # libffi vanagon component which is built before ruby. The ffi gem still - # vendors libffi, so we use the --enable-system-ffi option to ensure the ffi - # gem *always* uses the libffi.so we already built. Note the term "system" is - # misleading, because we override PKG_CONFIG_PATH below so that our libffi.so - # is preferred, not the one in /usr/lib. - if rb_major_minor_version > 2.7 - pkg.install do - "#{settings[:gem_install]} ffi-#{pkg.get_version}.gem -- --enable-system-ffi" - end - else - pkg.install do - "#{settings[:gem_install]} ffi-#{pkg.get_version}.gem -- --disable-system-ffi" - end - end - # Windows versions of the FFI gem have custom filenames, so we overwite the # defaults that _base-rubygem provides here, just for Windows for Ruby < 3.2 if platform.is_windows? && rb_major_minor_version < 3.2 # Pin this if lower than Ruby 2.7 pkg.version '1.9.25' if rb_major_minor_version < 2.7 + instance_eval File.read('configs/components/_base-rubygem.rb') + # Vanagon's `pkg.mirror` is additive, and the _base_rubygem sets the # non-Windows gem as the first mirror, which is incorrect. We need to unset # the list of mirrors before adding the Windows-appropriate ones here: @@ -81,6 +59,26 @@ pkg.install do "#{settings[:gem_install]} ffi-#{pkg.get_version}-#{platform.architecture}-mingw32.gem" end + else + # Prior to ruby 3.2, both ruby and the ffi gem vendored a version of libffi. + # If libffi happened to be installed in /usr/lib, then the ffi gem preferred + # that instead of building libffi itself. To ensure consistency, we use + # --disable-system-libffi so that the ffi gem *always* builds libffi, then + # builds the ffi_c native extension and links it against libffi.so. + # + # In ruby 3.2 and up, libffi is no longer vendored. So we created a separate + # libffi vanagon component which is built before ruby. The ffi gem still + # vendors libffi, so we use the --enable-system-libffi option to ensure the ffi + # gem *always* uses the libffi.so we already built. Note the term "system" is + # misleading, because we override PKG_CONFIG_PATH below so that our libffi.so + # is preferred, not the one in /usr/lib. + settings["#{pkg.get_name}_gem_install_options".to_sym] = + if rb_major_minor_version > 2.7 + "-- --enable-system-libffi" + else + "-- --disable-system-libffi" + end + instance_eval File.read('configs/components/_base-rubygem.rb') end # due to contrib/make_sunver.pl missing on solaris 11 we cannot compile libffi, so we provide the opencsw library @@ -157,4 +155,4 @@ %(#{platform[:sed]} -i '0,/ensure_required_ruby_version_met/b; /ensure_required_ruby_version_met/d' #{base_ruby}/rubygems/installer.rb) end end -end \ No newline at end of file +end diff --git a/configs/components/rubygem-nokogiri.rb b/configs/components/rubygem-nokogiri.rb index ae64f0906..79fd58038 100644 --- a/configs/components/rubygem-nokogiri.rb +++ b/configs/components/rubygem-nokogiri.rb @@ -1,14 +1,13 @@ -component 'rubygem-nokogiri' do |pkg, _settings, _platform| +component 'rubygem-nokogiri' do |pkg, settings, _platform| pkg.version '1.14.2' pkg.sha256sum 'c765a74aac6cf430a710bb0b6038b8ee11f177393cd6ae8dadc7a44a6e2658b6' - # On macOS when we are not cross compiling we need to use runtime's libxml2 and libxslt - if platform.is_macos? && !platform.is_cross_compiled? - settings[:gem_install_options] = "-- --use-system-libraries \ + + settings["#{pkg.get_name}_gem_install_options".to_sym] = "--platform=ruby -- \ + --use-system-libraries \ --with-xml2-lib=#{settings[:libdir]} \ --with-xml2-include=#{settings[:includedir]}/libxml2 \ --with-xslt-lib=#{settings[:libdir]} \ --with-xslt-include=#{settings[:includedir]}" - end instance_eval File.read('configs/components/_base-rubygem.rb') pkg.build_requires 'rubygem-mini_portile2' gem_home = settings[:gem_home] diff --git a/resources/files/ruby/patch-hostruby.rb b/resources/files/ruby/patch-hostruby.rb new file mode 100644 index 000000000..03194bf95 --- /dev/null +++ b/resources/files/ruby/patch-hostruby.rb @@ -0,0 +1,145 @@ +# When cross compiling we need to run gem install using the host ruby, but +# force ruby to use our overridden rbconfig.rb. To do that, we insert a +# require statement between the ruby executable and it's first argument, +# thereby hooking the ruby process. +# +# In the future we could use the --target-rbconfig= option to point +# to our rbconfig.rb. But that option is only available in newer ruby versions. +require 'rbconfig' +require 'tempfile' + +if ARGV.length < 2 + warn < + +example: patch-hostruby.rb 3.2.2 arm64-darwin +USAGE + exit(1) +end + +# target ruby versions (what we're trying to build) +target_ruby_version = ARGV[0] +target_triple = ARGV[1] +target_api_version = target_ruby_version.gsub(/\.\d*$/, '.0') + +# host ruby (the ruby we execute to build the target) +host_rubylibdir = RbConfig::CONFIG['rubylibdir'] +GEM_VERSION = Gem::Version.new(Gem::VERSION) + +# Rewrite the file in-place securely, yielding each line to the caller +def rewrite(file) + # create temp file in the same directory as the file we're patching, + # so rename doesn't cross filesystems + tmpfile = Tempfile.new(File.basename(file), File.dirname(file)) + begin + File.open("#{file}.orig", "w") do |orig| + File.open(file, 'r').readlines.each do |line| + orig.write(line) + yield line + tmpfile.write(line) + end + end + ensure + tmpfile.close + File.unlink(file) + File.rename(tmpfile.path, file) + tmpfile.unlink + end +end + +# Based on the RUBYGEMS version of the host ruby, the line and file that needs patching is different +# Note the RUBY version doesn't matter (for either the host or target ruby). +# +# Here we define different intervals. For each interval, we specify the regexp to match, what to +# replace it with, and which file to edit in-place. Note `\&` is a placeholder for whatever the regexp +# was, that way we can easily append to it. And since it's in a double quoted string, it's escaped +# as `\\&` +# +if GEM_VERSION <= Gem::Version.new('2.0.0') + # $ git show v2.0.0:lib/rubygems/ext/ext_conf_builder.rb + # cmd = "#{Gem.ruby} #{File.basename extension}" + regexp = /{Gem\.ruby}/ + replace = "\\& -r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb" + builder = 'rubygems/ext/ext_conf_builder.rb' +elsif GEM_VERSION < Gem::Version.new('3.0.0') # there weren't any tags between >= 2.7.11 and < 3.0.0 + # $ git show v2.0.1:lib/rubygems/ext/ext_conf_builder.rb + # cmd = [Gem.ruby, File.basename(extension), *args].join ' ' + # + # $ git show v2.7.11:lib/rubygems/ext/ext_conf_builder.rb + # cmd = [Gem.ruby, "-r", get_relative_path(siteconf.path), File.basename(extension), *args].join ' ' + regexp = /Gem\.ruby/ + replace = "\\&, '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb'" + builder = 'rubygems/ext/ext_conf_builder.rb' +elsif GEM_VERSION <= Gem::Version.new('3.4.8') + # $ git show v3.0.0:lib/rubygems/ext/ext_conf_builder.rb + # cmd = Gem.ruby.shellsplit << "-I" << File.expand_path("../../..", __FILE__) << + # + # $ git show v3.4.8:lib/rubygems/ext/ext_conf_builder.rb + # cmd = Gem.ruby.shellsplit << "-I" << File.expand_path("../..", __dir__) << File.basename(extension) + regexp = /Gem\.ruby\.shellsplit/ + replace = "\\& << '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb'" + builder = 'rubygems/ext/ext_conf_builder.rb' +elsif GEM_VERSION <= Gem::Version.new('3.4.14') + # NOTE: rubygems 3.4.9 moved the code to builder.rb + # + # $ git show v3.4.9:lib/rubygems/ext/builder.rb + # cmd = Gem.ruby.shellsplit + # + # $ git show v3.4.14:lib/rubygems/ext/builder.rb + # cmd = Gem.ruby.shellsplit + regexp = /Gem\.ruby\.shellsplit/ + replace = "\\& << '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb'" + builder = 'rubygems/ext/builder.rb' +elsif GEM_VERSION <= Gem::Version.new('3.5.10') + # $ git show v3.4.9:lib/rubygems/ext/builder.rb + # cmd = Shellwords.split(Gem.ruby) + # + # $ git show v3.5.10:lib/rubygems/ext/builder.rb + # cmd = Shellwords.split(Gem.ruby) + regexp = /Shellwords\.split\(Gem\.ruby\)/ + replace = "\\& << '-r/opt/puppetlabs/puppet/share/doc/rbconfig-#{target_ruby_version}-orig.rb'" + builder = 'rubygems/ext/builder.rb' +else + raise "We don't know how to patch rubygems #{GEM_VERSION}" +end + +# path to the builder file on the HOST ruby +builder = File.join(host_rubylibdir, builder) + +raise "We can't patch #{builder} because it doesn't exist" unless File.exist?(builder) + +# hook rubygems builder so it loads our rbconfig when building native gems +patched = false +rewrite(builder) do |line| + if line.gsub!(regexp, replace) + patched = true + end +end + +raise "Failed to patch rubygems hook, because we couldn't match #{regexp} in #{builder}" unless patched + +puts "Patched '#{regexp.inspect}' in #{builder}" + +# solaris 10 uses ruby 2.0 which doesn't install native extensions based on architecture +if RUBY_PLATFORM !~ /solaris2\.10$/ || RUBY_VERSION != '2.0.0' + # ensure native extensions are written to a directory that matches the + # architecture of the target ruby we're building for. To do that we + # patch the host ruby to pretend to be the target architecture. + triple_patched = false + api_version_patched = false + spec_file = "#{host_rubylibdir}/rubygems/basic_specification.rb" + rewrite(spec_file) do |line| + if line.gsub!(/Gem::Platform\.local\.to_s/, "'#{target_triple}'") + triple_patched = true + end + if line.gsub!(/Gem\.extension_api_version/, "'#{target_api_version}'") + api_version_patched = true + end + end + + raise "Failed to patch '#{target_triple}' in #{spec_file}" unless triple_patched + puts "Patched '#{target_triple}' in #{spec_file}" + + raise "Failed to patch '#{target_api_version}' in #{spec_file}" unless api_version_patched + puts "Patched '#{target_api_version}' in #{spec_file}" +end