From f1f59ae4c7c5b91c407e4fbc6516d07b1a0ab18d Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Thu, 16 Nov 2023 22:02:02 -0800 Subject: [PATCH 1/4] Refactor how libssp-0.dll is installed into primary ruby's bin We compile ruby 3.2 with strong stack protection. This causes ruby to be linked against libssp-0, which must be redistributed with the runtime: $ objdump -p ruby.exe | grep libssp DLL Name: libssp-0.dll Windows doesn't have an RPATH equivalent, so the dll must be copied into the same directory as ruby. This was partially addressed in commmit ba4c95513, however, the _base-ruby component is already responsible for installing Windows specific library dependencies when the main bin dir is different than ruby's bin dir, so move that logic there. --- configs/components/_base-ruby.rb | 30 +++++++++++++++++++++-------- configs/projects/_pdk-components.rb | 6 ------ 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/configs/components/_base-ruby.rb b/configs/components/_base-ruby.rb index a5d05f97d..5006040ba 100644 --- a/configs/components/_base-ruby.rb +++ b/configs/components/_base-ruby.rb @@ -114,15 +114,28 @@ [ "#{platform[:make]} -j$(shell expr $(shell #{platform[:num_cores]}) + 1) install" ] end +# For the pdk runtime, the ruby bin directory is different then the main bin +# directory. In order to run ruby *outside* of the normal pdk.bat then we need +# to copy all dlls that ruby depends on from the main bin directory to ruby's +# bin directory. This is because the main bin directory is not in our system +# PATH and Windows doesn't support RPATH. However, as mentioned in +# https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-search-order, +# Windows searches for DLLs in "The folder the calling process was loaded from +# (the executable's folder)." +# +# The agent runtime used to have the same issue prior to puppet 6, for example +# RE-7593. However, Windows paths were changed to match *nix in puppet 6, see +# commit 4b9d126dd5b. So only the pdk has this issue. if platform.is_windows? && settings[:bindir] != ruby_bindir - # As things stand right now, ssl should build under [INSTALLDIR]\Puppet\puppet on - # windows. However, if things need to run *outside* of the normal batch file runs - # (puppet.bat ,mco.bat etcc) the location of openssl away from where ruby is - # installed will cause a failure. Specifically this is meant to help services like - # mco that require openssl but don't have access to environment.bat. Refer to - # https://tickets.puppetlabs.com/browse/RE-7593 for details on why this causes - # failures and why these copies fix that. - # -Sean P. McDonald 07/01/2016 + # Ruby 3+ + if Gem::Version.new(pkg.get_version) >= Gem::Version.new('3.0') + pkg.install do + [ + "cp #{settings[:gcc_bindir]}/libssp-0.dll #{ruby_bindir}", + ] + end + end + if platform.architecture == "x64" gcc_postfix = 'seh' ssl_postfix = '-x64' @@ -131,6 +144,7 @@ ssl_postfix = '' end + # OpenSSL if Gem::Version.new(settings[:openssl_version]) >= Gem::Version.new('3.0') ssl_lib = "libssl-3#{ssl_postfix}.dll" crypto_lib = "libcrypto-3#{ssl_postfix}.dll" diff --git a/configs/projects/_pdk-components.rb b/configs/projects/_pdk-components.rb index fed6339f0..8d347e202 100644 --- a/configs/projects/_pdk-components.rb +++ b/configs/projects/_pdk-components.rb @@ -25,12 +25,6 @@ proj.component 'libxslt' unless platform.is_windows? proj.component "ruby-#{proj.ruby_version}" -# After installing ruby, we need to copy libssp to the ruby bindir on windows -if platform.is_windows? - ruby_component = @project.get_component "ruby-#{proj.ruby_version}" - ruby_component.install.push "cp '#{settings[:bindir]}/libssp-0.dll' '#{settings[:ruby_bindir]}/libssp-0.dll'" -end - proj.component 'ruby-augeas' unless platform.is_windows? proj.component 'ruby-selinux' if platform.is_el? || platform.is_fedora? proj.component 'ruby-stomp' From a2e5beab1a7eef8f1c7a85c993f99eba25c4cfb6 Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Thu, 16 Nov 2023 22:17:18 -0800 Subject: [PATCH 2/4] Don't install libffi-6 with ruby 3.2 Previously, we were copying libffi-6.dll to both ruby 2.7 and 3.2 bin directories on Windows: $ find . -type f -name "libffi*dll" ./2.7.8/bin/libffi-6.dll ./3.2.2/bin/libffi-6.dll However, we build libffi-8 with ruby 3.2, and it will be installed to the pdk bin directories in a future commit. So restrict libffi-6 to ruby 2. --- configs/components/runtime-pdk.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/components/runtime-pdk.rb b/configs/components/runtime-pdk.rb index 92645efaf..b09f8aa03 100644 --- a/configs/components/runtime-pdk.rb +++ b/configs/components/runtime-pdk.rb @@ -17,7 +17,7 @@ pkg.install_file "#{settings[:tools_root]}/bin/libgdbm-4.dll", "#{settings[:ruby_bindir]}/libgdbm-4.dll" pkg.install_file "#{settings[:tools_root]}/bin/libgdbm_compat-4.dll", "#{settings[:ruby_bindir]}/libgdbm_compat-4.dll" pkg.install_file "#{settings[:tools_root]}/bin/libiconv-2.dll", "#{settings[:ruby_bindir]}/libiconv-2.dll" - pkg.install_file "#{settings[:tools_root]}/bin/libffi-6.dll", "#{settings[:ruby_bindir]}/libffi-6.dll" + pkg.install_file "#{settings[:tools_root]}/bin/libffi-6.dll", "#{settings[:ruby_bindir]}/libffi-6.dll" if settings[:ruby_version].start_with?('2') # Copy the DLLs into additional ruby install bindirs as well. if settings.has_key?(:additional_rubies) @@ -26,7 +26,7 @@ pkg.install_file "#{settings[:tools_root]}/bin/libgdbm-4.dll", "#{local_settings[:ruby_bindir]}/libgdbm-4.dll" pkg.install_file "#{settings[:tools_root]}/bin/libgdbm_compat-4.dll", "#{local_settings[:ruby_bindir]}/libgdbm_compat-4.dll" pkg.install_file "#{settings[:tools_root]}/bin/libiconv-2.dll", "#{local_settings[:ruby_bindir]}/libiconv-2.dll" - pkg.install_file "#{settings[:tools_root]}/bin/libffi-6.dll", "#{local_settings[:ruby_bindir]}/libffi-6.dll" + pkg.install_file "#{settings[:tools_root]}/bin/libffi-6.dll", "#{local_settings[:ruby_bindir]}/libffi-6.dll" if rubyver.start_with?('2') end end From 887c920f702bcd7ebe5eaff816fe4ba766675a31 Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Thu, 16 Nov 2023 23:22:54 -0800 Subject: [PATCH 3/4] Install libyaml & libffi in ruby-specific bin dir Previously, the pdk-vanagon's gem prune command[1] failed on Windows with Ruby 3.2: $ GEM_PATH="C:/ProgramFiles64Folder/PuppetLabs/DevelopmentKit/private/puppet/ruby/3.2.0;C:/ProgramFiles64Folder/PuppetLabs/DevelopmentKit/share/cache/ruby/3.2.0" \ RUBYOPT="-Irubygems-prune" C:/ProgramFiles64Folder/PuppetLabs/DevelopmentKit/private/ruby/3.2.2/bin/gem.bat prune ... The specified module could not be found. - C:/ProgramFiles64Folder/PuppetLabs/DevelopmentKit/private/ruby/3.2.2/lib/ruby/3.2.0/x64-mingw32/psych.so (LoadError) The error is misleading, because it found psych.so, but couldn't load its dependency libyaml-0.2.dll. The reason this occurred was because the pdk-runtime installed libyaml into its prefix bin directory, which is different than its primary ruby bin directory: prefix C:/ProgramFiles64Folder/PuppetLabs/DevelopmentKit/bin/ ruby_dir C:/ProgramFiles64Folder/PuppetLabs/DevelopmentKit/private/ruby/3.2.2/bin However, the prefix bin directory is not in the PATH, so it couldn't load libyaml. This wasn't an issue for non-pdk runtimes, because the prefix and ruby_dir are the same. And on Windows the default Dll search path includes "The folder from which the application loaded."[2] This is not an issue for pdk runtimes on *nix, because we embed /opt/puppetlabs/pdk/lib into the RPATH: # objdump -p /opt/puppetlabs/pdk/private/ruby/3.2.2/lib/ruby/3.2.0/x86_64-linux/psych.so | grep -i path RUNPATH /opt/puppetlabs/pdk/lib:/opt/puppetlabs/pdk/private/ruby/3.2.2/lib So psych.so can load libyaml: # ldd /opt/puppetlabs/pdk/private/ruby/3.2.2/lib/ruby/3.2.0/x86_64-linux/psych.so | grep libyaml libyaml-0.so.2 => /opt/puppetlabs/pdk/lib/libyaml-0.so.2 This is not an issue when running the pdk on Windows, because the wrapper adds the directory to the PATH[3] The same issue existed when the ffi_c.so native extension tried to load libffi. Since libyaml and libffi may be depended on by more than just ruby 3.2, install the libraries in to the prefix directory and copy them into the ruby 3.2 bin directory. This was partially addressed in ba4c95513 and reverted in 811a384b2, but is implemented differently here. [1] https://github.com/puppetlabs/pdk-vanagon/blob/c7d35b03fa014f95570cee9c84fcfd2bbfc7df32/configs/components/gem-prune.rb#L37 [2] https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-search-order [3] https://github.com/puppetlabs/pdk-vanagon/blob/c7d35b03fa014f95570cee9c84fcfd2bbfc7df32/resources/files/windows/pdk.bat#L7 --- configs/components/_base-ruby.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configs/components/_base-ruby.rb b/configs/components/_base-ruby.rb index 5006040ba..0265a792d 100644 --- a/configs/components/_base-ruby.rb +++ b/configs/components/_base-ruby.rb @@ -132,6 +132,8 @@ pkg.install do [ "cp #{settings[:gcc_bindir]}/libssp-0.dll #{ruby_bindir}", + "cp #{settings[:bindir]}/libffi-8.dll #{ruby_bindir}", + "cp #{settings[:bindir]}/libyaml-0-2.dll #{ruby_bindir}" ] end end From 81059f6336c6b8f5f77ab9a3700eb0b2b58d5bcc Mon Sep 17 00:00:00 2001 From: Josh Cooper Date: Fri, 17 Nov 2023 14:55:29 -0800 Subject: [PATCH 4/4] DRY source and destination dirs --- configs/components/_base-ruby.rb | 12 +++++++----- configs/components/runtime-pdk.rb | 30 +++++++++++++++++------------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/configs/components/_base-ruby.rb b/configs/components/_base-ruby.rb index 0265a792d..1f9cd82dd 100644 --- a/configs/components/_base-ruby.rb +++ b/configs/components/_base-ruby.rb @@ -127,13 +127,15 @@ # RE-7593. However, Windows paths were changed to match *nix in puppet 6, see # commit 4b9d126dd5b. So only the pdk has this issue. if platform.is_windows? && settings[:bindir] != ruby_bindir + bindir = settings[:bindir] + # Ruby 3+ if Gem::Version.new(pkg.get_version) >= Gem::Version.new('3.0') pkg.install do [ "cp #{settings[:gcc_bindir]}/libssp-0.dll #{ruby_bindir}", - "cp #{settings[:bindir]}/libffi-8.dll #{ruby_bindir}", - "cp #{settings[:bindir]}/libyaml-0-2.dll #{ruby_bindir}" + "cp #{bindir}/libffi-8.dll #{ruby_bindir}", + "cp #{bindir}/libyaml-0-2.dll #{ruby_bindir}" ] end end @@ -160,9 +162,9 @@ pkg.install do [ - "cp #{settings[:bindir]}/libgcc_s_#{gcc_postfix}-1.dll #{ruby_bindir}", - "cp #{settings[:bindir]}/#{ssl_lib} #{ruby_bindir}", - "cp #{settings[:bindir]}/#{crypto_lib} #{ruby_bindir}", + "cp #{bindir}/libgcc_s_#{gcc_postfix}-1.dll #{ruby_bindir}", + "cp #{bindir}/#{ssl_lib} #{ruby_bindir}", + "cp #{bindir}/#{crypto_lib} #{ruby_bindir}" ] end diff --git a/configs/components/runtime-pdk.rb b/configs/components/runtime-pdk.rb index b09f8aa03..d3e60783a 100644 --- a/configs/components/runtime-pdk.rb +++ b/configs/components/runtime-pdk.rb @@ -7,26 +7,30 @@ pkg.install_file File.join(settings[:gcc_bindir], dll), File.join(settings[:bindir], dll) end - # Ruby needs zlib + tools_bindir = File.join(settings[:tools_root], 'bin') + + # curl, openssl, etc need zlib, so install in *main* bin dir pkg.build_requires "pl-zlib-#{platform.architecture}" - pkg.install_file "#{settings[:tools_root]}/bin/zlib1.dll", "#{settings[:bindir]}/zlib1.dll" + pkg.install_file "#{tools_bindir}/zlib1.dll", "#{settings[:bindir]}/zlib1.dll" # zlib, gdbm, yaml-cpp and iconv are all runtime dependancies of ruby, and their libraries need - # To exist inside our vendored ruby - pkg.install_file "#{settings[:tools_root]}/bin/zlib1.dll", "#{settings[:ruby_bindir]}/zlib1.dll" - pkg.install_file "#{settings[:tools_root]}/bin/libgdbm-4.dll", "#{settings[:ruby_bindir]}/libgdbm-4.dll" - pkg.install_file "#{settings[:tools_root]}/bin/libgdbm_compat-4.dll", "#{settings[:ruby_bindir]}/libgdbm_compat-4.dll" - pkg.install_file "#{settings[:tools_root]}/bin/libiconv-2.dll", "#{settings[:ruby_bindir]}/libiconv-2.dll" - pkg.install_file "#{settings[:tools_root]}/bin/libffi-6.dll", "#{settings[:ruby_bindir]}/libffi-6.dll" if settings[:ruby_version].start_with?('2') + # to exist inside our primary ruby's bin dir + ruby_bindir = settings[:ruby_bindir] + pkg.install_file "#{tools_bindir}/zlib1.dll", "#{ruby_bindir}/zlib1.dll" + pkg.install_file "#{tools_bindir}/libgdbm-4.dll", "#{ruby_bindir}/libgdbm-4.dll" + pkg.install_file "#{tools_bindir}/libgdbm_compat-4.dll", "#{ruby_bindir}/libgdbm_compat-4.dll" + pkg.install_file "#{tools_bindir}/libiconv-2.dll", "#{ruby_bindir}/libiconv-2.dll" + pkg.install_file "#{tools_bindir}/libffi-6.dll", "#{ruby_bindir}/libffi-6.dll" if settings[:ruby_version].start_with?('2') # Copy the DLLs into additional ruby install bindirs as well. if settings.has_key?(:additional_rubies) settings[:additional_rubies].each do |rubyver, local_settings| - pkg.install_file "#{settings[:tools_root]}/bin/zlib1.dll", "#{local_settings[:ruby_bindir]}/zlib1.dll" - pkg.install_file "#{settings[:tools_root]}/bin/libgdbm-4.dll", "#{local_settings[:ruby_bindir]}/libgdbm-4.dll" - pkg.install_file "#{settings[:tools_root]}/bin/libgdbm_compat-4.dll", "#{local_settings[:ruby_bindir]}/libgdbm_compat-4.dll" - pkg.install_file "#{settings[:tools_root]}/bin/libiconv-2.dll", "#{local_settings[:ruby_bindir]}/libiconv-2.dll" - pkg.install_file "#{settings[:tools_root]}/bin/libffi-6.dll", "#{local_settings[:ruby_bindir]}/libffi-6.dll" if rubyver.start_with?('2') + local_bindir = local_settings[:ruby_bindir] + pkg.install_file "#{tools_bindir}/zlib1.dll", "#{local_bindir}/zlib1.dll" + pkg.install_file "#{tools_bindir}/libgdbm-4.dll", "#{local_bindir}/libgdbm-4.dll" + pkg.install_file "#{tools_bindir}/libgdbm_compat-4.dll", "#{local_bindir}/libgdbm_compat-4.dll" + pkg.install_file "#{tools_bindir}/libiconv-2.dll", "#{local_bindir}/libiconv-2.dll" + pkg.install_file "#{tools_bindir}/libffi-6.dll", "#{local_bindir}/libffi-6.dll" if rubyver.start_with?('2') end end