From a1ed0f0aaed1daf14b697746a5dcc313a6ca03cd Mon Sep 17 00:00:00 2001 From: Nick Burgan-Illig Date: Fri, 22 Mar 2024 17:53:40 +0000 Subject: [PATCH 1/3] (PE-36494) Standardize pe-installer-runtime on shared format with more deps This standardizes the pe-installer-runtime definition to the format used by the other runtimes, using a shared definition for common things between 2021.7.x and main. It also expands the list of components so we are providing all dependencies to Bolt and Puppet. This is so that we don't get surprised when pe-installer-vanagon gem installs Bolt and pulls in dependencies that may break on the version of Ruby we're using. --- .../projects/_shared-pe-installer-runtime.rb | 156 ++++++++++++++++++ .../projects/pe-installer-runtime-2021.7.x.rb | 101 +----------- configs/projects/pe-installer-runtime-main.rb | 97 +---------- 3 files changed, 158 insertions(+), 196 deletions(-) create mode 100644 configs/projects/_shared-pe-installer-runtime.rb diff --git a/configs/projects/_shared-pe-installer-runtime.rb b/configs/projects/_shared-pe-installer-runtime.rb new file mode 100644 index 000000000..c9d2cd548 --- /dev/null +++ b/configs/projects/_shared-pe-installer-runtime.rb @@ -0,0 +1,156 @@ +platform = proj.get_platform +proj.version_from_git +proj.generate_archives true +proj.generate_packages false + +proj.description "The PE Installer runtime contains third-party components needed for PE Installer standalone packaging" +proj.license "See components" +proj.vendor "Puppet, Inc. " +proj.homepage "https://puppet.com" +proj.identifier "com.puppetlabs" + +# Used in component configurations to conditionally include dependencies +proj.setting(:runtime_project, 'pe-installer') + +proj.setting(:prefix, "/opt/puppetlabs/installer") +proj.setting(:ruby_dir, proj.prefix) +proj.setting(:bindir, File.join(proj.prefix, 'bin')) +proj.setting(:ruby_bindir, proj.bindir) +proj.setting(:libdir, File.join(proj.prefix, 'lib')) +proj.setting(:includedir, File.join(proj.prefix, "include")) +proj.setting(:datadir, File.join(proj.prefix, "share")) +proj.setting(:mandir, File.join(proj.datadir, "man")) + +proj.setting(:host_ruby, File.join(proj.ruby_bindir, "ruby")) +proj.setting(:host_gem, File.join(proj.ruby_bindir, "gem")) + +ruby_base_version = proj.ruby_version.gsub(/(\d+)\.(\d+)\.(\d+)/, '\1.\2.0') +proj.setting(:gem_home, File.join(proj.libdir, 'ruby', 'gems', ruby_base_version)) +proj.setting(:gem_install, "#{proj.host_gem} install --no-document --local --bindir=#{proj.ruby_bindir}") + +proj.setting(:artifactory_url, "https://artifactory.delivery.puppetlabs.net/artifactory") +proj.setting(:buildsources_url, "#{proj.artifactory_url}/generic/buildsources") + +# Define default CFLAGS and LDFLAGS for most platforms, and then +# tweak or adjust them as needed. +proj.setting(:cppflags, "-I#{proj.includedir} -I/opt/pl-build-tools/include") +proj.setting(:cflags, "#{proj.cppflags}") +proj.setting(:ldflags, "-L#{proj.libdir} -L/opt/pl-build-tools/lib -Wl,-rpath=#{proj.libdir}") + +# These flags are applied in addition to the defaults in configs/component/openssl.rb. +proj.setting(:openssl_extra_configure_flags, [ + 'no-dtls', + 'no-dtls1', + 'no-idea', + 'no-seed', + 'no-weak-ssl-ciphers', + '-DOPENSSL_NO_HEARTBEATS', +]) + +proj.setting(:augeas_version, '1.14.1') + +######## +# Load shared agent components +# When we want to run Bolt from the pe-installer package, we want our +# puppet to have the same gems as the default puppet agent install. +######## +instance_eval File.read('configs/projects/_shared-agent-components.rb') + +# pl-build-tools +proj.component 'runtime-pe-installer' + +# Below are copied from pe-bolt-server, since we're shipping much the same thing. +# Deps in _shared-agent-components are removed, and we do not include the +# agent gem. + +# R10k dependencies +proj.component('rubygem-gettext-setup') + +# hiera-eyaml and its dependencies +proj.component('rubygem-highline') +proj.component('rubygem-optimist') +proj.component('rubygem-hiera-eyaml') + +# faraday and its dependencies +proj.component('rubygem-faraday') +proj.component('rubygem-faraday-em_http') +proj.component('rubygem-faraday-em_synchrony') +proj.component('rubygem-faraday-excon') +proj.component('rubygem-faraday-httpclient') +proj.component('rubygem-faraday-multipart') +proj.component('rubygem-faraday-net_http') +proj.component('rubygem-faraday-net_http_persistent') +proj.component('rubygem-faraday-patron') +proj.component('rubygem-faraday-rack') +proj.component('rubygem-faraday-retry') +proj.component('rubygem-faraday_middleware') +proj.component('rubygem-ruby2_keywords') + +# Core dependencies +proj.component('rubygem-addressable') +proj.component('rubygem-aws-eventstream') +proj.component('rubygem-aws-partitions') +proj.component('rubygem-aws-sdk-core') +proj.component('rubygem-aws-sdk-ec2') +proj.component('rubygem-aws-sigv4') +proj.component('rubygem-bcrypt_pbkdf') +proj.component('rubygem-bindata') +proj.component('rubygem-builder') +proj.component('rubygem-CFPropertyList') +proj.component('rubygem-colored2') +proj.component('rubygem-concurrent-ruby') +proj.component('rubygem-connection_pool') +proj.component('rubygem-cri') +proj.component('rubygem-ed25519') +proj.component('rubygem-erubi') +proj.component('rubygem-facter') +proj.component('rubygem-ffi') +proj.component('rubygem-gssapi') +proj.component('rubygem-gyoku') +proj.component('rubygem-hiera') +proj.component('rubygem-httpclient') +proj.component('rubygem-jmespath') +proj.component('rubygem-jwt') +proj.component('rubygem-little-plugger') +proj.component('rubygem-log4r') +proj.component('rubygem-logging') +proj.component('rubygem-minitar') +proj.component('rubygem-molinillo') +proj.component('rubygem-multi_json') +proj.component('rubygem-multipart-post') +proj.component('rubygem-net-http-persistent') +proj.component('rubygem-net-scp') +proj.component('rubygem-net-ssh-krb') +proj.component('rubygem-nori') +proj.component('rubygem-rubyntlm') +proj.component('rubygem-orchestrator_client') +proj.component('rubygem-public_suffix') +proj.component('rubygem-paint') +proj.component('rubygem-puppet_forge') +proj.component('rubygem-puppet-resource_api') +proj.component('rubygem-puppet-strings') +proj.component('rubygem-puppetfile-resolver') +proj.component('rubygem-r10k') +proj.component('rubygem-rgen') +proj.component('rubygem-rubyntlm') +proj.component('rubygem-ruby_smb') +proj.component('rubygem-rubyzip') +proj.component('rubygem-scanf') +proj.component('rubygem-terminal-table') +proj.component('rubygem-thor') +proj.component('rubygem-unicode-display_width') +proj.component('rubygem-webrick') +proj.component('rubygem-yard') + +# Core Windows dependencies +proj.component('rubygem-windows_error') +proj.component('rubygem-winrm') +proj.component('rubygem-winrm-fs') + +# What to include in package? +proj.directory proj.prefix + +# Export the settings for the current project and platform as yaml during builds +proj.publish_yaml_settings + +proj.timeout 7200 if platform.is_windows? diff --git a/configs/projects/pe-installer-runtime-2021.7.x.rb b/configs/projects/pe-installer-runtime-2021.7.x.rb index 4cfa53466..662490ff2 100644 --- a/configs/projects/pe-installer-runtime-2021.7.x.rb +++ b/configs/projects/pe-installer-runtime-2021.7.x.rb @@ -1,8 +1,5 @@ project 'pe-installer-runtime-2021.7.x' do |proj| - # Used in component configurations to conditionally include dependencies - proj.setting(:runtime_project, 'pe-installer') proj.setting(:ruby_version, '2.7.8') - proj.setting(:augeas_version, '1.11.0') # We need to explicitly define 1.1.1k here to avoid # build dep conflicts between openssl-1.1.1 needed by curl # and krb5-devel @@ -11,101 +8,5 @@ else proj.setting(:openssl_version, '1.1.1') end - platform = proj.get_platform - - proj.version_from_git - proj.generate_archives true - proj.generate_packages false - - proj.description "The PE Installer runtime contains third-party components needed for PE Installer standalone packaging" - proj.license "See components" - proj.vendor "Puppet, Inc. " - proj.homepage "https://puppet.com" - proj.identifier "com.puppetlabs" - - proj.setting(:prefix, "/opt/puppetlabs/installer") - - proj.setting(:ruby_dir, proj.prefix) - proj.setting(:bindir, File.join(proj.prefix, 'bin')) - proj.setting(:ruby_bindir, proj.bindir) - proj.setting(:libdir, File.join(proj.prefix, 'lib')) - proj.setting(:includedir, File.join(proj.prefix, "include")) - proj.setting(:datadir, File.join(proj.prefix, "share")) - proj.setting(:mandir, File.join(proj.datadir, "man")) - - proj.setting(:host_ruby, File.join(proj.ruby_bindir, "ruby")) - proj.setting(:host_gem, File.join(proj.ruby_bindir, "gem")) - - ruby_base_version = proj.ruby_version.gsub(/(\d+)\.(\d+)\.(\d+)/, '\1.\2.0') - proj.setting(:gem_home, File.join(proj.libdir, 'ruby', 'gems', ruby_base_version)) - proj.setting(:gem_install, "#{proj.host_gem} install --no-document --local --bindir=#{proj.ruby_bindir}") - - proj.setting(:artifactory_url, "https://artifactory.delivery.puppetlabs.net/artifactory") - proj.setting(:buildsources_url, "#{proj.artifactory_url}/generic/buildsources") - - # Define default CFLAGS and LDFLAGS for most platforms, and then - # tweak or adjust them as needed. - proj.setting(:cppflags, "-I#{proj.includedir} -I/opt/pl-build-tools/include") - proj.setting(:cflags, "#{proj.cppflags}") - proj.setting(:ldflags, "-L#{proj.libdir} -L/opt/pl-build-tools/lib -Wl,-rpath=#{proj.libdir}") - - # These flags are applied in addition to the defaults in configs/component/openssl.rb. - proj.setting(:openssl_extra_configure_flags, [ - 'no-dtls', - 'no-dtls1', - 'no-idea', - 'no-seed', - 'no-weak-ssl-ciphers', - '-DOPENSSL_NO_HEARTBEATS', - ]) - - # What to build? - # -------------- - # - - # rubygem-net-ssh included in shared-agent-components - proj.setting(:rubygem_net_ssh_version, '5.2.0') - - ######## - # Load shared agent components - # When we want to run Bolt from the pe-installer package, we want our - # puppet to have the same gems as the default puppet agent install. - ######## - - instance_eval File.read(File.join(File.dirname(__FILE__), '_shared-agent-components.rb')) - - - # pl-build-tools - proj.component 'runtime-pe-installer' - - # R10k dependencies - proj.component 'rubygem-gettext-setup' - - # net-ssh dependencies for el8's OpenSSH default key format - proj.component 'rubygem-bcrypt_pbkdf' - proj.component 'rubygem-ed25519' - - # main puppet agent components - # boost and yaml-cpp omitted since we don't need - # pxp-agent deps - proj.component 'rubygem-concurrent-ruby' - proj.component 'rubygem-ffi' - proj.component 'rubygem-multi_json' - proj.component 'rubygem-optimist' - proj.component 'rubygem-highline' - proj.component 'rubygem-hiera-eyaml' - proj.component 'rubygem-httpclient' - proj.component 'rubygem-thor' - proj.component 'rubygem-sys-filesystem' - proj.component 'rubygem-prime' - proj.component 'rubygem-erubi' - proj.component 'rubygem-nori' - - # What to include in package? - proj.directory proj.prefix - - # Export the settings for the current project and platform as yaml during builds - proj.publish_yaml_settings - - proj.timeout 7200 if platform.is_windows? + instance_eval File.read(File.join(File.dirname(__FILE__), '_shared-pe-installer-runtime.rb')) end diff --git a/configs/projects/pe-installer-runtime-main.rb b/configs/projects/pe-installer-runtime-main.rb index 0f0bb17f4..ae441cb73 100644 --- a/configs/projects/pe-installer-runtime-main.rb +++ b/configs/projects/pe-installer-runtime-main.rb @@ -1,107 +1,12 @@ project 'pe-installer-runtime-main' do |proj| - # Used in component configurations to conditionally include dependencies - proj.setting(:runtime_project, 'pe-installer') proj.setting(:ruby_version, '3.2.3') - proj.setting(:augeas_version, '1.14.1') proj.setting(:openssl_version, '3.0') # NLTM uses MD4 unconditionally in its protocol, so legacy algos must be # enabled in OpenSSL >= 3.0 for Bolt's WinRM transport to work. # We DO NOT WANT legacy algos enabled for the Puppet Agent runtime. proj.setting(:use_legacy_openssl_algos, true) - platform = proj.get_platform - - proj.version_from_git - proj.generate_archives true - proj.generate_packages false - - proj.description "The PE Installer runtime contains third-party components needed for PE Installer standalone packaging" - proj.license "See components" - proj.vendor "Puppet, Inc. " - proj.homepage "https://puppet.com" - proj.identifier "com.puppetlabs" - - proj.setting(:prefix, "/opt/puppetlabs/installer") - - proj.setting(:ruby_dir, proj.prefix) - proj.setting(:bindir, File.join(proj.prefix, 'bin')) - proj.setting(:ruby_bindir, proj.bindir) - proj.setting(:libdir, File.join(proj.prefix, 'lib')) - proj.setting(:includedir, File.join(proj.prefix, "include")) - proj.setting(:datadir, File.join(proj.prefix, "share")) - proj.setting(:mandir, File.join(proj.datadir, "man")) - - proj.setting(:host_ruby, File.join(proj.ruby_bindir, "ruby")) - proj.setting(:host_gem, File.join(proj.ruby_bindir, "gem")) - - ruby_base_version = proj.ruby_version.gsub(/(\d+)\.(\d+)\.(\d+)/, '\1.\2.0') - proj.setting(:gem_home, File.join(proj.libdir, 'ruby', 'gems', ruby_base_version)) - proj.setting(:gem_install, "#{proj.host_gem} install --no-document --local --bindir=#{proj.ruby_bindir}") - - proj.setting(:artifactory_url, "https://artifactory.delivery.puppetlabs.net/artifactory") - proj.setting(:buildsources_url, "#{proj.artifactory_url}/generic/buildsources") - - # Define default CFLAGS and LDFLAGS for most platforms, and then - # tweak or adjust them as needed. - proj.setting(:cppflags, "-I#{proj.includedir} -I/opt/pl-build-tools/include") - proj.setting(:cflags, "#{proj.cppflags}") - proj.setting(:ldflags, "-L#{proj.libdir} -L/opt/pl-build-tools/lib -Wl,-rpath=#{proj.libdir}") - - # These flags are applied in addition to the defaults in configs/component/openssl.rb. - proj.setting(:openssl_extra_configure_flags, [ - 'no-dtls', - 'no-dtls1', - 'no-idea', - 'no-seed', - 'no-weak-ssl-ciphers', - '-DOPENSSL_NO_HEARTBEATS', - ]) - - # What to build? - # -------------- - # # rubygem-net-ssh included in shared-agent-components proj.setting(:rubygem_net_ssh_version, '7.0.1') - - ######## - # Load shared agent components - # When we want to run Bolt from the pe-installer package, we want our - # puppet to have the same gems as the default puppet agent install. - ######## - - instance_eval File.read(File.join(File.dirname(__FILE__), '_shared-agent-components.rb')) - - - # pl-build-tools - proj.component 'runtime-pe-installer' - - # R10k dependencies - proj.component 'rubygem-gettext-setup' - - # net-ssh dependencies for el8's OpenSSH default key format - proj.component 'rubygem-bcrypt_pbkdf' - proj.component 'rubygem-ed25519' - - # main puppet agent components - # boost and yaml-cpp omitted since we don't need - # pxp-agent deps - proj.component 'rubygem-concurrent-ruby' - proj.component 'rubygem-ffi' - proj.component 'rubygem-multi_json' - proj.component 'rubygem-optimist' - proj.component 'rubygem-highline' - proj.component 'rubygem-hiera-eyaml' - proj.component 'rubygem-httpclient' - proj.component 'rubygem-thor' - proj.component 'rubygem-sys-filesystem' - proj.component 'rubygem-prime' - proj.component 'rubygem-erubi' - - # What to include in package? - proj.directory proj.prefix - - # Export the settings for the current project and platform as yaml during builds - proj.publish_yaml_settings - - proj.timeout 7200 if platform.is_windows? + instance_eval File.read(File.join(File.dirname(__FILE__), '_shared-pe-installer-runtime.rb')) end From 08fe4ed289a68704a84d73d3a5914fa446c2b3ee Mon Sep 17 00:00:00 2001 From: Nick Burgan <13688219+nmburgan@users.noreply.github.com> Date: Tue, 26 Mar 2024 13:50:51 -0700 Subject: [PATCH 2/3] (maint) Add net-ssh version back into installer runtime 2021.7.x Accidentally removed this from the new pe-installer-runtime-2021.7.x, but since main and 2021.7.x use different versions this needs to stay in here. --- configs/projects/pe-installer-runtime-2021.7.x.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/configs/projects/pe-installer-runtime-2021.7.x.rb b/configs/projects/pe-installer-runtime-2021.7.x.rb index 662490ff2..9f820b608 100644 --- a/configs/projects/pe-installer-runtime-2021.7.x.rb +++ b/configs/projects/pe-installer-runtime-2021.7.x.rb @@ -8,5 +8,9 @@ else proj.setting(:openssl_version, '1.1.1') end + + # rubygem-net-ssh included in shared-agent-components + proj.setting(:rubygem_net_ssh_version, '5.2.0') + instance_eval File.read(File.join(File.dirname(__FILE__), '_shared-pe-installer-runtime.rb')) end From a2828322db923e9086596bde46f3a790600e41df Mon Sep 17 00:00:00 2001 From: donoghuc Date: Wed, 27 Mar 2024 09:12:57 -0700 Subject: [PATCH 3/3] (PE-36494) Ship puppet gem with installer runtime --- configs/projects/_shared-pe-installer-runtime.rb | 1 + configs/projects/pe-installer-runtime-2021.7.x.rb | 2 +- configs/projects/pe-installer-runtime-main.rb | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/configs/projects/_shared-pe-installer-runtime.rb b/configs/projects/_shared-pe-installer-runtime.rb index c9d2cd548..fec319e75 100644 --- a/configs/projects/_shared-pe-installer-runtime.rb +++ b/configs/projects/_shared-pe-installer-runtime.rb @@ -126,6 +126,7 @@ proj.component('rubygem-orchestrator_client') proj.component('rubygem-public_suffix') proj.component('rubygem-paint') +proj.component('rubygem-puppet') proj.component('rubygem-puppet_forge') proj.component('rubygem-puppet-resource_api') proj.component('rubygem-puppet-strings') diff --git a/configs/projects/pe-installer-runtime-2021.7.x.rb b/configs/projects/pe-installer-runtime-2021.7.x.rb index 9f820b608..04fe59f22 100644 --- a/configs/projects/pe-installer-runtime-2021.7.x.rb +++ b/configs/projects/pe-installer-runtime-2021.7.x.rb @@ -11,6 +11,6 @@ # rubygem-net-ssh included in shared-agent-components proj.setting(:rubygem_net_ssh_version, '5.2.0') - + proj.setting(:rubygem_puppet_version, '7.27.0') instance_eval File.read(File.join(File.dirname(__FILE__), '_shared-pe-installer-runtime.rb')) end diff --git a/configs/projects/pe-installer-runtime-main.rb b/configs/projects/pe-installer-runtime-main.rb index ae441cb73..f54ae8001 100644 --- a/configs/projects/pe-installer-runtime-main.rb +++ b/configs/projects/pe-installer-runtime-main.rb @@ -8,5 +8,6 @@ # rubygem-net-ssh included in shared-agent-components proj.setting(:rubygem_net_ssh_version, '7.0.1') + proj.setting(:rubygem_puppet_version, '8.3.1') instance_eval File.read(File.join(File.dirname(__FILE__), '_shared-pe-installer-runtime.rb')) end