From 7c345ecbabd3b7ef80ef6ae1a17ece68faf452f3 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Wed, 16 Jul 2025 11:34:56 -0400 Subject: [PATCH 01/14] [Isolated] Install cfn-dependencies only for AL2 --- .../recipes/install/cfn_bootstrap.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index cfe95c4ed5..7b5d8f0815 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -33,7 +33,7 @@ not_if { ::File.exist?("#{virtualenv_path}/bin/activate") } end -if aws_region.start_with?("us-iso") +if aws_region.start_with?("us-iso") && platform?('amazon') && node['platform_version'] == "2" remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cfn-dependencies.tgz" mode '0644' From a9724796b2e5139a437e94ce7d0ef7d0a96c9ad9 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Mon, 21 Jul 2025 09:51:55 -0400 Subject: [PATCH 02/14] Revert "[Isolated] Install cfn-dependencies only for AL2" This reverts commit 976d4799313634b3274be1da55ff768f1bd3503f. --- .../recipes/install/cfn_bootstrap.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index 7b5d8f0815..cfe95c4ed5 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -33,7 +33,7 @@ not_if { ::File.exist?("#{virtualenv_path}/bin/activate") } end -if aws_region.start_with?("us-iso") && platform?('amazon') && node['platform_version'] == "2" +if aws_region.start_with?("us-iso") remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cfn-dependencies.tgz" mode '0644' From 857638cde2e30084bfdabc25e1f0dd4dbcbcc93f Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Mon, 21 Jul 2025 09:52:42 -0400 Subject: [PATCH 03/14] [Isolated] USe latest cfn-dependencies --- .../recipes/install/cfn_bootstrap.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index cfe95c4ed5..ea4aa8fbf9 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -35,7 +35,7 @@ if aws_region.start_with?("us-iso") remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do - source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cfn-dependencies.tgz" + source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/pypi-cfn-dependencies-3.12-x86_64.tgz" mode '0644' retries 3 retry_delay 5 @@ -48,8 +48,8 @@ cwd "#{node['cluster']['base_dir']}" code <<-REQ set -e - tar xzf cfn-dependencies.tgz - cd cfn + tar xzf pypi-cfn-dependencies-3.12-x86_64.tgz + cd dependencies #{virtualenv_path}/bin/pip install * -f ./ --no-index REQ end From b56bb2cad26b2fd482050a8ac8b30e74c45e0b98 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Mon, 21 Jul 2025 09:56:17 -0400 Subject: [PATCH 04/14] [Isolated] Using Git REF for Uploading cookbook --- util/upload-cookbook.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/util/upload-cookbook.sh b/util/upload-cookbook.sh index c2169a4c3e..864ce56959 100755 --- a/util/upload-cookbook.sh +++ b/util/upload-cookbook.sh @@ -93,21 +93,22 @@ main() { # Create archive and md5 _cwd=$(pwd) pushd "${_srcdir}" > /dev/null || exit + GIT_REF=$(git rev-parse HEAD) _stashName=$(git stash create) - git archive --format tar --prefix="aws-parallelcluster-cookbook-${_version}/" "${_stashName:-HEAD}" | gzip > "${_cwd}/aws-parallelcluster-cookbook-${_version}.tgz" + git archive --format tar --prefix="aws-parallelcluster-cookbook-${_version}/" "${_stashName:-HEAD}" | gzip > "${_cwd}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz" #tar zcvf "${_cwd}/aws-parallelcluster-cookbook-${_version}.tgz" --transform "s,^aws-parallelcluster-cookbook/,aws-parallelcluster-cookbook-${_version}/," ../aws-parallelcluster-cookbook popd > /dev/null || exit - md5sum aws-parallelcluster-cookbook-${_version}.tgz > aws-parallelcluster-cookbook-${_version}.md5 + md5sum aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz > aws-parallelcluster-cookbook-${_version}-${GIT_REF}.md5 # upload packages _key_path="parallelcluster/${_version}/cookbooks" if [ -n "${_scope}" ]; then _key_path="${_key_path}/${_scope}" fi - aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}.tgz s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz || _error_exit 'Failed to push cookbook to S3' - aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}.md5 s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}.md5 || _error_exit 'Failed to push cookbook md5 to S3' - aws ${_profile} --region "${_region}" s3api head-object --bucket ${_bucket} --key ${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz --output text --query LastModified > aws-parallelcluster-cookbook-${_version}.tgz.date || _error_exit 'Failed to fetch LastModified date' - aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}.tgz.date s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz.date || _error_exit 'Failed to push cookbook date' + aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz || _error_exit 'Failed to push cookbook to S3' + aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}-${GIT_REF}.md5 s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.md5 || _error_exit 'Failed to push cookbook md5 to S3' + aws ${_profile} --region "${_region}" s3api head-object --bucket ${_bucket} --key ${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz --output text --query LastModified > aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz.date || _error_exit 'Failed to fetch LastModified date' + aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz.date s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz.date || _error_exit 'Failed to push cookbook date' _bucket_region=$(aws ${_profile} s3api get-bucket-location --bucket ${_bucket} --output text) if [ ${_bucket_region} = "None" ]; then From e49d4a17ec4ada4aefb9f8a1f5b66f6d695e6d86 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Tue, 22 Jul 2025 14:02:30 -0400 Subject: [PATCH 05/14] [Isolated] Rename the cfn-dependecies files --- .../recipes/install/cfn_bootstrap.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index ea4aa8fbf9..dad69ad964 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -48,7 +48,7 @@ cwd "#{node['cluster']['base_dir']}" code <<-REQ set -e - tar xzf pypi-cfn-dependencies-3.12-x86_64.tgz + tar xzf cfn-dependencies.tgz cd dependencies #{virtualenv_path}/bin/pip install * -f ./ --no-index REQ From dd33ac100d207a1ca78e193972cda384d59ccc74 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Thu, 24 Jul 2025 10:37:38 -0400 Subject: [PATCH 06/14] [Isolated] Chnage the name of Cookbook Dependencies and the folder name inside the Tar --- .../recipes/install/cookbook_virtualenv.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb index cdc557df5f..2845ec9c29 100644 --- a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb +++ b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb @@ -12,8 +12,10 @@ # limitations under the License. virtualenv_path = cookbook_virtualenv_path -pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/pypi-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}.tgz" +dependency_package_name = "pypi-cookbook-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}" +pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{dependency_package_name}.tgz" if platform?('amazon') && node['platform_version'] == "2" + dependency_package_name = "dependencies" pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz" end @@ -46,7 +48,7 @@ code <<-REQ set -e tar xzf cookbook-dependencies.tgz - cd dependencies + cd #{dependency_package_name} #{virtualenv_path}/bin/pip install * -f ./ --no-index REQ end From 506bb335e71f6676535f1f5491e98a2b37007362 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Thu, 24 Jul 2025 12:33:12 -0400 Subject: [PATCH 07/14] [Isolated] Chnage the name of CFN Dependencies and the folder name inside the Tar --- .../recipes/install/cfn_bootstrap.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index dad69ad964..0eee866fee 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -34,8 +34,14 @@ end if aws_region.start_with?("us-iso") + dependency_package_name = "pypi-cfn-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}" + dependency_folder_name = dependency_package_name + if platform?('amazon') && node['platform_version'] == "2" + dependency_package_name = "cfn-dependencies" + dependency_folder_name = "cfn" + end remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do - source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/pypi-cfn-dependencies-3.12-x86_64.tgz" + source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz" mode '0644' retries 3 retry_delay 5 @@ -49,7 +55,7 @@ code <<-REQ set -e tar xzf cfn-dependencies.tgz - cd dependencies + cd #{dependency_folder_name} #{virtualenv_path}/bin/pip install * -f ./ --no-index REQ end From 5a30c1a3ebb131cde77fe2391d286d90ae57781f Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Tue, 29 Jul 2025 16:31:12 -0400 Subject: [PATCH 08/14] [Isolated] Installing Cfn-bootstrap using `--no-build-isolation` as 3.12.8 uses setup.py based installation where it uses a isolated build instead of looking at existing site-packages --- .../recipes/install/custom_parallelcluster_node.rb | 8 +++++++- .../recipes/install/cfn_bootstrap.rb | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb index 247823c173..0cc5e36bed 100644 --- a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb +++ b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb @@ -19,6 +19,12 @@ # TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the # bash block to a recipe that uses the pyenv resource. +if aws_region.start_with?("us-iso") + command = "pip install . --no-build-isolation" +else + command = "pip install ." +end + if aws_region.start_with?("us-iso") && platform?('amazon') && node['platform_version'] == "2" remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/node-dependencies.tgz" @@ -59,7 +65,7 @@ mkdir aws-parallelcluster-custom-node tar -xzf aws-parallelcluster-node.tgz --directory aws-parallelcluster-custom-node cd aws-parallelcluster-custom-node/*aws-parallelcluster-node* - pip install . + #{command} deactivate NODE end diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index 0eee866fee..8270a62ff7 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -79,11 +79,16 @@ retry_delay 5 end +if aws_region.start_with?("us-iso") + command = "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package} --no-build-isolation" +else + command = "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package}" +end bash "Install CloudFormation helpers from #{cfnbootstrap_package}" do user 'root' group 'root' cwd '/tmp' - code "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package}" + code command creates "#{virtualenv_path}/bin/cfn-hup" end From fc056aa8ac7e27dc90fc96877f74f621f52b0850 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Fri, 1 Aug 2025 16:03:50 -0400 Subject: [PATCH 09/14] [Isolated] Install efs-proxy cargo dependecies for isolated environment --- .../resources/efs/efs_redhat8.rb | 20 ++++++++++++++++++- .../efs/partial/_install_from_tar.rb | 14 +++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb index b2ab45bc1b..67747e8859 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb @@ -18,6 +18,24 @@ use 'partial/_get_package_version_rpm' use 'partial/_common' -use 'partial/_redhat_based' +# use 'partial/_redhat_based' use 'partial/_install_from_tar' use 'partial/_mount_umount' + +def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) + <<-EFSUTILSINSTALL + set -e + tar xf #{efs_utils_tarball} + mv efs-proxy-dependencies-#{efs_utils_version}.tar.gz efs-utils-#{efs_utils_version}/src/proxy/ + cd efs-utils-#{efs_utils_version}/src/proxy/ + tar -xf efs-proxy-dependencies-#{efs_utils_version}.tar.gz + cargo build --offline + cd ../.. + make rpm + yum -y install ./build/#{efs_utils_package}*rpm + EFSUTILSINSTALL +end + +def prerequisites + %w(rpm-build make rust cargo openssl-devel) +end diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb index cf65440cf0..112a6ce094 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb @@ -50,6 +50,20 @@ action :create_if_missing end + if aws_region.start_with?("us-iso") && platform?('redhat') && node['platform_version'] == "8" + efs_proxy_deps = "efs-proxy-dependencies-#{package_version}.tar.gz" + efs_proxy_deps_tarball = "#{node['cluster']['sources_dir']}/#{efs_proxy_deps}" + efs_proxy_deps_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/#{efs_proxy_deps}" + remote_file efs_proxy_deps_tarball do + source efs_proxy_deps_url + mode '0644' + retries 3 + retry_delay 5 + checksum new_resource.efs_utils_checksum + action :create_if_missing + end + end + # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html bash "install efs utils" do cwd node['cluster']['sources_dir'] From 38a75777c12b12fdba357724b1eb2b5397c343d8 Mon Sep 17 00:00:00 2001 From: Helena Greebe Date: Mon, 11 Aug 2025 20:21:58 -0400 Subject: [PATCH 10/14] [Isolated] Install new node pypi dependencies and move efs-proxy installation --- .../install/custom_parallelcluster_node.rb | 13 ++++- .../resources/efs/efs_redhat8.rb | 58 ++++++++++++++++++- .../efs/partial/_install_from_tar.rb | 14 ----- .../recipes/install/cookbook_virtualenv.rb | 26 +++++---- 4 files changed, 81 insertions(+), 30 deletions(-) diff --git a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb index 0cc5e36bed..af868a80b0 100644 --- a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb +++ b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb @@ -25,9 +25,16 @@ command = "pip install ." end -if aws_region.start_with?("us-iso") && platform?('amazon') && node['platform_version'] == "2" +if aws_region.start_with?("us-iso") + dependency_package_name = "pypi-node-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}" + dependency_folder_name = dependency_package_name + if platform?('amazon') && node['platform_version'] == "2" + dependency_package_name = "node-dependencies" + dependency_folder_name = "node" + end + remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do - source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/node-dependencies.tgz" + source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz" mode '0644' retries 3 retry_delay 5 @@ -41,7 +48,7 @@ code <<-REQ set -e tar xzf node-dependencies.tgz - cd node + cd #{dependency_folder_name} #{node_virtualenv_path}/bin/pip install * -f ./ --no-index REQ end diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb index 67747e8859..12193f73cb 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb @@ -19,7 +19,7 @@ use 'partial/_get_package_version_rpm' use 'partial/_common' # use 'partial/_redhat_based' -use 'partial/_install_from_tar' +# use 'partial/_install_from_tar' use 'partial/_mount_umount' def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) @@ -39,3 +39,59 @@ def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) def prerequisites %w(rpm-build make rust cargo openssl-devel) end + +action :install_utils do + package_repos 'update package repositories' do + action :update + end + + package prerequisites do + retries 3 + retry_delay 5 + end + + directory node['cluster']['sources_dir'] do + recursive true + end + + return if redhat_on_docker? + + package_name = "amazon-efs-utils" + package_version = new_resource.efs_utils_version + efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" + efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz" + + # Do not install efs-utils if a same or newer version is already installed. + return if already_installed?(package_name, package_version) + + # On all OSes but Amazon Linux 2, amazon-efs-utils and stunnel are installed from source, + # because their OS repos do not have amazon-efs-utils and new stunnel + + # Get EFS Utils tarball + remote_file efs_utils_tarball do + source efs_utils_url + mode '0644' + retries 3 + retry_delay 5 + checksum new_resource.efs_utils_checksum + action :create_if_missing + end + + efs_proxy_deps = "efs-proxy-dependencies-#{package_version}.tar.gz" + efs_proxy_deps_tarball = "#{node['cluster']['sources_dir']}/#{efs_proxy_deps}" + efs_proxy_deps_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/#{efs_proxy_deps}" + remote_file efs_proxy_deps_tarball do + source efs_proxy_deps_url + mode '0644' + retries 3 + retry_delay 5 + action :create_if_missing + end + + # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html + bash "install efs utils" do + cwd node['cluster']['sources_dir'] + code install_script_code(efs_utils_tarball, package_name, package_version) + end + action_increase_poll_interval +end diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb index 112a6ce094..cf65440cf0 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb @@ -50,20 +50,6 @@ action :create_if_missing end - if aws_region.start_with?("us-iso") && platform?('redhat') && node['platform_version'] == "8" - efs_proxy_deps = "efs-proxy-dependencies-#{package_version}.tar.gz" - efs_proxy_deps_tarball = "#{node['cluster']['sources_dir']}/#{efs_proxy_deps}" - efs_proxy_deps_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/#{efs_proxy_deps}" - remote_file efs_proxy_deps_tarball do - source efs_proxy_deps_url - mode '0644' - retries 3 - retry_delay 5 - checksum new_resource.efs_utils_checksum - action :create_if_missing - end - end - # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html bash "install efs utils" do cwd node['cluster']['sources_dir'] diff --git a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb index 2845ec9c29..8717b06a69 100644 --- a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb +++ b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb @@ -33,22 +33,24 @@ not_if { ::File.exist?("#{cookbook_virtualenv_path}/bin/activate") } end -remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do - source pypi_s3_uri - mode '0644' - retries 3 - retry_delay 5 - action :create_if_missing -end +if aws_region.start_with?("us-iso") + remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do + source pypi_s3_uri + mode '0644' + retries 3 + retry_delay 5 + action :create_if_missing + end -bash 'pip install' do - user 'root' - group 'root' - cwd "#{node['cluster']['base_dir']}" - code <<-REQ + bash 'pip install' do + user 'root' + group 'root' + cwd "#{node['cluster']['base_dir']}" + code <<-REQ set -e tar xzf cookbook-dependencies.tgz cd #{dependency_package_name} #{virtualenv_path}/bin/pip install * -f ./ --no-index REQ + end end From 42788435d613a808b5760cd3d066f95edd68a693 Mon Sep 17 00:00:00 2001 From: Helena Greebe Date: Tue, 12 Aug 2025 11:19:20 -0400 Subject: [PATCH 11/14] [Isolated] Only install efs-proxy deps when in adc regions --- .../resources/efs/efs_redhat8.rb | 67 +++++++------------ .../efs/partial/_install_from_tar.rb | 22 ++++-- 2 files changed, 38 insertions(+), 51 deletions(-) diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb index 12193f73cb..0d16d45c6b 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb @@ -22,7 +22,7 @@ # use 'partial/_install_from_tar' use 'partial/_mount_umount' -def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) +def adc_install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) <<-EFSUTILSINSTALL set -e tar xf #{efs_utils_tarball} @@ -40,58 +40,37 @@ def prerequisites %w(rpm-build make rust cargo openssl-devel) end -action :install_utils do - package_repos 'update package repositories' do - action :update - end - - package prerequisites do - retries 3 - retry_delay 5 - end - - directory node['cluster']['sources_dir'] do - recursive true - end - - return if redhat_on_docker? +action :install_efs_utils do package_name = "amazon-efs-utils" package_version = new_resource.efs_utils_version efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" - efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz" - # Do not install efs-utils if a same or newer version is already installed. - return if already_installed?(package_name, package_version) + if aws_region.start_with?("us-iso") - # On all OSes but Amazon Linux 2, amazon-efs-utils and stunnel are installed from source, - # because their OS repos do not have amazon-efs-utils and new stunnel + efs_proxy_deps = "efs-proxy-dependencies-#{package_version}.tar.gz" + efs_proxy_deps_tarball = "#{node['cluster']['sources_dir']}/#{efs_proxy_deps}" + efs_proxy_deps_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/#{efs_proxy_deps}" + remote_file efs_proxy_deps_tarball do + source efs_proxy_deps_url + mode '0644' + retries 3 + retry_delay 5 + action :create_if_missing + end - # Get EFS Utils tarball - remote_file efs_utils_tarball do - source efs_utils_url - mode '0644' - retries 3 - retry_delay 5 - checksum new_resource.efs_utils_checksum - action :create_if_missing - end + bash "install efs utils" do + cwd node['cluster']['sources_dir'] + code adc_install_script_code(efs_utils_tarball, package_name, package_version) + end - efs_proxy_deps = "efs-proxy-dependencies-#{package_version}.tar.gz" - efs_proxy_deps_tarball = "#{node['cluster']['sources_dir']}/#{efs_proxy_deps}" - efs_proxy_deps_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/#{efs_proxy_deps}" - remote_file efs_proxy_deps_tarball do - source efs_proxy_deps_url - mode '0644' - retries 3 - retry_delay 5 - action :create_if_missing + else + # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html + bash "install efs utils" do + cwd node['cluster']['sources_dir'] + code install_script_code(efs_utils_tarball, package_name, package_version) + end end - # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html - bash "install efs utils" do - cwd node['cluster']['sources_dir'] - code install_script_code(efs_utils_tarball, package_name, package_version) - end action_increase_poll_interval end diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb index cf65440cf0..000b91e76a 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb @@ -12,6 +12,10 @@ # or in the "LICENSE.txt" file accompanying this file. # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. +package_name = "amazon-efs-utils" +package_version = new_resource.efs_utils_version +efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" +efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz" action :install_utils do package_repos 'update package repositories' do @@ -29,11 +33,6 @@ return if redhat_on_docker? - package_name = "amazon-efs-utils" - package_version = new_resource.efs_utils_version - efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" - efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz" - # Do not install efs-utils if a same or newer version is already installed. return if already_installed?(package_name, package_version) @@ -50,10 +49,19 @@ action :create_if_missing end + action_install_efs_utils + # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html + # bash "install efs utils" do + # cwd node['cluster']['sources_dir'] + # code install_script_code(efs_utils_tarball, package_name, package_version) + # end + action_increase_poll_interval +end + +action :install_efs_utils do bash "install efs utils" do cwd node['cluster']['sources_dir'] code install_script_code(efs_utils_tarball, package_name, package_version) end - action_increase_poll_interval -end +end \ No newline at end of file From ab36358caed44cf2f203ab6957bdf4f4c8076e22 Mon Sep 17 00:00:00 2001 From: Helena Greebe Date: Wed, 13 Aug 2025 13:31:29 -0400 Subject: [PATCH 12/14] [Isolated] Only install efs-proxy-deps in adc --- .../resources/efs/efs_redhat8.rb | 4 ++-- .../resources/efs/partial/_install_from_tar.rb | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb index 0d16d45c6b..4e9cd02dd9 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb @@ -18,8 +18,8 @@ use 'partial/_get_package_version_rpm' use 'partial/_common' -# use 'partial/_redhat_based' -# use 'partial/_install_from_tar' +use 'partial/_redhat_based' +use 'partial/_install_from_tar' use 'partial/_mount_umount' def adc_install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb index 000b91e76a..15357374dc 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb @@ -13,11 +13,12 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. package_name = "amazon-efs-utils" -package_version = new_resource.efs_utils_version -efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" -efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz" action :install_utils do + package_version = new_resource.efs_utils_version + efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" + efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz" + package_repos 'update package repositories' do action :update end @@ -60,6 +61,9 @@ end action :install_efs_utils do + package_version = new_resource.efs_utils_version + efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" + bash "install efs utils" do cwd node['cluster']['sources_dir'] code install_script_code(efs_utils_tarball, package_name, package_version) From 29ba943b63f79dea834cca3bb00de9b44942dcfe Mon Sep 17 00:00:00 2001 From: Helena Greebe Date: Fri, 15 Aug 2025 21:12:35 -0400 Subject: [PATCH 13/14] [Isolated] Fix unit tests --- .../recipes/install/custom_parallelcluster_node.rb | 10 +++++----- .../recipes/install/cfn_bootstrap.rb | 10 +++++----- .../resources/efs/efs_redhat8.rb | 1 - .../resources/efs/partial/_install_from_tar.rb | 2 +- .../spec/unit/recipes/cookbook_virtualenv_spec.rb | 8 -------- 5 files changed, 11 insertions(+), 20 deletions(-) diff --git a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb index af868a80b0..788a715f33 100644 --- a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb +++ b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb @@ -19,11 +19,11 @@ # TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the # bash block to a recipe that uses the pyenv resource. -if aws_region.start_with?("us-iso") - command = "pip install . --no-build-isolation" -else - command = "pip install ." -end +command = if aws_region.start_with?("us-iso") + "pip install . --no-build-isolation" + else + "pip install ." + end if aws_region.start_with?("us-iso") dependency_package_name = "pypi-node-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}" diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index 8270a62ff7..e795e70943 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -79,11 +79,11 @@ retry_delay 5 end -if aws_region.start_with?("us-iso") - command = "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package} --no-build-isolation" -else - command = "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package}" -end +command = if aws_region.start_with?("us-iso") + "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package} --no-build-isolation" + else + "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package}" + end bash "Install CloudFormation helpers from #{cfnbootstrap_package}" do user 'root' group 'root' diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb index 4e9cd02dd9..abe333a933 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb @@ -41,7 +41,6 @@ def prerequisites end action :install_efs_utils do - package_name = "amazon-efs-utils" package_version = new_resource.efs_utils_version efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb index 15357374dc..c3e320c906 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb @@ -68,4 +68,4 @@ cwd node['cluster']['sources_dir'] code install_script_code(efs_utils_tarball, package_name, package_version) end -end \ No newline at end of file +end diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb index 65b83d509e..1234953b68 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb @@ -32,14 +32,6 @@ expect(node.default['cluster']['cookbook_virtualenv_path']).to eq(virtualenv_path) is_expected.to write_node_attributes('dump node attributes') end - - it 'installs python packages' do - is_expected.to run_bash("pip install").with( - user: 'root', - group: 'root', - cwd: "#{node['cluster']['base_dir']}" - ).with_code(/tar xzf cookbook-dependencies.tgz/) - end end end end From ddbc975959d80893f59bf156dc0db502946e558b Mon Sep 17 00:00:00 2001 From: Helena Greebe Date: Mon, 18 Aug 2025 16:49:31 -0400 Subject: [PATCH 14/14] [Isolated] Test python pacakges are installed when in an ADC region --- .../spec/unit/recipes/cookbook_virtualenv_spec.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb index 1234953b68..0ce79250cc 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb @@ -6,12 +6,15 @@ cached(:python_version) { 'python_version' } cached(:system_pyenv_root) { 'system_pyenv_root' } cached(:virtualenv_path) { 'system_pyenv_root/versions/python_version/envs/cookbook_virtualenv' } + cached(:aws_region) { 'us-iso-test' } context "when cookbook virtualenv not installed yet" do cached(:chef_run) do runner = runner(platform: platform, version: version) do |node| + allow_any_instance_of(Object).to receive(:aws_region).and_return(aws_region) node.override['cluster']['system_pyenv_root'] = system_pyenv_root node.override['cluster']['python-version'] = python_version + node.override['cluster']['region'] = aws_region end runner.converge(described_recipe) end @@ -32,6 +35,15 @@ expect(node.default['cluster']['cookbook_virtualenv_path']).to eq(virtualenv_path) is_expected.to write_node_attributes('dump node attributes') end + context "when in isolated region" do + it 'installs python packages' do + is_expected.to run_bash("pip install").with( + user: 'root', + group: 'root', + cwd: "#{node['cluster']['base_dir']}" + ).with_code(/tar xzf cookbook-dependencies.tgz/) + end + end end end end