diff --git a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb index 247823c173..788a715f33 100644 --- a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb +++ b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb @@ -19,9 +19,22 @@ # TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the # bash block to a recipe that uses the pyenv resource. -if aws_region.start_with?("us-iso") && platform?('amazon') && node['platform_version'] == "2" +command = if aws_region.start_with?("us-iso") + "pip install . --no-build-isolation" + else + "pip install ." + end + +if aws_region.start_with?("us-iso") + dependency_package_name = "pypi-node-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}" + dependency_folder_name = dependency_package_name + if platform?('amazon') && node['platform_version'] == "2" + dependency_package_name = "node-dependencies" + dependency_folder_name = "node" + end + remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do - source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/node-dependencies.tgz" + source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz" mode '0644' retries 3 retry_delay 5 @@ -35,7 +48,7 @@ code <<-REQ set -e tar xzf node-dependencies.tgz - cd node + cd #{dependency_folder_name} #{node_virtualenv_path}/bin/pip install * -f ./ --no-index REQ end @@ -59,7 +72,7 @@ mkdir aws-parallelcluster-custom-node tar -xzf aws-parallelcluster-node.tgz --directory aws-parallelcluster-custom-node cd aws-parallelcluster-custom-node/*aws-parallelcluster-node* - pip install . + #{command} deactivate NODE end diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index cfe95c4ed5..e795e70943 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -34,8 +34,14 @@ end if aws_region.start_with?("us-iso") + dependency_package_name = "pypi-cfn-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}" + dependency_folder_name = dependency_package_name + if platform?('amazon') && node['platform_version'] == "2" + dependency_package_name = "cfn-dependencies" + dependency_folder_name = "cfn" + end remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do - source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cfn-dependencies.tgz" + source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz" mode '0644' retries 3 retry_delay 5 @@ -49,7 +55,7 @@ code <<-REQ set -e tar xzf cfn-dependencies.tgz - cd cfn + cd #{dependency_folder_name} #{virtualenv_path}/bin/pip install * -f ./ --no-index REQ end @@ -73,11 +79,16 @@ retry_delay 5 end +command = if aws_region.start_with?("us-iso") + "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package} --no-build-isolation" + else + "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package}" + end bash "Install CloudFormation helpers from #{cfnbootstrap_package}" do user 'root' group 'root' cwd '/tmp' - code "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package}" + code command creates "#{virtualenv_path}/bin/cfn-hup" end diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb index b2ab45bc1b..abe333a933 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb @@ -21,3 +21,55 @@ use 'partial/_redhat_based' use 'partial/_install_from_tar' use 'partial/_mount_umount' + +def adc_install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) + <<-EFSUTILSINSTALL + set -e + tar xf #{efs_utils_tarball} + mv efs-proxy-dependencies-#{efs_utils_version}.tar.gz efs-utils-#{efs_utils_version}/src/proxy/ + cd efs-utils-#{efs_utils_version}/src/proxy/ + tar -xf efs-proxy-dependencies-#{efs_utils_version}.tar.gz + cargo build --offline + cd ../.. + make rpm + yum -y install ./build/#{efs_utils_package}*rpm + EFSUTILSINSTALL +end + +def prerequisites + %w(rpm-build make rust cargo openssl-devel) +end + +action :install_efs_utils do + package_name = "amazon-efs-utils" + package_version = new_resource.efs_utils_version + efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" + + if aws_region.start_with?("us-iso") + + efs_proxy_deps = "efs-proxy-dependencies-#{package_version}.tar.gz" + efs_proxy_deps_tarball = "#{node['cluster']['sources_dir']}/#{efs_proxy_deps}" + efs_proxy_deps_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/#{efs_proxy_deps}" + remote_file efs_proxy_deps_tarball do + source efs_proxy_deps_url + mode '0644' + retries 3 + retry_delay 5 + action :create_if_missing + end + + bash "install efs utils" do + cwd node['cluster']['sources_dir'] + code adc_install_script_code(efs_utils_tarball, package_name, package_version) + end + + else + # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html + bash "install efs utils" do + cwd node['cluster']['sources_dir'] + code install_script_code(efs_utils_tarball, package_name, package_version) + end + end + + action_increase_poll_interval +end diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb index cf65440cf0..c3e320c906 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb @@ -12,8 +12,13 @@ # or in the "LICENSE.txt" file accompanying this file. # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. +package_name = "amazon-efs-utils" action :install_utils do + package_version = new_resource.efs_utils_version + efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" + efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz" + package_repos 'update package repositories' do action :update end @@ -29,11 +34,6 @@ return if redhat_on_docker? - package_name = "amazon-efs-utils" - package_version = new_resource.efs_utils_version - efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" - efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz" - # Do not install efs-utils if a same or newer version is already installed. return if already_installed?(package_name, package_version) @@ -50,10 +50,22 @@ action :create_if_missing end + action_install_efs_utils + # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html + # bash "install efs utils" do + # cwd node['cluster']['sources_dir'] + # code install_script_code(efs_utils_tarball, package_name, package_version) + # end + action_increase_poll_interval +end + +action :install_efs_utils do + package_version = new_resource.efs_utils_version + efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" + bash "install efs utils" do cwd node['cluster']['sources_dir'] code install_script_code(efs_utils_tarball, package_name, package_version) end - action_increase_poll_interval end diff --git a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb index cdc557df5f..8717b06a69 100644 --- a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb +++ b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb @@ -12,8 +12,10 @@ # limitations under the License. virtualenv_path = cookbook_virtualenv_path -pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/pypi-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}.tgz" +dependency_package_name = "pypi-cookbook-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}" +pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{dependency_package_name}.tgz" if platform?('amazon') && node['platform_version'] == "2" + dependency_package_name = "dependencies" pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz" end @@ -31,22 +33,24 @@ not_if { ::File.exist?("#{cookbook_virtualenv_path}/bin/activate") } end -remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do - source pypi_s3_uri - mode '0644' - retries 3 - retry_delay 5 - action :create_if_missing -end +if aws_region.start_with?("us-iso") + remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do + source pypi_s3_uri + mode '0644' + retries 3 + retry_delay 5 + action :create_if_missing + end -bash 'pip install' do - user 'root' - group 'root' - cwd "#{node['cluster']['base_dir']}" - code <<-REQ + bash 'pip install' do + user 'root' + group 'root' + cwd "#{node['cluster']['base_dir']}" + code <<-REQ set -e tar xzf cookbook-dependencies.tgz - cd dependencies + cd #{dependency_package_name} #{virtualenv_path}/bin/pip install * -f ./ --no-index REQ + end end diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb index 65b83d509e..0ce79250cc 100644 --- a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb +++ b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/cookbook_virtualenv_spec.rb @@ -6,12 +6,15 @@ cached(:python_version) { 'python_version' } cached(:system_pyenv_root) { 'system_pyenv_root' } cached(:virtualenv_path) { 'system_pyenv_root/versions/python_version/envs/cookbook_virtualenv' } + cached(:aws_region) { 'us-iso-test' } context "when cookbook virtualenv not installed yet" do cached(:chef_run) do runner = runner(platform: platform, version: version) do |node| + allow_any_instance_of(Object).to receive(:aws_region).and_return(aws_region) node.override['cluster']['system_pyenv_root'] = system_pyenv_root node.override['cluster']['python-version'] = python_version + node.override['cluster']['region'] = aws_region end runner.converge(described_recipe) end @@ -32,13 +35,14 @@ expect(node.default['cluster']['cookbook_virtualenv_path']).to eq(virtualenv_path) is_expected.to write_node_attributes('dump node attributes') end - - it 'installs python packages' do - is_expected.to run_bash("pip install").with( - user: 'root', - group: 'root', - cwd: "#{node['cluster']['base_dir']}" - ).with_code(/tar xzf cookbook-dependencies.tgz/) + context "when in isolated region" do + it 'installs python packages' do + is_expected.to run_bash("pip install").with( + user: 'root', + group: 'root', + cwd: "#{node['cluster']['base_dir']}" + ).with_code(/tar xzf cookbook-dependencies.tgz/) + end end end end diff --git a/util/upload-cookbook.sh b/util/upload-cookbook.sh index c2169a4c3e..864ce56959 100755 --- a/util/upload-cookbook.sh +++ b/util/upload-cookbook.sh @@ -93,21 +93,22 @@ main() { # Create archive and md5 _cwd=$(pwd) pushd "${_srcdir}" > /dev/null || exit + GIT_REF=$(git rev-parse HEAD) _stashName=$(git stash create) - git archive --format tar --prefix="aws-parallelcluster-cookbook-${_version}/" "${_stashName:-HEAD}" | gzip > "${_cwd}/aws-parallelcluster-cookbook-${_version}.tgz" + git archive --format tar --prefix="aws-parallelcluster-cookbook-${_version}/" "${_stashName:-HEAD}" | gzip > "${_cwd}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz" #tar zcvf "${_cwd}/aws-parallelcluster-cookbook-${_version}.tgz" --transform "s,^aws-parallelcluster-cookbook/,aws-parallelcluster-cookbook-${_version}/," ../aws-parallelcluster-cookbook popd > /dev/null || exit - md5sum aws-parallelcluster-cookbook-${_version}.tgz > aws-parallelcluster-cookbook-${_version}.md5 + md5sum aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz > aws-parallelcluster-cookbook-${_version}-${GIT_REF}.md5 # upload packages _key_path="parallelcluster/${_version}/cookbooks" if [ -n "${_scope}" ]; then _key_path="${_key_path}/${_scope}" fi - aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}.tgz s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz || _error_exit 'Failed to push cookbook to S3' - aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}.md5 s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}.md5 || _error_exit 'Failed to push cookbook md5 to S3' - aws ${_profile} --region "${_region}" s3api head-object --bucket ${_bucket} --key ${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz --output text --query LastModified > aws-parallelcluster-cookbook-${_version}.tgz.date || _error_exit 'Failed to fetch LastModified date' - aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}.tgz.date s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz.date || _error_exit 'Failed to push cookbook date' + aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz || _error_exit 'Failed to push cookbook to S3' + aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}-${GIT_REF}.md5 s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.md5 || _error_exit 'Failed to push cookbook md5 to S3' + aws ${_profile} --region "${_region}" s3api head-object --bucket ${_bucket} --key ${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz --output text --query LastModified > aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz.date || _error_exit 'Failed to fetch LastModified date' + aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz.date s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz.date || _error_exit 'Failed to push cookbook date' _bucket_region=$(aws ${_profile} s3api get-bucket-location --bucket ${_bucket} --output text) if [ ${_bucket_region} = "None" ]; then