Revert "Upgrade python version to 3.12 for all OSs except AL2" (#2886)

* Revert "[Test] Fix Python version in kitchen tests related to AWS Batch and CFN Boostrap scripts. In particular, we expect all Oses to use Python 3.12.8, except for AL2 which uses 3.9.20."

This reverts commit 4574ee5.

* Revert "Upgrade python version to 3.12 for all OSs except AL2 and remove unneccesary python dep installation (#2869)"

This reverts commit 45431e6.

---------

Co-authored-by: Himani Anil Deshpande <[email protected]>

Author: himani2411

CHANGELOG.md

@@ -9,7 +9,6 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 - Add support for Ubuntu24.
 
 **CHANGES**
-- Upgrade Python to 3.12.8 for all OSs except AL2 (from 3.9.20).
 - On Ubuntu 22.04, install the Nvidia driver with the same compiler version used to compile the kernel.
 - Upgrade `aws-cfn-bootstrap` to version 2.0-32.
 - Upgrade amazon-efs-utils to version 2.1.0.

cookbooks/aws-parallelcluster-awsbatch/recipes/awsbatch_virtualenv.rb

@@ -28,5 +28,25 @@
   not_if { ::File.exist?("#{virtualenv_path}/bin/activate") }
 end
 
+remote_file "#{node['cluster']['base_dir']}/awsbatch-dependencies.tgz" do
+  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/awsbatch-dependencies.tgz"
+  mode '0644'
+  retries 3
+  retry_delay 5
+  action :create_if_missing
+end
+
+bash 'pip install' do
+  user 'root'
+  group 'root'
+  cwd "#{node['cluster']['base_dir']}"
+  code <<-REQ
+    set -e
+    tar xzf awsbatch-dependencies.tgz
+    cd awsbatch
+    #{virtualenv_path}/bin/pip install * -f ./ --no-index
+    REQ
+end
+
 node.default['cluster']['awsbatch_virtualenv_path'] = virtualenv_path
 node_attributes "dump node attributes"

cookbooks/aws-parallelcluster-awsbatch/test/controls/awsbatch_virtualenv_spec.rb

@@ -9,11 +9,11 @@
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
+python_version = '3.9.20'
 base_dir = "/opt/parallelcluster"
 pyenv_dir = "#{base_dir}/pyenv"
 
 control 'tag:install_awsbatch_virtualenv_created' do
-  python_version = os_properties.alinux2? ? '3.9.20' : '3.12.8'
   title "awsbatch virtualenv should be created on #{python_version}"
   only_if { !os_properties.redhat? }
 

cookbooks/aws-parallelcluster-computefleet/kitchen.computefleet-config.yml

@@ -31,7 +31,7 @@ suites:
     attributes:
       cluster:
         custom_node_package: https://github.com/aws/aws-parallelcluster-node/archive/develop.tar.gz
-        python-version: 3.12.8
+        python-version: 3.9.20
         node_virtualenv_path: /opt/parallelcluster/pyenv/versions/node_virtualenv
   - name: fleet_status
     run_list:

cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb

@@ -19,6 +19,25 @@
 
 # TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the
 # bash block to a recipe that uses the pyenv resource.
+remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do
+  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/node-dependencies.tgz"
+  mode '0644'
+  retries 3
+  retry_delay 5
+  action :create_if_missing
+end
+
+bash 'pip install' do
+  user 'root'
+  group 'root'
+  cwd "#{node['cluster']['base_dir']}"
+  code <<-REQ
+    set -e
+    tar xzf node-dependencies.tgz
+    cd node
+    #{node_virtualenv_path}/bin/pip install * -f ./ --no-index
+    REQ
+end
 
 bash "install custom aws-parallelcluster-node" do
   cwd Chef::Config[:file_cache_path]

cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb

@@ -15,7 +15,7 @@
 virtualenv_name = 'cfn_bootstrap_virtualenv'
 pyenv_root = node['cluster']['system_pyenv_root']
 # FIXME: Python Version cfn_bootstrap_virtualenv due to a bug with cfn-hup
-python_version = node['cluster']['python-version']
+python_version = '3.9.20'
 virtualenv_path = "#{pyenv_root}/versions/#{python_version}/envs/#{virtualenv_name}"
 
 node.default['cluster']['cfn_bootstrap_virtualenv_path'] = virtualenv_path
@@ -33,6 +33,26 @@
   not_if { ::File.exist?("#{virtualenv_path}/bin/activate") }
 end
 
+remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do
+  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cfn-dependencies.tgz"
+  mode '0644'
+  retries 3
+  retry_delay 5
+  action :create_if_missing
+end
+
+bash 'pip install' do
+  user 'root'
+  group 'root'
+  cwd "#{node['cluster']['base_dir']}"
+  code <<-REQ
+    set -e
+    tar xzf cfn-dependencies.tgz
+    cd cfn
+    #{virtualenv_path}/bin/pip install * -f ./ --no-index
+    REQ
+end
+
 cfnbootstrap_version = '2.0-32'
 cfnbootstrap_package = "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz"
 

cookbooks/aws-parallelcluster-environment/spec/unit/recipes/cfn_bootstrap_spec.rb

@@ -5,7 +5,7 @@
     context "on #{platform}#{version}" do
       cached(:cfnbootstrap_version) { '2.0-32' }
       cached(:cfnbootstrap_package) { "aws-cfn-bootstrap-py3-#{cfnbootstrap_version}.tar.gz" }
-      cached(:python_version) { "#{node['cluster']['python-version']}" }
+      cached(:python_version) { '3.9.20' }
       cached(:system_pyenv_root) { 'system_pyenv_root' }
       cached(:virtualenv_path) { "system_pyenv_root/versions/#{python_version}/envs/cfn_bootstrap_virtualenv" }
       cached(:timeout) { 1800 }
@@ -32,6 +32,14 @@
           )
         end
 
+        it 'installs python packages' do
+          is_expected.to run_bash("pip install").with(
+            user: 'root',
+            group: 'root',
+            cwd: "#{node['cluster']['base_dir']}"
+          )
+        end
+
         it 'sets virtualenv path' do
           expect(node.default['cluster']['cfn_bootstrap_virtualenv_path']).to eq(virtualenv_path)
           is_expected.to write_node_attributes('dump node attributes')

cookbooks/aws-parallelcluster-environment/test/controls/cfn_bootstrap_spec.rb

@@ -9,11 +9,11 @@
 # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
+cfn_python_version = '3.9.20'
 base_dir = "/opt/parallelcluster"
 pyenv_dir = "#{base_dir}/pyenv"
 
 control 'tag:install_cfnbootstrap_virtualenv_created' do
-  cfn_python_version = os_properties.alinux2? ? '3.9.20' : '3.12.8'
   title "cfnbootstrap virtualenv should be created on #{cfn_python_version}"
   only_if { !os_properties.redhat_on_docker? }
 

cookbooks/aws-parallelcluster-platform/recipes/install/awscli.rb

@@ -20,7 +20,7 @@
 
 file_cache_path = Chef::Config[:file_cache_path]
 region = aws_region
-awscli_url = "https://awscli.#{aws_domain}/awscli-exe-linux-#{node['kernel']['machine']}.zip"
+awscli_url = "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip"
 
 if region.start_with?("us-iso-")
   awscli_url = "https://aws-sdk-common-infra-dca-prod-deployment-bucket.s3.#{aws_region}.#{aws_domain}/aws-cli-v2/linux/x86_64/awscli-exe-linux-x86_64.zip"
@@ -42,11 +42,10 @@
   overwrite true
 end
 
-bash 'install awscli' do
-  code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin"
-end
-
 if region.start_with?("us-iso")
+  bash 'install awscli' do
+    code "#{file_cache_path}/awscli/aws/install -i /usr/local/aws -b /usr/local/bin"
+  end
 
   cookbook_file "#{node['cluster']['scripts_dir']}/iso-ca-bundle-config.sh" do
     source 'isolated/iso-ca-bundle-config.sh'
@@ -60,4 +59,8 @@
   execute "patch ca bundle" do
     command "sh #{node['cluster']['scripts_dir']}/iso-ca-bundle-config.sh"
   end
+else
+  bash 'install awscli' do
+    code "#{cookbook_virtualenv_path}/bin/python #{file_cache_path}/awscli/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws"
+  end
 end

cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb

@@ -12,10 +12,6 @@
 # limitations under the License.
 
 virtualenv_path = cookbook_virtualenv_path
-pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/pypi-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}.tgz"
-if platform?('amazon') && node['platform_version'] == "2"
-  pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz"
-end
 
 node.default['cluster']['cookbook_virtualenv_path'] = virtualenv_path
 node_attributes "dump node attributes"
@@ -32,7 +28,7 @@
 end
 
 remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do
-  source pypi_s3_uri
+  source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz"
   mode '0644'
   retries 3
   retry_delay 5