[B200] Add support for P6-B200 instance type.

gmarciani · gmarciani · commit 253fc5fc9bfe · 2025-09-16T20:36:13.000-04:00
In particular, it consists in installing Infiniband packages, load Infiniband kernel modules at boot time, ann install NVIDIA NVLink Subnet Manager.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,7 @@ This file is used to list changes made in each version of the AWS ParallelCluste
 
 **ENHANCEMENTS**
 - Add support for P6e-GB200 instances. ParallelCluster sets up Slurm topology plugin to handle P6e-GB200 UltraServers. See limitations section for important additional setup requirements.
+- Add support for P6-B200 instances for all OSs except AL2.
 - Add `build-image` support for Amazon Linux 2023 AMIs based on kernel 6.12 (in addition to 6.1).
 
 **LIMITATIONS**
diff --git a/cookbooks/aws-parallelcluster-platform/attributes/platform.rb b/cookbooks/aws-parallelcluster-platform/attributes/platform.rb
@@ -27,6 +27,9 @@
 default['cluster']['nvidia']['imex']['shared_dir'] = "#{node['cluster']['shared_dir']}/nvidia-imex"
 default['cluster']['nvidia']['imex']['force_configuration'] = false
 
+# NVIDIA NVLSM
+default['cluster']['nvidia']['nvlsm']['enabled'] = true
+
 # DCV
 default['cluster']['dcv']['authenticator']['user'] = "dcvextauth"
 default['cluster']['dcv']['authenticator']['user_id'] = node['cluster']['reserved_base_uid'] + 3
diff --git a/cookbooks/aws-parallelcluster-platform/files/infiniband/infiniband.conf b/cookbooks/aws-parallelcluster-platform/files/infiniband/infiniband.conf
@@ -0,0 +1 @@
+ib_umad
diff --git a/cookbooks/aws-parallelcluster-platform/libraries/nvidia.rb b/cookbooks/aws-parallelcluster-platform/libraries/nvidia.rb
@@ -27,11 +27,12 @@ def get_nvswitch_count(device_id)
 end
 
 def get_device_ids
-  #  A100 (P4), H100(P5), B200(P6) and GB200()p6e) systems have NVSwitches
+  #  A100 (P4), H100(P5), B200(P6) and GB200(p6e) systems have NVSwitches
   # NVSwitch device id is 10de:1af1 for P4 instance
   # NVSwitch device id is 10de:22a3 for P5 instance
+  # NVSwitch device id is 10de:2901 for P6 instance
   # NVSwitch device id is 10de:2941 for P6e instance
-  { 'a100' => '10de:1af1', 'h100' => '10de:22a3', 'gb200' => '10de:2941' }
+  { 'a100' => '10de:1af1', 'h100' => '10de:22a3', 'b200' => '10de:2901', 'gb200' => '10de:2941' }
 end
 
 def is_gb200_node?
diff --git a/cookbooks/aws-parallelcluster-platform/recipes/install/nvidia_install.rb b/cookbooks/aws-parallelcluster-platform/recipes/install/nvidia_install.rb
@@ -21,6 +21,8 @@
 
 gdrcopy 'Install Nvidia gdrcopy'
 
+nvidia_nvlsm 'Install Nvidia NVLink Subnet Manager'
+
 fabric_manager 'Install Nvidia Fabric Manager'
 
 nvidia_dcgm 'install Nvidia datacenter-gpu-manager'
diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/nvidia_nvlsm_alinux2023.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/nvidia_nvlsm_alinux2023.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# Copyright:: 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+provides :nvidia_nvlsm, platform: 'amazon' do |node|
+  node['platform_version'].to_i == 2023
+end
+
+use 'partial/_nvidia_nvlsm_common.rb'
+use 'partial/_nvidia_nvlsm_rhel.rb'
+
+def platform
+  "amzn#{node['platform_version'].to_i}"
+end
diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/nvidia_nvlsm_amazon2.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/nvidia_nvlsm_amazon2.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# Copyright:: 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+provides :nvidia_nvlsm, platform: 'amazon', platform_version: '2'
+
+use 'partial/_nvidia_nvlsm_common.rb'
+use 'partial/_nvidia_nvlsm_rhel.rb'
+
+def nvlsm_installation_enabled?
+  # NVLSM is not supported on Amazon Linux 2.
+  false
+end
+
+def platform
+  "amzn#{node['platform_version'].to_i}"
+end
diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/nvidia_nvlsm_redhat8.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/nvidia_nvlsm_redhat8.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# Copyright:: 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+provides :nvidia_nvlsm, platform: 'redhat' do |node|
+  node['platform_version'].to_i >= 8
+end
+
+use 'partial/_nvidia_nvlsm_common.rb'
+use 'partial/_nvidia_nvlsm_rhel.rb'
+
+def platform
+  "rhel#{node['platform_version'].to_i}"
+end
diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/nvidia_nvlsm_rocky8.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/nvidia_nvlsm_rocky8.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# Copyright:: 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+provides :nvidia_nvlsm, platform: 'rocky' do |node|
+  node['platform_version'].to_i >= 8
+end
+
+use 'partial/_nvidia_nvlsm_common.rb'
+use 'partial/_nvidia_nvlsm_rhel.rb'
+
+def platform
+  "rhel#{node['platform_version'].to_i}"
+end
diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/nvidia_nvlsm_ubuntu22+.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/nvidia_nvlsm_ubuntu22+.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# Copyright:: 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+provides :nvidia_nvlsm, platform: 'ubuntu' do |node|
+  node['platform_version'].to_i >= 22
+end
+
+use 'partial/_nvidia_nvlsm_common.rb'
+use 'partial/_nvidia_nvlsm_debian.rb'
+
+def platform
+  "ubuntu#{node['platform_version'].delete('.')}"
+end
diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/partial/_nvidia_nvlsm_common.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/partial/_nvidia_nvlsm_common.rb
@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+#
+# Copyright:: 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+unified_mode true
+default_action :install
+
+action :install do
+  return unless nvlsm_installation_enabled?
+
+  action_install_nvlsm_dependencies
+  action_install_nvlsm
+end
+
+action :install_nvlsm_dependencies do
+  bash "Install nvlsm dependencies" do
+    user 'root'
+    code <<-CODE
+    set -ex
+    #{nvidia_nvlsm_install_dependencies_commands}
+    CODE
+    retries 3
+    retry_delay 5
+  end
+
+  # Make sure kernel module for Infiniband is loaded at instance boot time
+  cookbook_file 'infiniband.conf' do
+    source 'infiniband/infiniband.conf'
+    path '/etc/modules-load.d/parallelcluster-infiniband.conf'
+    owner 'root'
+    group 'root'
+    mode '0644'
+  end
+end
+
+action :install_nvlsm do
+  remote_file "#{node['cluster']['sources_dir']}/#{nvidia_nvlsm_package_full_name}" do
+    source nvidia_nvlsm_url
+    checksum nvidia_nvlsm_checksum
+    mode '0644'
+    retries 3
+    retry_delay 5
+    action :create_if_missing
+  end
+
+  bash "Install nvlsm" do
+    user 'root'
+    cwd node['cluster']['sources_dir']
+    code <<-CODE
+    set -ex
+    #{nvidia_nvlsm_install_commands}
+    CODE
+    retries 3
+    retry_delay 5
+  end
+end
+
+def nvidia_nvlsm_package
+  "nvlsm"
+end
+
+def nvidia_nvlsm_version
+  "2025.03.9-1"
+end
+
+def nvidia_nvlsm_url
+  "#{node['cluster']['artifacts_s3_url']}/dependencies/nvidia_nvlsm/#{platform}/#{nvidia_nvlsm_package_full_name}"
+end
+
+def nvidia_nvlsm_package_full_name
+  # OS dependent
+end
+
+def nvidia_nvlsm_checksum
+  # OS dependent
+end
+
+def nvidia_nvlsm_install_commands
+  # OS dependent
+end
+
+def nvidia_nvlsm_install_dependencies_commands
+  # OS dependent
+end
+
+def nvlsm_installation_enabled?
+  if on_docker? ||
+     node['cluster']['nvidia']['nvlsm']['enabled'] == false ||
+     !nvidia_enabled? ||
+     nvlsm_installed?
+    false
+  else
+    true
+  end
+end
+
+def nvlsm_installed?
+  ::File.exist?("/opt/nvidia/nvlsm/sbin/nvlsm")
+end
diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/partial/_nvidia_nvlsm_debian.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/partial/_nvidia_nvlsm_debian.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+#
+# Copyright:: 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+def nvidia_nvlsm_package_full_name
+  "#{nvidia_nvlsm_package}_#{nvidia_nvlsm_version}_#{arch_suffix}.deb"
+end
+
+def arch_suffix
+  arm_instance? ? 'arm64' : 'amd64'
+end
+
+def nvidia_nvlsm_checksum
+  if arm_instance?
+    '6bb405a3494d9fcb6dad6e641d02afb71fd4e2f6a2b4ed3d5cf6cbff22964eb3'
+  else
+    '61f280e469624c43eecb0e08305452887e02f73e4763252a41f728d1843f1cc5'
+  end
+end
+
+def nvidia_nvlsm_install_commands
+  "dpkg -i #{nvidia_nvlsm_package_full_name} && apt-mark hold #{nvidia_nvlsm_package}"
+end
+
+def nvidia_nvlsm_install_dependencies_commands
+  "apt install -y infiniband-diags ibutils"
+end
diff --git a/cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/partial/_nvidia_nvlsm_rhel.rb b/cookbooks/aws-parallelcluster-platform/resources/nvidia_nvlsm/partial/_nvidia_nvlsm_rhel.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+#
+# Copyright:: 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+def nvidia_nvlsm_package_full_name
+  "#{nvidia_nvlsm_package}-#{nvidia_nvlsm_version}.#{arch_suffix}.rpm"
+end
+
+def arch_suffix
+  arm_instance? ? 'aarch64' : 'x86_64'
+end
+
+def nvidia_nvlsm_checksum
+  if arm_instance?
+    'f21f14843c11ce64136fd1c3fa763b7511e18f160695f54b2a8d763776313539'
+  else
+    '88d5e52183bb5ee763eb864bbd119b591e7f45af32c52bd7ba0aa8f74fc19057'
+  end
+end
+
+def nvidia_nvlsm_install_commands
+  "yum install -y #{nvidia_nvlsm_package_full_name} && yum versionlock #{nvidia_nvlsm_package}"
+end
+
+def nvidia_nvlsm_install_dependencies_commands
+  "yum install -y infiniband-diags libibumad"
+end
diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/nvidia_install_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/recipes/nvidia_install_spec.rb
@@ -1,7 +1,6 @@
 require 'spec_helper'
 
 describe 'aws-parallelcluster-platform::nvidia_install' do
-
   for_all_oses do |platform, version|
     context "on #{platform}#{version}" do
       cached(:chef_run) do
@@ -22,6 +21,10 @@
         is_expected.to setup_gdrcopy('Install Nvidia gdrcopy')
       end
 
+      it 'installs nvidia_nvlsm' do
+        is_expected.to install_nvidia_nvlsm('Install Nvidia NVLink Subnet Manager')
+      end
+
       it 'installs fabric_manager' do
         is_expected.to setup_fabric_manager('Install Nvidia Fabric Manager')
       end
diff --git a/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_nvlsm_spec.rb b/cookbooks/aws-parallelcluster-platform/spec/unit/resources/nvidia_nvlsm_spec.rb
