hpc-cluster-amd-slurmv5.yaml

# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

---
blueprint_name: hpc-cluster-amd

vars:
  project_id:  ## Set GCP Project ID Here ##
  deployment_name: amd-v5
  region: us-east4
  zone: us-east4-c

deployment_groups:
- group: primary
  modules:
  - id: network1
    source: modules/network/vpc

  - id: homefs
    source: modules/file-system/filestore
    use: [network1]
    settings:
      local_mount: /home

  - id: swfs
    source: modules/file-system/filestore
    use: [network1]
    settings:
      local_mount: /sw

  - id: spack
    source: community/modules/scripts/spack-install
    settings:
      install_dir: /sw/spack
      spack_ref: v0.18.1
      log_file: /var/log/spack.log
      configs:
      - type: file
        scope: defaults
        content: |
          config:
            build_stage:
              - /opt/spack_build_stage
      - type: file
        scope: defaults
        content: |
          modules:
            default:
              tcl:
                hash_length: 0
                all:
                  conflict:
                    - '{name}'
                projections:
                  all: '{name}/{version}-{compiler.name}-{compiler.version}'
      - type: file
        scope: site
        content: |
          packages:
            slurm:
              externals:
                - spec: slurm@22-05-8
                  prefix: /usr/local
              buildable: False
      - type: file
        scope: site
        content: |
          concretizer:
            targets:
              host_compatible: false
      compilers:
      # gcc 12.1.0 is known to have runtime failures with OpenFOAM 8
      # gcc 10.3.0 is the earliest copy of gcc with Zen 3 support
      - gcc@10.3.0 %gcc@4.8.5 target=x86_64
      environments:
      - name: openfoam
        type: file
        content: |
          spack:
            definitions:
            - compilers:
              - gcc@10.3.0
            - mpis:
              - openmpi@4.1.3+legacylaunchers+pmi fabrics=none schedulers=slurm
            - packages:
              - flex@2.6.4
            - mpi_packages:
              - openfoam-org@8 ^flex@2.6.4 target=zen3
            specs:
            - matrix:
              - - $mpis
              - - $%compilers
            - matrix:
              - - $packages
              - - $%compilers
            - matrix:
              - - $mpi_packages
              - - $%compilers
              - - $^mpis
            concretizer:
              unify: when_possible

  - id: spack-startup
    source: modules/scripts/startup-script
    settings:
      runners:
      - $(spack.install_spack_deps_runner)
      - $(spack.install_spack_runner)
      - type: shell
        destination: shutdown.sh
        content: |
          #!/bin/bash
          if [ ! -f /etc/block_auto_shutdown ]; then
                  touch /etc/block_auto_shutdown
                  shutdown -h +1
          fi

  - id: slurm_startup
    source: modules/scripts/startup-script
    settings:
      runners:
      - $(spack.setup_spack_runner)
      # the following installation of AOCC may be automated in the future
      # with a clear direction to the user to read the EULA at
      # https://developer.amd.com/aocc-compiler-eula/
      - type: data
        destination: /var/tmp/install_aocc.sh
        content: |
          #!/bin/bash
          source /sw/spack/share/spack/setup-env.sh
          spack install aocc@3.2.0 +license-agreed
          spack load aocc@3.2.0
          spack compiler find --scope site
          spack -d install -v openmpi@4.1.3 %aocc@3.2.0 +legacylaunchers +pmi schedulers=slurm
      - type: data
        destination: /var/tmp/openfoam_test.sh
        content: |
          #!/bin/bash
          # the following line works around a problem activating environments
          # before directory is accessed
          ls -lha /sw/spack/var/spack/environments/openfoam/ &>/dev/null
          spack env activate openfoam
          DIR=$HOME/openfoam_test
          mkdir -p $DIR
          cd $DIR
          cp -fr $WM_PROJECT_DIR/tutorials/incompressible/simpleFoam/motorBike .
          cd motorBike
          ./Allrun

  - id: spack_builder
    source: modules/compute/vm-instance
    use: [network1, swfs, spack-startup]
    settings:
      name_prefix: spack-builder
      machine_type: c2d-standard-16
      disable_public_ips: true
      instance_image:
        # these images must match the images used by Slurm modules below because
        # we are building OpenMPI with PMI support in libaries contained in
        # Slurm installation
        family: schedmd-v5-slurm-22-05-8-hpc-centos-7
        project: schedmd-slurm-public

  - id: low_cost_node_group
    source: community/modules/compute/schedmd-slurm-gcp-v5-node-group
    settings:
      machine_type: c2d-standard-4
      node_count_dynamic_max: 10

  - id: low_cost_partition
    source: community/modules/compute/schedmd-slurm-gcp-v5-partition
    use:
    - network1
    - low_cost_node_group
    settings:
      partition_name: lowcost
      enable_placement: false

  - id: compute_node_group
    source: community/modules/compute/schedmd-slurm-gcp-v5-node-group
    settings:
      machine_type: c2d-standard-112
      node_count_dynamic_max: 50

  # because is_default is set to true, jobs will run on this partition unless an
  # alternative partition is specified using, for example, "srun -p lowcost"
  - id: compute_partition
    source: community/modules/compute/schedmd-slurm-gcp-v5-partition
    use:
    - network1
    - compute_node_group
    settings:
      partition_name: compute
      enable_placement: true
      is_default: true

  - id: slurm_controller
    source: community/modules/scheduler/schedmd-slurm-gcp-v5-controller
    use:
    - network1
    - homefs
    - swfs
    - low_cost_partition
    - compute_partition
    settings:
      machine_type: c2d-standard-4

  - id: slurm_login
    source: community/modules/scheduler/schedmd-slurm-gcp-v5-login
    use:
    - network1
    - slurm_controller
    - slurm_startup
    settings:
      # need at least 8 physical cores to run OpenFOAM test
      machine_type: c2d-standard-16