From 5c5a11451490c88f2f7b75e15e02be6128d4c86f Mon Sep 17 00:00:00 2001 From: Srikanth Ramakrishna Date: Thu, 16 Jan 2025 18:52:46 +0000 Subject: [PATCH 1/8] validate on v1.19.0 stack --- examples/kubernetes/Chart.yaml | 23 ++- ...multi-node-multi-card-lora-clm-values.yaml | 142 ++++++++++++++++++ .../kubernetes/templates/mpijob-helm.yaml | 72 +++++++++ 3 files changed, 233 insertions(+), 4 deletions(-) create mode 100644 examples/kubernetes/ci/multi-node-multi-card-lora-clm-values.yaml create mode 100644 examples/kubernetes/templates/mpijob-helm.yaml diff --git a/examples/kubernetes/Chart.yaml b/examples/kubernetes/Chart.yaml index dc0400ccb0..7d892c0e30 100644 --- a/examples/kubernetes/Chart.yaml +++ b/examples/kubernetes/Chart.yaml @@ -1,12 +1,27 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + apiVersion: v2 -name: optimum-habana-example-chart -description: This Helm chart deploys example jobs using Optimum for Intel® Gaudi® Accelerators to a Kubernetes cluster. +name: helm-training-demo-chart +description: This Helm chart deploys a multi-node, multi-card helm job on a Kubernetes cluster. # Compatible Kubernetes versions -kubeVersion: 1.27-1.29 +kubeVersion: v1.28.7 # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) version: 0.1.0 - diff --git a/examples/kubernetes/ci/multi-node-multi-card-lora-clm-values.yaml b/examples/kubernetes/ci/multi-node-multi-card-lora-clm-values.yaml new file mode 100644 index 0000000000..22d9fad7a5 --- /dev/null +++ b/examples/kubernetes/ci/multi-node-multi-card-lora-clm-values.yaml @@ -0,0 +1,142 @@ +# Default values for examples. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + # -- Determines when the kubelet will pull the image to the worker nodes. Choose from: `IfNotPresent`, `Always`, or `Never`. If updates to the image have been made, use `Always` to ensure the newest image is used. + pullPolicy: Always + cleanPodPolicy: Running + # -- Repository and name of the docker image + repository: + # -- Tag of the docker image + tag: + +imagePullSecrets: [] + +# # -- Pod [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) to attach metadata to the job +podAnnotations: {} + +# # -- Specify a pod security context to run as a non-root user +# podSecurityContext: +# fsGroup: 1000 + +# securityContext: +# # -- Run as privileged or unprivileged. Certain deployments may require running as privileged, check with your system admin. + privileged: false + +# -- The default 64MB of shared memory for docker containers can be insufficient when using more than one HPU. Setting hostIPC: true allows reusing the host's shared memory space inside the container. +hostIPC: true + +# -- Define a config map's data as container environment variables +envFrom: [] + +# -- Define environment variables to set in the container +env: +- name: LOGLEVEL + value: INFO + +secret: + # -- Hugging Face token encoded using base64. + encodedToken: + # -- If a token is provided, specify a mount path that will be used to set HF_TOKEN_PATH + secretMountPath: /tmp/hf_token + +storage: + # -- Name of the storage class to use for the persistent volume claim. To list the available storage classes use: `kubectl get storageclass`. + storageClassName: csi-wekafs-fs + # -- [Access modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) for the persistent volume. + accessModes: + - "ReadWriteMany" + # -- Storage [resources](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#resources) + resources: + requests: + storage: 30Gi + # -- Locaton where the PVC will be mounted in the pods + pvcMountPath: &pvcMountPath /tmp/pvc-mount + # -- A data access pod will be deployed when set to true + deployDataAccessPod: false + +resources: + limits: + # -- Specify the number of Gaudi card(s) + cpu: 16 + habana.ai/gaudi: 2 + # -- Specify [Memory limits](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory) requests for the job + memory: 64Gi + # -- Specify hugepages-2Mi requests for the job + hugepages-2Mi: 4400Mi + requests: + # -- Specify the number of Gaudi card(s) + cpu: 16 + habana.ai/gaudi: 2 + # -- Specify [Memory resource](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory) requests for the job + memory: 64Gi + # -- Specify hugepages-2Mi requests for the job + hugepages-2Mi: 4400Mi + + +# -- Number of Gaudi nodes to be used +numNodes: 2 +# -- Number of Gaudi cards to be used per one node +numCards: 1 +# -- Number of slots per worker +slotsPerWorker: 1 + + +# Define the command to run in the container +command: +# python command to supply mpi run commands: + - python + - /optimum-habana/examples/language-modeling/run_lora_clm.py + - --model_name_or_path + - huggyllama/llama-7b + - --dataset_name + - tatsu-lab/alpaca + - --bf16 + - --output_dir + - *pvcMountPath + - --num_train_epochs + - "3" + - --per_device_train_batch_size + - "12" + - --evaluation_strategy + - "no" + - --save_strategy + - "no" + - --learning_rate + - "1e-4" + - --warmup_ratio + - "0.03" + - --lr_scheduler_type + - "constant" + - --max_grad_norm + - "0.3" + - --logging_steps + - "1" + - --do_train + - --do_eval + - --use_habana + - --use_lazy_mode + - --throughput_warmup_steps + - "3" + - --lora_rank + - "8" + - --lora_alpha=16 + - --lora_dropout=0.05 + - --lora_target_modules + - "q_proj" + - "v_proj" + - --dataset_concatenation + - --max_seq_length=512 + - --low_cpu_mem_usage=True + - --validation_split_percentage=4 + - --adam_epsilon=1e-08 + +# # -- Optionally specify a [node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) with labels the determine which node your worker pod will land on +nodeSelector: {} + +# # -- Optionally specify [tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) to allow the worker pod to land on a node with a taint. +tolerations: [] + +# # -- Optionally provide node [affinities](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) to constrain which node your worker pod will be scheduled on +affinity: {} diff --git a/examples/kubernetes/templates/mpijob-helm.yaml b/examples/kubernetes/templates/mpijob-helm.yaml new file mode 100644 index 0000000000..33d3da7544 --- /dev/null +++ b/examples/kubernetes/templates/mpijob-helm.yaml @@ -0,0 +1,72 @@ +{{- if and .Values.numNodes (gt (int .Values.numNodes) 1) }} +apiVersion: kubeflow.org/v2beta1 +kind: MPIJob +metadata: + name: {{ .Release.Name }}-mpijob +spec: + slotsPerWorker: {{ .Values.slotsPerWorker }} + runPolicy: + cleanPodPolicy: {{ .Values.image.cleanPodPolicy }} + mpiReplicaSpecs: + Launcher: + replicas: 1 + template: + spec: + hostIPC: {{ .Values.hostIPC }} + containers: + - name: {{ .Release.Name }}-mpijob-container + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/bin/bash", "-c"] + args: + - >- + /usr/bin/ssh-keygen -A; + /usr/sbin/sshd; + HOSTSFILE=$OMPI_MCA_orte_default_hostfile; + MASTER_ADDR="$(head -n 1 $HOSTSFILE | sed -n s/[[:space:]]slots.*//p)"; + echo $MASTER_ADDR; + NUM_NODES=$(wc -l < $HOSTSFILE); + CARDS_PER_NODE={{ .Values.numCards }}; + N_CARDS=$((NUM_NODES*CARDS_PER_NODE)); + + MODEL_PATH=/optimum-habana/examples/language-modeling; + cd $MODEL_PATH; + mpirun -np $N_CARDS --npernode $CARDS_PER_NODE \ + --allow-run-as-root \ + --bind-to core \ + --map-by ppr:$CARDS_PER_NODE:node:PE=6 \ + -rank-by core --report-bindings \ + --tag-output \ + --merge-stderr-to-stdout --prefix $MPI_ROOT \ + -x MASTER_ADDR=$MASTER_ADDR \ + -mca btl_tcp_if_include eth0 \ + -mca oob_tcp_if_include eth0 \ + -mca plm_rsh_no_tree_spawn 1 \ + {{ .Values.command | join " " }}; + resources: + limits: + cpu: 16 + memory: 64Gi + hugepages-2Mi: 4400Mi + requests: + cpu: 16 + memory: 64Gi + hugepages-2Mi: 4400Mi + Worker: + replicas: {{ .Values.numNodes }} + template: + spec: + hostIPC: {{ .Values.hostIPC }} + containers: + - name: {{ .Release.Name }}-mpijob-container + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/bin/bash", "-c"] + args: + - >- + /usr/bin/ssh-keygen -A; + /usr/sbin/sshd; + sleep 365d; + resources: + {{- toYaml .Values.resources | nindent 16 }} +{{- end }} From e38c59e281eb62485e0931a9397850e0c4e08bc5 Mon Sep 17 00:00:00 2001 From: Srikanth Ramakrishna Date: Thu, 16 Jan 2025 18:57:42 +0000 Subject: [PATCH 2/8] change only kubeversion --- examples/kubernetes/Chart.yaml | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/examples/kubernetes/Chart.yaml b/examples/kubernetes/Chart.yaml index 7d892c0e30..cb7fd22653 100644 --- a/examples/kubernetes/Chart.yaml +++ b/examples/kubernetes/Chart.yaml @@ -1,22 +1,6 @@ -# Copyright (c) 2024 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 - apiVersion: v2 -name: helm-training-demo-chart -description: This Helm chart deploys a multi-node, multi-card helm job on a Kubernetes cluster. +name: optimum-habana-example-chart +description: This Helm chart deploys example jobs using Optimum for Intel® Gaudi® Accelerators to a Kubernetes # Compatible Kubernetes versions kubeVersion: v1.28.7 From 8754c17e2e948bec36d51479a2711ff8b36d1c25 Mon Sep 17 00:00:00 2001 From: Srikanth Ramakrishna Date: Thu, 16 Jan 2025 18:58:27 +0000 Subject: [PATCH 3/8] add cluster --- examples/kubernetes/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kubernetes/Chart.yaml b/examples/kubernetes/Chart.yaml index cb7fd22653..622de3af48 100644 --- a/examples/kubernetes/Chart.yaml +++ b/examples/kubernetes/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: optimum-habana-example-chart -description: This Helm chart deploys example jobs using Optimum for Intel® Gaudi® Accelerators to a Kubernetes +description: This Helm chart deploys example jobs using Optimum for Intel® Gaudi® Accelerators to a Kubernetes cluster # Compatible Kubernetes versions kubeVersion: v1.28.7 From a35e643943008ef52d93656a932447ad1f5ba234 Mon Sep 17 00:00:00 2001 From: Srikanth Ramakrishna Date: Fri, 17 Jan 2025 18:24:59 +0000 Subject: [PATCH 4/8] install pre-requisites --- .../kubernetes/templates/mpijob-helm.yaml | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/examples/kubernetes/templates/mpijob-helm.yaml b/examples/kubernetes/templates/mpijob-helm.yaml index 33d3da7544..0b0f59caca 100644 --- a/examples/kubernetes/templates/mpijob-helm.yaml +++ b/examples/kubernetes/templates/mpijob-helm.yaml @@ -28,7 +28,26 @@ spec: NUM_NODES=$(wc -l < $HOSTSFILE); CARDS_PER_NODE={{ .Values.numCards }}; N_CARDS=$((NUM_NODES*CARDS_PER_NODE)); - + + SETUP_CMD="git clone --single-branch --branch v1.15.0 https://github.com/huggingface/optimum-habana.git; \ + pip install -r optimum-habana/examples/language-modeling/requirements.txt; + + eval $SETUP_CMD; + + mpirun --npernode 1 \ + --tag-output \ + --allow-run-as-root \ + --prefix $MPI_ROOT \ + -mca routed direct \ + git clone --single-branch --branch v1.15.0 https://github.com/huggingface/optimum-habana.git; + + mpirun --npernode 1 \ + --tag-output \ + --allow-run-as-root \ + --prefix $MPI_ROOT \ + -mca routed direct \ + pip install -r optimum-habana/examples/language-modeling/requirements.txt; + MODEL_PATH=/optimum-habana/examples/language-modeling; cd $MODEL_PATH; mpirun -np $N_CARDS --npernode $CARDS_PER_NODE \ From 31dc74a55ea94bb53e18e20010494b831eb2a7b3 Mon Sep 17 00:00:00 2001 From: Srikanth Ramakrishna Date: Fri, 17 Jan 2025 18:41:02 +0000 Subject: [PATCH 5/8] update readme --- examples/kubernetes/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md index 06f4f01d09..03b5e073d9 100644 --- a/examples/kubernetes/README.md +++ b/examples/kubernetes/README.md @@ -136,6 +136,7 @@ Validated use cases can be found in the `ci` directory: | [`ci/multi-card-glue-values.yaml`](ci/multi-card-glue-values.yaml) | 2 | Uses 2 HPUs from a single node with the [`gaudi_spawn.py`](../gaudi_spawn.py) script to [fine tune BERT large](../text-classification/README.md#multi-card-training) (with whole word masking) on the text classification MRPC task using `run_glue.py`. | [`ci/single-card-lora-clm-values.yaml`](ci/single-card-lora-clm-values.yaml) | 1 | Uses a single card to [fine tune Llama1-7B](../language-modeling/README.md#peft) with LoRA using the `run_lora_clm.py` script. | [`ci/multi-card-lora-clm-values.yaml`](ci/multi-card-lora-clm-values.yaml) | 8 | Uses 8 HPUs from a single node with the [`gaudi_spawn.py`](../gaudi_spawn.py) script to [fine tune Llama1-7B](../language-modeling/README.md#peft) with LoRA using the `run_lora_clm.py` script. +| [`ci/multi-card-lora-clm-values.yaml`](ci/multi-card-lora-clm-values.yaml) | 8 | Uses 8 HPUs from a single node with the [`gaudi_spawn.py`](../gaudi_spawn.py) script to [fine tune Llama1-7B](../language-modeling/README.md#peft) with LoRA using the `run_lora_clm.py` script. ### Deploy job to the cluster From b7ee19487df2dae373f4ead7156be56b6d0964f4 Mon Sep 17 00:00:00 2001 From: Srikanth Ramakrishna Date: Fri, 17 Jan 2025 18:43:35 +0000 Subject: [PATCH 6/8] restore chart description --- examples/kubernetes/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kubernetes/Chart.yaml b/examples/kubernetes/Chart.yaml index 622de3af48..4f134e6eaa 100644 --- a/examples/kubernetes/Chart.yaml +++ b/examples/kubernetes/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: optimum-habana-example-chart -description: This Helm chart deploys example jobs using Optimum for Intel® Gaudi® Accelerators to a Kubernetes cluster +description: This Helm chart deploys example jobs using Optimum for Intel® Gaudi® Accelerators to a Kubernetes cluster. # Compatible Kubernetes versions kubeVersion: v1.28.7 From 0297a75524a25bb5531c044f7ce9cd9662c62e13 Mon Sep 17 00:00:00 2001 From: Srikanth Ramakrishna Date: Fri, 17 Jan 2025 22:13:58 +0000 Subject: [PATCH 7/8] update based on review comments --- examples/kubernetes/README.md | 2 +- .../kubernetes/ci/multi-node-multi-card-lora-clm-values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/kubernetes/README.md b/examples/kubernetes/README.md index 03b5e073d9..dd7d13fca2 100644 --- a/examples/kubernetes/README.md +++ b/examples/kubernetes/README.md @@ -136,7 +136,7 @@ Validated use cases can be found in the `ci` directory: | [`ci/multi-card-glue-values.yaml`](ci/multi-card-glue-values.yaml) | 2 | Uses 2 HPUs from a single node with the [`gaudi_spawn.py`](../gaudi_spawn.py) script to [fine tune BERT large](../text-classification/README.md#multi-card-training) (with whole word masking) on the text classification MRPC task using `run_glue.py`. | [`ci/single-card-lora-clm-values.yaml`](ci/single-card-lora-clm-values.yaml) | 1 | Uses a single card to [fine tune Llama1-7B](../language-modeling/README.md#peft) with LoRA using the `run_lora_clm.py` script. | [`ci/multi-card-lora-clm-values.yaml`](ci/multi-card-lora-clm-values.yaml) | 8 | Uses 8 HPUs from a single node with the [`gaudi_spawn.py`](../gaudi_spawn.py) script to [fine tune Llama1-7B](../language-modeling/README.md#peft) with LoRA using the `run_lora_clm.py` script. -| [`ci/multi-card-lora-clm-values.yaml`](ci/multi-card-lora-clm-values.yaml) | 8 | Uses 8 HPUs from a single node with the [`gaudi_spawn.py`](../gaudi_spawn.py) script to [fine tune Llama1-7B](../language-modeling/README.md#peft) with LoRA using the `run_lora_clm.py` script. +| [`ci/multi-node-multi-card-lora-clm-values.yaml`](ci/multi-node-multi-card-lora-clm-values.yaml) | 2 | Uses 1 HPU each from two nodes with the [`gaudi_spawn.py`](../gaudi_spawn.py) script to [fine tune Llama1-7B](../language-modeling/README.md#peft) with LoRA using the `run_lora_clm.py` script. ### Deploy job to the cluster diff --git a/examples/kubernetes/ci/multi-node-multi-card-lora-clm-values.yaml b/examples/kubernetes/ci/multi-node-multi-card-lora-clm-values.yaml index 22d9fad7a5..3b223c1add 100644 --- a/examples/kubernetes/ci/multi-node-multi-card-lora-clm-values.yaml +++ b/examples/kubernetes/ci/multi-node-multi-card-lora-clm-values.yaml @@ -43,7 +43,7 @@ secret: storage: # -- Name of the storage class to use for the persistent volume claim. To list the available storage classes use: `kubectl get storageclass`. - storageClassName: csi-wekafs-fs + storageClassName: nfs-client # -- [Access modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) for the persistent volume. accessModes: - "ReadWriteMany" From ebbe21a6b0c6449c0ae506f36dcf1d03c209c652 Mon Sep 17 00:00:00 2001 From: Srikanth Ramakrishna Date: Tue, 21 Jan 2025 15:10:07 +0000 Subject: [PATCH 8/8] comment security context --- .../kubernetes/ci/multi-node-multi-card-lora-clm-values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/kubernetes/ci/multi-node-multi-card-lora-clm-values.yaml b/examples/kubernetes/ci/multi-node-multi-card-lora-clm-values.yaml index 3b223c1add..94dd05de87 100644 --- a/examples/kubernetes/ci/multi-node-multi-card-lora-clm-values.yaml +++ b/examples/kubernetes/ci/multi-node-multi-card-lora-clm-values.yaml @@ -22,7 +22,7 @@ podAnnotations: {} # securityContext: # # -- Run as privileged or unprivileged. Certain deployments may require running as privileged, check with your system admin. - privileged: false + # privileged: false # -- The default 64MB of shared memory for docker containers can be insufficient when using more than one HPU. Setting hostIPC: true allows reusing the host's shared memory space inside the container. hostIPC: true