Remove virtual-env, change folder name

aws-ami-creation/hcl2-files/build.pkr.hcl → infrastructure/ami/hcl2-files/build.pkr.hcl

@@ -4,9 +4,9 @@ build {
     "source.amazon-ebs.ubuntu"
   ]
   provisioner "shell" {
-    script = "scripts/validate_neuron_exist.sh"
+    script = "scripts/setup-neuron.sh"
   }
   provisioner "shell" {
-    script = "scripts/install_huggingface_libraries.sh"
+    script = "scripts/install-huggingface-libraries.sh"
   }
 }

aws-ami-creation/hcl2-files/sources.pkr.hcl → infrastructure/ami/hcl2-files/sources.pkr.hcl

@@ -1,13 +1,13 @@
 source "amazon-ebs" "ubuntu" {
-  ami_name      = "huggingface-neuron-dl-ami-ubuntu-22.04_{{timestamp}}"
+  ami_name      = "huggingface-neuron-dl-ami-ubuntu-20.04_{{timestamp}}"
   instance_type = var.instance_type
   region        = var.region
   source_ami    = var.source_ami
   ssh_username  = var.ssh_username
   launch_block_device_mappings {
     device_name           = "/dev/sda1"
     encrypted             = true
-    volume_size           = 100
+    volume_size           = 512
     volume_type           = "gp2"
     delete_on_termination = true
   }

aws-ami-creation/hcl2-files/variables.pkr.hcl → infrastructure/ami/hcl2-files/variables.pkr.hcl

@@ -11,7 +11,7 @@ variable "instance_type" {
 }
 
 variable "source_ami" {
-  default     = "ami-0328507805a665ec1"
+  default     = "ami-077399be2d0ae7e0c"
   description = "Base Image"
   type        = string
 }

aws-ami-creation/scripts/install_huggingface_libraries.sh → infrastructure/ami/scripts/install-hugginface-libraries.sh

@@ -2,10 +2,7 @@
 
 echo "Step: install-hugging-face-libraries"
 
-echo "Activating the virtual-env"
-source /opt/aws_neuron_venv_pytorch/bin/activate
-
-pip install --upgrade --no-cache-dir \
+sudo -H -u ubuntu bash -c 'pip install --upgrade --no-cache-dir \
     "transformers[sklearn,sentencepiece,vision]==4.36.2" \
     "optimum-neuron==0.0.17" \
     "datasets==2.16.1" \
@@ -16,22 +13,23 @@ pip install --upgrade --no-cache-dir \
     "notebook==7.0.6" \
     "markupsafe==2.1.1" \
     "jinja2==3.1.2" \
-    "attrs==23.1.0"
+    "attrs==23.1.0"'
 
 echo 'export PATH="${HOME}/.local/bin:$PATH"' >> "${HOME}/.bashrc"
 
 
 echo "Step: copy-optimum-neuron-examples"
 git clone -b "v0.0.17" https://github.com/huggingface/optimum-neuron.git
-mkdir /home/ubuntu/huggingface-neuron-samples/
+mkdir /home/ubuntu/huggingface-neuron-samples/ /home/ubuntu/huggingface-neuron-notebooks/
 mv optimum-neuron/examples/* /home/ubuntu/huggingface-neuron-samples/
+mv optimum-neuron/notebooks/* /home/ubuntu/huggingface-neuron-notebooks/
 rm -rf optimum-neuron
-chown -R ubuntu:ubuntu /home/ubuntu/huggingface-neuron-samples
+chown -R ubuntu:ubuntu /home/ubuntu/huggingface-neuron-samples /home/ubuntu/huggingface-neuron-notebooks
 
 
 echo "Step: clean-apt-cache"
 sudo apt autoremove -y
 sudo apt clean -y
 
 echo "Step: validate-imports-of-huggingface-libraries"
-python -c 'import transformers;import datasets;import accelerate;import evaluate;import tensorboard;'
+sudo -H -u ubuntu bash -c 'python -c "import transformers;import datasets;import accelerate;import evaluate;import tensorboard; import torch;"'

infrastructure/ami/scripts/setup-neuron.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+echo "Step: validate-neuron-devices"
+neuron-ls
+
+echo "Step: install-ubuntu-packages"
+sudo apt-get update
+sudo apt-get -y upgrade --only-upgrade systemd
+sudo apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    cmake \
+    curl \
+    emacs \
+    git \
+    git-lfs \
+    jq \
+    software-properties-common \
+    unzip \
+    vim \
+    wget \
+    zlib1g-dev \
+    libsndfile1-dev \
+    ffmpeg \
+    g++
+
+sudo apt-get update # Update to fix broken packages
+sudo apt-get install -y --no-install-recommends python-is-python3 python3-pip python3-dev python3-virtualenv
+
+echo "Step: install-neuron-python-packages"
+sudo -H -u ubuntu bash -c 'pip install --upgrade --no-cache-dir \
+    --extra-index-url https://pip.repos.neuron.amazonaws.com \
+    "protobuf==3.20.2" \
+    torch-neuronx=="1.13.1.1.13.0" \
+    "transformers-neuronx==0.9.474" \
+    "neuronx_distributed==0.6.0" \
+    "tensorboard-plugin-neuronx" \
+    "torchvision==0.14.*"'
+
+sudo -H -u ubuntu bash -c 'echo 'export PATH="${HOME}/.local/bin:$PATH"' >> ${HOME}/.bashrc'
+sudo -H -u ubuntu bash -c 'python -c "import torch_neuronx"'