Remove files, add tag as parameter

.github/workflows/build-ami.yml

@@ -6,10 +6,16 @@ on:
         - main
       paths:
         - 'aws-ami-creation/**'
+    pull_request:
+      branches:
+        - main
+      paths:
+        - 'aws-ami-creation/**'
   workflow_dispatch:
     inputs: 
       tag: 
-        description: 'Tag to use for the AMI'
+        description: 'Tag to use for the AMI build'
+        default: 'main'
   schedule:
     # Schedule the workflow to run every second day at midnight UTC
     - cron: '0 0 */2 * *'
@@ -21,42 +27,38 @@ jobs:
             working-directory: infrastructure/ami
         runs-on: ubuntu-latest
         steps:
+          - name: Checkout
+            uses: actions/checkout@v3
+            with: 
+              ref: ${{ github.event.inputs.tag}}
+
+          - name: Setup Packer
+          - uses: hashicorp/setup-packer@main
 
-        - uses: actions/checkout@v3
-        - uses: hashicorp/setup-packer@main
-
-        - name: Determine tag
-          id: determine-tag
-          run: echo "::set-output name=TAG::${{ github.event.inputs.tag || github.ref }}"
-
-        - name: Use specific tag
-          run: echo "Building tag ${{ steps.determine-tag.outputs.TAG }}"
-
-        - name: configure aws credentials
-          uses: aws-actions/configure-aws-credentials@v1
-          with:
-            aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-            aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-            aws-region: us-east-1
-
-        - name: Packer format
-          id: format
-          run: packer fmt hcl2-files
-          continue-on-error: true
-
-        - name: Packer Init
-          id: init
-          run: packer init hcl2-files
-          continue-on-error: true
-
-        - name: Packer Validate
-          id: validate
-          run: packer validate hcl2-files
-          continue-on-error: true
-
-        - name: Packer Build
-          id: build
-          run:  | 
-            SOURCE_AMI=ami-077399be2d0ae7e0c
-            packer build -var "source_ami=$SOURCE_AMI" -var "tag_name=${{ steps.determine-tag.outputs.TAG }}" hcl2-files
+          - name: configure aws credentials
+            uses: aws-actions/configure-aws-credentials@v1
+            with:
+              aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+              aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+              aws-region: us-east-1
+
+          - name: Packer format
+            id: format
+            run: packer fmt hcl2-files
+            continue-on-error: true
+
+          - name: Packer Init
+            id: init
+            run: packer init hcl2-files
+            continue-on-error: true
+
+          - name: Packer Validate
+            id: validate
+            run: packer validate hcl2-files
+            continue-on-error: true
+
+          - name: Packer Build
+            id: build
+            run:  | 
+              packer build -var "optimum_neuron_tag=${{ steps.determine-tag.outputs.TAG }}" hcl2-files
         

infrastructure/ami/README.md

@@ -3,7 +3,6 @@
 This directory contains the files for building AMI using [Packer](https://github.com/hashicorp/packer) that is later published as a AWS Marketplace asset.
 
 
-
 ## Folder Structure
 
 - [hcl2-files](./hcl2-files/) - Includes different files which are used by a Packer pipeline to build an AMI. The files are:
@@ -34,11 +33,11 @@ Using Environment Variables:
 ```bash
 export AWS_ACCESS_KEY_ID=<access_key>
 export AWS_SECRET_ACCESS_KEY=<secret_key>
-````
+```
 
 Using AWS CLI:
 ```bash
-aws configure
+aws configure sso
 ```
 
 There are other ways to configure AWS credentials. You can read more about it [here](https://github.com/aws/aws-cli?tab=readme-ov-file#configuration).
@@ -62,19 +61,15 @@ packer fmt ./hcl2-files
 You can validate the syntax and configuration of your files locally. This command will return a zero exit status on success, and a non-zero exit status on failure. 
 
 ```bash
-packer validate ./hcl2-files
+packer validate -var 'region=us-west-2' -var 'optimum_neuron_version=v0.0.17' ./hcl2-files
 ```
 
 #### Run Packer build
 You can run Packer locally. This command will build the AMI and upload it to AWS.
 
+You need to set variables with no default values using `-var` flag. For example:
 ```bash
-packer build ./hcl2-files
-```
-
-You can also override variables using `-var` flag. For example, to override `region` variable:
-```bash
-packer build -var 'region=us-west-2' ./hcl2-files
+packer build -var 'region=us-west-2' -var 'optimum_neuron_version=v0.0.17' ./hcl2-files
 ```
 
 To trigger a github action workflow manually, you can use GitHub CLI:

infrastructure/ami/hcl2-files/build.pkr.hcl

@@ -4,9 +4,16 @@ build {
     "source.amazon-ebs.ubuntu"
   ]
   provisioner "shell" {
-    script = "scripts/setup-neuron.sh"
+    script = "scripts/validate-neuron.sh"
   }
   provisioner "shell" {
     script = "scripts/install-huggingface-libraries.sh"
+    environment_vars = [
+      "TRANSFORMERS_VERSION=${var.transformers_version}",
+      "OPTIMUM_NEURON_TAG=${var.optimum_neuron_tag}",
+    ]
+  }
+  provisioner "shell" {
+    inline = ["echo 'source /opt/aws_neuron_venv_pytorch/bin/activate' >> /home/ubuntu/.bashrc"]
   }
 }

infrastructure/ami/hcl2-files/sources.pkr.hcl

@@ -6,12 +6,11 @@ source "amazon-ebs" "ubuntu" {
   ssh_username  = var.ssh_username
   launch_block_device_mappings {
     device_name           = "/dev/sda1"
-    encrypted             = true
+    // encrypted             = true
     volume_size           = 512
     volume_type           = "gp2"
     delete_on_termination = true
   }
-  tags = {
-    tag_name = var.tag_name
-  }
+  ami_users   = var.ami_users
+  ami_regions = var.ami_regions
 }

infrastructure/ami/hcl2-files/variables.pkr.hcl

@@ -1,6 +1,5 @@
 variable "region" {
-  default     = "us-east-1"
-  description = "The AWS region you're using"
+  description = "The AWS region"
   type        = string
 }
 
@@ -11,7 +10,7 @@ variable "instance_type" {
 }
 
 variable "source_ami" {
-  default     = "ami-077399be2d0ae7e0c"
+  default     = "ami-0fbea04d7389bcd4e"
   description = "Base Image"
   type        = string
 }
@@ -22,8 +21,26 @@ variable "ssh_username" {
   type        = string
 }
 
-variable "tag_name" {
-  default     = "develop"
-  description = "Name tag for AMI"
+variable "optimum_neuron_tag" {
+  default     = "v0.0.17"
+  description = "Optimum Neuron version to install"
   type        = string
+}
+
+variable "transformers_version" {
+  default     = "4.36.2"
+  description = "Transformers version to install"
+  type        = string
+}
+
+variable "ami_users" {
+  default     = ["754289655784", "558105141721"]
+  description = "AWS accounts to share AMI with"
+  type        = list(string)
+}
+
+variable "ami_regions" {
+  default     = ["eu-west-1"]
+  description = "AWS regions to share AMI with"
+  type        = list(string)
 }

infrastructure/ami/scripts/install-huggingface-libraries.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Activate the neuron virtual environment
+source /opt/aws_neuron_venv_pytorch/bin/activate
+
+echo "Step: install-hugging-face-libraries"
+
+echo "TRANSFORMERS_VERSION: $TRANSFORMERS_VERSION"
+echo "OPTIMUM_NEURON_TAG: $OPTIMUM_NEURON_TAG"
+
+pip install --upgrade --no-cache-dir \
+    "transformers[sklearn,sentencepiece,vision]==$TRANSFORMERS_VERSION" \
+    "datasets==2.16.1" \
+    "accelerate==0.23.0" \
+    "diffusers==0.25.0" \
+    "evaluate==0.4.1" \
+    "requests==2.31.0" \
+    "notebook==7.0.6" \
+    "markupsafe==2.1.1" \
+    "jinja2==3.1.2" \
+    "attrs==23.1.0"
+
+echo 'export PATH="${HOME}/.local/bin:$PATH"' >> "${HOME}/.bashrc"
+
+echo "Step: install-and-copy-optimum-neuron-examples"
+git clone -b $OPTIMUM_NEURON_TAG https://github.com/huggingface/optimum-neuron.git
+
+cd optimum-neuron
+python setup.py install
+cd ..
+
+mkdir /home/ubuntu/huggingface-neuron-samples/ /home/ubuntu/huggingface-neuron-notebooks/
+mv optimum-neuron/examples/* /home/ubuntu/huggingface-neuron-samples/
+mv optimum-neuron/notebooks/* /home/ubuntu/huggingface-neuron-notebooks/
+rm -rf optimum-neuron
+chmod -R 777 /home/ubuntu/huggingface-neuron-samples /home/ubuntu/huggingface-neuron-notebooks
+
+echo "Step: validate-imports-of-huggingface-libraries"
+bash -c 'python -c "import transformers;import datasets;import accelerate;import evaluate;import tensorboard; import torch;"'

infrastructure/ami/scripts/validate-neuron.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+echo "Step: validate-neuron-devices"
+neuron-ls
+
+# Activate the neuron virtual environment
+source /opt/aws_neuron_venv_pytorch/bin/activate
+
+python -c 'import torch'
+python -c 'import torch_neuronx'
+
+echo "Installing Tensorboard Plugin for Neuron"
+pip install --upgrade --no-cache-dir \
+    "tensorboard-plugin-neuronx"