Skip to content

Commit

Permalink
Merge pull request #59 from Azure/main-dec31
Browse files Browse the repository at this point in the history
Merge main-dec31 into main
  • Loading branch information
sdonohoo authored Jan 5, 2023
2 parents c91788c + 8757b00 commit 664496b
Show file tree
Hide file tree
Showing 52 changed files with 1,235 additions and 140 deletions.
42 changes: 22 additions & 20 deletions .github/workflows/create-compute.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ on:
required: true
type: string
cluster_tier:
required: true
required: false
type: string
secrets:
creds:
Expand All @@ -40,24 +40,26 @@ jobs:
run: az extension add -n ml -y
- name: update-extension
run: az extension update -n ml
- name: create-compute
- name: create-compute-default-tier
if: ${{ inputs.cluster_tier == ''}}
run: |
az ml compute create --name ${{ inputs.cluster_name }} \
--type AmlCompute \
--tier "dedicated" \
--size ${{ inputs.size }} \
--min-instances ${{ inputs.min_instances }} \
--max-instances ${{ inputs.max_instances }} \
--resource-group ${{ inputs.resource_group }} \
--workspace-name ${{ inputs.workspace_name }}
- name: create-compute-using-tier
if: ${{ inputs.cluster_tier != ''}}
run: |
compute_name=$(az ml compute show -n ${{ inputs.cluster_name }}\
az ml compute create --name ${{ inputs.cluster_name }} \
--type AmlCompute \
--tier ${{ inputs.cluster_tier }} \
--size ${{ inputs.size }} \
--min-instances ${{ inputs.min_instances }} \
--max-instances ${{ inputs.max_instances }} \
--resource-group ${{ inputs.resource_group }} \
--workspace-name ${{ inputs.workspace_name }} \
--query name -o tsv)
if [[ -z "$compute_name" ]]
then
echo "Compute does not exist. Creating the cluster..."
az ml compute create --name ${{ inputs.cluster_name }} \
--type amlcompute \
--size ${{ inputs.size }} \
--min-instances ${{ inputs.min_instances }} \
--max-instances ${{ inputs.max_instances }} \
--tier ${{ inputs.cluster_tier }} \
--resource-group ${{ inputs.resource_group }} \
--workspace-name ${{ inputs.workspace_name }}
else
echo "Compute exists. Skipping cluster creation."
exit 0
fi
--workspace-name ${{ inputs.workspace_name }}
88 changes: 70 additions & 18 deletions .github/workflows/read-yaml.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,30 @@
name: read-yaml

on:
on:
workflow_call:
inputs:
file_name:
required: true
type: string
outputs:
location:
description: "The resource group to use"
value: ${{ jobs.read-yaml.outputs.location }}
namespace:
description: "Namespace of the resources"
value: ${{ jobs.read-yaml.outputs.namespace }}
postfix:
description: "Postfix for the resource names"
value: ${{ jobs.read-yaml.outputs.postfix }}
environment:
description: "Environment"
value: ${{ jobs.read-yaml.outputs.environment }}
enable_aml_computecluster:
description: "Boolean to decide whether to deploy an AML compute cluster or not"
value: ${{ jobs.read-yaml.outputs.enable_aml_computecluster }}
enable_monitoring:
description: "Boolean to decide whether to enable monitoring"
value: ${{ jobs.read-yaml.outputs.enable_monitoring }}
resource_group:
description: "The resource group to use"
value: ${{ jobs.read-yaml.outputs.resource_group }}
Expand All @@ -19,21 +37,55 @@ on:
oep:
description: "The online endpoint"
value: ${{ jobs.read-yaml.outputs.oep }}
terraform_version:
description: "Terraform version"
value: ${{ jobs.read-yaml.outputs.terraform_version }}
terraform_workingdir:
description: "Terraform State working directory"
value: ${{ jobs.read-yaml.outputs.terraform_workingdir }}
terraform_st_location:
description: "Terraform State location"
value: ${{ jobs.read-yaml.outputs.terraform_st_location }}
terraform_st_resource_group:
description: "Terraform State Resource Group"
value: ${{ jobs.read-yaml.outputs.terraform_st_resource_group }}
terraform_st_storage_account:
description: "Terraform State Storage Account"
value: ${{ jobs.read-yaml.outputs.terraform_st_storage_account }}
terraform_st_container_name:
description: "Terraform State Container name"
value: ${{ jobs.read-yaml.outputs.terraform_st_container_name }}
terraform_st_key:
description: "Terraform State Key"
value: ${{ jobs.read-yaml.outputs.terraform_st_key }}
jobs:
read-yaml:
runs-on: ubuntu-latest
outputs:
resource_group: ${{ steps.read_action_js.outputs.resource_group }}
aml_workspace: ${{ steps.read_action_js.outputs.aml_workspace }}
bep: ${{ steps.read_action_js.outputs.bep }}
oep: ${{ steps.read_action_js.outputs.oep }}
steps:
- name: Checking out repo
uses: actions/checkout@v2
- name: Read YAML
uses: Azure/mlops-templates/read_yaml_action@main
id: read_action_js
with:
config: ${{ github.workspace }}/${{ inputs.file_name }}
- name: Echo step output
run: echo "${{ steps.read_action_js.outputs.resource_group }} ${{ steps.read_action_js.outputs.aml_workspace }} ${{ steps.read_action_js.outputs.bep }} ${{ steps.read_action_js.outputs.oep }} "
read-yaml:
runs-on: ubuntu-latest
outputs:
location: ${{ steps.read_action_js.outputs.location }}
namespace: ${{ steps.read_action_js.outputs.namespace }}
postfix: ${{ steps.read_action_js.outputs.postfix }}
environment: ${{ steps.read_action_js.outputs.environment }}
enable_aml_computecluster: ${{ steps.read_action_js.outputs.enable_aml_computecluster }}
enable_monitoring: ${{ steps.read_action_js.outputs.enable_monitoring }}
resource_group: ${{ steps.read_action_js.outputs.resource_group }}
aml_workspace: ${{ steps.read_action_js.outputs.aml_workspace }}
bep: ${{ steps.read_action_js.outputs.bep }}
oep: ${{ steps.read_action_js.outputs.oep }}
terraform_version: ${{ steps.read_action_js.outputs.terraform_version }}
terraform_workingdir: ${{ steps.read_action_js.outputs.terraform_workingdir }}
terraform_st_location: ${{ steps.read_action_js.outputs.terraform_st_location }}
terraform_st_resource_group: ${{ steps.read_action_js.outputs.terraform_st_resource_group }}
terraform_st_storage_account: ${{ steps.read_action_js.outputs.terraform_st_storage_account }}
terraform_st_container_name: ${{ steps.read_action_js.outputs.terraform_st_container_name }}
terraform_st_key: ${{ steps.read_action_js.outputs.terraform_st_key }}
steps:
- name: Checking out repo
uses: actions/checkout@v2
- name: Read test
uses: Azure/mlops-templates/read_yaml_action@main
id: read_action_js
with:
config: ${{ github.workspace }}/${{ inputs.file_name }}
- name: Echo step output
run: echo "${{ steps.read_action_js.outputs.resource_group }} ${{ steps.read_action_js.outputs.aml_workspace }}"; echo "${{ steps.read_action_js.outputs.bep }} ${{ steps.read_action_js.outputs.oep }}"; echo "${{ steps.read_action_js.outputs.terraform_version }} ${{ steps.read_action_js.outputs.terraform_workingdir }}"; echo "${{ steps.read_action_js.outputs.terraform_st_resource_group }} ${{ steps.read_action_js.outputs.terraform_st_storage_account }} " echo "${{ steps.read_action_js.outputs.terraform_st_container_name }} ${{ steps.read_action_js.outputs.terraform_st_key }} "
9 changes: 9 additions & 0 deletions .github/workflows/register-environment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ on:
required: false
default: ''
type: string
dockerfile-location:
required: false
default: ''
type: string
secrets:
creds:
required: true
Expand Down Expand Up @@ -46,3 +50,8 @@ jobs:
run: |
az ml environment create --file ${{ github.workspace }}/${{ inputs.environment_file }} --resource-group ${{ inputs.resource_group }} \
--workspace-name ${{ inputs.workspace_name }}
- name: create-environment-from-image
if: ${{ inputs.dockerfile-location != ''}}
run: |
az ml environment create --file ${{ github.workspace }}/${{ inputs.environment_file }} --resource-group ${{ inputs.resource_group }} \
--workspace-name ${{ inputs.workspace_name }} --build-context ${{ github.workspace }}/${{ inputs.dockerfile-location }}
195 changes: 195 additions & 0 deletions .github/workflows/tf-gha-install-terraform.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
name: tf-gha-install-terraform.yml

on:
workflow_call:
inputs:
TFAction:
type: string
description: 'Terraform Action- Apply or Destroy'
required: true
default: 'apply'
dply_environment:
type: string
required: true
location:
description: 'Location'
type: string
required: true
namespace:
description: 'Namespace'
type: string
required: true
postfix:
description: 'postfix'
type: string
required: true
environment:
description: 'Enviroment of the AML workspace'
type: string
required: true
enable_aml_computecluster:
description: 'Boolean to decide whether to deploy an AML compute cluster or not'
type: boolean
required: true
enable_monitoring:
description: 'Boolean to decide whether to enable monitoring'
type: boolean
required: true
terraform_version:
description: 'Terraform version'
type: string
required: true
terraform_workingdir:
description: 'Terraform State working directory'
type: string
required: true
terraform_st_location:
description: 'Terraform version'
type: string
required: true
terraform_st_storage_account:
type: string
description: 'name of the terraform state storage account to create'
required: true
terraform_st_resource_group:
type: string
description: 'name of the terraform resource group to host the terraform state resources'
required: true
terraform_st_container_name:
type: string
description: 'name of the terraform state container'
required: true
terraform_st_key:
type: string
description: 'name of the terraform state key'
required: true
terraform_plan_location:
type: string
description: 'location of Terraform plan'
required: true
terraform_plan_vnet:
type: string
description: 'vnet of Terraform plan'
required: true
secrets:
azure_creds:
required: true
clientId:
required: true
clientSecret:
required: true
tenantId:
required: true
subscriptionId:
required: true
jobs:
create-tfstate-resources:
name: Create Resources for Terraform State
runs-on: ubuntu-latest
defaults:
run:
shell: bash
steps:
- name: azure-login
uses: azure/login@v1
with:
creds: ${{ env.AZURE_CREDENTIALS }}
# environment: 'TBD' - default to azurecloud
env:
AZURE_CREDENTIALS: ${{ secrets.azure_creds }}
- name: create-tfstate-resource-group
uses: azure/CLI@v1
with:
azcliversion: 2.30.0
#
inlineScript: |
az account set -s ${{ env.ARM_SUBSCRIPTION_ID }}
az group create --location ${{ inputs.terraform_st_location }} --name ${{ inputs.terraform_st_resource_group }}
env:
ARM_CLIENT_ID: ${{ secrets.clientId }}
ARM_CLIENT_SECRET: ${{ secrets.clientSecret }}
ARM_SUBSCRIPTION_ID: ${{ secrets.subscriptionId }}
ARM_TENANT_ID: ${{ secrets.tenantId }}
- name: create-tfstate-storage-account
uses: azure/CLI@v1
with:
azcliversion: 2.30.0
#
inlineScript: |
az account set -s ${{ env.ARM_SUBSCRIPTION_ID }}
az storage account create -n ${{ inputs.terraform_st_storage_account }} -g ${{ inputs.terraform_st_resource_group }} -l ${{ inputs.terraform_st_location }} --sku Standard_LRS
env:
ARM_CLIENT_ID: ${{ secrets.clientId }}
ARM_CLIENT_SECRET: ${{ secrets.clientSecret }}
ARM_SUBSCRIPTION_ID: ${{ secrets.subscriptionId }}
ARM_TENANT_ID: ${{ secrets.tenantId }}
- name: create-tfstate-storage-container
uses: azure/CLI@v1
with:
azcliversion: 2.30.0
#
inlineScript: |
az account set -s ${{ env.ARM_SUBSCRIPTION_ID }}
az storage container create --account-name ${{ inputs.terraform_st_storage_account }} --name ${{ inputs.terraform_st_container_name }}
env:
ARM_CLIENT_ID: ${{ secrets.clientId }}
ARM_CLIENT_SECRET: ${{ secrets.clientSecret }}
ARM_SUBSCRIPTION_ID: ${{ secrets.subscriptionId }}
ARM_TENANT_ID: ${{ secrets.tenantId }}
install-terraform:
name: 'Install Terraform'
runs-on: ubuntu-latest
needs: create-tfstate-resources
environment: ${{ inputs.dply_environment }}
defaults:
run:
shell: bash
steps:
- name: checkout
uses: actions/checkout@v3
- name: Setup Terraform
uses: hashicorp/setup-terraform@v2
- name: Terraform Init
run: echo ${{ env.ARM_SUBSCRIPTION_ID }};
echo "all env:${{ env.ARM_CLIENT_ID }}::${{ env.ARM_SUBSCRIPTION_ID }}";
terraform -chdir=${{ inputs.terraform_workingdir }} init
-backend-config='storage_account_name=${{ inputs.terraform_st_storage_account }}'
-backend-config='container_name=${{ inputs.terraform_st_container_name }}'
-backend-config='key=${{ inputs.terraform_st_key }}'
-backend-config='resource_group_name=${{ inputs.terraform_st_resource_group }}';
env:
ARM_CLIENT_ID: ${{ secrets.clientId }}
ARM_CLIENT_SECRET: ${{ secrets.clientSecret }}
ARM_SUBSCRIPTION_ID: ${{ secrets.subscriptionId }}
ARM_TENANT_ID: ${{ secrets.tenantId }}
- name: Terraform Validate
run: terraform -chdir=${{ inputs.terraform_workingdir }} validate
env:
ARM_CLIENT_ID: ${{ secrets.clientId }}
ARM_CLIENT_SECRET: ${{ secrets.clientSecret }}
ARM_SUBSCRIPTION_ID: ${{ secrets.subscriptionId }}
ARM_TENANT_ID: ${{ secrets.tenantId }}
- name: Terraform Plan
run: terraform -chdir=${{ inputs.terraform_workingdir }} plan -out=plan.tfplan -input=false -var "location=${{ inputs.terraform_plan_location }}" -var "prefix=${{ inputs.namespace }}" -var "postfix=${{ inputs.postfix }}" -var "environment=${{ inputs.environment }}"
-var "enable_aml_computecluster=${{ inputs.enable_aml_computecluster }}" -var "enable_monitoring=${{ inputs.enable_monitoring }}" -var "client_secret=${{ env.ARM_CLIENT_SECRET }}"
env:
ARM_CLIENT_ID: ${{ secrets.clientId }}
ARM_CLIENT_SECRET: ${{ secrets.clientSecret }}
ARM_SUBSCRIPTION_ID: ${{ secrets.subscriptionId }}
ARM_TENANT_ID: ${{ secrets.tenantId }}
- name: Terraform Apply
if: ${{ inputs.TFAction == 'apply'}}
run: terraform -chdir=${{ inputs.terraform_workingdir }} apply -input=false -auto-approve plan.tfplan
env:
ARM_CLIENT_ID: ${{ secrets.clientId }}
ARM_CLIENT_SECRET: ${{ secrets.clientSecret }}
ARM_SUBSCRIPTION_ID: ${{ secrets.subscriptionId }}
ARM_TENANT_ID: ${{ secrets.tenantId }}
- name: Terraform Destroy
if: ${{ inputs.TFAction == 'destroy' }}
run: terraform -chdir=${{ inputs.terraform_workingdir }} destroy -input=false -auto-approve
env:
ARM_CLIENT_ID: ${{ secrets.clientId }}
ARM_CLIENT_SECRET: ${{ secrets.clientSecret }}
ARM_SUBSCRIPTION_ID: ${{ secrets.subscriptionId }}
ARM_TENANT_ID: ${{ secrets.tenantId }}
Loading

0 comments on commit 664496b

Please sign in to comment.