diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index c5885689f69..d1e876eae83 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -132,6 +132,7 @@ steps: SIG_IMAGE_NAME="$(cat vhdbuilder/packer/settings.json | grep "sig_image_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \ BUILD_LOCATION="$(cat vhdbuilder/packer/settings.json | grep "location" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \ ARTIFACT_ID=$(date +%s) && \ + echo "##vso[task.setvariable variable=DESTINATION_STORAGE_CONTAINER]${CLASSIC_BLOB}" && \ echo "##vso[task.setvariable variable=ARTIFACT_ID]${ARTIFACT_ID}" && \ echo "##vso[task.setvariable variable=PACKER_BUILD_LOCATION]${BUILD_LOCATION}" && \ echo "##vso[task.setvariable variable=PACKER_VNET_RESOURCE_GROUP_NAME]${PACKER_VNET_RESOURCE_GROUP_NAME}" && \ @@ -160,6 +161,36 @@ steps: condition: always() displayName: Set Post-Build Variables + - task: AzureCLI@2 + inputs: + azureSubscription: $(VHD_ARM_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + echo "##vso[task.setvariable variable=DESTINATION_STORAGE_CONTAINER]${CLASSIC_BLOB_STAGING}" && \ + STAGING_CONTAINER_EXISTS=$(az storage container exists --account-name ${STORAGE_ACCOUNT_NAME} --name $VHD_STAGING_CONTAINER_NAME --auth-mode login | jq -r '.exists') + if [ "${STAGING_CONTAINER_EXISTS,,}" = "false" ]; then + echo "Creating staging container $VHD_STAGING_CONTAINER_NAME in storage account $STORAGE_ACCOUNT_NAME" + az storage container create --account-name "$STORAGE_ACCOUNT_NAME" --name "$VHD_STAGING_CONTAINER_NAME" --auth-mode login || exit 1 + else + echo "Staging container $VHD_STAGING_CONTAINER_NAME already exists in storage account $STORAGE_ACCOUNT_NAME" + fi + echo "DESTINATION_STORAGE_CONTAINER is set to ${CLASSIC_BLOB_STAGING}" + condition: | + or( + and( + succeeded(), + eq(variables.GENERATE_PUBLISHING_INFO, 'True'), + eq(variables.RUN_PREFETCH_OPTIMIZATION, 'False'), + eq(variables.ENVIRONMENT, 'tme') + ), + and( + eq(variables.BUILDER, 'imagecustomizer'), + eq(variables.ENVIRONMENT, 'tme') + ) + ) + displayName: Set Destination Storage Container to Staging + - task: AzureCLI@2 inputs: azureSubscription: $(VHD_ARM_SERVICE_CONNECTION) @@ -173,9 +204,6 @@ steps: env: RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME) GENERATE_PUBLISHING_INFO: $(GENERATE_PUBLISHING_INFO) - CLASSIC_BLOB_STAGING: $(CLASSIC_BLOB_STAGING) - VHD_CONTAINER_NAME: $(VHD_CONTAINER_NAME) - VHD_STAGING_CONTAINER_NAME: $(VHD_STAGING_CONTAINER_NAME) - task: AzureCLI@2 inputs: @@ -316,6 +344,34 @@ steps: env: RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME) + - task: AzureCLI@2 + inputs: + azureSubscription: $(VHD_ARM_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + echo "Copying ${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd to immutable storage container" + export AZCOPY_AUTO_LOGIN_TYPE="AZCLI" + export AZCOPY_CONCURRENCY_VALUE="AUTO" + az storage blob copy start --account-name "$STORAGE_ACCOUNT_NAME" --destination-blob "${CAPTURED_SIG_VERSION}.vhd" --destination-container "$VHD_CONTAINER_NAME" --source-uri "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --auth-mode login || exit 1 + echo "Successfully copied to immutable container" + # Remove old VHD from staging container after copy to immutable container is complete + azcopy remove "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --recursive=true + condition: | + or( + and( + succeeded(), + eq(variables.GENERATE_PUBLISHING_INFO, 'True'), + eq(variables.RUN_PREFETCH_OPTIMIZATION, 'False'), + eq(variables.ENVIRONMENT, 'tme') + ), + and( + eq(variables.BUILDER, 'imagecustomizer'), + eq(variables.ENVIRONMENT, 'tme') + ) + ) + displayName: Copy to Immutable Storage Container + - task: DownloadPipelineArtifact@2 displayName: Download Build Performance Program condition: and(succeeded(), eq(variables.ENVIRONMENT, 'tme')) diff --git a/vhdbuilder/packer/convert-sig-to-classic-storage-account-blob.sh b/vhdbuilder/packer/convert-sig-to-classic-storage-account-blob.sh index 86da1b92d74..a4afd315e63 100755 --- a/vhdbuilder/packer/convert-sig-to-classic-storage-account-blob.sh +++ b/vhdbuilder/packer/convert-sig-to-classic-storage-account-blob.sh @@ -129,7 +129,7 @@ if [ "$sas" = "None" ]; then fi capture_benchmark "${SCRIPT_NAME}_grant_access_to_disk" -echo "Uploading $disk_resource_id to ${CLASSIC_BLOB}/${CAPTURED_SIG_VERSION}.vhd" +echo "Uploading $disk_resource_id to ${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" echo "Setting azcopy environment variables with pool identity: $AZURE_MSI_RESOURCE_STRING" export AZCOPY_AUTO_LOGIN_TYPE="AZCLI" @@ -139,7 +139,7 @@ export AZCOPY_JOB_PLAN_LOCATION="$(pwd)/azcopy-job-plan-files/" mkdir -p "${AZCOPY_LOG_LOCATION}" mkdir -p "${AZCOPY_JOB_PLAN_LOCATION}" -if ! azcopy copy "${sas}" "${CLASSIC_BLOB}/${CAPTURED_SIG_VERSION}.vhd" --recursive=true ; then +if ! azcopy copy "${sas}" "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --recursive=true ; then azExitCode=$? # loop through azcopy log files shopt -s nullglob @@ -159,7 +159,7 @@ if ! azcopy copy "${sas}" "${CLASSIC_BLOB}/${CAPTURED_SIG_VERSION}.vhd" --recurs exit $azExitCode fi -echo "Uploaded $disk_resource_id to ${CLASSIC_BLOB}/${CAPTURED_SIG_VERSION}.vhd" +echo "Uploaded $disk_resource_id to ${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" capture_benchmark "${SCRIPT_NAME}_upload_disk_to_blob" if ! az disk revoke-access --ids $disk_resource_id; then diff --git a/vhdbuilder/packer/imagecustomizer/scripts/publish-imagecustomizer-image.sh b/vhdbuilder/packer/imagecustomizer/scripts/publish-imagecustomizer-image.sh index a9c819e5fae..fcbb14e044d 100755 --- a/vhdbuilder/packer/imagecustomizer/scripts/publish-imagecustomizer-image.sh +++ b/vhdbuilder/packer/imagecustomizer/scripts/publish-imagecustomizer-image.sh @@ -45,20 +45,6 @@ export AZCOPY_JOB_PLAN_LOCATION="$(pwd)/azcopy-job-plan-files/" mkdir -p "${AZCOPY_LOG_LOCATION}" mkdir -p "${AZCOPY_JOB_PLAN_LOCATION}" -if [ "${ENVIRONMENT,,}" = "tme" ]; then - # If environment is TME, we use a staging container in order to later copy the blob to an immutable container. - DESTINATION_STORAGE_CONTAINER=${CLASSIC_BLOB_STAGING} - STAGING_CONTAINER_EXISTS=$(az storage container exists --account-name ${STORAGE_ACCOUNT_NAME} --name $VHD_STAGING_CONTAINER_NAME --auth-mode login | jq -r '.exists') - if [ "${STAGING_CONTAINER_EXISTS,,}" = "false" ]; then - echo "Creating staging container $VHD_STAGING_CONTAINER_NAME in storage account $STORAGE_ACCOUNT_NAME" - az storage container create --account-name "$STORAGE_ACCOUNT_NAME" --name "$VHD_STAGING_CONTAINER_NAME" --auth-mode login || exit 1 - else - echo "Staging container $VHD_STAGING_CONTAINER_NAME already exists in storage account $STORAGE_ACCOUNT_NAME" - fi -else - DESTINATION_STORAGE_CONTAINER=${CLASSIC_BLOB} -fi - echo "Uploading ${OUT_DIR}/${CONFIG}.vhd to ${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" if ! azcopy copy "${OUT_DIR}/${CONFIG}.vhd" "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --recursive=true; then azExitCode=$? @@ -83,15 +69,6 @@ fi echo "Uploaded ${OUT_DIR}/${CONFIG}.vhd to ${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" -if [ "${ENVIRONMENT,,}" = "tme" ] && [ "${GENERATE_PUBLISHING_INFO,,}" = "true" ]; then - echo "Copying ${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd to immutable storage container" - az storage blob copy start --account-name "$STORAGE_ACCOUNT_NAME" --destination-blob "${CAPTURED_SIG_VERSION}.vhd" --destination-container "$VHD_CONTAINER_NAME" --source-uri "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --auth-mode login || exit 1 - echo "Successfully copied to immutable container" -else - echo "GENERATE_PUBLISHING_INFO is false or we are in a testing / prod environment, skipping copying ${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd to immutable storage container" -fi -capture_benchmark "${SCRIPT_NAME}_upload_vhd_to_blob" - GALLERY_RESOURCE_ID=/subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_NAME}/providers/Microsoft.Compute/galleries/${SIG_GALLERY_NAME} SIG_IMAGE_RESOURCE_ID="${GALLERY_RESOURCE_ID}/images/${SIG_IMAGE_NAME}/versions/${CAPTURED_SIG_VERSION}" MANAGED_IMAGE_RESOURCE_ID="/subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_NAME}/providers/Microsoft.Compute/images/${IMAGE_NAME}" @@ -126,7 +103,7 @@ az sig image-version create \ --target-regions ${TARGET_REGIONS} capture_benchmark "${SCRIPT_NAME}_create_sig_image_version" -if [ "${ENVIRONMENT,,}" = "tme" ] || [ "${GENERATE_PUBLISHING_INFO,,}" = "false" ]; then +if [ "${GENERATE_PUBLISHING_INFO,,}" = "false" ]; then azcopy remove "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --recursive=true fi