feat: ensure successful blob upload without prefetch optimization enabled

.pipelines/templates/.builder-release-template.yaml

@@ -125,6 +125,23 @@ steps:
       BUILD_ID: $(Build.BuildId)
       BUILD_DEFINITION_NAME: $(Build.DefinitionName)
 
+  - bash: |
+      if [ "${ENVIRONMENT,,}" = "tme" ] && [ "${GENERATE_PUBLISHING_INFO,,}" = "true" ] && [ ${RUN_PREFETCH_OPTIMIZATION,,} = "false" ] || [ ${BUILDER} = "imagecustomizer" ]; then
+          DESTINATION_STORAGE_CONTAINER=${CLASSIC_BLOB_STAGING}
+          STAGING_CONTAINER_EXISTS=$(az storage container exists --account-name ${STORAGE_ACCOUNT_NAME} --name $VHD_STAGING_CONTAINER_NAME --auth-mode login | jq -r '.exists')
+          if [ "${STAGING_CONTAINER_EXISTS,,}" = "false" ]; then
+            echo "Creating staging container $VHD_STAGING_CONTAINER_NAME in storage account $STORAGE_ACCOUNT_NAME"
+            az storage container create --account-name "$STORAGE_ACCOUNT_NAME" --name "$VHD_STAGING_CONTAINER_NAME" --auth-mode login || exit 1
+          else
+            echo "Staging container $VHD_STAGING_CONTAINER_NAME already exists in storage account $STORAGE_ACCOUNT_NAME"
+          fi
+      else
+          DESTINATION_STORAGE_CONTAINER=${CLASSIC_BLOB}
+      fi
+      echo "##vso[task.setvariable variable=DESTINATION_STORAGE_CONTAINER]${DESTINATION_STORAGE_CONTAINER}"
+    condition: always()
+    displayName: Set Destination Storage Container
+
   - bash: |
       PACKER_VNET_RESOURCE_GROUP_NAME="$(cat vhdbuilder/packer/settings.json | grep "vnet_resource_group_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
       PACKER_VNET_NAME="$(cat vhdbuilder/packer/settings.json | grep "vnet_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
@@ -316,6 +333,29 @@ steps:
     env:
       RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME)
 
+  - task: AzureCLI@2
+    inputs:
+      azureSubscription: $(VHD_ARM_SERVICE_CONNECTION)
+      scriptType: bash
+      scriptLocation: inlineScript
+      inlineScript: |
+        echo "Copying ${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd to immutable storage container"
+        az storage blob copy start --account-name "$STORAGE_ACCOUNT_NAME" --destination-blob "${CAPTURED_SIG_VERSION}.vhd" --destination-container "$VHD_CONTAINER_NAME" --source-uri "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --auth-mode login || exit 1
+        echo "Successfully copied to immutable container"
+        # Remove old VHD from staging container after copy to immutable container is complete
+        azcopy remove "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --recursive=true
+    condition: |
+      or(
+        and(
+          succeeded(),
+          eq(variables.GENERATE_PUBLISHING_INFO, 'True'),
+          eq(variables.RUN_PREFETCH_OPTIMIZATION, 'False'),
+          eq(variables.ENVIRONMENT, 'tme')
+        ),
+          eq(variables.BUILDER, 'imagecustomizer')
+      )
+    displayName: Copy to Immutable Storage Container
+
   - task: DownloadPipelineArtifact@2
     displayName: Download Build Performance Program
     condition: and(succeeded(), eq(variables.ENVIRONMENT, 'tme'))

vhdbuilder/packer/convert-sig-to-classic-storage-account-blob.sh

@@ -139,7 +139,7 @@ export AZCOPY_JOB_PLAN_LOCATION="$(pwd)/azcopy-job-plan-files/"
 mkdir -p "${AZCOPY_LOG_LOCATION}"
 mkdir -p "${AZCOPY_JOB_PLAN_LOCATION}"
 
-if ! azcopy copy "${sas}" "${CLASSIC_BLOB}/${CAPTURED_SIG_VERSION}.vhd" --recursive=true ; then
+if ! azcopy copy "${sas}" "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --recursive=true ; then
   azExitCode=$?
   # loop through azcopy log files
   shopt -s nullglob
@@ -159,7 +159,7 @@ if ! azcopy copy "${sas}" "${CLASSIC_BLOB}/${CAPTURED_SIG_VERSION}.vhd" --recurs
   exit $azExitCode
 fi
 
-echo "Uploaded $disk_resource_id to ${CLASSIC_BLOB}/${CAPTURED_SIG_VERSION}.vhd"
+echo "Uploaded $disk_resource_id to ${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd"
 capture_benchmark "${SCRIPT_NAME}_upload_disk_to_blob"
 
 if ! az disk revoke-access --ids $disk_resource_id; then

vhdbuilder/packer/imagecustomizer/scripts/publish-imagecustomizer-image.sh

@@ -45,20 +45,6 @@ export AZCOPY_JOB_PLAN_LOCATION="$(pwd)/azcopy-job-plan-files/"
 mkdir -p "${AZCOPY_LOG_LOCATION}"
 mkdir -p "${AZCOPY_JOB_PLAN_LOCATION}"
 
-if [ "${ENVIRONMENT,,}" = "tme" ]; then
-    # If environment is TME, we use a staging container in order to later copy the blob to an immutable container.
-    DESTINATION_STORAGE_CONTAINER=${CLASSIC_BLOB_STAGING}
-    STAGING_CONTAINER_EXISTS=$(az storage container exists --account-name ${STORAGE_ACCOUNT_NAME} --name $VHD_STAGING_CONTAINER_NAME --auth-mode login | jq -r '.exists')
-    if [ "${STAGING_CONTAINER_EXISTS,,}" = "false" ]; then
-        echo "Creating staging container $VHD_STAGING_CONTAINER_NAME in storage account $STORAGE_ACCOUNT_NAME"
-        az storage container create --account-name "$STORAGE_ACCOUNT_NAME" --name "$VHD_STAGING_CONTAINER_NAME" --auth-mode login || exit 1
-    else
-        echo "Staging container $VHD_STAGING_CONTAINER_NAME already exists in storage account $STORAGE_ACCOUNT_NAME"
-    fi
-else
-    DESTINATION_STORAGE_CONTAINER=${CLASSIC_BLOB}
-fi
-
 echo "Uploading ${OUT_DIR}/${CONFIG}.vhd to ${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd"
 if ! azcopy copy "${OUT_DIR}/${CONFIG}.vhd" "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --recursive=true; then
     azExitCode=$?
@@ -83,15 +69,6 @@ fi
 
 echo "Uploaded ${OUT_DIR}/${CONFIG}.vhd to ${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd"
 
-if [ "${ENVIRONMENT,,}" = "tme"  ] && [ "${GENERATE_PUBLISHING_INFO,,}" = "true" ]; then
-    echo "Copying ${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd to immutable storage container"
-    az storage blob copy start --account-name "$STORAGE_ACCOUNT_NAME" --destination-blob "${CAPTURED_SIG_VERSION}.vhd" --destination-container "$VHD_CONTAINER_NAME" --source-uri "${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" --auth-mode login || exit 1
-    echo "Successfully copied to immutable container"
-else
-    echo "GENERATE_PUBLISHING_INFO is false or we are in a testing / prod environment, skipping copying ${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd to immutable storage container"
-fi
-capture_benchmark "${SCRIPT_NAME}_upload_vhd_to_blob"
-
 GALLERY_RESOURCE_ID=/subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_NAME}/providers/Microsoft.Compute/galleries/${SIG_GALLERY_NAME}
 SIG_IMAGE_RESOURCE_ID="${GALLERY_RESOURCE_ID}/images/${SIG_IMAGE_NAME}/versions/${CAPTURED_SIG_VERSION}"
 MANAGED_IMAGE_RESOURCE_ID="/subscriptions/${SUBSCRIPTION_ID}/resourceGroups/${RESOURCE_GROUP_NAME}/providers/Microsoft.Compute/images/${IMAGE_NAME}"