test: validate the existence of VHD cached file (Azure#332)

* Add VHD content test after VHD building * fix and add Makefile target * add `--no-wait` to reduce waiting time cost * do not download resource * add fixes * add run-test.sh execute permission * fix credential issue * fix comments * use acs-mirror
mainred · Sep 23, 2020 · 4faab19 · 4faab19
1 parent 3d2dc2c
commit 4faab19
Show file tree

Hide file tree

Showing 6 changed files with 300 additions and 2 deletions.
diff --git a/.pipelines/.vsts-vhd-builder-windows.yaml b/.pipelines/.vsts-vhd-builder-windows.yaml
@@ -84,6 +84,22 @@ jobs:
       -e SIG_IMAGE_VERSION=${SIG_IMAGE_VERSION} \
       ${DEIS_GO_DEV_IMAGE} make -f packer.mk run-packer-windows
     displayName: Building windows VHD
+  - script: |
+      OS_DISK_URI="$(cat packer-output | grep "OSDiskUri:" | cut -d " " -f 2)" && \
+      docker run --rm \
+      -v ${PWD}:/go/src/github.com/Azure/AgentBaker \
+      -w /go/src/github.com/Azure/AgentBaker \
+      -e CLIENT_ID=${BUILD_CLIENT_ID} \
+      -e CLIENT_SECRET="$(BUILD_CLIENT_SECRET)" \
+      -e TENANT_ID=${BUILD_TENANT_ID} \
+      -e SUBSCRIPTION_ID="${BUILD_SUBSCRIPTION_ID}" \
+      -e OS_DISK_URI=${OS_DISK_URI} \
+      -e AZURE_LOCATION=${BUILD_AZURE_LOCATION} \
+      -e CONTAINER_RUNTIME=$(CONTAINER_RUNTIME) \
+      -e WINDOWS_SKU=$(WINDOWS_SKU) \
+      -e OS_TYPE="Windows" \
+      ${DEIS_GO_DEV_IMAGE} make -f packer.mk test-building-vhd
+    displayName: Run VHD cache test
   - task: PublishPipelineArtifact@1
     inputs:
       artifact: 'vhd-release-notes'

diff --git a/packer.mk b/packer.mk
@@ -49,3 +49,7 @@ convert-sig-to-classic-storage-account-blob: az-login
 
 windows-vhd-publishing-info: az-login
 	@./vhdbuilder/packer/generate-windows-vhd-publishing-info.sh
+
+test-building-vhd: az-login
+	@./vhdbuilder/packer/test/run-test.sh
+
diff --git a/test.mk b/test.mk
@@ -27,3 +27,4 @@ test-kubernetes:
 
 test-azure-constants:
 	./scripts/azure-const.sh
+
diff --git a/vhdbuilder/packer/configure-windows-vhd.ps1 b/vhdbuilder/packer/configure-windows-vhd.ps1
@@ -6,13 +6,15 @@
         This script is used by packer to produce Windows AKS images.
 #>
 
-param()
+param (
+    $containerRuntime
+)
 
 $ErrorActionPreference = "Stop"
 
 filter Timestamp { "$(Get-Date -Format o): $_" }
 
-$global:containerdPackageUrl = "https://marosset.blob.core.windows.net/pub/containerd/containerd-0.0.87-public.zip"
+$global:containerdPackageUrl = "https://acs-mirror.azureedge.net/containerd/ms/0.0.11-1/binaries/containerd-windows-0.0.11-1.zip"
 
 function Write-Log($Message) {
     $msg = $message | Timestamp

diff --git a/vhdbuilder/packer/test/run-test.sh b/vhdbuilder/packer/test/run-test.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+set -eux
+
+WIN_SCRIPT_PATH="vhd-content-test.ps1"
+TEST_RESOURCE_PREFIX="vhd-test"
+
+RESOURCE_GROUP_NAME="$TEST_RESOURCE_PREFIX-$(date +%s)"
+az group create --name $RESOURCE_GROUP_NAME --location ${AZURE_LOCATION} --tags 'source=AgentBaker'
+
+# defer function to cleanup resource group
+function cleanup {
+    az group delete --name $RESOURCE_GROUP_NAME --yes --no-wait
+}
+trap cleanup EXIT
+
+DISK_NAME="${TEST_RESOURCE_PREFIX}-disk"
+VM_NAME="${TEST_RESOURCE_PREFIX}-vm"
+az disk create --resource-group $RESOURCE_GROUP_NAME \
+    --name $DISK_NAME \
+    --source "${OS_DISK_URI}"  \
+    --query id
+az vm create --name $VM_NAME \
+    --resource-group $RESOURCE_GROUP_NAME \
+    --attach-os-disk $DISK_NAME \
+    --os-type $OS_TYPE  \
+    --public-ip-address ""
+time az vm wait -g $RESOURCE_GROUP_NAME -n $VM_NAME --created
+
+FULL_PATH=$(realpath $0)
+CDIR=$(dirname $FULL_PATH)
+if [ "$OS_TYPE" == "Windows" ]; then
+    SCRIPT_PATH="$CDIR/$WIN_SCRIPT_PATH"
+    ret=$(az vm run-command invoke --command-id RunPowerShellScript \
+        --name $VM_NAME \
+        --resource-group $RESOURCE_GROUP_NAME  \
+        --scripts  @$SCRIPT_PATH \
+        --output json \
+        --parameters "containerRuntime=${CONTAINER_RUNTIME}")
+fi
+# An example of failed run-command output:
+# {
+#   "value": [
+#     {
+#       "code": "ComponentStatus/StdOut/succeeded",
+#       "displayStatus": "Provisioning succeeded",
+#       "level": "Info",
+#       "message": "c:\akse-cache\containerd\containerd-0.0.87-public.zip is cached as expected
+# c:\akse-cache\win-vnet-cni\azure-vnet-cni-singletenancy-windows-amd64-v1.1.2.zip is cached as expected
+# ... ...
+# "
+#       "time": null
+#     },
+#     {
+#       "code": "ComponentStatus/StdErr/succeeded",
+#       "displayStatus": "Provisioning succeeded",
+#       "level": "Info",
+#       "message": "Test-FilesToCacheOnVHD : File c:\akse-cache\win-k8s\v1.15.10-azs-1int.zip does not exist
+# At C:\Packages\Plugins\Microsoft.CPlat.Core.RunCommandWindows\1.1.5\Downloads\script0.ps1:146 char:1
+# + Test-FilesToCacheOnVHD
+# + ~~~~~~~~~~~~~~~~~~~~~~
+#     + CategoryInfo          : NotSpecified: (:) [Write-Error], WriteErrorException
+#     + FullyQualifiedErrorId : Microsoft.PowerShell.Commands.WriteErrorException,Test-FilesToCacheOnVHD
+#  ",
+#       "time": null
+#     }
+#   ]
+# }
+
+# we have to use `-E` to disable interpretation of backslash escape sequences, for jq cannot process string 
+# with a range of control characters not escaped as shown in the error below:
+#   Invalid string: control characters from U+0000 through U+001F must be escaped
+errMsg=$(echo -E $ret | jq '.value[]  | select(.code == "ComponentStatus/StdErr/succeeded") | .message')
+# a successful errMsg should be '""' after parsed by `jq`
+if [[ $errMsg != \"\" ]]; then
+    exit 1
+fi
diff --git a/vhdbuilder/packer/test/vhd-content-test.ps1 b/vhdbuilder/packer/test/vhd-content-test.ps1
@@ -0,0 +1,199 @@
+<#
+    .SYNOPSIS
+        verify the content of Windows image built
+    .DESCRIPTION
+        This script is used to verify the content of Windows image built
+#>
+
+param (
+    $containerRuntime
+)
+
+# TODO(qinhao): we can share the variables from configure-windows-vhd.ps1
+$global:containerdPackageUrl = "https://acs-mirror.azureedge.net/containerd/ms/0.0.11-1/binaries/containerd-windows-0.0.11-1.zip"
+
+function Compare-AllowedSecurityProtocols
+{
+
+    $allowedProtocols = @()
+    $insecureProtocols = @([System.Net.SecurityProtocolType]::SystemDefault, [System.Net.SecurityProtocolType]::Ssl3)
+
+    foreach ($protocol in [System.Enum]::GetValues([System.Net.SecurityProtocolType]))
+    {
+        if ($insecureProtocols -notcontains $protocol)
+        {
+            $allowedProtocols += $protocol
+        }
+    }
+    if([System.Net.ServicePointManager]::SecurityProtocol -ne $allowedProtocols) {
+        Write-Error "allowedSecurityProtocols '$([System.Net.ServicePointManager]::SecurityProtocol)', expecting '$allowedProtocols'"
+        exit 1
+    }
+}
+
+function Test-FilesToCacheOnVHD
+{
+    # TODO(qinhao): share this map variable with `configure-windows-vhd.ps1`
+    $map = @{
+        "c:\akse-cache\" = @(
+            "https://github.com/Azure/AgentBaker/raw/master/vhdbuilder/scripts/windows/collect-windows-logs.ps1",
+            "https://github.com/Microsoft/SDN/raw/master/Kubernetes/flannel/l2bridge/cni/win-bridge.exe",
+            "https://github.com/microsoft/SDN/raw/master/Kubernetes/windows/debug/collectlogs.ps1",
+            "https://github.com/microsoft/SDN/raw/master/Kubernetes/windows/debug/dumpVfpPolicies.ps1",
+            "https://github.com/microsoft/SDN/raw/master/Kubernetes/windows/debug/portReservationTest.ps1",
+            "https://github.com/microsoft/SDN/raw/master/Kubernetes/windows/debug/starthnstrace.cmd",
+            "https://github.com/microsoft/SDN/raw/master/Kubernetes/windows/debug/startpacketcapture.cmd",
+            "https://github.com/microsoft/SDN/raw/master/Kubernetes/windows/debug/stoppacketcapture.cmd",
+            "https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/debug/VFP.psm1",
+            "https://github.com/microsoft/SDN/raw/master/Kubernetes/windows/helper.psm1",
+            "https://github.com/Microsoft/SDN/raw/master/Kubernetes/windows/hns.psm1",
+            "https://globalcdn.nuget.org/packages/microsoft.applicationinsights.2.11.0.nupkg",
+            "https://acs-mirror.azureedge.net/aks-engine/windows/provisioning/signedscripts-v0.0.2.zip"
+        );
+        "c:\akse-cache\containerd\" = @(
+            $global:containerdPackageUrl
+        );
+
+        "c:\akse-cache\win-k8s\" = @(
+            "https://acs-mirror.azureedge.net/kubernetes/v1.15.11-azs/windowszip/v1.15.11-azs-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.15.12-azs/windowszip/v1.15.12-azs-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.16.13-azs/windowszip/v1.16.13-azs-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.16.14-azs/windowszip/v1.16.14-azs-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.17.9-azs/windowszip/v1.17.9-azs-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.17.11-azs/windowszip/v1.17.11-azs-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.15.10/windowszip/v1.15.10-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.15.11/windowszip/v1.15.11-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.15.11-hotfix.20200714/windowszip/v1.15.11-hotfix.20200714-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.15.11-hotfix.20200817/windowszip/v1.15.11-hotfix.20200817-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.15.12/windowszip/v1.15.12-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.15.12-hotfix.20200714/windowszip/v1.15.12-hotfix.20200714-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.15.12-hotfix.20200623/windowszip/v1.15.12-hotfix.20200623-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.15.12-hotfix.20200817/windowszip/v1.15.12-hotfix.20200817-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.16.10-hotfix.20200817/windowszip/v1.16.10-hotfix.20200817-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.16.12/windowszip/v1.16.12-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.16.13/windowszip/v1.16.13-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.16.13-hotfix.20200714/windowszip/v1.16.13-hotfix.20200714-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.16.13-hotfix.20200817/windowszip/v1.16.13-hotfix.20200817-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.16.13-hotfix.20200824/windowszip/v1.16.13-hotfix.20200824-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.16.14/windowszip/v1.16.14-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.16.15-hotfix.20200903/windowszip/v1.16.15-hotfix.20200903-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.17.7-hotfix.20200817/windowszip/v1.17.7-hotfix.20200817-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.17.8/windowszip/v1.17.8-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.17.9/windowszip/v1.17.9-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.17.9-hotfix.20200714/windowszip/v1.17.9-hotfix.20200714-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.17.9-hotfix.20200817/windowszip/v1.17.9-hotfix.20200817-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.17.9-hotfix.20200824/windowszip/v1.17.9-hotfix.20200824-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.17.11/windowszip/v1.17.11-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.17.11-hotfix.20200901/windowszip/v1.17.11-hotfix.20200901-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.18.4-hotfix.20200626/windowszip/v1.18.4-hotfix.20200626-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.18.5/windowszip/v1.18.5-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.18.6/windowszip/v1.18.6-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.18.6-hotfix.20200723/windowszip/v1.18.6-hotfix.20200723-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.18.8/windowszip/v1.18.8-1int.zip",
+            "https://acs-mirror.azureedge.net/kubernetes/v1.19.0/windowszip/v1.19.0-1int.zip"
+        );
+        "c:\akse-cache\win-vnet-cni\" = @(
+            "https://acs-mirror.azureedge.net/azure-cni/v1.1.2/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.2.zip",
+            "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip"
+            "https://acs-mirror.azureedge.net/azure-cni/v1.1.6/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.6.zip"
+        )
+    }
+
+    $emptyFiles = @()
+    $missingPaths = @()
+    foreach ($dir in $map.Keys)
+    {
+        if(!(Test-Path $dir))
+        {
+            Write-Error "Directory $dir does not exit"
+            $missingPaths = $missingPaths + $dir
+            continue
+        }
+
+        foreach ($URL in $map[$dir])
+        {
+            $fileName = [IO.Path]::GetFileName($URL)
+            $dest = [IO.Path]::Combine($dir, $fileName)
+
+            if(![System.IO.File]::Exists($dest))
+            {
+                Write-Error "File $dest does not exist"
+                $emptyFiles = $emptyFiles + $dest
+                continue
+            }
+
+            # NOTE(qinhao): tried to download all the files and compare file MD5 but as it takes
+            #               too long(hours) for the whole process, so check the file size temporarily
+            #               until we have a better way to validate these cached files
+            if ((Get-Item $dest).length -eq 0kb) {
+                Write-Error "File $dest is with size 0kb"
+                $emptyFiles = $emptyFiles + $dest
+            }
+
+            Write-Output "$dest is cached as expected"
+        }
+    }
+    if ($emptyFiles.count -gt 0 -Or $missingPaths.count -gt 0)
+    {
+        Write-Error "cache files base paths $missingPaths or(and) cached files $emptyFiles do not exist"
+        exit 1
+    }
+
+}
+
+function Test-PatchInstalled
+{
+    # patchIDs contains a list of hotfixes patched in "configure-windows-vhd.ps1", like "kb4558998"
+    $patchIDs = @()
+    $hotfix = Get-HotFix
+    $currenHotfixes = @()
+    foreach($hotfixID in $hotfix.HotFixID)
+    {
+        $currenHotfixes += $hotfixID
+    }
+
+    $lostPatched = @($patchIDs | Where-Object {$currenHotfixes -notcontains $_})
+    if($lostPatched.count -ne 0)
+    {
+        Write-Error "$lostPatched is(are) not installed"
+        exit 1
+    }
+    Write-Output "All pathced $patchIDs are installed"
+}
+
+function Test-ImagesPulled
+{
+    param (
+        $containerRuntime
+    )
+    $imagesToPull = @(
+        "mcr.microsoft.com/windows/servercore:ltsc2019",
+        "mcr.microsoft.com/windows/nanoserver:1809",
+        "mcr.microsoft.com/oss/kubernetes/pause:1.4.0",
+        "mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.0.1-alpha.1-windows-1809-amd64",
+        "mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v1.2.1-alpha.1-windows-1809-amd64")
+
+    if ($containerRuntime -eq 'containerd') {
+        $pulledImages = ctr.exe -n k8s.io -q
+    }
+    elseif ($containerRuntime -eq 'docker') {
+        $pulledImages = docker images --format "{{.Repository}}:{{.Tag}}"
+    }
+    else {
+        Write-Error "unsupported container runtime $containerRuntime"
+    }
+
+    Write-Output "Container runtime: $containerRuntime"
+    if(Compare-Object $imagesToPull $pulledImages) {
+        Write-Error "images to pull do not equal images cached $imagesToPull != $pulledImages"
+        exit 1
+    }
+    else {
+        Write-Output "images are cached as expected"
+    }
+}
+
+Compare-AllowedSecurityProtocols
+Test-FilesToCacheOnVHD
+Test-PatchInstalled
+Test-ImagesPulled  -containerRuntime $containerRuntime
Original file line number	Diff line number	Diff line change
Expand Up		@@ -27,3 +27,4 @@ test-kubernetes:

		test-azure-constants:
		./scripts/azure-const.sh