[DO NOT MERGE] debug pmc or afd network issues in pipeline jobs

.pipelines/templates/PackageBuild.yml

@@ -226,7 +226,7 @@ steps:
         max_cascading_rebuilds_arg="MAX_CASCADING_REBUILDS=${{ parameters.maxCascadingRebuilds }}"
       fi
 
-      sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" build-packages -j$(nproc) \
+      sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" pre-cache -j$(nproc) \
         CONCURRENT_PACKAGE_BUILDS=${{ parameters.concurrentPackageBuilds }} \
         CONFIG_FILE="" \
         MAX_CPU="${{ parameters.maxCPU }}" \
@@ -244,7 +244,9 @@ steps:
         $quick_rebuild_packages_arg \
         $run_check_arg \
         $toolchain_archive_arg \
-        $use_ccache_arg
+        $use_ccache_arg || true
+
+      sudo ${{ parameters.buildRepoRoot }}/testcurl.sh ${{ parameters.buildRepoRoot }}/build/logs/precache/precache.log
     displayName: "Build packages"
 
   - ${{ if parameters.outputArtifactsFolder }}:

.pipelines/templates/ToolchainBuild.yml

@@ -27,7 +27,9 @@ steps:
     parameters:
       buildRepoRoot: ${{ parameters.buildRepoRoot }}
 
-  - bash: sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" "-j$(nproc)" toolchain QUICK_REBUILD=y
+  - script: |
+      sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" "-j$(nproc)" pre-cache REBUILD_TOOLS=y DAILY_BUILD_ID=lkg || true
+      sudo ${{ parameters.buildRepoRoot }}/testcurl.sh ${{ parameters.buildRepoRoot }}/build/logs/precache/precache.log
     displayName: "Build toolchain"
 
   - bash: |
@@ -54,4 +56,3 @@ steps:
       tar -C "${{ parameters.buildRepoRoot }}/build/logs/toolchain" -czf "$published_logs_dir/toolchain.logs.tar.gz" .
     condition: always()
     displayName: "Copy logs for publishing"
-

testcurl.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+LOGFILE=$1
+
+# We have a log file, some lines will contain "Failed to download". From each of these lines, extract the URL that failed to download.
+# The URLs will be in the format "https://*.rpm".
+
+# time="2025-01-13T15:53:34Z" level=info msg="Attempt 1/8: Failed to download (https://packages.microsoft.com/azurelinux/3.0/prod/base/x86_64/Packages/l/libnuma-2.0.16-1.azl3.x86_64.rpm) with error: (download error:\nrequest failed:\nGet \"https://packages.microsoft.com/azurelinux/3.0/prod/base/x86_64/Packages/l/libnuma-2.0.16-1.azl3.x86_64.rpm\": dial tcp 13.107.246.40:443: i/o timeout)"
+# We would want to extract the URL "https://packages.microsoft.com/azurelinux/3.0/prod/base/x86_64/Packages/l/libnuma-2.0.16-1.azl3.x86_64.rpm"
+# There are several errors that have "i/o timeout", those are transient errors, so we can ignore them.
+urls=$(cat $LOGFILE | grep "Failed to download" | grep -v "i/o timeout" | grep -oP 'https://[^ \)]*.rpm' | sort | uniq)
+
+printf '\n\n\n###################### Errors: ######################\n\n\n'
+# Print the error lines here for reference
+cat $LOGFILE | grep "Failed to download" | grep -v "i/o timeout"
+
+# For each url, debug the issue by running curl -v <url>. If the curl command fails, print the error message, but continue to the next URL.
+# Save the files in a directory called "debug" in the current directory.
+for url in $urls; do
+    printf '\n\n\n###################### Debugging %s ######################\n\n\n' "$url"
+    curl -vO "$url" || true
+done
+
+echo "Done debugging"
+exit 1

toolkit/tools/internal/network/network.go

@@ -148,6 +148,14 @@ func DownloadFile(url, dst string, caCerts *x509.CertPool, tlsCerts []tls.Certif
 	}
 
 	response, err := client.Get(url)
+
+	if response != nil {
+		//DEBUG Dump the response headers to the log
+		logger.Log.Warnf("#### DEBUG #### URL: %s, STATUS: %s, HEADER: %v", url, response.Status, response.Header)
+	} else {
+		logger.Log.Warnf("#### DEBUG #### URL: %s, STATUS: <nil>, HEADER: <nil>", url)
+	}
+
 	if err != nil {
 		return fmt.Errorf("%w:\nrequest failed:\n%w", ErrDownloadFileOther, err)
 	}