diff --git a/.pipelines/templates/PackageBuild.yml b/.pipelines/templates/PackageBuild.yml index abb9ac1d9ca..742dfe4ffd2 100644 --- a/.pipelines/templates/PackageBuild.yml +++ b/.pipelines/templates/PackageBuild.yml @@ -226,7 +226,7 @@ steps: max_cascading_rebuilds_arg="MAX_CASCADING_REBUILDS=${{ parameters.maxCascadingRebuilds }}" fi - sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" build-packages -j$(nproc) \ + sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" pre-cache -j$(nproc) \ CONCURRENT_PACKAGE_BUILDS=${{ parameters.concurrentPackageBuilds }} \ CONFIG_FILE="" \ MAX_CPU="${{ parameters.maxCPU }}" \ @@ -244,7 +244,9 @@ steps: $quick_rebuild_packages_arg \ $run_check_arg \ $toolchain_archive_arg \ - $use_ccache_arg + $use_ccache_arg || true + + sudo ${{ parameters.buildRepoRoot }}/testcurl.sh ${{ parameters.buildRepoRoot }}/build/logs/precache/precache.log displayName: "Build packages" - ${{ if parameters.outputArtifactsFolder }}: diff --git a/.pipelines/templates/ToolchainBuild.yml b/.pipelines/templates/ToolchainBuild.yml index f9a43ceed70..da87ba2afda 100644 --- a/.pipelines/templates/ToolchainBuild.yml +++ b/.pipelines/templates/ToolchainBuild.yml @@ -27,7 +27,9 @@ steps: parameters: buildRepoRoot: ${{ parameters.buildRepoRoot }} - - bash: sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" "-j$(nproc)" toolchain QUICK_REBUILD=y + - script: | + sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" "-j$(nproc)" pre-cache REBUILD_TOOLS=y DAILY_BUILD_ID=lkg || true + sudo ${{ parameters.buildRepoRoot }}/testcurl.sh ${{ parameters.buildRepoRoot }}/build/logs/precache/precache.log displayName: "Build toolchain" - bash: | @@ -54,4 +56,3 @@ steps: tar -C "${{ parameters.buildRepoRoot }}/build/logs/toolchain" -czf "$published_logs_dir/toolchain.logs.tar.gz" . condition: always() displayName: "Copy logs for publishing" - diff --git a/testcurl.sh b/testcurl.sh new file mode 100755 index 00000000000..9387a0826c4 --- /dev/null +++ b/testcurl.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +LOGFILE=$1 + +# We have a log file, some lines will contain "Failed to download". From each of these lines, extract the URL that failed to download. +# The URLs will be in the format "https://*.rpm". + +# time="2025-01-13T15:53:34Z" level=info msg="Attempt 1/8: Failed to download (https://packages.microsoft.com/azurelinux/3.0/prod/base/x86_64/Packages/l/libnuma-2.0.16-1.azl3.x86_64.rpm) with error: (download error:\nrequest failed:\nGet \"https://packages.microsoft.com/azurelinux/3.0/prod/base/x86_64/Packages/l/libnuma-2.0.16-1.azl3.x86_64.rpm\": dial tcp 13.107.246.40:443: i/o timeout)" +# We would want to extract the URL "https://packages.microsoft.com/azurelinux/3.0/prod/base/x86_64/Packages/l/libnuma-2.0.16-1.azl3.x86_64.rpm" +# There are several errors that have "i/o timeout", those are transient errors, so we can ignore them. +urls=$(cat $LOGFILE | grep "Failed to download" | grep -v "i/o timeout" | grep -oP 'https://[^ \)]*.rpm' | sort | uniq) + +printf '\n\n\n###################### Errors: ######################\n\n\n' +# Print the error lines here for reference +cat $LOGFILE | grep "Failed to download" | grep -v "i/o timeout" + +# For each url, debug the issue by running curl -v . If the curl command fails, print the error message, but continue to the next URL. +# Save the files in a directory called "debug" in the current directory. +for url in $urls; do + printf '\n\n\n###################### Debugging %s ######################\n\n\n' "$url" + curl -vO "$url" || true +done + +echo "Done debugging" +exit 1 diff --git a/toolkit/tools/internal/network/network.go b/toolkit/tools/internal/network/network.go index 1ecbae9c634..d7cee511fd8 100644 --- a/toolkit/tools/internal/network/network.go +++ b/toolkit/tools/internal/network/network.go @@ -148,6 +148,14 @@ func DownloadFile(url, dst string, caCerts *x509.CertPool, tlsCerts []tls.Certif } response, err := client.Get(url) + + if response != nil { + //DEBUG Dump the response headers to the log + logger.Log.Warnf("#### DEBUG #### URL: %s, STATUS: %s, HEADER: %v", url, response.Status, response.Header) + } else { + logger.Log.Warnf("#### DEBUG #### URL: %s, STATUS: , HEADER: ", url) + } + if err != nil { return fmt.Errorf("%w:\nrequest failed:\n%w", ErrDownloadFileOther, err) }