Skip to content

WIP #144257

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 18 commits into
base: master
Choose a base branch
from
Open

WIP #144257

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,12 @@ jobs:
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
fi

# Always print disk usage, even for jobs that don't free up disk space.
- name: print disk usage
run: |
echo "disk usage:"
df -h

- name: disable git crlf conversion
run: git config --global core.autocrlf false

Expand All @@ -117,14 +123,17 @@ jobs:
with:
fetch-depth: 2

# Free up disk space on Linux by removing preinstalled components that
# Free up disk space on Linux and Windows by removing preinstalled components that
# we do not need. We do this to enable some of the less resource
# intensive jobs to run on free runners, which however also have
# less disk space.
- name: free up disk space
run: src/ci/scripts/free-disk-space.sh
if: matrix.free_disk

- name: early exit
run: exit 1

# Rust Log Analyzer can't currently detect the PR number of a GitHub
# Actions build on its own, so a hint in the log message is needed to
# point it in the right direction.
Expand Down
13 changes: 2 additions & 11 deletions src/ci/github-actions/jobs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,20 +31,11 @@ runners:
<<: *base-job

- &job-windows
os: windows-2022
<<: *base-job

# NOTE: windows-2025 has less disk space available than windows-2022,
# because the D drive is missing.
- &job-windows-25
os: windows-2025
free_disk: true
<<: *base-job

- &job-windows-8c
os: windows-2022-8core-32gb
<<: *base-job

- &job-windows-25-8c
os: windows-2025-8core-32gb
<<: *base-job

Expand Down Expand Up @@ -655,7 +646,7 @@ auto:
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
DIST_REQUIRE_ALL_TOOLS: 1
CODEGEN_BACKENDS: llvm,cranelift
<<: *job-windows-25-8c
<<: *job-windows-8c

- name: dist-i686-msvc
env:
Expand Down
265 changes: 265 additions & 0 deletions src/ci/scripts/free-disk-space-linux.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,265 @@
#!/bin/bash
set -euo pipefail

# Free disk space on Linux GitHub action runners
# Script inspired by https://github.com/jlumbroso/free-disk-space

isX86() {
local arch
arch=$(uname -m)
if [ "$arch" = "x86_64" ]; then
return 0
else
return 1
fi
}

# Check if we're on a GitHub hosted runner.
# In aws codebuild, the variable RUNNER_ENVIRONMENT is "self-hosted".
isGitHubRunner() {
# `:-` means "use the value of RUNNER_ENVIRONMENT if it exists, otherwise use an empty string".
if [[ "${RUNNER_ENVIRONMENT:-}" == "github-hosted" ]]; then
return 0
else
return 1
fi
}

# print a line of the specified character
printSeparationLine() {
for ((i = 0; i < 80; i++)); do
printf "%s" "$1"
done
printf "\n"
}

# compute available space
# REF: https://unix.stackexchange.com/a/42049/60849
# REF: https://stackoverflow.com/a/450821/408734
getAvailableSpace() {
df -a | awk 'NR > 1 {avail+=$4} END {print avail}'
}

# make Kb human readable (assume the input is Kb)
# REF: https://unix.stackexchange.com/a/44087/60849
formatByteCount() {
numfmt --to=iec-i --suffix=B --padding=7 "${1}000"
}

# macro to output saved space
printSavedSpace() {
# Disk space before the operation
local before=${1}
local title=${2:-}

local after
after=$(getAvailableSpace)
local saved=$((after - before))

if [ "$saved" -lt 0 ]; then
echo "::warning::Saved space is negative: $saved. Using '0' as saved space."
saved=0
fi

echo ""
printSeparationLine "*"
if [ -n "${title}" ]; then
echo "=> ${title}: Saved $(formatByteCount "$saved")"
else
echo "=> Saved $(formatByteCount "$saved")"
fi
printSeparationLine "*"
echo ""
}

# macro to print output of df with caption
printDF() {
local caption=${1}

printSeparationLine "="
echo "${caption}"
echo ""
echo "$ df -h"
echo ""
df -h
printSeparationLine "="
}

removeUnusedFilesAndDirs() {
local to_remove=(
"/usr/share/java"
)

if isGitHubRunner; then
to_remove+=(
"/usr/local/aws-sam-cli"
"/usr/local/doc/cmake"
"/usr/local/julia"*
"/usr/local/lib/android"
"/usr/local/share/chromedriver-"*
"/usr/local/share/chromium"
"/usr/local/share/cmake-"*
"/usr/local/share/edge_driver"
"/usr/local/share/emacs"
"/usr/local/share/gecko_driver"
"/usr/local/share/icons"
"/usr/local/share/powershell"
"/usr/local/share/vcpkg"
"/usr/local/share/vim"
"/usr/share/apache-maven-"*
"/usr/share/gradle-"*
"/usr/share/kotlinc"
"/usr/share/miniconda"
"/usr/share/php"
"/usr/share/ri"
"/usr/share/swift"

# binaries
"/usr/local/bin/azcopy"
"/usr/local/bin/bicep"
"/usr/local/bin/ccmake"
"/usr/local/bin/cmake-"*
"/usr/local/bin/cmake"
"/usr/local/bin/cpack"
"/usr/local/bin/ctest"
"/usr/local/bin/helm"
"/usr/local/bin/kind"
"/usr/local/bin/kustomize"
"/usr/local/bin/minikube"
"/usr/local/bin/packer"
"/usr/local/bin/phpunit"
"/usr/local/bin/pulumi-"*
"/usr/local/bin/pulumi"
"/usr/local/bin/stack"

# Haskell runtime
"/usr/local/.ghcup"

# Azure
"/opt/az"
"/usr/share/az_"*
)

if [ -n "${AGENT_TOOLSDIRECTORY:-}" ]; then
# Environment variable set by GitHub Actions
to_remove+=(
"${AGENT_TOOLSDIRECTORY}"
)
else
echo "::warning::AGENT_TOOLSDIRECTORY is not set. Skipping removal."
fi
else
# Remove folders and files present in AWS CodeBuild
to_remove+=(
# binaries
"/usr/local/bin/ecs-cli"
"/usr/local/bin/eksctl"
"/usr/local/bin/kubectl"

"${HOME}/.gradle"
"${HOME}/.dotnet"
"${HOME}/.goenv"
"${HOME}/.phpenv"

)
fi

for element in "${to_remove[@]}"; do
if [ ! -e "$element" ]; then
# The file or directory doesn't exist.
# Maybe it was removed in a newer version of the runner or it's not present in a
# specific architecture (e.g. ARM).
echo "::warning::Directory or file $element does not exist, skipping."
fi
done

# Remove all files and directories at once to save time.
sudo rm -rf "${to_remove[@]}"
}

execAndMeasureSpaceChange() {
local operation=${1} # Function to execute
local title=${2}

local before
before=$(getAvailableSpace)
$operation

printSavedSpace "$before" "$title"
}

# Remove large packages
# REF: https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh
cleanPackages() {
local packages=(
'^aspnetcore-.*'
'^dotnet-.*'
'^llvm-.*'
'^mongodb-.*'
'firefox'
'libgl1-mesa-dri'
'mono-devel'
'php.*'
)

if isGitHubRunner; then
packages+=(
azure-cli
)

if isX86; then
packages+=(
'google-chrome-stable'
'google-cloud-cli'
'google-cloud-sdk'
'powershell'
)
fi
else
packages+=(
'google-chrome-stable'
)
fi

sudo apt-get -qq remove -y --fix-missing "${packages[@]}"

sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed"
sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed failed"
}

# Remove Docker images.
# Ubuntu 22 runners have docker images already installed.
# They aren't present in ubuntu 24 runners.
cleanDocker() {
echo "=> Removing the following docker images:"
sudo docker image ls
echo "=> Removing docker images..."
sudo docker image prune --all --force || true
}

# Remove Swap storage
cleanSwap() {
sudo swapoff -a || true
sudo rm -rf /mnt/swapfile || true
free -h
}

# Display initial disk space stats

AVAILABLE_INITIAL=$(getAvailableSpace)

printDF "BEFORE CLEAN-UP:"
echo ""
execAndMeasureSpaceChange cleanPackages "Unused packages"
execAndMeasureSpaceChange cleanDocker "Docker images"
execAndMeasureSpaceChange cleanSwap "Swap storage"
execAndMeasureSpaceChange removeUnusedFilesAndDirs "Unused files and directories"

# Output saved space statistic
echo ""
printDF "AFTER CLEAN-UP:"

echo ""
echo ""

printSavedSpace "$AVAILABLE_INITIAL" "Total saved"
36 changes: 36 additions & 0 deletions src/ci/scripts/free-disk-space-windows.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Free disk space on Windows GitHub action runners.

$ErrorActionPreference = 'Stop'

Get-Volume | Out-String | Write-Output

$available = $(Get-Volume C).SizeRemaining

$dirs = 'Does not exit',
'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Tools\Llvm',
'C:\rtools45', 'C:\ghcup', 'C:\Program Files (x86)\Android',
'C:\Program Files\Google\Chrome', 'C:\Program Files (x86)\Microsoft\Edge',
'C:\Program Files\Mozilla Firefox', 'C:\Program Files\MySQL', 'C:\Julia',
'C:\Program Files\MongoDB', 'C:\Program Files\Azure Cosmos DB Emulator',
'C:\Program Files\PostgreSQL', 'C:\Program Files\Unity Hub',
'C:\Strawberry', 'C:\hostedtoolcache\windows\Java_Temurin-Hotspot_jdk',
'C:\does not exist', 'oh no'

foreach ($dir in $dirs) {
Start-ThreadJob -InputObject $dir {
Remove-Item -Recurse -Force -LiteralPath $input
} | Out-Null
}

foreach ($job in Get-Job) {
Wait-Job $job | Out-Null
if ($job.Error) {
Write-Output "::warning file=$PSCommandPath::$($job.Error)"
}
Remove-Job $job
}

Get-Volume | Out-String | Write-Output

$saved = ($(Get-Volume C).SizeRemaining - $available) / 1gb
Write-Output "total space saved $saved GB"
Loading
Loading