Skip to content

Commit 6ec219a

Browse files
rust-bors[bot]rust-bors[bot]
authored
Auto merge of #144257 - ChrisDenton:free-diskspace, r=<try>
WIP try-job: x86_64-msvc-1
2 parents 35487a2 + 77c5f45 commit 6ec219a

File tree

5 files changed

+317
-274
lines changed

5 files changed

+317
-274
lines changed

‎.github/workflows/ci.yml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,12 @@ jobs:
109109
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
110110
fi
111111
112+
# Always print disk usage, even for jobs that don't free up disk space.
113+
- name: print disk usage
114+
run: |
115+
echo "disk usage:"
116+
df -h
117+
112118
- name: disable git crlf conversion
113119
run: git config --global core.autocrlf false
114120

@@ -117,14 +123,17 @@ jobs:
117123
with:
118124
fetch-depth: 2
119125

120-
# Free up disk space on Linux by removing preinstalled components that
126+
# Free up disk space on Linux and Windows by removing preinstalled components that
121127
# we do not need. We do this to enable some of the less resource
122128
# intensive jobs to run on free runners, which however also have
123129
# less disk space.
124130
- name: free up disk space
125131
run: src/ci/scripts/free-disk-space.sh
126132
if: matrix.free_disk
127133

134+
- name: early exit
135+
run: exit 1
136+
128137
# Rust Log Analyzer can't currently detect the PR number of a GitHub
129138
# Actions build on its own, so a hint in the log message is needed to
130139
# point it in the right direction.

‎src/ci/github-actions/jobs.yml

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,20 +31,11 @@ runners:
3131
<<: *base-job
3232

3333
- &job-windows
34-
os: windows-2022
35-
<<: *base-job
36-
37-
# NOTE: windows-2025 has less disk space available than windows-2022,
38-
# because the D drive is missing.
39-
- &job-windows-25
4034
os: windows-2025
35+
free_disk: true
4136
<<: *base-job
4237

4338
- &job-windows-8c
44-
os: windows-2022-8core-32gb
45-
<<: *base-job
46-
47-
- &job-windows-25-8c
4839
os: windows-2025-8core-32gb
4940
<<: *base-job
5041

@@ -655,7 +646,7 @@ auto:
655646
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
656647
DIST_REQUIRE_ALL_TOOLS: 1
657648
CODEGEN_BACKENDS: llvm,cranelift
658-
<<: *job-windows-25-8c
649+
<<: *job-windows-8c
659650

660651
- name: dist-i686-msvc
661652
env:

‎src/ci/scripts/free-disk-space-linux.sh

Lines changed: 265 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,265 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
# Free disk space on Linux GitHub action runners
5+
# Script inspired by https://github.com/jlumbroso/free-disk-space
6+
7+
isX86() {
8+
local arch
9+
arch=$(uname -m)
10+
if [ "$arch" = "x86_64" ]; then
11+
return 0
12+
else
13+
return 1
14+
fi
15+
}
16+
17+
# Check if we're on a GitHub hosted runner.
18+
# In aws codebuild, the variable RUNNER_ENVIRONMENT is "self-hosted".
19+
isGitHubRunner() {
20+
# `:-` means "use the value of RUNNER_ENVIRONMENT if it exists, otherwise use an empty string".
21+
if [[ "${RUNNER_ENVIRONMENT:-}" == "github-hosted" ]]; then
22+
return 0
23+
else
24+
return 1
25+
fi
26+
}
27+
28+
# print a line of the specified character
29+
printSeparationLine() {
30+
for ((i = 0; i < 80; i++)); do
31+
printf "%s" "$1"
32+
done
33+
printf "\n"
34+
}
35+
36+
# compute available space
37+
# REF: https://unix.stackexchange.com/a/42049/60849
38+
# REF: https://stackoverflow.com/a/450821/408734
39+
getAvailableSpace() {
40+
df -a | awk 'NR > 1 {avail+=$4} END {print avail}'
41+
}
42+
43+
# make Kb human readable (assume the input is Kb)
44+
# REF: https://unix.stackexchange.com/a/44087/60849
45+
formatByteCount() {
46+
numfmt --to=iec-i --suffix=B --padding=7 "${1}000"
47+
}
48+
49+
# macro to output saved space
50+
printSavedSpace() {
51+
# Disk space before the operation
52+
local before=${1}
53+
local title=${2:-}
54+
55+
local after
56+
after=$(getAvailableSpace)
57+
local saved=$((after - before))
58+
59+
if [ "$saved" -lt 0 ]; then
60+
echo "::warning::Saved space is negative: $saved. Using '0' as saved space."
61+
saved=0
62+
fi
63+
64+
echo ""
65+
printSeparationLine "*"
66+
if [ -n "${title}" ]; then
67+
echo "=> ${title}: Saved $(formatByteCount "$saved")"
68+
else
69+
echo "=> Saved $(formatByteCount "$saved")"
70+
fi
71+
printSeparationLine "*"
72+
echo ""
73+
}
74+
75+
# macro to print output of df with caption
76+
printDF() {
77+
local caption=${1}
78+
79+
printSeparationLine "="
80+
echo "${caption}"
81+
echo ""
82+
echo "$ df -h"
83+
echo ""
84+
df -h
85+
printSeparationLine "="
86+
}
87+
88+
removeUnusedFilesAndDirs() {
89+
local to_remove=(
90+
"/usr/share/java"
91+
)
92+
93+
if isGitHubRunner; then
94+
to_remove+=(
95+
"/usr/local/aws-sam-cli"
96+
"/usr/local/doc/cmake"
97+
"/usr/local/julia"*
98+
"/usr/local/lib/android"
99+
"/usr/local/share/chromedriver-"*
100+
"/usr/local/share/chromium"
101+
"/usr/local/share/cmake-"*
102+
"/usr/local/share/edge_driver"
103+
"/usr/local/share/emacs"
104+
"/usr/local/share/gecko_driver"
105+
"/usr/local/share/icons"
106+
"/usr/local/share/powershell"
107+
"/usr/local/share/vcpkg"
108+
"/usr/local/share/vim"
109+
"/usr/share/apache-maven-"*
110+
"/usr/share/gradle-"*
111+
"/usr/share/kotlinc"
112+
"/usr/share/miniconda"
113+
"/usr/share/php"
114+
"/usr/share/ri"
115+
"/usr/share/swift"
116+
117+
# binaries
118+
"/usr/local/bin/azcopy"
119+
"/usr/local/bin/bicep"
120+
"/usr/local/bin/ccmake"
121+
"/usr/local/bin/cmake-"*
122+
"/usr/local/bin/cmake"
123+
"/usr/local/bin/cpack"
124+
"/usr/local/bin/ctest"
125+
"/usr/local/bin/helm"
126+
"/usr/local/bin/kind"
127+
"/usr/local/bin/kustomize"
128+
"/usr/local/bin/minikube"
129+
"/usr/local/bin/packer"
130+
"/usr/local/bin/phpunit"
131+
"/usr/local/bin/pulumi-"*
132+
"/usr/local/bin/pulumi"
133+
"/usr/local/bin/stack"
134+
135+
# Haskell runtime
136+
"/usr/local/.ghcup"
137+
138+
# Azure
139+
"/opt/az"
140+
"/usr/share/az_"*
141+
)
142+
143+
if [ -n "${AGENT_TOOLSDIRECTORY:-}" ]; then
144+
# Environment variable set by GitHub Actions
145+
to_remove+=(
146+
"${AGENT_TOOLSDIRECTORY}"
147+
)
148+
else
149+
echo "::warning::AGENT_TOOLSDIRECTORY is not set. Skipping removal."
150+
fi
151+
else
152+
# Remove folders and files present in AWS CodeBuild
153+
to_remove+=(
154+
# binaries
155+
"/usr/local/bin/ecs-cli"
156+
"/usr/local/bin/eksctl"
157+
"/usr/local/bin/kubectl"
158+
159+
"${HOME}/.gradle"
160+
"${HOME}/.dotnet"
161+
"${HOME}/.goenv"
162+
"${HOME}/.phpenv"
163+
164+
)
165+
fi
166+
167+
for element in "${to_remove[@]}"; do
168+
if [ ! -e "$element" ]; then
169+
# The file or directory doesn't exist.
170+
# Maybe it was removed in a newer version of the runner or it's not present in a
171+
# specific architecture (e.g. ARM).
172+
echo "::warning::Directory or file $element does not exist, skipping."
173+
fi
174+
done
175+
176+
# Remove all files and directories at once to save time.
177+
sudo rm -rf "${to_remove[@]}"
178+
}
179+
180+
execAndMeasureSpaceChange() {
181+
local operation=${1} # Function to execute
182+
local title=${2}
183+
184+
local before
185+
before=$(getAvailableSpace)
186+
$operation
187+
188+
printSavedSpace "$before" "$title"
189+
}
190+
191+
# Remove large packages
192+
# REF: https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh
193+
cleanPackages() {
194+
local packages=(
195+
'^aspnetcore-.*'
196+
'^dotnet-.*'
197+
'^llvm-.*'
198+
'^mongodb-.*'
199+
'firefox'
200+
'libgl1-mesa-dri'
201+
'mono-devel'
202+
'php.*'
203+
)
204+
205+
if isGitHubRunner; then
206+
packages+=(
207+
azure-cli
208+
)
209+
210+
if isX86; then
211+
packages+=(
212+
'google-chrome-stable'
213+
'google-cloud-cli'
214+
'google-cloud-sdk'
215+
'powershell'
216+
)
217+
fi
218+
else
219+
packages+=(
220+
'google-chrome-stable'
221+
)
222+
fi
223+
224+
sudo apt-get -qq remove -y --fix-missing "${packages[@]}"
225+
226+
sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed"
227+
sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed failed"
228+
}
229+
230+
# Remove Docker images.
231+
# Ubuntu 22 runners have docker images already installed.
232+
# They aren't present in ubuntu 24 runners.
233+
cleanDocker() {
234+
echo "=> Removing the following docker images:"
235+
sudo docker image ls
236+
echo "=> Removing docker images..."
237+
sudo docker image prune --all --force || true
238+
}
239+
240+
# Remove Swap storage
241+
cleanSwap() {
242+
sudo swapoff -a || true
243+
sudo rm -rf /mnt/swapfile || true
244+
free -h
245+
}
246+
247+
# Display initial disk space stats
248+
249+
AVAILABLE_INITIAL=$(getAvailableSpace)
250+
251+
printDF "BEFORE CLEAN-UP:"
252+
echo ""
253+
execAndMeasureSpaceChange cleanPackages "Unused packages"
254+
execAndMeasureSpaceChange cleanDocker "Docker images"
255+
execAndMeasureSpaceChange cleanSwap "Swap storage"
256+
execAndMeasureSpaceChange removeUnusedFilesAndDirs "Unused files and directories"
257+
258+
# Output saved space statistic
259+
echo ""
260+
printDF "AFTER CLEAN-UP:"
261+
262+
echo ""
263+
echo ""
264+
265+
printSavedSpace "$AVAILABLE_INITIAL" "Total saved"

‎src/ci/scripts/free-disk-space-windows.ps1

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Free disk space on Windows GitHub action runners.
2+
3+
$ErrorActionPreference = 'Stop'
4+
5+
Get-Volume | Out-String | Write-Output
6+
7+
$available = $(Get-Volume C).SizeRemaining
8+
9+
$dirs = 'Does not exit',
10+
'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Tools\Llvm',
11+
'C:\rtools45', 'C:\ghcup', 'C:\Program Files (x86)\Android',
12+
'C:\Program Files\Google\Chrome', 'C:\Program Files (x86)\Microsoft\Edge',
13+
'C:\Program Files\Mozilla Firefox', 'C:\Program Files\MySQL', 'C:\Julia',
14+
'C:\Program Files\MongoDB', 'C:\Program Files\Azure Cosmos DB Emulator',
15+
'C:\Program Files\PostgreSQL', 'C:\Program Files\Unity Hub',
16+
'C:\Strawberry', 'C:\hostedtoolcache\windows\Java_Temurin-Hotspot_jdk',
17+
'C:\does not exist', 'oh no'
18+
19+
foreach ($dir in $dirs) {
20+
Start-ThreadJob { Remove-Item -Recurse -Force $dir }
21+
}
22+
23+
foreach ($job in Get-Job) {
24+
Wait-Job $job > $null
25+
if ($job.Error) {
26+
Write-Output "::warning file=$PSCommandPath::$($job.Error)"
27+
}
28+
Remove-Job $job
29+
}
30+
31+
Get-Volume | Out-String | Write-Output
32+
33+
$saved = ($(Get-Volume C).SizeRemaining - $available) / 1gb
34+
Write-Output "total space saved $saved GB"

0 commit comments

Comments
 (0)