Skip to content

Commit 459e592

Browse files
weliveindetailweliveindetail
authored
[windows] Prepare toolchain build to run tests for LLDB (#76894)
The LLDB test suite is not very stable on Windows. In order to improve the situation, however, we need to get it tested continuously. This patch attempts to solve the chicken-egg problem. It introduces a simple override for unstable tests: list them in file `utils/windows-llvm-lit-test-overrides.txt` and SwiftCI will skip/xfail them. This approach has a number of benefits: * overrides don't need to be in sync with https://github.com/swiftlang/llvm-project * overrides are tracked in one place and are not spread across the LLDB test suite * overrides are swiftlang-specific, which clearly states the differences to upstream LLVM * we can enable continuous testing (and get reports for new failures) without fixing the world first Once the remaining subset of tests passes reliably, we can iterate to enable the others and increase coverage. This change implements the infrastructure. There is a separate PR to enable tests in CI, which can be reverted with fewer side-effects.
1 parent e06a6c7 commit 459e592

File tree

2 files changed

+264
-2
lines changed

2 files changed

+264
-2
lines changed

‎utils/build.ps1

+80-2
Original file line numberDiff line numberDiff line change
@@ -725,6 +725,14 @@ function Fetch-Dependencies {
725725

726726
DownloadAndVerify $PinnedBuild "$BinaryCache\$PinnedToolchain.exe" $PinnedSHA256
727727

728+
if ($Test -contains "lldb") {
729+
# The make tool isn't part of MSYS
730+
$GnuWin32MakeURL = "https://downloads.sourceforge.net/project/ezwinports/make-4.4.1-without-guile-w32-bin.zip"
731+
$GnuWin32MakeHash = "fb66a02b530f7466f6222ce53c0b602c5288e601547a034e4156a512dd895ee7"
732+
DownloadAndVerify $GnuWin32MakeURL "$BinaryCache\GnuWin32Make-4.4.1.zip" $GnuWin32MakeHash
733+
Extract-ZipFile GnuWin32Make-4.4.1.zip $BinaryCache GnuWin32Make-4.4.1
734+
}
735+
728736
# TODO(compnerd) stamp/validate that we need to re-extract
729737
New-Item -ItemType Directory -ErrorAction Ignore $BinaryCache\toolchains | Out-Null
730738
Extract-Toolchain "$PinnedToolchain.exe" $BinaryCache $PinnedToolchain
@@ -772,6 +780,28 @@ function Fetch-Dependencies {
772780
Write-Output "Installing 'setuptools-75.1.0-py3-none-any.whl' ..."
773781
Invoke-Program -OutNull $Python '-I' -m pip install "$BinaryCache\python\setuptools-75.1.0-py3-none-any.whl" --disable-pip-version-check
774782
}
783+
if ($Test -contains "lldb") {
784+
# 'psutil' is required for testing LLDB
785+
try {
786+
Invoke-Program -OutNull $Python -c 'import psutil' *> $null
787+
} catch {
788+
$WheelURL = "https://files.pythonhosted.org/packages/11/91/87fa6f060e649b1e1a7b19a4f5869709fbf750b7c8c262ee776ec32f3028/psutil-6.1.0-cp37-abi3-win_amd64.whl"
789+
$WheelHash = "a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be"
790+
DownloadAndVerify $WheelURL "$BinaryCache\python\psutil-6.1.0-cp37-abi3-win_amd64.whl" $WheelHash
791+
Write-Output "Installing 'psutil-6.1.0-cp37-abi3-win_amd64.whl' ..."
792+
Invoke-Program -OutNull $Python '-I' -m pip install "$BinaryCache\python\psutil-6.1.0-cp37-abi3-win_amd64.whl" --disable-pip-version-check
793+
}
794+
# 'unittest2' is required for testing LLDB
795+
try {
796+
Invoke-Program -OutNull $Python -c 'import unittest2' *> $null
797+
} catch {
798+
$WheelURL = "https://files.pythonhosted.org/packages/72/20/7f0f433060a962200b7272b8c12ba90ef5b903e218174301d0abfd523813/unittest2-1.1.0-py2.py3-none-any.whl"
799+
$WheelHash = "13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8"
800+
DownloadAndVerify $WheelURL "$BinaryCache\python\unittest2-1.1.0-py2.py3-none-any.whl" $WheelHash
801+
Write-Output "Installing 'unittest2-1.1.0-py2.py3-none-any.whl' ..."
802+
Invoke-Program -OutNull $Python '-I' -m pip install "$BinaryCache\python\unittest2-1.1.0-py2.py3-none-any.whl" --disable-pip-version-check
803+
}
804+
}
775805
}
776806

777807
Download-Python $HostArchName
@@ -1215,6 +1245,13 @@ function Build-CMakeProject {
12151245
} elseif ($UsePinnedCompilers.Contains("Swift")) {
12161246
$env:Path = "$(Get-PinnedToolchainRuntime);${env:Path}"
12171247
}
1248+
1249+
if ($ToBatch) {
1250+
Write-Output ""
1251+
Write-Output "echo cmake.exe $cmakeGenerateArgs"
1252+
} else {
1253+
Write-Host "cmake.exe $cmakeGenerateArgs"
1254+
}
12181255
Invoke-Program cmake.exe @cmakeGenerateArgs
12191256

12201257
# Build all requested targets
@@ -1471,11 +1508,50 @@ function Build-Compilers() {
14711508
SWIFT_NATIVE_SWIFT_TOOLS_PATH = "";
14721509
}
14731510

1511+
if ($TestLLVM) { $Targets += @("check-llvm") }
14741512
if ($TestClang) { $Targets += @("check-clang") }
14751513
if ($TestLLD) { $Targets += @("check-lld") }
1476-
if ($TestLLDB) { $Targets += @("check-lldb") }
1477-
if ($TestLLVM) { $Targets += @("check-llvm") }
14781514
if ($TestSwift) { $Targets += @("check-swift", "SwiftCompilerPlugin") }
1515+
if ($TestLLDB) {
1516+
$Targets += @("check-lldb")
1517+
1518+
function Select-LitTestOverrides {
1519+
param([string] $TestStatus)
1520+
1521+
$MatchingLines=(Get-Content $PSScriptRoot/windows-llvm-lit-test-overrides.txt | Select-String -Pattern "`^${TestStatus}.*$")
1522+
$TestNames=$MatchingLines | ForEach-Object { ($_ -replace $TestStatus,"").Trim() }
1523+
return $TestNames
1524+
}
1525+
1526+
# Override some test results with llvm-lit.
1527+
$TestsToXFail=Select-LitTestOverrides "xfail"
1528+
$TestsToSkip=Select-LitTestOverrides "skip"
1529+
$env:LIT_XFAIL=$TestsToXFail -join ";"
1530+
$env:LIT_FILTER_OUT="($($TestsToSkip -join '|'))"
1531+
1532+
# Transitive dependency of _lldb.pyd
1533+
$RuntimeBinaryCache = Get-TargetProjectBinaryCache $Arch Runtime
1534+
cp $RuntimeBinaryCache\bin\swiftCore.dll "$CompilersBinaryCache\lib\site-packages\lldb"
1535+
1536+
# Runtime dependencies of repl_swift.exe
1537+
$SwiftRTSubdir = "lib\swift\windows"
1538+
Write-Host "Copying '$RuntimeBinaryCache\$SwiftRTSubdir\$($Arch.LLVMName)\swiftrt.obj' to '$CompilersBinaryCache\$SwiftRTSubdir'"
1539+
cp "$RuntimeBinaryCache\$SwiftRTSubdir\$($Arch.LLVMName)\swiftrt.obj" "$CompilersBinaryCache\$SwiftRTSubdir"
1540+
Write-Host "Copying '$RuntimeBinaryCache\bin\swiftCore.dll' to '$CompilersBinaryCache\bin'"
1541+
cp "$RuntimeBinaryCache\bin\swiftCore.dll" "$CompilersBinaryCache\bin"
1542+
1543+
$TestingDefines += @{
1544+
LLDB_INCLUDE_TESTS = "YES";
1545+
# Check for required Python modules in CMake
1546+
LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS = "YES";
1547+
# No watchpoint support on windows: https://github.com/llvm/llvm-project/issues/24820
1548+
LLDB_TEST_USER_ARGS = "--skip-category=watchpoint";
1549+
# gtest sharding breaks llvm-lit's --xfail and LIT_XFAIL inputs: https://github.com/llvm/llvm-project/issues/102264
1550+
LLVM_LIT_ARGS = "-v --no-gtest-sharding --show-xfail";
1551+
# LLDB Unit tests link against this library
1552+
LLVM_UNITTEST_LINK_FLAGS = "$($Arch.SDKInstallRoot)\usr\lib\swift\windows\$($Arch.LLVMName)\swiftCore.lib";
1553+
}
1554+
}
14791555
} else {
14801556
$Targets = @("distribution", "install-distribution")
14811557
$TestingDefines = @{
@@ -1517,9 +1593,11 @@ function Build-Compilers() {
15171593
LLDB_PYTHON_EXT_SUFFIX = ".pyd";
15181594
LLDB_PYTHON_RELATIVE_PATH = "lib/site-packages";
15191595
LLDB_TABLEGEN = (Join-Path -Path $BuildTools -ChildPath "lldb-tblgen.exe");
1596+
LLDB_TEST_MAKE = "$BinaryCache\GnuWin32Make-4.4.1\bin\make.exe";
15201597
LLVM_CONFIG_PATH = (Join-Path -Path $BuildTools -ChildPath "llvm-config.exe");
15211598
LLVM_EXTERNAL_SWIFT_SOURCE_DIR = "$SourceCache\swift";
15221599
LLVM_NATIVE_TOOL_DIR = $BuildTools;
1600+
LLVM_HOST_TRIPLE = $BuildArch.LLVMTarget;
15231601
LLVM_TABLEGEN = (Join-Path $BuildTools -ChildPath "llvm-tblgen.exe");
15241602
LLVM_USE_HOST_TOOLS = "NO";
15251603
Python3_EXECUTABLE = (Get-PythonExecutable);

‎utils/windows-llvm-lit-test-overrides.txt

+184
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
# build.ps1 overrides the status of each test in this file when LLVM tests are run with lit.
2+
#
3+
# Prefer `xfail` for tests that fail reliably. Use `skip` for flaky tests.
4+
# Test summaries call them "excluded". Please find the right "Tests that ..."
5+
# category for new entries.
6+
7+
### Tests that fail reliably ###
8+
9+
xfail lldb-api :: lang/cpp/unique-types4/TestUniqueTypes4.py
10+
xfail lldb-shell :: Recognizer/verbose_trap.test
11+
xfail lldb-shell :: Settings/TestEchoCommands.test
12+
xfail lldb-shell :: Swift/MissingVFSOverlay.test
13+
xfail lldb-shell :: Swift/No.swiftmodule.test
14+
xfail lldb-shell :: Swift/ToolchainMismatch.test
15+
xfail lldb-shell :: Swift/global.test
16+
xfail lldb-shell :: Swift/runtime-initialization.test
17+
xfail lldb-shell :: SwiftREPL/BreakpointSimple.test
18+
xfail lldb-shell :: SwiftREPL/ClosureScope.test
19+
xfail lldb-shell :: SwiftREPL/ExistentialTypes.test
20+
xfail lldb-shell :: SwiftREPL/LookupAfterImport.test
21+
xfail lldb-shell :: SwiftREPL/LookupWithAttributedImport.test
22+
xfail lldb-shell :: SwiftREPL/OpenClass.test
23+
xfail lldb-shell :: SwiftREPL/OptionalUnowned.test
24+
xfail lldb-shell :: SwiftREPL/RedirectInputUnreadable.test
25+
xfail lldb-shell :: SwiftREPL/SwiftInterface.test
26+
xfail lldb-shell :: SymbolFile/DWARF/x86/dead-code-filtering.yaml
27+
28+
### Tests that pass locally, but fail in CI reliably ###
29+
30+
# https://github.com/swiftlang/llvm-project/issues/9539
31+
xfail lldb-shell :: SwiftREPL/Basic.test
32+
xfail lldb-shell :: SwiftREPL/Class.test
33+
xfail lldb-shell :: SwiftREPL/ComputedProperties.test
34+
xfail lldb-shell :: SwiftREPL/Deadlock.test
35+
xfail lldb-shell :: SwiftREPL/DiagnosticOptions.test
36+
xfail lldb-shell :: SwiftREPL/Dict.test
37+
xfail lldb-shell :: SwiftREPL/ErrorReturn.test
38+
xfail lldb-shell :: SwiftREPL/ExclusivityREPL.test
39+
xfail lldb-shell :: SwiftREPL/GenericTypealias.test
40+
xfail lldb-shell :: SwiftREPL/Generics.test
41+
xfail lldb-shell :: SwiftREPL/ImportError.test
42+
xfail lldb-shell :: SwiftREPL/MetatypeRepl.test
43+
xfail lldb-shell :: SwiftREPL/Optional.test
44+
xfail lldb-shell :: SwiftREPL/PropertyWrapperTopLevel.test
45+
xfail lldb-shell :: SwiftREPL/RecursiveClass.test
46+
xfail lldb-shell :: SwiftREPL/Redefinition.test
47+
xfail lldb-shell :: SwiftREPL/RedirectInput.test
48+
xfail lldb-shell :: SwiftREPL/RedirectInputNoSuchFile.test
49+
xfail lldb-shell :: SwiftREPL/Regex.test
50+
xfail lldb-shell :: SwiftREPL/SimpleExpressions.test
51+
xfail lldb-shell :: SwiftREPL/Struct.test
52+
xfail lldb-shell :: SwiftREPL/Subclassing.test
53+
xfail lldb-shell :: SwiftREPL/SwiftInterfaceForceModuleLoadMode.test
54+
xfail lldb-shell :: SwiftREPL/SwiftTypeLookup.test
55+
xfail lldb-shell :: SwiftREPL/SyntaxError.test
56+
xfail lldb-shell :: SwiftREPL/UninitVariables.test
57+
xfail lldb-shell :: SwiftREPL/ZeroSizeStruct.test
58+
xfail lldb-shell :: SwiftREPL/enum-singlecase.test
59+
xfail lldb-shell :: SwiftREPL/one-char-string.test
60+
61+
xfail lldb-api :: commands/apropos/with-process/TestAproposWithProcess.py
62+
xfail lldb-api :: commands/command/nested_alias/TestNestedAlias.py
63+
xfail lldb-api :: commands/expression/entry-bp/TestExprEntryBP.py
64+
xfail lldb-api :: commands/memory/write/TestMemoryWrite.py
65+
xfail lldb-api :: functionalities/breakpoint/address_breakpoints/TestAddressBreakpoints.py
66+
xfail lldb-api :: functionalities/breakpoint/auto_continue/TestBreakpointAutoContinue.py
67+
xfail lldb-api :: functionalities/breakpoint/breakpoint_conditions/TestBreakpointConditions.py
68+
xfail lldb-api :: functionalities/breakpoint/breakpoint_options/TestBreakpointOptions.py
69+
xfail lldb-api :: functionalities/breakpoint/step_over_breakpoint/TestStepOverBreakpoint.py
70+
xfail lldb-api :: functionalities/conditional_break/TestConditionalBreak.py
71+
xfail lldb-api :: functionalities/memory/find/TestMemoryFind.py
72+
xfail lldb-api :: lang/c/anonymous/TestAnonymous.py
73+
xfail lldb-api :: lang/c/array_types/TestArrayTypes.py
74+
xfail lldb-api :: lang/c/enum_types/TestEnumTypes.py
75+
xfail lldb-api :: lang/c/forward/TestForwardDeclaration.py
76+
xfail lldb-api :: lang/c/function_types/TestFunctionTypes.py
77+
xfail lldb-api :: lang/c/register_variables/TestRegisterVariables.py
78+
xfail lldb-api :: lang/c/set_values/TestSetValues.py
79+
xfail lldb-api :: lang/c/shared_lib/TestSharedLib.py
80+
xfail lldb-api :: lang/cpp/class_types/TestClassTypes.py
81+
xfail lldb-api :: lang/cpp/inlines/TestInlines.py
82+
xfail lldb-api :: python_api/compile_unit/TestCompileUnitAPI.py
83+
xfail lldb-api :: python_api/thread/TestThreadAPI.py
84+
xfail lldb-api :: source-manager/TestSourceManager.py
85+
xfail lldb-shell :: Driver/TestConvenienceVariables.test
86+
87+
# https://github.com/swiftlang/llvm-project/issues/9620
88+
xfail lldb-shell :: Swift/expression-diagnostics.test
89+
90+
# https://github.com/swiftlang/llvm-project/issues/9540
91+
xfail lldb-shell :: SymbolFile/NativePDB/local-variables.cpp
92+
xfail lldb-shell :: SymbolFile/NativePDB/stack_unwinding01.cpp
93+
xfail lldb-api :: lang/c/local_variables/TestLocalVariables.py
94+
95+
# https://github.com/swiftlang/llvm-project/issues/9637
96+
xfail lldb-api :: python_api/address_range/TestAddressRange.py
97+
98+
# https://github.com/swiftlang/llvm-project/issues/9643
99+
xfail lldb-shell :: Commands/command-process-launch-user-entry.test
100+
101+
# Skip SymbolTests because we cannot xfail unittests by name. We would need to
102+
# specify their indexes, but these are subject to change. Right now, failures
103+
# are:
104+
# FAIL: lldb-unit :: Symbol/./SymbolTests.exe/24/27 (1 of 27)
105+
# FAIL: lldb-unit :: Symbol/./SymbolTests.exe/21/27 (3 of 27)
106+
# FAIL: lldb-unit :: Symbol/./SymbolTests.exe/22/27 (9 of 27)
107+
# FAIL: lldb-unit :: Symbol/./SymbolTests.exe/25/27 (11 of 27)
108+
#
109+
# Respective tests are:
110+
# lldb-unit :: Symbol/./SymbolTests.exe/ClangArgs/UniquingCollisionWithAddedFlags
111+
# lldb-unit :: Symbol/./SymbolTests.exe/ClangArgs/UniquingCollisionWithExistingFlags
112+
# lldb-unit :: Symbol/./SymbolTests.exe/TestSwiftASTContext/ApplyWorkingDir
113+
# lldb-unit :: Symbol/./SymbolTests.exe/TestSwiftASTContext/PluginPath
114+
#
115+
skip lldb-unit :: Symbol/./SymbolTests.exe
116+
117+
### Tests that fail occasionally ###
118+
119+
# https://github.com/swiftlang/llvm-project/issues/9705
120+
skip lldb-api :: python_api/section/TestSectionAPI.py
121+
122+
# Passes upstream in bot lldb-aarch64-windows; it fails in swiftlang
123+
# since https://github.com/swiftlang/llvm-project/pull/9493 relanded
124+
skip lldb-api :: functionalities/breakpoint/same_cu_name/TestFileBreakpoinsSameCUName.py
125+
126+
# Disable 37 tests: 17 unsupported, 6 usually failed, 14 usually passed (but flaky)
127+
# https://github.com/swiftlang/llvm-project/issues/9099
128+
# https://github.com/swiftlang/llvm-project/issues/9100
129+
# https://github.com/swiftlang/llvm-project/issues/9101
130+
skip lldb-api :: tools/lldb-server
131+
132+
### Tests that time out occasionally ###
133+
134+
skip lldb-api :: commands/dwim-print/TestDWIMPrint.py
135+
skip lldb-api :: commands/thread/backtrace/TestThreadBacktraceRepeat.py
136+
skip lldb-api :: functionalities/inferior-crashing/TestInferiorCrashing.py
137+
skip lldb-api :: functionalities/step_scripted/TestStepScripted.py
138+
139+
### Tests that pass occasionally ###
140+
141+
# Fixed upstream: https://github.com/llvm/llvm-project/commit/ec009994a06338995dfb6431c943b299f9327fd2
142+
# But patches don't apply on stable branch due to downstream changes.
143+
skip lldb-api :: functionalities/archives/TestBSDArchives.py
144+
skip lldb-api :: macosx/duplicate-archive-members/TestDuplicateMembers.py
145+
146+
### Tests that pass accidentally ###
147+
148+
https://github.com/llvm/llvm-project/issues/116972
149+
skip lldb-api :: api/command-return-object/TestSBCommandReturnObject.py
150+
skip lldb-api :: api/multiple-targets/TestMultipleTargets.py
151+
152+
### Tests that fail during setup and thus show up as UNRESOLVED ***
153+
154+
# https://github.com/swiftlang/llvm-project/issues/9887
155+
skip lldb-api :: lang/swift/async/tasks/TestSwiftTaskBacktrace.py
156+
skip lldb-api :: lang/swift/async/tasks/TestSwiftTaskSelect.py
157+
158+
skip lldb-api :: functionalities/breakpoint/breakpoint_command/TestBreakpointCommandsFromPython.py
159+
skip lldb-api :: tools/lldb-dap/instruction-breakpoint/TestDAP_instruction_breakpoint.py
160+
skip lldb-api :: tools/lldb-dap/output/TestDAP_output.py
161+
162+
### Untriaged ###
163+
164+
skip lldb-api :: commands/statistics/basic/TestStats.py
165+
skip lldb-api :: lang/BoundsSafety/array_of_ptrs/TestArrayOfBoundsSafetyPointers.py
166+
skip lldb-api :: lang/BoundsSafety/out_of_bounds_pointer/TestOutOfBoundsPointer.py
167+
skip lldb-shell :: Commands/command-expr-diagnostics.test
168+
skip lldb-unit :: Host/./HostTests.exe
169+
170+
# https://github.com/llvm/llvm-project/issues/62983
171+
skip lldb-api :: functionalities/var_path/TestVarPath.py
172+
173+
# https://github.com/swiftlang/llvm-project/issues/9073
174+
skip lldb-api :: lang/c/trampoline_stepping/TestTrampolineStepping.py
175+
176+
# https://github.com/swiftlang/llvm-project/issues/9072
177+
skip lldb-api :: lang/cpp/bitfields/TestCppBitfields.py
178+
179+
skip lldb-shell :: Swift/astcontext_error.test
180+
skip lldb-shell :: Swift/cond-breakpoint.test
181+
182+
skip lldb-api :: functionalities/data-formatter/data-formatter-cpp/TestDataFormatterCpp.py
183+
skip lldb-api :: lang/cpp/frame-var-depth-and-elem-count/TestFrameVarDepthAndElemCount.py
184+
skip lldb-api :: functionalities/inferior-crashing/recursive-inferior/TestRecursiveInferiorStep.py

0 commit comments

Comments
 (0)