Skip to content

Expand static Test262 execution and exception behavior (#486) #1185

Expand static Test262 execution and exception behavior (#486)

Expand static Test262 execution and exception behavior (#486) #1185

Workflow file for this run

name: CI
on:
push:
branches: [main]
pull_request:
jobs:
tests:
name: test (${{ matrix.flavor }}, ${{ matrix.shard }}/10)
# arm64 macOS: same clang/libc the differential + number-formatting
# oracles were verified against.
runs-on: macos-15
strategy:
fail-fast: false
matrix:
flavor: [plain, san]
shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
env:
# Capped workers: the shared runner's cores oversubscribe under the
# corpus compile fan-out, and vitest's worker RPC times out ("Timeout
# calling onTaskUpdate") with every test green — the same contention
# artifact the harness README documents for local parallel suites.
SCRIPTC_TEST_WORKERS: "4"
# san = ASan + RC audit over the same suite; anything but "1" is plain.
SCRIPTC_SAN: ${{ matrix.flavor == 'san' && '1' || '' }}
# Within-file partition for the large case-driven harnesses
# (differential, llvm-differential, npm, server, Test262, coverage, and
# Vercel E2E): they hash stable case names into slice i of 10
# (tests/harness/shard.ts), so the ten shards' unions run every case
# exactly once. vitest's own --shard is FILE-granular and cannot split
# these monster files (differential.test.ts alone is ~9 min
# sequential), hence the case-sharded steps below.
SCRIPTC_TEST_SHARD: ${{ matrix.shard }}/10
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 11.1.3
- uses: actions/setup-node@v4
with:
node-version-file: .node-version
cache: pnpm
- run: brew install llvm@22
- run: pnpm install --frozen-lockfile
- run: pnpm --filter @scriptc/llvm-darwin-arm64 build:native
- run: pnpm --filter @scriptc/runtime-darwin-arm64 build:native
- run: pnpm build
# Separate vitest invocations because the shard axes must not mix: a file
# lands in exactly ONE --shard slice, so an env-sharded file behind
# --shard would run only one of its ten case slices and silently
# drop the rest. Files split by --shard here; cases split by env below.
- name: Tests (file-sharded, case-sharded files excluded)
run: >-
pnpm test --shard=${{ matrix.shard }}/10
--exclude=tests/harness/differential.test.ts
--exclude=tests/harness/llvm-differential.test.ts
--exclude=tests/harness/npm.test.ts
--exclude=tests/harness/server.test.ts
--exclude=tests/harness/test262.test.ts
--exclude=tests/harness/coverage.test.ts
--exclude=tests/harness/vercel-e2e.test.ts
--exclude=packages/compiler/src/backend/native-toolchain.test.ts
# Cache invalidation tests mutate compiler inputs process-wide and must
# stay serial within a worker. They do not consume SCRIPTC_SAN, so divide
# them across the ten plain jobs instead of running the 6+ minute file
# in full in one shard of both flavors.
- name: Tests (native-cache correctness, slice ${{ matrix.shard }}/10)
if: matrix.flavor == 'plain'
env:
SCRIPTC_CACHE_TEST_SHARD: ${{ matrix.shard }}/10
SCRIPTC_TEST_STABLE_TOOLCHAIN: "0"
run: pnpm test packages/compiler/src/backend/native-toolchain.test.ts
- name: Tests (case-sharded, slice ${{ matrix.shard }}/10)
run: >-
pnpm test
tests/harness/differential.test.ts
tests/harness/llvm-differential.test.ts
tests/harness/npm.test.ts
tests/harness/server.test.ts
tests/harness/test262.test.ts
tests/harness/vercel-e2e.test.ts
# Coverage analysis is frontend-only: SCRIPTC_SAN cannot change its
# result, so the ten plain slices cover the full corpus once.
- name: Tests (coverage sweep, slice ${{ matrix.shard }}/10)
if: matrix.flavor == 'plain'
run: pnpm test tests/harness/coverage.test.ts
# A real host-clang/glibc build, not the zig cross-target lane: catches
# Linux feature-test macro omissions at compile time and separate-libm
# omissions/order errors at link time.
linux_host_clang:
name: test (linux host clang/glibc)
runs-on: ubuntu-24.04
timeout-minutes: 10
env:
# The verdict must come from a live clang invocation, never a cached
# binary copied from a previous build.
SCRIPTC_NO_CACHE: "1"
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 11.1.3
- uses: actions/setup-node@v4
with:
node-version-file: .node-version
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: pnpm build
- name: Host clang static-build regression
run: >-
pnpm test packages/compiler/test/cc-driver.test.ts
--testNamePattern "host-native clang static build"
# The TS7 parity sweep exercises the compiler frontend, so it can run
# alongside the Linux LLVM/runtime package build instead of following it.
ts7_parity:
name: test (TypeScript 7 parity)
runs-on: ubuntu-24.04
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 11.1.3
- uses: actions/setup-node@v4
with:
node-version-file: .node-version
cache: pnpm
- run: pnpm install --frozen-lockfile
- run: pnpm build
- name: TypeScript 7 full parity sweep
run: pnpm test:ts7
llvm_artifacts_linux_x64:
name: test (Linux x64 LLVM artifacts, no clang)
runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 11.1.3
- uses: actions/setup-node@v4
with:
node-version-file: .node-version
cache: pnpm
- name: Install pinned LLVM development distribution
run: |
sudo apt-get update
sudo apt-get install --yes g++ zlib1g-dev libzstd-dev
curl -fL --retry 3 -o "$RUNNER_TEMP/llvm.tar.xz" https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/LLVM-22.1.8-Linux-X64.tar.xz
tar -xJf "$RUNNER_TEMP/llvm.tar.xz" -C "$RUNNER_TEMP"
mv "$RUNNER_TEMP/LLVM-22.1.8-Linux-X64" "$RUNNER_TEMP/llvm-22.1.8"
echo "LLVM_DIR=$RUNNER_TEMP/llvm-22.1.8/lib/cmake/llvm" >> "$GITHUB_ENV"
echo "$RUNNER_TEMP/llvm-22.1.8/bin" >> "$GITHUB_PATH"
- run: pnpm install --frozen-lockfile
- run: pnpm --filter @scriptc/llvm-linux-x64-gnu build:native
- uses: vercel-labs/setup-zig@v1
with:
version: 0.16.0
- run: CC=zig AR=zig pnpm --filter @scriptc/runtime-linux-x64-gnu build:native
- name: Build and verify Linux x64 musl release packages
run: |
pnpm --filter @scriptc/llvm-linux-x64-musl build:native
CC=zig AR=zig pnpm --filter @scriptc/runtime-linux-x64-musl build:native
helper=packages/llvm-linux-x64-musl/bin/scriptc-llvm-codegen
readelf -l "$helper" > "$RUNNER_TEMP/musl-helper-program-headers"
if grep -q INTERP "$RUNNER_TEMP/musl-helper-program-headers"; then
echo "musl helper must be a static executable" >&2
exit 1
fi
docker run --rm --volume "$GITHUB_WORKSPACE:/work:ro" alpine:3.22 \
/work/$helper version --format=json | node -e '
const value = JSON.parse(require("node:fs").readFileSync(0, "utf8"));
if (value.default_target !== "x86_64-unknown-linux-musl" || !value.supported_targets?.includes("wasm32-unknown-wasi")) process.exit(1);
'
node packages/runtime-linux-x64-musl/scripts/verify.mjs
- run: pnpm --filter @scriptc/runtime-wasm32-wasi build:native
- run: pnpm build
- name: Linux helper object format and no-clang output contract
run: |
PATH="$RUNNER_TEMP/traps:$PATH"
mkdir -p "$RUNNER_TEMP/traps"
for tool in clang cc gcc ar ld; do printf '#!/bin/sh\nexit 97\n' > "$RUNNER_TEMP/traps/$tool"; chmod +x "$RUNNER_TEMP/traps/$tool"; done
node packages/cli/dist/main.js build tests/corpus/001-hello.ts --emit=obj -o "$RUNNER_TEMP/hello.o"
file "$RUNNER_TEMP/hello.o" | grep 'ELF 64-bit LSB relocatable, x86-64'
- name: Linux helper/runtime-pack executable differential smoke
env:
SCRIPTC_NO_CACHE: "1"
SCRIPTC_LEGACY_C_PIPELINE: "0"
run: |
node packages/cli/dist/main.js build tests/corpus/001-hello.ts -o "$RUNNER_TEMP/hello"
test "$("$RUNNER_TEMP/hello")" = 'hello world'
- name: Debian 12 glibc 2.36 runtime-pack smoke
run: |
docker run --rm --volume "$GITHUB_WORKSPACE:/work:ro" --workdir /work node:24-bookworm-slim sh -ec '
apt-get update
apt-get install --yes --no-install-recommends clang
SCRIPTC_NO_CACHE=1 SCRIPTC_LEGACY_C_PIPELINE=0 node packages/cli/dist/main.js build tests/corpus/001-hello.ts -o /tmp/hello
test "$(/tmp/hello)" = "hello world"
'
- name: WASI helper object/runtime-pack smoke
run: pnpm test packages/cli/test/wasi-runtime-pack.test.ts
llvm_artifacts_macos_arm64:
name: test (macOS arm64 LLVM artifacts, no clang, ${{ matrix.shard }}/5)
runs-on: macos-15
# Shard 1 additionally runs package/install and link-parity contracts
# before its LLVM differential slice. The slice itself can take ~10
# minutes on a cold hosted runner, so 20 minutes cancels an otherwise
# healthy job before the differential completes.
timeout-minutes: 35
strategy:
fail-fast: false
matrix:
shard: [1, 2, 3, 4, 5]
env:
# The helper differential uses the harness's stable per-case sharding,
# keeping each macOS job below its timeout while the matrix union still
# exercises every LLVM-tier program.
SCRIPTC_TEST_SHARD: ${{ matrix.shard }}/5
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 11.1.3
- uses: actions/setup-node@v4
with:
node-version-file: .node-version
cache: pnpm
- run: brew install llvm@22
- run: pnpm install --frozen-lockfile
- run: pnpm --filter @scriptc/llvm-darwin-arm64 build:native
- run: pnpm --filter @scriptc/runtime-darwin-arm64 build:native
- run: pnpm build
- name: No-clang assembly/object contract
if: matrix.shard == 1
run: pnpm test packages/cli/test/native-output.test.ts
- name: Helper diagnostics and object/link parity
if: matrix.shard == 1
run: >-
pnpm test
packages/compiler/test/native-codegen-integration.test.ts
packages/compiler/src/backend/runtime-pack.test.ts
packages/cli/test/native-link-info.test.ts
packages/cli/test/runtime-pack.test.ts
tests/harness/native-object-example.test.ts
- name: Default helper/runtime-pack executable does not compile C
if: matrix.shard == 1
env:
SCRIPTC_LEGACY_C_PIPELINE: "0"
run: >-
pnpm test packages/cli/test/runtime-pack.test.ts
--testNamePattern "legacy C pipeline disabled"
- name: LLVM-tier helper object differential (${{ matrix.shard }}/5)
env:
SCRIPTC_LLVM_HELPER_ONLY: "1"
SCRIPTC_TEST_WORKERS: "4"
run: pnpm test tests/harness/llvm-differential.test.ts
- name: Packaged helper contract
if: matrix.shard == 1
run: |
TARBALL=$(pnpm --dir packages/llvm-darwin-arm64 pack --pack-destination "$RUNNER_TEMP" --silent)
node scripts/verify-llvm-package.mjs "$RUNNER_TEMP/$(basename "$TARBALL")"
- name: Packed npm installation smoke
if: matrix.shard == 1
run: |
pnpm --dir packages/runtime pack --pack-destination "$RUNNER_TEMP" --silent
pnpm --dir packages/runtime-darwin-arm64 pack --pack-destination "$RUNNER_TEMP" --silent
pnpm --dir packages/compiler pack --pack-destination "$RUNNER_TEMP" --silent
pnpm --dir packages/cli pack --pack-destination "$RUNNER_TEMP" --silent
PREFIX="$RUNNER_TEMP/installed-scriptc"
npm install --prefix "$PREFIX" --ignore-scripts \
"$RUNNER_TEMP/scriptc-runtime-$(node -p "require('./packages/runtime/package.json').version").tgz" \
"$RUNNER_TEMP/scriptc-runtime-darwin-arm64-$(node -p "require('./packages/runtime-darwin-arm64/package.json').version").tgz" \
"$RUNNER_TEMP/scriptc-llvm-darwin-arm64-$(node -p "require('./packages/llvm-darwin-arm64/package.json').version").tgz" \
"$RUNNER_TEMP/scriptc-compiler-$(node -p "require('./packages/compiler/package.json').version").tgz" \
"$RUNNER_TEMP/scriptc-$(node -p "require('./packages/cli/package.json').version").tgz"
"$PREFIX/node_modules/.bin/scriptc" build tests/corpus/001-hello.ts \
--emit=obj -o "$RUNNER_TEMP/installed.o"
file "$RUNNER_TEMP/installed.o" | grep 'Mach-O 64-bit object arm64'
"$PREFIX/node_modules/.bin/scriptc" build tests/corpus/001-hello.ts \
--print=native-link-info -o "$RUNNER_TEMP/installed-link.o" \
> "$RUNNER_TEMP/installed-link.json"
node examples/native-object/link.mjs cc \
"$RUNNER_TEMP/installed-link.json" "$RUNNER_TEMP/installed-program"
test "$("$RUNNER_TEMP/installed-program")" = 'hello world'
llvm_artifacts_macos_x64:
name: test (macOS x64 LLVM artifacts, no clang)
runs-on: macos-15-intel
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 11.1.3
- uses: actions/setup-node@v4
with:
node-version-file: .node-version
cache: pnpm
- run: brew install llvm@22
- run: pnpm install --frozen-lockfile
- run: LLVM_DIR=/usr/local/opt/llvm@22/lib/cmake/llvm pnpm --filter @scriptc/llvm-darwin-x64 build:native
- run: pnpm --filter @scriptc/runtime-darwin-x64 build:native
- run: pnpm build
- name: Intel Mach-O no-clang object contract
run: |
mkdir -p "$RUNNER_TEMP/traps"
for tool in clang cc gcc ar ld; do printf '#!/bin/sh\nexit 97\n' > "$RUNNER_TEMP/traps/$tool"; chmod +x "$RUNNER_TEMP/traps/$tool"; done
PATH="$RUNNER_TEMP/traps:$PATH" node packages/cli/dist/main.js build tests/corpus/001-hello.ts --emit=obj -o "$RUNNER_TEMP/hello.o"
file "$RUNNER_TEMP/hello.o" | grep 'Mach-O 64-bit object x86_64'
- name: Intel helper/runtime-pack executable smoke
env:
SCRIPTC_NO_CACHE: "1"
SCRIPTC_LEGACY_C_PIPELINE: "0"
run: |
node packages/cli/dist/main.js build tests/corpus/001-hello.ts -o "$RUNNER_TEMP/hello"
test "$($RUNNER_TEMP/hello)" = 'hello world'
# Exercises the supported Windows GNU target and the built CLI end to end:
# TS7 must open its synthetic project, ambient files must resolve across
# slash styles, and the default executable must use the .exe suffix.
windows_cli:
name: test (windows CLI smoke)
runs-on: windows-latest
timeout-minutes: 15
env:
SCRIPTC_NO_CACHE: "1"
SCRIPTC_CC: zigcc
SCRIPTC_TARGET: x86_64-windows-gnu
steps:
- uses: actions/checkout@v4
- uses: vercel-labs/setup-zig@v1
with:
version: 0.16.0
- uses: pnpm/action-setup@v4
with:
version: 11.1.3
- uses: actions/setup-node@v4
with:
node-version-file: .node-version
cache: pnpm
- name: Install dependencies
shell: pwsh
run: |
pnpm install --frozen-lockfile
if ($LASTEXITCODE -ne 0) {
Write-Host "pnpm exited without a diagnostic; retrying once"
pnpm install --frozen-lockfile
}
- run: pnpm build
- name: Windows path regressions
run: pnpm exec vitest run packages/compiler/test/ts7/program-adapter.test.ts packages/compiler/test/ts7/resolver-parity.test.ts packages/cli/test/paths.test.ts
- name: Windows Sandbox path regression
run: >-
pnpm exec vitest run tests/harness/worktree-files.test.ts
--testNamePattern "workspace reset keeps Linux Sandbox paths POSIX"
- name: Windows scriptc run smoke
shell: pwsh
run: |
zig version
$output = & node packages/cli/dist/main.js run tests/corpus/001-hello.ts --backend c
if ($LASTEXITCODE -ne 0) {
exit $LASTEXITCODE
}
$actual = ($output -join "`n").Trim()
if ($actual -ne "hello world") {
throw "unexpected scriptc output: $actual"
}
- name: Windows PE subsystem selection
env:
SCRIPTC_NO_CACHE: "0"
run: pnpm exec vitest run packages/cli/test/windows-subsystem.test.ts
- name: Windows CA-store differential
shell: pwsh
run: |
$nodeOutput = & node tests/corpus/2557-tls-ca-store.ts
if ($LASTEXITCODE -ne 0) {
exit $LASTEXITCODE
}
$nativeOutput = & node packages/cli/dist/main.js run tests/corpus/2557-tls-ca-store.ts --backend c
if ($LASTEXITCODE -ne 0) {
exit $LASTEXITCODE
}
$expected = ($nodeOutput -join "`n").Trim()
$actual = ($nativeOutput -join "`n").Trim()
if ($actual -ne $expected) {
throw "Windows CA-store output differed:`nNODE:`n$expected`nNATIVE:`n$actual"
}
- name: Windows CA EKU policy
run: >-
pnpm exec vitest run packages/compiler/test/cc-driver.test.ts
--testNamePattern "Windows CA roots honor"
llvm_artifacts_windows_x64:
name: test (Windows x64 MSVC LLVM artifacts)
# The official LLVM 22 Windows archives are MSVC-built and the helper
# selects CMake's VS 2022 generator. `windows-latest` can move to an
# image without that generator, so retain the hosted VS 2022 contract.
runs-on: windows-2022
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 11.1.3
- uses: actions/setup-node@v4
with:
node-version-file: .node-version
cache: pnpm
- name: Install pinned LLVM development distribution
shell: pwsh
run: |
# windows-2022 already provides CMake, Ninja, VS 2022, and 7-Zip.
# Use the complete archive because it supplies LLVM's exported CMake
# targets and static libraries. Windows tar.exe takes 25+ minutes to
# materialize it; two-stage 7-Zip extraction takes seconds.
$archive = "$env:RUNNER_TEMP\clang+llvm-22.1.8-x86_64-pc-windows-msvc.tar.xz"
$llvm = "$env:RUNNER_TEMP\llvm-22.1.8"
$expanded = "$env:RUNNER_TEMP\llvm-expanded"
$sevenZip = "$env:ProgramFiles\7-Zip\7z.exe"
if (-not (Test-Path $sevenZip)) {
choco install 7zip --no-progress -y
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
}
if (-not (Test-Path $sevenZip)) { throw "7-Zip is unavailable" }
curl.exe -fL --retry 3 -o $archive https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/clang%2Bllvm-22.1.8-x86_64-pc-windows-msvc.tar.xz
& $sevenZip x -y "-o$expanded" $archive
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
& $sevenZip x -y "-o$expanded" "$expanded\clang+llvm-22.1.8-x86_64-pc-windows-msvc.tar"
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
Move-Item "$expanded\clang+llvm-22.1.8-x86_64-pc-windows-msvc" $llvm
echo "LLVM_DIR=$llvm\lib\cmake\llvm" >> $env:GITHUB_ENV
echo "$llvm\bin" >> $env:GITHUB_PATH
- run: pnpm install --frozen-lockfile
- name: Build Windows helper
run: pnpm --filter @scriptc/llvm-win32-x64-msvc build:native
- uses: vercel-labs/setup-zig@v1
with:
version: 0.16.0
- name: Build Windows runtime pack
run: pnpm --filter @scriptc/runtime-win32-x64-msvc build:native
- run: pnpm build
- name: Windows COFF no-clang object contract
shell: pwsh
run: |
node packages/cli/dist/main.js build tests/corpus/001-hello.ts --emit=obj -o "$env:RUNNER_TEMP\hello.obj"
$bytes = [System.IO.File]::ReadAllBytes("$env:RUNNER_TEMP\hello.obj")
if ($bytes[0] -ne 0x64 -or $bytes[1] -ne 0x86) { throw "expected AMD64 COFF object" }
# Retain the pre-matrix "test" check for branch protection and badges. It
# passes only when every matrix shard and independent integration job does.
test:
needs: [tests, linux_host_clang, ts7_parity, llvm_artifacts_linux_x64, llvm_artifacts_macos_arm64, llvm_artifacts_macos_x64, windows_cli, llvm_artifacts_windows_x64]
if: always()
runs-on: ubuntu-latest
steps:
- name: All lanes green
run: |
test "${{ needs.tests.result }}" = success
test "${{ needs.linux_host_clang.result }}" = success
test "${{ needs.ts7_parity.result }}" = success
test "${{ needs.llvm_artifacts_linux_x64.result }}" = success
test "${{ needs.llvm_artifacts_macos_arm64.result }}" = success
test "${{ needs.llvm_artifacts_macos_x64.result }}" = success
test "${{ needs.windows_cli.result }}" = success
test "${{ needs.llvm_artifacts_windows_x64.result }}" = success