Skip to content

Commit ff674d0

Browse files
QiliangCuiQiliangCui
committed
Move "skip" out of test command.
Signed-off-by: Qiliang Cui <[email protected]>
1 parent 5644ffb commit ff674d0

File tree

2 files changed

+50
-100
lines changed

2 files changed

+50
-100
lines changed

‎.buildkite/pipeline_jax.yml‎

Lines changed: 46 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -19,34 +19,24 @@ steps:
1919
- label: "E2E MLPerf tests for JAX models with quantization"
2020
key: test_1
2121
soft_fail: true
22+
if: build.env("NIGHTLY") == "1"
2223
env:
2324
QUANTIZATION: "True"
2425
agents:
2526
queue: tpu_v6e_queue
2627
commands:
27-
- |
28-
if [[ "$$NIGHTLY" == "1" ]]; then
29-
.buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
30-
else
31-
echo "Skipping: NIGHTLY environment variable not set"
32-
exit 0
33-
fi
28+
- .buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
3429

3530
- label: "E2E MLPerf tests for JAX new models"
3631
key: test_2
3732
soft_fail: true
33+
if: build.env("NIGHTLY") == "1"
3834
env:
3935
NEW_MODEL_DESIGN: "1"
4036
agents:
4137
queue: tpu_v6e_queue
4238
commands:
43-
- |
44-
if [[ "$$NIGHTLY" == "1" ]]; then
45-
.buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
46-
else
47-
echo "Skipping: NIGHTLY environment variable not set"
48-
exit 0
49-
fi
39+
- .buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
5040

5141
- label: "E2E MLPerf tests for JAX + vLLM models on single chip"
5242
key: test_3
@@ -61,52 +51,35 @@ steps:
6151
- label: "E2E MLperf tests for Llama4 models"
6252
key: test_4
6353
soft_fail: true
54+
if: build.env("NIGHTLY") == "1"
6455
env:
6556
NEW_MODEL_DESIGN: "1"
6657
USE_V6E8_QUEUE: "True"
6758
agents:
6859
queue: tpu_v6e_8_queue
6960
commands:
70-
- |
71-
if [[ "$$NIGHTLY" == "1" ]]; then
72-
.buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
73-
else
74-
echo "Skipping: NIGHTLY environment variable not set"
75-
exit 0
76-
fi
77-
61+
- .buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
7862

7963
- label: "E2E multi modality test"
8064
key: test_5
8165
soft_fail: true
66+
if: build.env("NIGHTLY") == "1"
8267
agents:
8368
queue: tpu_v6e_queue
8469
commands:
85-
- |
86-
if [[ "$$NIGHTLY" == "1" ]]; then
87-
.buildkite/scripts/run_in_docker.sh \
88-
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/e2e/test_multi_modal_inference.py && \
89-
bash /workspace/tpu_inference/tests/e2e/benchmarking/mm_bench.sh'
90-
else
91-
echo "Skipping: NIGHTLY environment variable not set"
92-
exit 0
93-
fi
70+
- .buildkite/scripts/run_in_docker.sh \
71+
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/e2e/test_multi_modal_inference.py && \
72+
bash /workspace/tpu_inference/tests/e2e/benchmarking/mm_bench.sh'
9473

9574
- label: "E2E speculative decoding test"
9675
key: test_6
9776
soft_fail: true
77+
if: build.env("NIGHTLY") == "1"
9878
agents:
9979
queue: tpu_v6e_queue
10080
commands:
101-
- |
102-
if [[ "$$NIGHTLY" == "1" ]]; then
103-
.buildkite/scripts/run_in_docker.sh \
104-
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/e2e/test_speculative_decoding.py'
105-
else
106-
echo "Skipping: NIGHTLY environment variable not set"
107-
exit 0
108-
fi
109-
81+
- .buildkite/scripts/run_in_docker.sh \
82+
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/e2e/test_speculative_decoding.py'
11083

11184
- label: "JAX unit tests"
11285
key: test_7
@@ -143,71 +116,60 @@ steps:
143116
- label: "JAX unit tests - kernels"
144117
key: test_8
145118
soft_fail: true
119+
if: |
120+
build.env("NIGHTLY") == "1" ||
121+
buildkite.git.hasChangedFiles("tpu_inference/kernels/**") ||
122+
buildkite.git.hasChangedFiles("tests/kernels/**") ||
123+
buildkite.git.hasChangedFiles("requirements.txt")
146124
agents:
147125
queue: tpu_v6e_queue
148126
commands:
149-
- |
150-
if [[ "$$NIGHTLY" == "1" ]] || git diff --name-only HEAD~1 | grep -qE '^(tpu_inference/kernels|tests/kernels|requirements\.txt)'; then
151-
.buildkite/scripts/run_in_docker.sh \
152-
python3 -m pytest -s -v -x /workspace/tpu_inference/tests/kernels \
153-
--ignore=/workspace/tpu_inference/tests/kernels/ragged_paged_attention_kernel_v2_test.py \
154-
--ignore=/workspace/tpu_inference/tests/kernels/ragged_kv_cache_update_v2_test.py \
155-
--ignore=/workspace/tpu_inference/tests/kernels/collectives \
156-
--ignore=/workspace/tpu_inference/tests/kernels/fused_moe_v1_test.py
157-
else
158-
echo "Skipping: no changes detected in kernels, tests/kernels, or requirements.txt"
159-
exit 0
160-
fi
127+
- .buildkite/scripts/run_in_docker.sh \
128+
python3 -m pytest -s -v -x /workspace/tpu_inference/tests/kernels \
129+
--ignore=/workspace/tpu_inference/tests/kernels/ragged_paged_attention_kernel_v2_test.py \
130+
--ignore=/workspace/tpu_inference/tests/kernels/ragged_kv_cache_update_v2_test.py \
131+
--ignore=/workspace/tpu_inference/tests/kernels/collectives \
132+
--ignore=/workspace/tpu_inference/tests/kernels/fused_moe_v1_test.py
161133

162134
- label: "JAX unit tests - collective kernels"
163135
key: test_9
164136
soft_fail: true
137+
if: |
138+
build.env("NIGHTLY") == "1" ||
139+
buildkite.git.hasChangedFiles("tpu_inference/kernels/collectives/**") ||
140+
buildkite.git.hasChangedFiles("tests/kernels/collectives/**") ||
141+
buildkite.git.hasChangedFiles("requirements.txt")
165142
agents:
166-
queue: tpu_v6e_8_queue
143+
queue: tpu_v6e_8_queue
167144
commands:
168-
- |
169-
if [[ "$$NIGHTLY" == "1" ]] || git diff --name-only HEAD~1 | grep -qE '^(tpu_inference/kernels/collectives|tests/kernels/collectives|requirements\.txt)'; then
170-
.buildkite/scripts/run_in_docker.sh \
171-
python3 -m pytest -s -v -x /workspace/tpu_inference/tests/kernels/collectives
172-
else
173-
echo "Skipping: no changes detected in kernels/collectives, tests/kernels/collectives, or requirements.txt"
174-
exit 0
175-
fi
145+
- .buildkite/scripts/run_in_docker.sh \
146+
python3 -m pytest -s -v -x /workspace/tpu_inference/tests/kernels/collectives
176147

177148
- label: "lora e2e tests for JAX + vLLM models single chip"
178149
key: test_10
179150
soft_fail: true
151+
if: build.env("NIGHTLY") == "1"
180152
agents:
181153
queue: tpu_v6e_queue
182154
commands:
183-
- |
184-
if [[ "$$NIGHTLY" == "1" ]]; then
185-
.buildkite/scripts/run_in_docker.sh \
186-
bash -c 'MODEL_IMPL_TYPE=vllm TPU_BACKEND_TYPE=jax python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_lora.py'
187-
else
188-
echo "Skipping: NIGHTLY environment variable not set"
189-
exit 0
190-
fi
155+
- .buildkite/scripts/run_in_docker.sh \
156+
bash -c 'MODEL_IMPL_TYPE=vllm TPU_BACKEND_TYPE=jax python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_lora.py'
191157

192158
- label: "E2E MLPerf tests for JAX + vLLM models on multiple chips"
193159
key: test_11
194160
soft_fail: true
161+
if: build.env("NIGHTLY") == "1"
195162
env:
196163
MODEL_IMPL_TYPE: "vllm"
197164
agents:
198165
queue: tpu_v6e_8_queue
199166
commands:
200-
- |
201-
if [[ "$$NIGHTLY" == "1" ]]; then
202-
.buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
203-
else
204-
echo "Skipping: NIGHTLY environment variable not set"
205-
exit 0
206-
fi
167+
- .buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
207168

208169
- label: "E2E MLperf tests for DeepSeek-R1 (no accuracy, 12-decoder layers only)"
209170
key: test_12
210171
soft_fail: true
172+
if: build.env("NIGHTLY") == "1"
211173
env:
212174
NEW_MODEL_DESIGN: "1"
213175
USE_V6E8_QUEUE: "True"
@@ -216,32 +178,20 @@ steps:
216178
agents:
217179
queue: tpu_v6e_8_queue
218180
commands:
219-
- |
220-
if [[ "$$NIGHTLY" == "1" ]]; then
221-
.buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh -m deepseek-ai/DeepSeek-R1-0528 --use-dummy-weights
222-
else
223-
echo "Skipping: NIGHTLY environment variable not set"
224-
exit 0
225-
fi
181+
- .buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh -m deepseek-ai/DeepSeek-R1-0528 --use-dummy-weights
226182

227183
- label: "lora e2e tests for JAX + vLLM models multi chips"
228184
key: test_13
229185
soft_fail: true
186+
if: build.env("NIGHTLY") == "1"
230187
env:
231188
USE_V6E8_QUEUE: "True"
232189
VLLM_LOG_LEVEL: "INFO"
233190
agents:
234191
queue: tpu_v6e_8_queue
235192
commands:
236-
- |
237-
if [[ "$$NIGHTLY" == "1" ]]; then
238-
.buildkite/scripts/run_in_docker.sh \
239-
bash -c 'MODEL_IMPL_TYPE=vllm TPU_BACKEND_TYPE=jax python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_lora.py'
240-
else
241-
echo "Skipping: NIGHTLY environment variable not set"
242-
exit 0
243-
fi
244-
193+
- .buildkite/scripts/run_in_docker.sh \
194+
bash -c 'MODEL_IMPL_TYPE=vllm TPU_BACKEND_TYPE=jax python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_lora.py'
245195

246196
- label: "lora unit tests on single chip"
247197
key: test_15
@@ -257,20 +207,16 @@ steps:
257207
- label: "lora unit tests on multi chips"
258208
key: test_16
259209
soft_fail: true
210+
if: build.env("NIGHTLY") == "1"
260211
env:
261212
USE_V6E8_QUEUE: "True"
262213
VLLM_LOG_LEVEL: "INFO"
263214
agents:
264215
queue: tpu_v6e_8_queue
265216
commands:
266-
- |
267-
if [[ "$$NIGHTLY" == "1" ]]; then
268-
.buildkite/scripts/run_in_docker.sh \
269-
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_layers.py'
270-
else
271-
echo "Skipping: NIGHTLY environment variable not set"
272-
exit 0
273-
fi
217+
- .buildkite/scripts/run_in_docker.sh \
218+
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_layers.py'
219+
274220
# -----------------------------------------------------------------
275221
# NOTIFICATION STEP
276222
# -----------------------------------------------------------------

‎.buildkite/scripts/check_results.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@ echo "--- Checking Test Outcomes"
1414

1515
for KEY in "$@"; do
1616
OUTCOME=$(buildkite-agent step get "outcome" --step "${KEY}" || echo "skipped")
17+
if [ -z "$OUTCOME" ]; then
18+
OUTCOME="skipped"
19+
fi
20+
1721
echo "Step ${KEY} outcome: ${OUTCOME}"
1822

1923
if [ "${OUTCOME}" != "passed" ] && [ "${OUTCOME}" != "skipped" ] ; then

0 commit comments

Comments
 (0)