Skip to content

Commit 329f346

Browse files
QiliangCuiQiliangCui
committed
Move "skip" out of test command.
Signed-off-by: Qiliang Cui <[email protected]>
1 parent 5644ffb commit 329f346

File tree

2 files changed

+52
-100
lines changed

2 files changed

+52
-100
lines changed

‎.buildkite/pipeline_jax.yml‎

Lines changed: 48 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -19,34 +19,24 @@ steps:
1919
- label: "E2E MLPerf tests for JAX models with quantization"
2020
key: test_1
2121
soft_fail: true
22+
if: build.env("NIGHTLY") == "1"
2223
env:
2324
QUANTIZATION: "True"
2425
agents:
2526
queue: tpu_v6e_queue
2627
commands:
27-
- |
28-
if [[ "$$NIGHTLY" == "1" ]]; then
29-
.buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
30-
else
31-
echo "Skipping: NIGHTLY environment variable not set"
32-
exit 0
33-
fi
28+
- .buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
3429

3530
- label: "E2E MLPerf tests for JAX new models"
3631
key: test_2
3732
soft_fail: true
33+
if: build.env("NIGHTLY") == "1"
3834
env:
3935
NEW_MODEL_DESIGN: "1"
4036
agents:
4137
queue: tpu_v6e_queue
4238
commands:
43-
- |
44-
if [[ "$$NIGHTLY" == "1" ]]; then
45-
.buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
46-
else
47-
echo "Skipping: NIGHTLY environment variable not set"
48-
exit 0
49-
fi
39+
- .buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
5040

5141
- label: "E2E MLPerf tests for JAX + vLLM models on single chip"
5242
key: test_3
@@ -61,52 +51,35 @@ steps:
6151
- label: "E2E MLperf tests for Llama4 models"
6252
key: test_4
6353
soft_fail: true
54+
if: build.env("NIGHTLY") == "1"
6455
env:
6556
NEW_MODEL_DESIGN: "1"
6657
USE_V6E8_QUEUE: "True"
6758
agents:
6859
queue: tpu_v6e_8_queue
6960
commands:
70-
- |
71-
if [[ "$$NIGHTLY" == "1" ]]; then
72-
.buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
73-
else
74-
echo "Skipping: NIGHTLY environment variable not set"
75-
exit 0
76-
fi
77-
61+
- .buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
7862

7963
- label: "E2E multi modality test"
8064
key: test_5
8165
soft_fail: true
66+
if: build.env("NIGHTLY") == "1"
8267
agents:
8368
queue: tpu_v6e_queue
8469
commands:
85-
- |
86-
if [[ "$$NIGHTLY" == "1" ]]; then
87-
.buildkite/scripts/run_in_docker.sh \
88-
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/e2e/test_multi_modal_inference.py && \
89-
bash /workspace/tpu_inference/tests/e2e/benchmarking/mm_bench.sh'
90-
else
91-
echo "Skipping: NIGHTLY environment variable not set"
92-
exit 0
93-
fi
70+
- .buildkite/scripts/run_in_docker.sh \
71+
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/e2e/test_multi_modal_inference.py && \
72+
bash /workspace/tpu_inference/tests/e2e/benchmarking/mm_bench.sh'
9473

9574
- label: "E2E speculative decoding test"
9675
key: test_6
9776
soft_fail: true
77+
if: build.env("NIGHTLY") == "1"
9878
agents:
9979
queue: tpu_v6e_queue
10080
commands:
101-
- |
102-
if [[ "$$NIGHTLY" == "1" ]]; then
103-
.buildkite/scripts/run_in_docker.sh \
104-
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/e2e/test_speculative_decoding.py'
105-
else
106-
echo "Skipping: NIGHTLY environment variable not set"
107-
exit 0
108-
fi
109-
81+
- .buildkite/scripts/run_in_docker.sh \
82+
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/e2e/test_speculative_decoding.py'
11083

11184
- label: "JAX unit tests"
11285
key: test_7
@@ -143,71 +116,62 @@ steps:
143116
- label: "JAX unit tests - kernels"
144117
key: test_8
145118
soft_fail: true
119+
if: |
120+
build.env("NIGHTLY") == "1"
121+
if_changed:
122+
- "tpu_inference/kernels/**/*"
123+
- "tests/kernels/**/*"
124+
- "requirements.txt"
146125
agents:
147126
queue: tpu_v6e_queue
148127
commands:
149-
- |
150-
if [[ "$$NIGHTLY" == "1" ]] || git diff --name-only HEAD~1 | grep -qE '^(tpu_inference/kernels|tests/kernels|requirements\.txt)'; then
151-
.buildkite/scripts/run_in_docker.sh \
152-
python3 -m pytest -s -v -x /workspace/tpu_inference/tests/kernels \
153-
--ignore=/workspace/tpu_inference/tests/kernels/ragged_paged_attention_kernel_v2_test.py \
154-
--ignore=/workspace/tpu_inference/tests/kernels/ragged_kv_cache_update_v2_test.py \
155-
--ignore=/workspace/tpu_inference/tests/kernels/collectives \
156-
--ignore=/workspace/tpu_inference/tests/kernels/fused_moe_v1_test.py
157-
else
158-
echo "Skipping: no changes detected in kernels, tests/kernels, or requirements.txt"
159-
exit 0
160-
fi
128+
- .buildkite/scripts/run_in_docker.sh \
129+
python3 -m pytest -s -v -x /workspace/tpu_inference/tests/kernels \
130+
--ignore=/workspace/tpu_inference/tests/kernels/ragged_paged_attention_kernel_v2_test.py \
131+
--ignore=/workspace/tpu_inference/tests/kernels/ragged_kv_cache_update_v2_test.py \
132+
--ignore=/workspace/tpu_inference/tests/kernels/collectives \
133+
--ignore=/workspace/tpu_inference/tests/kernels/fused_moe_v1_test.py
161134

162135
- label: "JAX unit tests - collective kernels"
163136
key: test_9
164137
soft_fail: true
138+
if: |
139+
build.env("NIGHTLY") == "1"
140+
if_changed:
141+
- "tpu_inference/kernels/collectives/**/*"
142+
- "tests/kernels/collectives/**/*"
143+
- "requirements.txt"
165144
agents:
166-
queue: tpu_v6e_8_queue
145+
queue: tpu_v6e_8_queue
167146
commands:
168-
- |
169-
if [[ "$$NIGHTLY" == "1" ]] || git diff --name-only HEAD~1 | grep -qE '^(tpu_inference/kernels/collectives|tests/kernels/collectives|requirements\.txt)'; then
170-
.buildkite/scripts/run_in_docker.sh \
171-
python3 -m pytest -s -v -x /workspace/tpu_inference/tests/kernels/collectives
172-
else
173-
echo "Skipping: no changes detected in kernels/collectives, tests/kernels/collectives, or requirements.txt"
174-
exit 0
175-
fi
147+
- .buildkite/scripts/run_in_docker.sh \
148+
python3 -m pytest -s -v -x /workspace/tpu_inference/tests/kernels/collectives
176149

177150
- label: "lora e2e tests for JAX + vLLM models single chip"
178151
key: test_10
179152
soft_fail: true
153+
if: build.env("NIGHTLY") == "1"
180154
agents:
181155
queue: tpu_v6e_queue
182156
commands:
183-
- |
184-
if [[ "$$NIGHTLY" == "1" ]]; then
185-
.buildkite/scripts/run_in_docker.sh \
186-
bash -c 'MODEL_IMPL_TYPE=vllm TPU_BACKEND_TYPE=jax python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_lora.py'
187-
else
188-
echo "Skipping: NIGHTLY environment variable not set"
189-
exit 0
190-
fi
157+
- .buildkite/scripts/run_in_docker.sh \
158+
bash -c 'MODEL_IMPL_TYPE=vllm TPU_BACKEND_TYPE=jax python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_lora.py'
191159

192160
- label: "E2E MLPerf tests for JAX + vLLM models on multiple chips"
193161
key: test_11
194162
soft_fail: true
163+
if: build.env("NIGHTLY") == "1"
195164
env:
196165
MODEL_IMPL_TYPE: "vllm"
197166
agents:
198167
queue: tpu_v6e_8_queue
199168
commands:
200-
- |
201-
if [[ "$$NIGHTLY" == "1" ]]; then
202-
.buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
203-
else
204-
echo "Skipping: NIGHTLY environment variable not set"
205-
exit 0
206-
fi
169+
- .buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh
207170

208171
- label: "E2E MLperf tests for DeepSeek-R1 (no accuracy, 12-decoder layers only)"
209172
key: test_12
210173
soft_fail: true
174+
if: build.env("NIGHTLY") == "1"
211175
env:
212176
NEW_MODEL_DESIGN: "1"
213177
USE_V6E8_QUEUE: "True"
@@ -216,32 +180,20 @@ steps:
216180
agents:
217181
queue: tpu_v6e_8_queue
218182
commands:
219-
- |
220-
if [[ "$$NIGHTLY" == "1" ]]; then
221-
.buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh -m deepseek-ai/DeepSeek-R1-0528 --use-dummy-weights
222-
else
223-
echo "Skipping: NIGHTLY environment variable not set"
224-
exit 0
225-
fi
183+
- .buildkite/scripts/run_in_docker.sh bash /workspace/tpu_inference/tests/e2e/benchmarking/mlperf.sh -m deepseek-ai/DeepSeek-R1-0528 --use-dummy-weights
226184

227185
- label: "lora e2e tests for JAX + vLLM models multi chips"
228186
key: test_13
229187
soft_fail: true
188+
if: build.env("NIGHTLY") == "1"
230189
env:
231190
USE_V6E8_QUEUE: "True"
232191
VLLM_LOG_LEVEL: "INFO"
233192
agents:
234193
queue: tpu_v6e_8_queue
235194
commands:
236-
- |
237-
if [[ "$$NIGHTLY" == "1" ]]; then
238-
.buildkite/scripts/run_in_docker.sh \
239-
bash -c 'MODEL_IMPL_TYPE=vllm TPU_BACKEND_TYPE=jax python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_lora.py'
240-
else
241-
echo "Skipping: NIGHTLY environment variable not set"
242-
exit 0
243-
fi
244-
195+
- .buildkite/scripts/run_in_docker.sh \
196+
bash -c 'MODEL_IMPL_TYPE=vllm TPU_BACKEND_TYPE=jax python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_lora.py'
245197

246198
- label: "lora unit tests on single chip"
247199
key: test_15
@@ -257,20 +209,16 @@ steps:
257209
- label: "lora unit tests on multi chips"
258210
key: test_16
259211
soft_fail: true
212+
if: build.env("NIGHTLY") == "1"
260213
env:
261214
USE_V6E8_QUEUE: "True"
262215
VLLM_LOG_LEVEL: "INFO"
263216
agents:
264217
queue: tpu_v6e_8_queue
265218
commands:
266-
- |
267-
if [[ "$$NIGHTLY" == "1" ]]; then
268-
.buildkite/scripts/run_in_docker.sh \
269-
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_layers.py'
270-
else
271-
echo "Skipping: NIGHTLY environment variable not set"
272-
exit 0
273-
fi
219+
- .buildkite/scripts/run_in_docker.sh \
220+
bash -c 'python3 -m pytest -s -v -x /workspace/tpu_inference/tests/lora/test_layers.py'
221+
274222
# -----------------------------------------------------------------
275223
# NOTIFICATION STEP
276224
# -----------------------------------------------------------------

‎.buildkite/scripts/check_results.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@ echo "--- Checking Test Outcomes"
1414

1515
for KEY in "$@"; do
1616
OUTCOME=$(buildkite-agent step get "outcome" --step "${KEY}" || echo "skipped")
17+
if [ -z "$OUTCOME" ]; then
18+
OUTCOME="skipped"
19+
fi
20+
1721
echo "Step ${KEY} outcome: ${OUTCOME}"
1822

1923
if [ "${OUTCOME}" != "passed" ] && [ "${OUTCOME}" != "skipped" ] ; then

0 commit comments

Comments
 (0)