TST: dask tests are extremely flaky (#263)

Reviewed at #263

Author: crusaderky

.github/workflows/array-api-tests-dask.yml

@@ -10,4 +10,9 @@ jobs:
       package-version: '>= 2024.9.0'
       module-name: dask.array
       extra-requires: numpy
-      pytest-extra-args: --disable-deadline --max-examples=5
+      # Dask is substantially slower then other libraries on unit tests.
+      # Reduce the number of examples to speed up CI, even though this means that this
+      # workflow is barely more than a smoke test, and one should expect extreme 
+      # flakiness. Before changes to dask-xfails.txt or dask-skips.txt, please run
+      # the full test suite with at least 200 examples.
+      pytest-extra-args: --max-examples=5

.github/workflows/array-api-tests.yml

@@ -33,7 +33,7 @@ on:
         description: "Multiline string of environment variables to set for the test run."
 
 env:
-  PYTEST_ARGS: "--max-examples 200 -v -rxXfE --ci ${{ inputs.pytest-extra-args }} --hypothesis-disable-deadline"
+  PYTEST_ARGS: "--max-examples 200 -v -rxXfE --ci ${{ inputs.pytest-extra-args }} --hypothesis-disable-deadline --durations 10"
 
 jobs:
   tests:

dask-skips.txt

@@ -1,5 +1,9 @@
-# slow and not implemented in dask
-array_api_tests/test_linalg.py::test_matrix_power
+# NOTE: dask tests run on a very small number of examples in CI due to
+# slowness. This causes very high flakiness in the tests.
+# Before changing this file, please run with at least 200 examples.
 
-# hangs on 2024.12
+# Passes, but extremely slow
+array_api_tests/test_linalg.py::test_outer
+
+# Hangs
 array_api_tests/test_creation_functions.py::test_eye

dask-xfails.txt

@@ -1,3 +1,11 @@
+# NOTE: dask tests run on a very small number of examples in CI due to
+# slowness. This causes very high flakiness in the tests.
+# Before changing this file, please run with at least 200 examples.
+
+# Broken edge case with shape 0
+# https://github.com/dask/dask/issues/11800
+array_api_tests/test_array_object.py::test_setitem
+
 # Various indexing errors
 array_api_tests/test_array_object.py::test_getitem_masking
 
@@ -11,6 +19,9 @@ array_api_tests/test_data_type_functions.py::test_finfo[float32]
 # (I think the test is not forcing the op to be computed?)
 array_api_tests/test_creation_functions.py::test_linspace
 
+# Shape mismatch
+array_api_tests/test_indexing_functions.py::test_take
+
 # Array methods and attributes not already on da.Array cannot be wrapped
 array_api_tests/test_has_names.py::test_has_names[array_method-__array_namespace__]
 array_api_tests/test_has_names.py::test_has_names[array_method-to_device]
@@ -31,7 +42,7 @@ array_api_tests/test_set_functions.py::test_unique_values
 # fails for ndim > 2
 array_api_tests/test_linalg.py::test_svdvals
 
-# dtype mismatch got uint64, but should be uint8, NPY_PROMOTION_STATE=weak doesn't help :(
+# dtype mismatch got uint64, but should be uint8; NPY_PROMOTION_STATE=weak doesn't help
 array_api_tests/test_linalg.py::test_tensordot
 
 # AssertionError: out.dtype=uint64, but should be uint8 [tensordot(uint8, uint8)]
@@ -89,29 +100,28 @@ array_api_tests/meta/test_hypothesis_helpers.py::test_symmetric_matrices
 # https://github.com/dask/dask/issues/11706
 array_api_tests/test_creation_functions.py::test_arange
 
+# da.searchsorted with a sorter argument is not supported
+array_api_tests/test_searching_functions.py::test_searchsorted
+
 # 2023.12 support
 array_api_tests/test_manipulation_functions.py::test_repeat
 
 # 2024.12 support
-array_api_tests/test_array_object.py::test_setitem
 array_api_tests/test_array_object.py::test_getitem_arrays_and_ints_1[1]
 array_api_tests/test_array_object.py::test_getitem_arrays_and_ints_1[None]
 array_api_tests/test_array_object.py::test_getitem_arrays_and_ints_2[1]
 array_api_tests/test_array_object.py::test_getitem_arrays_and_ints_2[None]
 array_api_tests/test_has_names.py::test_has_names[indexing-take_along_axis]
+array_api_tests/test_signatures.py::test_func_signature[count_nonzero]
 array_api_tests/test_signatures.py::test_func_signature[take_along_axis]
 
 array_api_tests/test_linalg.py::test_cholesky
 array_api_tests/test_linalg.py::test_linalg_matmul
+array_api_tests/test_linalg.py::test_matmul
 array_api_tests/test_linalg.py::test_matrix_norm
 array_api_tests/test_linalg.py::test_qr
-array_api_tests/test_manipulation_functions.py::test_concat
 array_api_tests/test_manipulation_functions.py::test_roll
-array_api_tests/test_operators_and_elementwise_functions.py::test_add[add(x1, x2)]
-array_api_tests/test_operators_and_elementwise_functions.py::test_bitwise_left_shift[bitwise_left_shift(x1, x2)]
-array_api_tests/test_operators_and_elementwise_functions.py::test_bitwise_right_shift[bitwise_right_shift(x1, x2)]
-array_api_tests/test_operators_and_elementwise_functions.py::test_greater[__gt__(x1, x2)]
-array_api_tests/test_signatures.py::test_func_signature[count_nonzero]
+
 array_api_tests/test_special_cases.py::test_binary[floor_divide(x1_i is +infinity and isfinite(x2_i) and x2_i > 0) -> +infinity]
 array_api_tests/test_special_cases.py::test_binary[floor_divide(x1_i is +infinity and isfinite(x2_i) and x2_i < 0) -> -infinity]
 array_api_tests/test_special_cases.py::test_binary[floor_divide(x1_i is -infinity and isfinite(x2_i) and x2_i > 0) -> -infinity]
@@ -131,5 +141,3 @@ array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(x1_i is -infinity
 array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(isfinite(x1_i) and x1_i > 0 and x2_i is -infinity) -> -0]
 array_api_tests/test_special_cases.py::test_iop[__ifloordiv__(isfinite(x1_i) and x1_i < 0 and x2_i is +infinity) -> -0]
 array_api_tests/test_special_cases.py::test_nan_propagation[cumulative_prod]
-
-