ci: Add doctests to CI again. (#767)

Co-authored-by: Willow Ahrens <[email protected]>

Author: hameerabbasi

.github/workflows/ci.yml

@@ -50,7 +50,12 @@ jobs:
           pip install -e '.[tests]'
       - name: Run tests
         run: |
-          SPARSE_BACKEND=Numba pytest --pyargs sparse --cov-report=xml:coverage_Numba.xml -n auto -vvv
+          if [ $(python -c 'import numpy as np; print(np.lib.NumpyVersion(np.__version__) >= "2.0.0a1")') = 'True' ]; then
+            pytest --pyargs sparse --doctest-modules --cov-report=xml:coverage_Numba.xml -n auto -vvv
+          else
+            pytest --pyargs sparse --cov-report=xml:coverage_Numba.xml -n auto -vvv
+          fi
+          python -c 'import finch'
           SPARSE_BACKEND=Finch pytest --pyargs sparse/tests --cov-report=xml:coverage_Finch.xml -n auto -vvv
           SPARSE_BACKEND=MLIR pytest --pyargs sparse/mlir_backend --cov-report=xml:coverage_MLIR.xml -n auto -vvv
       - uses: codecov/codecov-action@v5
@@ -126,6 +131,9 @@ jobs:
         SPARSE_BACKEND: ${{ matrix.backend }}
       run: |
         cd ${GITHUB_WORKSPACE}/array-api-tests
+        if [ "${SPARSE_BACKEND}" = "Finch" ]; then
+            python -c 'import finch'
+        fi
         pytest array_api_tests -v -c pytest.ini -n auto --max-examples=2 --derandomize --disable-deadline -o xfail_strict=True --xfails-file ${GITHUB_WORKSPACE}/ci/${{ matrix.backend }}-array-api-xfails.txt --skips-file ${GITHUB_WORKSPACE}/ci/${{ matrix.backend }}-array-api-skips.txt
 
 on:

.gitignore

@@ -49,6 +49,7 @@ coverage.xml
 test_results/
 junit/
 .hypothesis/
+coverage_*.xml
 
 # Translations
 *.mo
@@ -85,4 +86,3 @@ docs/examples_ipynb/
 
 # Pixi envs
 .pixi/
-pixi.lock

ci/environment.yml

@@ -5,14 +5,14 @@ channels:
 dependencies:
   - python
   - pip
-  - numpy
-  - numba
-  - scipy
-  - dask
-  - pytest
-  - pytest-cov
-  - pytest-xdist
   - pip:
     - finch-tensor>=0.2.2
     - finch-mlir>=0.0.2
     - pytest-codspeed
+    - numpy
+    - numba
+    - scipy
+    - dask
+    - pytest
+    - pytest-cov
+    - pytest-xdist

conftest.py

@@ -1,3 +1,7 @@
+import pathlib
+
+import sparse
+
 import pytest
 
 
@@ -7,8 +11,18 @@ def add_doctest_modules(doctest_namespace):
 
     import numpy as np
 
-    if sparse._BackendType.Numba != sparse._BACKEND:
-        pass  # TODO: pytest.skip() skips Finch and MLIR tests
-
     doctest_namespace["np"] = np
     doctest_namespace["sparse"] = sparse
+
+
+def pytest_ignore_collect(collection_path: pathlib.Path, config: pytest.Config) -> bool | None:
+    if "numba_backend" in collection_path.parts and sparse._BackendType.Numba != sparse._BACKEND:
+        return True
+
+    if "mlir_backend" in collection_path.parts and sparse._BackendType.MLIR != sparse._BACKEND:
+        return True
+
+    if "finch_backend" in collection_path.parts and sparse._BackendType.Finch != sparse._BACKEND:
+        return True
+
+    return None

pixi.lock

@@ -12,29 +12,28 @@ numpy = ">=1.17"
 [dependencies]
 python = ">=3.10,<3.13"
 
-[feature.extras.pypi-dependencies]
+[feature.extra.pypi-dependencies]
 dask = { version = ">=2024", extras = ["array"] }
 scipy = ">=0.19"
 scikit-learn = "*"
 
-[feature.docs.pypi-dependencies]
+[feature.doc.pypi-dependencies]
 mkdocs-material = "*"
 mkdocstrings = { version = "*", extras = ["python"] }
 mkdocs-gen-files = "*"
 mkdocs-literate-nav = "*"
 mkdocs-section-index = "*"
 mkdocs-jupyter = "*"
 
-[feature.tests.tasks]
-test = "pytest --pyargs sparse -n auto"
-test-mlir = { cmd = "pytest --pyargs sparse.mlir_backend -v" }
+[feature.test.tasks]
+test = "pytest -n auto --doctest-modules"
+test-mlir = "pytest --pyargs sparse.mlir_backend -v"
 test-finch = { cmd = "pytest --pyargs sparse/tests -n auto -v", depends-on = ["precompile"] }
 
-[feature.tests.dependencies]
+[feature.test.pypi-dependencies]
 pytest = ">=3.5"
 pytest-cov = "*"
 pytest-xdist = "*"
-pre-commit = "*"
 pytest-codspeed = "*"
 
 [feature.notebooks.pypi-dependencies]
@@ -51,6 +50,7 @@ matrepr = "*"
 precompile = "python -c 'import finch'"
 
 [feature.finch.dependencies]
+python = ">=3.10"
 juliaup = ">=1.17.10"
 
 [feature.finch.pypi-dependencies]
@@ -63,16 +63,20 @@ SPARSE_BACKEND = "Finch"
 [feature.finch.target.osx-arm64.activation.env]
 PYTHONFAULTHANDLER = "${HOME}/faulthandler.log"
 
+[feature.mlir.dependencies]
+python = ">=3.10"
+
 [feature.mlir.pypi-dependencies]
 scipy = ">=0.19"
 finch-mlir = ">=0.0.2"
+"PyYAML" = "*"
 
 [feature.mlir.activation.env]
 SPARSE_BACKEND = "MLIR"
 
 [environments]
-tests = ["tests", "extras"]
-docs = ["docs", "extras"]
-mlir-dev = {features = ["tests", "mlir"], no-default-feature = true}
-finch-dev = {features = ["tests", "finch"], no-default-feature = true}
-notebooks = ["extras", "mlir", "finch", "notebooks"]
+test = ["test", "extra"]
+doc = ["doc", "extra"]
+mlir-dev = {features = ["test", "mlir"], no-default-feature = true}
+finch-dev = {features = ["test", "finch"], no-default-feature = true}
+notebooks = ["extra", "mlir", "finch", "notebooks"]

pixi.toml

@@ -1,5 +1,5 @@
 [pytest]
-addopts = --cov-report term-missing --cov-report html --cov-report=term:skip-covered --cov sparse --cov-config .coveragerc --junitxml=junit/test-results.xml
+addopts = --cov-report term-missing --cov-report html --cov-report=term:skip-covered --cov sparse --cov-config .coveragerc
 filterwarnings =
     ignore::PendingDeprecationWarning
 testpaths =

pytest.ini

@@ -9,9 +9,6 @@
 import numpy as np
 import scipy.sparse as sps
 
-if sparse._BACKEND != sparse._BackendType.MLIR:
-    pytest.skip("skipping MLIR tests", allow_module_level=True)
-
 parametrize_dtypes = pytest.mark.parametrize(
     "dtype",
     [

sparse/mlir_backend/tests/__init__.py

@@ -1176,13 +1176,13 @@ def _parse_einsum_input(operands):
     Examples
     --------
     The operand list is simplified to reduce printing:
-    >>> np.random.seed(123)
-    >>> a = np.random.rand(4, 4)
-    >>> b = np.random.rand(4, 4, 4)
-    >>> _parse_einsum_input(("...a,...a->...", a, b))
-    ('za,xza', 'xz', [a, b]) # may vary
-    >>> _parse_einsum_input((a, [Ellipsis, 0], b, [Ellipsis, 0]))
-    ('za,xza', 'xz', [a, b]) # may vary
+    >>> rng = np.random.default_rng(42)
+    >>> a = rng.random((4, 4))
+    >>> b = rng.random((4, 4, 4))
+    >>> _parse_einsum_input(("...a,...a->...", a, b))  # doctest: +SKIP
+    ('za,xza', 'xz', [a, b])
+    >>> _parse_einsum_input((a, [Ellipsis, 0], b, [Ellipsis, 0]))  # doctest: +SKIP
+    ('za,xza', 'xz', [a, b])
     """
 
     if len(operands) == 0:
@@ -2061,7 +2061,7 @@ def asarray(obj, /, *, dtype=None, format="coo", copy=False, device=None):
     Examples
     --------
     >>> x = np.eye(8, dtype="i8")
-    >>> sparse.asarray(x, format="COO")
+    >>> sparse.asarray(x, format="coo")
     <COO: shape=(8, 8), dtype=int64, nnz=8, fill_value=0>
     """
 

sparse/mlir_backend/tests/test_simple.py

@@ -819,7 +819,7 @@ def _prune(self):
         --------
         >>> coords = np.array([[0, 1, 2, 3]])
         >>> data = np.array([1, 0, 1, 2])
-        >>> s = COO(coords, data).asformat("gcxs")
+        >>> s = COO(coords, data, shape=(4,)).asformat("gcxs")
         >>> s._prune()
         >>> s.nnz
         3

sparse/numba_backend/_common.py

@@ -901,7 +901,7 @@ def diagonalize(a, axis=0):
     >>> a = sparse.random((3, 3, 3, 3, 3), density=0.3)
     >>> a_diag = sparse.diagonalize(a, axis=2)
     >>> (sparse.diagonal(a_diag, axis1=2, axis2=5) == a.transpose([0, 1, 3, 4, 2])).all()
-    True
+    np.True_
 
     Returns
     -------
@@ -944,7 +944,7 @@ def isposinf(x, out=None):
     --------
     [`numpy.isposinf`][] : The NumPy equivalent
     """
-    from .core import elemwise
+    from sparse import elemwise
 
     return elemwise(lambda x, out=None, dtype=None: np.isposinf(x, out=out), x, out=out)
 
@@ -971,7 +971,7 @@ def isneginf(x, out=None):
     --------
     [`numpy.isneginf`][] : The NumPy equivalent
     """
-    from .core import elemwise
+    from sparse import elemwise
 
     return elemwise(lambda x, out=None, dtype=None: np.isneginf(x, out=out), x, out=out)
 
@@ -1234,7 +1234,7 @@ def unique_values(x, /):
     >>> import sparse
     >>> x = sparse.COO.from_numpy([1, 0, 2, 1, 2, -3])
     >>> sparse.unique_values(x)
-    array([-3, 0, 1, 2])
+    array([-3,  0,  1,  2])
     """
 
     x = _validate_coo_input(x)
@@ -1279,9 +1279,9 @@ def sort(x, /, *, axis=-1, descending=False, stable=False):
     >>> import sparse
     >>> x = sparse.COO.from_numpy([1, 0, 2, 0, 2, -3])
     >>> sparse.sort(x).todense()
-    array([-3, 0, 0, 1, 2, 2])
+    array([-3,  0,  0,  1,  2,  2])
     >>> sparse.sort(x, descending=True).todense()
-    array([ 2, 2, 1, 0, 0, -3])
+    array([ 2,  2,  1,  0,  0, -3])
 
     """
     from .._common import moveaxis

sparse/numba_backend/_compressed/compressed.py

@@ -156,7 +156,7 @@ class COO(SparseArray, NDArrayOperatorsMixin):  # lgtm [py/missing-equals]
     >>> rows = [0, 1, 2, 3, 4]
     >>> cols = [0, 0, 0, 1, 1]
     >>> data = [10, 20, 30, 40, 50]
-    >>> z = COO((data, (rows, cols)))
+    >>> z = COO((data, (rows, cols)), shape=(5, 2))
     >>> z.todense()  # doctest: +NORMALIZE_WHITESPACE
     array([[10,  0],
            [20,  0],
@@ -168,10 +168,10 @@ class COO(SparseArray, NDArrayOperatorsMixin):  # lgtm [py/missing-equals]
     indices imply summation:
 
     >>> d = {(0, 0, 0): 1, (1, 2, 3): 2, (1, 1, 0): 3}
-    >>> COO(d)
+    >>> COO(d, shape=(2, 3, 4))
     <COO: shape=(2, 3, 4), dtype=int64, nnz=3, fill_value=0>
     >>> L = [((0, 0), 1), ((1, 1), 2), ((0, 0), 3)]
-    >>> COO(L).todense()  # doctest: +NORMALIZE_WHITESPACE
+    >>> COO(L, shape=(2, 2)).todense()  # doctest: +NORMALIZE_WHITESPACE
     array([[4, 0],
            [0, 2]])
 
@@ -440,6 +440,7 @@ def from_scipy_sparse(cls, x, /, *, fill_value=None):
 
         Examples
         --------
+        >>> import scipy.sparse
         >>> x = scipy.sparse.rand(6, 3, density=0.2)
         >>> s = COO.from_scipy_sparse(x)
         >>> np.array_equal(x.todense(), s.todense())
@@ -459,7 +460,7 @@ def from_scipy_sparse(cls, x, /, *, fill_value=None):
         )
 
     @classmethod
-    def from_iter(cls, x, shape=None, fill_value=None, dtype=None):
+    def from_iter(cls, x, shape, fill_value=None, dtype=None):
         """
         Converts an iterable in certain formats to a [`sparse.COO`][] array. See examples
         for details.
@@ -468,7 +469,7 @@ def from_iter(cls, x, shape=None, fill_value=None, dtype=None):
         ----------
         x : Iterable or Iterator
             The iterable to convert to [`sparse.COO`][].
-        shape : tuple[int], optional
+        shape : tuple[int]
             The shape of the array.
         fill_value : scalar
             The fill value for this array.
@@ -486,31 +487,31 @@ def from_iter(cls, x, shape=None, fill_value=None, dtype=None):
         Here, the first part represents the coordinate and the second part represents the value.
 
         >>> x = [((0, 0), 1), ((1, 1), 1)]
-        >>> s = COO.from_iter(x)
+        >>> s = COO.from_iter(x, shape=(2, 2))
         >>> s.todense()
         array([[1, 0],
                [0, 1]])
 
         You can also have a similar format with a dictionary.
 
         >>> x = {(0, 0): 1, (1, 1): 1}
-        >>> s = COO.from_iter(x)
+        >>> s = COO.from_iter(x, shape=(2, 2))
         >>> s.todense()
         array([[1, 0],
                [0, 1]])
 
         The third supported format is ``(data, (..., row, col))``.
 
         >>> x = ([1, 1], ([0, 1], [0, 1]))
-        >>> s = COO.from_iter(x)
+        >>> s = COO.from_iter(x, shape=(2, 2))
         >>> s.todense()
         array([[1, 0],
                [0, 1]])
 
         You can also pass in a [`collections.abc.Iterator`][] object.
 
         >>> x = [((0, 0), 1), ((1, 1), 1)].__iter__()
-        >>> s = COO.from_iter(x)
+        >>> s = COO.from_iter(x, shape=(2, 2))
         >>> s.todense()
         array([[1, 0],
                [0, 1]])
@@ -1293,7 +1294,7 @@ def _sort_indices(self):
         --------
         >>> coords = np.array([[1, 2, 0]], dtype=np.uint8)
         >>> data = np.array([4, 1, 3], dtype=np.uint8)
-        >>> s = COO(coords, data)
+        >>> s = COO(coords, data, shape=(3,))
         >>> s._sort_indices()
         >>> s.coords  # doctest: +NORMALIZE_WHITESPACE
         array([[0, 1, 2]], dtype=uint8)
@@ -1321,7 +1322,7 @@ def _sum_duplicates(self):
         --------
         >>> coords = np.array([[0, 1, 1, 2]], dtype=np.uint8)
         >>> data = np.array([6, 5, 2, 2], dtype=np.uint8)
-        >>> s = COO(coords, data)
+        >>> s = COO(coords, data, shape=(3,))
         >>> s._sum_duplicates()
         >>> s.coords  # doctest: +NORMALIZE_WHITESPACE
         array([[0, 1, 2]], dtype=uint8)
@@ -1354,7 +1355,7 @@ def _prune(self):
         --------
         >>> coords = np.array([[0, 1, 2, 3]])
         >>> data = np.array([1, 0, 1, 2])
-        >>> s = COO(coords, data)
+        >>> s = COO(coords, data, shape=(4,))
         >>> s._prune()
         >>> s.nnz
         3