Support NumPy 2.5.0 (#2308)

Author: kounelisagis

.github/workflows/daily-test-build-numpy.yml

@@ -28,12 +28,13 @@ jobs:
           - { python-version: "3.10", numpy-version: "1.25.2" }
           - { python-version: "3.10", numpy-version: "2.2.6" }
           - { python-version: "3.11", numpy-version: "1.25.2" }
-          - { python-version: "3.11", numpy-version: "2.3.4" }
+          - { python-version: "3.11", numpy-version: "2.4.6" }
           - { python-version: "3.12", numpy-version: "1.26.4" }
-          - { python-version: "3.12", numpy-version: "2.3.4" }
+          - { python-version: "3.12", numpy-version: "2.5.0" }
           - { python-version: "3.13", numpy-version: "2.1.3" }
-          - { python-version: "3.13", numpy-version: "2.3.4" }
-          - { python-version: "3.14", numpy-version: "2.3.4" }
+          - { python-version: "3.13", numpy-version: "2.5.0" }
+          - { python-version: "3.14", numpy-version: "2.3.5" }
+          - { python-version: "3.14", numpy-version: "2.5.0" }
       fail-fast: false
     env:
       TILEDB_VERSION: ${{ inputs.libtiledb_version }}

tiledb/array.py

@@ -1187,9 +1187,7 @@ def set_query(self, serialized_query):
 
         out = OrderedDict()
         for name in results.keys():
-            arr = results[name][0]
-            arr.dtype = q.buffer_dtype(name)
-            out[name] = arr
+            out[name] = results[name][0].view(q.buffer_dtype(name))
         return out
 
     # pickling support: this is a lightweight pickle for distributed use.

tiledb/core.cc

@@ -1447,8 +1447,14 @@ class PyQuery {
                 auto arr = py::array(py::dtype("int8"), size, data_ptr);
                 o = py::memoryview(arr);
             } else {
-                o = py::array(py::dtype("uint8"), size, data_ptr);
-                o.attr("dtype") = dtype;
+                // `size` is the cell's byte length and must be a whole number
+                // of dtype elements; a partial element means corrupt offsets.
+                if (size % dtype.itemsize() != 0) {
+                    TPY_ERROR_LOC(
+                        "internal error: buffer size is not a multiple of the "
+                        "attribute itemsize");
+                }
+                o = py::array(dtype, size / dtype.itemsize(), data_ptr);
             }
 
             result_p[i - 1] = o;

tiledb/dataframe_.py

@@ -372,9 +372,13 @@ def create_dim(dtype, values, full_domain, tile, **kwargs):
         if np.issubdtype(dtype, np.datetime64):
             date_unit = np.datetime_data(dtype)[0]
             dim_min = np.datetime64(dtype_min, date_unit)
-            tile_max = np.iinfo(np.uint64).max - tile
-            if np.uint64(dtype_max - dtype_min) > tile_max:
-                dim_max = np.datetime64(dtype_max - tile, date_unit)
+            # Subtract in Python ints; a full-domain datetime64 difference
+            # overflows int64.
+            dtype_min_i = int(dtype_min.astype(np.int64))
+            dtype_max_i = int(dtype_max.astype(np.int64))
+            tile_max = int(np.iinfo(np.uint64).max) - int(tile)
+            if (dtype_max_i - dtype_min_i) > tile_max:
+                dim_max = np.datetime64(dtype_max_i - int(tile), date_unit)
         elif np.issubdtype(dtype, np.integer):
             tile_max = np.iinfo(np.uint64).max - tile
             if np.uint64(dtype_max - dtype_min) > tile_max:
@@ -383,9 +387,14 @@ def create_dim(dtype, values, full_domain, tile, **kwargs):
         dim_min, dim_max = None, None
 
     # TODO: simplify this logic and/or move to DataType.cast_tile_extent
-    if np.issubdtype(dtype, np.integer) or np.issubdtype(dtype, np.datetime64):
+    if np.issubdtype(dtype, np.integer):
         # we can't make a tile larger than the dimension range or lower than 1
         tile = max(1, min(tile, 1 + np.uint64(dim_max - dim_min)))
+    elif np.issubdtype(dtype, np.datetime64):
+        # Clamp as for integers, computing the range in Python ints since a
+        # full-domain datetime64 difference overflows int64.
+        dim_range = int(dim_max.astype(np.int64)) - int(dim_min.astype(np.int64))
+        tile = max(1, min(tile, 1 + dim_range))
     elif np.issubdtype(dtype, np.floating):
         # this difference can be inf
         with np.errstate(over="ignore"):

tiledb/dense_array.py

@@ -345,7 +345,7 @@ def _read_dense_subarray(
                     # Note: fixed blobs are always 1 byte per cell
                     arr = arr.view("S1")
                 else:
-                    arr.dtype = dtype
+                    arr = arr.view(dtype)
                 if len(arr) == 0:
                     # special case: the C API returns 0 len for blank arrays
                     arr = np.zeros(output_shape, dtype=dtype)
@@ -357,13 +357,11 @@ def _read_dense_subarray(
                     )
 
                 if layout == lt.LayoutType.ROW_MAJOR:
-                    arr.shape = output_shape
-                    arr = np.require(arr, requirements="C")
+                    arr = np.require(arr.reshape(output_shape), requirements="C")
                 elif layout == lt.LayoutType.COL_MAJOR:
-                    arr.shape = output_shape
-                    arr = np.require(arr, requirements="F")
+                    arr = np.require(arr.reshape(output_shape), requirements="F")
                 else:
-                    arr.shape = np.prod(output_shape)
+                    arr = arr.reshape(np.prod(output_shape))
 
                 out[name] = arr
 
@@ -837,13 +835,12 @@ def read_subarray(self, subarray):
             if len(item[1]) > 0:
                 arr = pyquery.unpack_buffer(name, item[0], item[1])
             else:
-                arr = item[0]
-                arr.dtype = (
+                arr = item[0].view(
                     self.schema.attr_or_dim_dtype(name)
                     if not self.schema.has_dim_label(name)
                     else self.schema.dim_label(name).dtype
                 )
-            arr.shape = result_shape
+            arr = arr.reshape(result_shape)
             result_dict[name if name != "__attr" else ""] = arr
 
         return result_dict

tiledb/multirange_indexing.py

@@ -409,7 +409,7 @@ def _run_query(self) -> Dict[str, np.ndarray]:
             for name, arr in result_dict.items():
                 # TODO check/test layout
                 if not self.array.schema.has_dim_label(name):
-                    arr.shape = self.result_shape
+                    result_dict[name] = arr.reshape(self.result_shape)
         return result_dict
 
 
@@ -825,8 +825,7 @@ def _get_pyquery_results(pyquery: PyQuery, array: Array) -> Dict[str, np.ndarray
         if len(item[1]) > 0:
             arr = pyquery.unpack_buffer(name, item[0], item[1])
         else:
-            arr = item[0]
-            arr.dtype = (
+            arr = item[0].view(
                 schema.attr_or_dim_dtype(name)
                 if not schema.has_dim_label(name)
                 else schema.dim_label(name).dtype

tiledb/npbuffer.cc

@@ -471,22 +471,18 @@ class NumpyConvert {
     NumpyConvert(py::array input) {
         // require a flat buffer
         if (input.ndim() != 1) {
-            // try to take a 1D view on the input
-            auto v = input.attr("view")();
-            // this will throw if the shape cannot be modified zero-copy,
-            // which is what we want
-            try {
-                v.attr("shape") = py::int_(input.size());
-            } catch (py::error_already_set& e) {
-                if (e.matches(PyExc_AttributeError)) {
-                    use_iter_ = true;
-                } else {
-                    throw;
-                }
-            } catch (std::exception& e) {
-                std::cout << e.what() << std::endl;
+            // reshape(-1) returns a zero-copy view when the array can be
+            // flattened in place; the bulk paths need that view, so use it
+            // when the buffer address is unchanged. Otherwise the array is
+            // not contiguous: discard the copy and iterate the original
+            // element-wise instead.
+            py::array flat = input.attr("reshape")(py::int_(-1));
+            if (flat.data() == input.data()) {
+                input_ = flat;
+            } else {
+                use_iter_ = true;
+                input_ = input;
             }
-            input_ = v;
         } else {
             input_ = input;
         }

tiledb/sparse_array.py

@@ -439,8 +439,7 @@ def read_subarray(self, subarray):
             if len(item[1]) > 0:
                 arr = pyquery.unpack_buffer(name, item[0], item[1])
             else:
-                arr = item[0]
-                arr.dtype = (
+                arr = item[0].view(
                     self.schema.attr_or_dim_dtype(name)
                     if not self.schema.has_dim_label(name)
                     else self.schema.dim_label(name).dtype
@@ -597,8 +596,7 @@ def _read_sparse_subarray(self, subarray, attr_names: list, cond, layout):
                 if len(results[name][1]) > 0:  # note: len(offsets) > 0
                     arr = q.unpack_buffer(name, results[name][0], results[name][1])
                 else:
-                    arr = results[name][0]
-                    arr.dtype = self.schema.attr_or_dim_dtype(name)
+                    arr = results[name][0].view(self.schema.attr_or_dim_dtype(name))
                 out[final_name] = arr
             else:
                 arr = results[name][0]
@@ -623,8 +621,7 @@ def _read_sparse_subarray(self, subarray, attr_names: list, cond, layout):
                     elif el_dtype == np.dtype("U0"):
                         out[final_name] = ""
                     else:
-                        arr.dtype = el_dtype
-                        out[final_name] = arr
+                        out[final_name] = arr.view(el_dtype)
 
             if self.schema.has_attr(final_name) and self.attr(final_name).isnullable:
                 out[final_name] = np.ma.array(

tiledb/tests/test_core.py

@@ -37,8 +37,7 @@ def test_pyquery_basic(self):
             subarray.add_ranges([[(0, 3)]])
             q2.set_subarray(subarray)
             q2.submit()
-            res = q2.results()[""][0]
-            res.dtype = np.double
+            res = q2.results()[""][0].view(np.double)
             assert_array_equal(res, a[:])
 
     def test_pyquery_init(self):

tiledb/tests/test_libtiledb.py

@@ -2697,7 +2697,11 @@ def test_dense_datetime_vector(self):
                 (np.datetime64("2010-11-01") - start) / np.timedelta64(1, "D")
             )
             read_ndays = int(
-                (np.datetime64("2011-01-31") - np.datetime64("2010-11-01") + 1)
+                (
+                    np.datetime64("2011-01-31")
+                    - np.datetime64("2010-11-01")
+                    + np.timedelta64(1, "D")
+                )
                 / np.timedelta64(1, "D")
             )
             expected = a1_vals[read_offset : read_offset + read_ndays]
@@ -2712,7 +2716,11 @@ def test_dense_datetime_vector(self):
                 (np.datetime64("2010-01-01") - start) / np.timedelta64(1, "D")
             )
             read_ndays = int(
-                (np.datetime64("2011-01-01") - np.datetime64("2010-01-01") + 1)
+                (
+                    np.datetime64("2011-01-01")
+                    - np.datetime64("2010-01-01")
+                    + np.timedelta64(1, "D")
+                )
                 / np.timedelta64(1, "D")
             )
             expected = a1_vals[read_offset : read_offset + read_ndays]
@@ -2725,7 +2733,11 @@ def test_dense_datetime_vector(self):
                 (np.datetime64("2010-01-01") - start) / np.timedelta64(1, "D")
             )
             read_ndays = int(
-                (np.datetime64("2011-01-31") - np.datetime64("2010-01-01") + 1)
+                (
+                    np.datetime64("2011-01-31")
+                    - np.datetime64("2010-01-01")
+                    + np.timedelta64(1, "D")
+                )
                 / np.timedelta64(1, "D")
             )
             expected = a1_vals[read_offset : read_offset + read_ndays]