diff --git a/.github/workflows/daily-test-build-numpy.yml b/.github/workflows/daily-test-build-numpy.yml index 0cc133df33..e3056647b7 100644 --- a/.github/workflows/daily-test-build-numpy.yml +++ b/.github/workflows/daily-test-build-numpy.yml @@ -28,12 +28,13 @@ jobs: - { python-version: "3.10", numpy-version: "1.25.2" } - { python-version: "3.10", numpy-version: "2.2.6" } - { python-version: "3.11", numpy-version: "1.25.2" } - - { python-version: "3.11", numpy-version: "2.3.4" } + - { python-version: "3.11", numpy-version: "2.4.6" } - { python-version: "3.12", numpy-version: "1.26.4" } - - { python-version: "3.12", numpy-version: "2.3.4" } + - { python-version: "3.12", numpy-version: "2.5.0" } - { python-version: "3.13", numpy-version: "2.1.3" } - - { python-version: "3.13", numpy-version: "2.3.4" } - - { python-version: "3.14", numpy-version: "2.3.4" } + - { python-version: "3.13", numpy-version: "2.5.0" } + - { python-version: "3.14", numpy-version: "2.3.5" } + - { python-version: "3.14", numpy-version: "2.5.0" } fail-fast: false env: TILEDB_VERSION: ${{ inputs.libtiledb_version }} diff --git a/tiledb/array.py b/tiledb/array.py index cd9b3c72b4..d235ebb2c9 100644 --- a/tiledb/array.py +++ b/tiledb/array.py @@ -1187,9 +1187,7 @@ def set_query(self, serialized_query): out = OrderedDict() for name in results.keys(): - arr = results[name][0] - arr.dtype = q.buffer_dtype(name) - out[name] = arr + out[name] = results[name][0].view(q.buffer_dtype(name)) return out # pickling support: this is a lightweight pickle for distributed use. diff --git a/tiledb/core.cc b/tiledb/core.cc index 05887b3004..a7c2152b2f 100644 --- a/tiledb/core.cc +++ b/tiledb/core.cc @@ -1447,8 +1447,14 @@ class PyQuery { auto arr = py::array(py::dtype("int8"), size, data_ptr); o = py::memoryview(arr); } else { - o = py::array(py::dtype("uint8"), size, data_ptr); - o.attr("dtype") = dtype; + // `size` is the cell's byte length and must be a whole number + // of dtype elements; a partial element means corrupt offsets. + if (size % dtype.itemsize() != 0) { + TPY_ERROR_LOC( + "internal error: buffer size is not a multiple of the " + "attribute itemsize"); + } + o = py::array(dtype, size / dtype.itemsize(), data_ptr); } result_p[i - 1] = o; diff --git a/tiledb/dataframe_.py b/tiledb/dataframe_.py index 50755eccad..d7983853b9 100644 --- a/tiledb/dataframe_.py +++ b/tiledb/dataframe_.py @@ -372,9 +372,13 @@ def create_dim(dtype, values, full_domain, tile, **kwargs): if np.issubdtype(dtype, np.datetime64): date_unit = np.datetime_data(dtype)[0] dim_min = np.datetime64(dtype_min, date_unit) - tile_max = np.iinfo(np.uint64).max - tile - if np.uint64(dtype_max - dtype_min) > tile_max: - dim_max = np.datetime64(dtype_max - tile, date_unit) + # Subtract in Python ints; a full-domain datetime64 difference + # overflows int64. + dtype_min_i = int(dtype_min.astype(np.int64)) + dtype_max_i = int(dtype_max.astype(np.int64)) + tile_max = int(np.iinfo(np.uint64).max) - int(tile) + if (dtype_max_i - dtype_min_i) > tile_max: + dim_max = np.datetime64(dtype_max_i - int(tile), date_unit) elif np.issubdtype(dtype, np.integer): tile_max = np.iinfo(np.uint64).max - tile if np.uint64(dtype_max - dtype_min) > tile_max: @@ -383,9 +387,14 @@ def create_dim(dtype, values, full_domain, tile, **kwargs): dim_min, dim_max = None, None # TODO: simplify this logic and/or move to DataType.cast_tile_extent - if np.issubdtype(dtype, np.integer) or np.issubdtype(dtype, np.datetime64): + if np.issubdtype(dtype, np.integer): # we can't make a tile larger than the dimension range or lower than 1 tile = max(1, min(tile, 1 + np.uint64(dim_max - dim_min))) + elif np.issubdtype(dtype, np.datetime64): + # Clamp as for integers, computing the range in Python ints since a + # full-domain datetime64 difference overflows int64. + dim_range = int(dim_max.astype(np.int64)) - int(dim_min.astype(np.int64)) + tile = max(1, min(tile, 1 + dim_range)) elif np.issubdtype(dtype, np.floating): # this difference can be inf with np.errstate(over="ignore"): diff --git a/tiledb/dense_array.py b/tiledb/dense_array.py index 71e086aaa4..76ea6fef2b 100644 --- a/tiledb/dense_array.py +++ b/tiledb/dense_array.py @@ -345,7 +345,7 @@ def _read_dense_subarray( # Note: fixed blobs are always 1 byte per cell arr = arr.view("S1") else: - arr.dtype = dtype + arr = arr.view(dtype) if len(arr) == 0: # special case: the C API returns 0 len for blank arrays arr = np.zeros(output_shape, dtype=dtype) @@ -357,13 +357,11 @@ def _read_dense_subarray( ) if layout == lt.LayoutType.ROW_MAJOR: - arr.shape = output_shape - arr = np.require(arr, requirements="C") + arr = np.require(arr.reshape(output_shape), requirements="C") elif layout == lt.LayoutType.COL_MAJOR: - arr.shape = output_shape - arr = np.require(arr, requirements="F") + arr = np.require(arr.reshape(output_shape), requirements="F") else: - arr.shape = np.prod(output_shape) + arr = arr.reshape(np.prod(output_shape)) out[name] = arr @@ -837,13 +835,12 @@ def read_subarray(self, subarray): if len(item[1]) > 0: arr = pyquery.unpack_buffer(name, item[0], item[1]) else: - arr = item[0] - arr.dtype = ( + arr = item[0].view( self.schema.attr_or_dim_dtype(name) if not self.schema.has_dim_label(name) else self.schema.dim_label(name).dtype ) - arr.shape = result_shape + arr = arr.reshape(result_shape) result_dict[name if name != "__attr" else ""] = arr return result_dict diff --git a/tiledb/multirange_indexing.py b/tiledb/multirange_indexing.py index 537caf22dd..b4004cefaf 100644 --- a/tiledb/multirange_indexing.py +++ b/tiledb/multirange_indexing.py @@ -409,7 +409,7 @@ def _run_query(self) -> Dict[str, np.ndarray]: for name, arr in result_dict.items(): # TODO check/test layout if not self.array.schema.has_dim_label(name): - arr.shape = self.result_shape + result_dict[name] = arr.reshape(self.result_shape) return result_dict @@ -825,8 +825,7 @@ def _get_pyquery_results(pyquery: PyQuery, array: Array) -> Dict[str, np.ndarray if len(item[1]) > 0: arr = pyquery.unpack_buffer(name, item[0], item[1]) else: - arr = item[0] - arr.dtype = ( + arr = item[0].view( schema.attr_or_dim_dtype(name) if not schema.has_dim_label(name) else schema.dim_label(name).dtype diff --git a/tiledb/npbuffer.cc b/tiledb/npbuffer.cc index 0ee7c39bab..0b6b30bc76 100644 --- a/tiledb/npbuffer.cc +++ b/tiledb/npbuffer.cc @@ -471,22 +471,18 @@ class NumpyConvert { NumpyConvert(py::array input) { // require a flat buffer if (input.ndim() != 1) { - // try to take a 1D view on the input - auto v = input.attr("view")(); - // this will throw if the shape cannot be modified zero-copy, - // which is what we want - try { - v.attr("shape") = py::int_(input.size()); - } catch (py::error_already_set& e) { - if (e.matches(PyExc_AttributeError)) { - use_iter_ = true; - } else { - throw; - } - } catch (std::exception& e) { - std::cout << e.what() << std::endl; + // reshape(-1) returns a zero-copy view when the array can be + // flattened in place; the bulk paths need that view, so use it + // when the buffer address is unchanged. Otherwise the array is + // not contiguous: discard the copy and iterate the original + // element-wise instead. + py::array flat = input.attr("reshape")(py::int_(-1)); + if (flat.data() == input.data()) { + input_ = flat; + } else { + use_iter_ = true; + input_ = input; } - input_ = v; } else { input_ = input; } diff --git a/tiledb/sparse_array.py b/tiledb/sparse_array.py index 70be2c86a8..fba1c66fb9 100644 --- a/tiledb/sparse_array.py +++ b/tiledb/sparse_array.py @@ -439,8 +439,7 @@ def read_subarray(self, subarray): if len(item[1]) > 0: arr = pyquery.unpack_buffer(name, item[0], item[1]) else: - arr = item[0] - arr.dtype = ( + arr = item[0].view( self.schema.attr_or_dim_dtype(name) if not self.schema.has_dim_label(name) else self.schema.dim_label(name).dtype @@ -597,8 +596,7 @@ def _read_sparse_subarray(self, subarray, attr_names: list, cond, layout): if len(results[name][1]) > 0: # note: len(offsets) > 0 arr = q.unpack_buffer(name, results[name][0], results[name][1]) else: - arr = results[name][0] - arr.dtype = self.schema.attr_or_dim_dtype(name) + arr = results[name][0].view(self.schema.attr_or_dim_dtype(name)) out[final_name] = arr else: arr = results[name][0] @@ -623,8 +621,7 @@ def _read_sparse_subarray(self, subarray, attr_names: list, cond, layout): elif el_dtype == np.dtype("U0"): out[final_name] = "" else: - arr.dtype = el_dtype - out[final_name] = arr + out[final_name] = arr.view(el_dtype) if self.schema.has_attr(final_name) and self.attr(final_name).isnullable: out[final_name] = np.ma.array( diff --git a/tiledb/tests/test_core.py b/tiledb/tests/test_core.py index b236a0f879..88d9b6ba4e 100644 --- a/tiledb/tests/test_core.py +++ b/tiledb/tests/test_core.py @@ -37,8 +37,7 @@ def test_pyquery_basic(self): subarray.add_ranges([[(0, 3)]]) q2.set_subarray(subarray) q2.submit() - res = q2.results()[""][0] - res.dtype = np.double + res = q2.results()[""][0].view(np.double) assert_array_equal(res, a[:]) def test_pyquery_init(self): diff --git a/tiledb/tests/test_libtiledb.py b/tiledb/tests/test_libtiledb.py index 07d852ba5b..06fa2f0107 100644 --- a/tiledb/tests/test_libtiledb.py +++ b/tiledb/tests/test_libtiledb.py @@ -2697,7 +2697,11 @@ def test_dense_datetime_vector(self): (np.datetime64("2010-11-01") - start) / np.timedelta64(1, "D") ) read_ndays = int( - (np.datetime64("2011-01-31") - np.datetime64("2010-11-01") + 1) + ( + np.datetime64("2011-01-31") + - np.datetime64("2010-11-01") + + np.timedelta64(1, "D") + ) / np.timedelta64(1, "D") ) expected = a1_vals[read_offset : read_offset + read_ndays] @@ -2712,7 +2716,11 @@ def test_dense_datetime_vector(self): (np.datetime64("2010-01-01") - start) / np.timedelta64(1, "D") ) read_ndays = int( - (np.datetime64("2011-01-01") - np.datetime64("2010-01-01") + 1) + ( + np.datetime64("2011-01-01") + - np.datetime64("2010-01-01") + + np.timedelta64(1, "D") + ) / np.timedelta64(1, "D") ) expected = a1_vals[read_offset : read_offset + read_ndays] @@ -2725,7 +2733,11 @@ def test_dense_datetime_vector(self): (np.datetime64("2010-01-01") - start) / np.timedelta64(1, "D") ) read_ndays = int( - (np.datetime64("2011-01-31") - np.datetime64("2010-01-01") + 1) + ( + np.datetime64("2011-01-31") + - np.datetime64("2010-01-01") + + np.timedelta64(1, "D") + ) / np.timedelta64(1, "D") ) expected = a1_vals[read_offset : read_offset + read_ndays] diff --git a/tiledb/tests/test_subarray.py b/tiledb/tests/test_subarray.py index 42fa2d2349..5e6455e1f9 100644 --- a/tiledb/tests/test_subarray.py +++ b/tiledb/tests/test_subarray.py @@ -202,8 +202,7 @@ def test_add_fixed_sized_point_ranges(self): q.submit() results = q.results() - arr = results["a"][0] - arr.dtype = np.int32 + arr = results["a"][0].view(np.int32) assert arr.shape == (2,) assert np.array_equal(arr, np.array([10, 30], dtype=np.int32)) @@ -234,7 +233,6 @@ def test_add_var_sized_point_ranges(self): q.submit() results = q.results() - arr = results["a"][0] - arr.dtype = np.int32 + arr = results["a"][0].view(np.int32) assert arr.shape == (2,) assert np.array_equal(arr, np.array([10, 30], dtype=np.int32))