Skip to content

Commit b84f28d

Browse files
committed
fix: make dask_property picklable
property keeps fget/fset/fdel in C slots and has no reduction of its own, so behavior classes defined in __main__ or a notebook could not be sent to a distributed worker. Assisted-by: ClaudeCode:claude-opus-4.8
1 parent 8d3bf51 commit b84f28d

2 files changed

Lines changed: 54 additions & 0 deletions

File tree

src/coffea/util.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,12 @@ def impl(*args, **kwargs):
369369
return descriptor
370370

371371

372+
def _rebuild_dask_property(fget, fset, fdel, doc, dask_get):
373+
prop = _DaskProperty(fget, fset, fdel, doc)
374+
prop._dask_get = dask_get
375+
return prop
376+
377+
372378
class _DaskProperty(property):
373379
_dask_get = None
374380

@@ -377,6 +383,16 @@ def dask(self, func):
377383
self._dask_get = _make_dask_descriptor(func)
378384
return self
379385

386+
def __reduce__(self):
387+
# property keeps fget/fset/fdel in C slots and offers no reduction, so
388+
# a behavior class carrying a dask_property cannot be pickled by value
389+
# (which is what cloudpickle does for classes defined in __main__ or a
390+
# notebook) unless we provide one.
391+
return (
392+
_rebuild_dask_property,
393+
(self.fget, self.fset, self.fdel, self.__doc__, self._dask_get),
394+
)
395+
380396

381397
def _adapt_naive_dask_get(func):
382398
def wrapper(self, dask_array, *args, **kwargs):

tests/test_util.py

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,3 +71,41 @@ def output(x):
7171
finally:
7272
if os.path.exists(filename):
7373
os.remove(filename)
74+
75+
76+
def test_dask_property_is_picklable():
77+
"""Behavior classes defined outside an importable module get pickled by
78+
value (e.g. by cloudpickle, when a dask graph goes to a distributed worker),
79+
which walks the class dict -- and plain property objects cannot be pickled.
80+
"""
81+
cloudpickle = pytest.importorskip("cloudpickle")
82+
83+
from coffea.util import dask_property
84+
85+
class Thing:
86+
def __init__(self, x):
87+
self.x = x
88+
89+
@dask_property
90+
def doubled(self):
91+
"""twice x"""
92+
return 2 * self.x
93+
94+
@doubled.dask
95+
def doubled(self, dask_array):
96+
return 20 * dask_array.x
97+
98+
@dask_property(no_dispatch=True)
99+
def tripled(self):
100+
return 3 * self.x
101+
102+
# defined in a function body, so this has to go by value
103+
unpickled = cloudpickle.loads(cloudpickle.dumps(Thing))
104+
105+
assert unpickled(1).doubled == 2
106+
assert unpickled(1).tripled == 3
107+
108+
doubled = unpickled.__dict__["doubled"]
109+
assert doubled.__doc__ == "twice x"
110+
assert doubled._dask_get(unpickled(1), unpickled, Thing(3)) == 60
111+
assert unpickled.__dict__["tripled"]._dask_get(unpickled(2), unpickled, None) == 6

0 commit comments

Comments
 (0)