Skip to content

Commit f8d2203

Browse files
authored
Use wkspawn from ConcurrentUtilities instead of Threads.spawn (#469)
Should fix #336. For more context, see the [same fix](JuliaData/CSV.jl@077e177) we made for this in CSV.jl. Basically, objects interpolated into or returned from spawned tasks can be unexpectedly kept alive long after the task has finished and the object should have been garbage-collected due to individual threads holding the most recent task as a reference. Using `@wkspawn` ensures tasks themselves don't hold references to any of these once they're done executing.
1 parent fc7cc2d commit f8d2203

File tree

3 files changed

+6
-6
lines changed

3 files changed

+6
-6
lines changed

src/append.jl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ function append(
169169
# start message writing from channel
170170
threaded = ntasks > 1
171171
tsk =
172-
threaded ? (Threads.@spawn for msg in msgs
172+
threaded ? (@wkspawn for msg in msgs
173173
Base.write(io, msg, blocks, sch, alignment)
174174
end) : (@async for msg in msgs
175175
Base.write(io, msg, blocks, sch, alignment)
@@ -191,7 +191,7 @@ function append(
191191
end
192192

193193
if threaded
194-
Threads.@spawn process_partition(
194+
@wkspawn process_partition(
195195
tbl_cols,
196196
dictencodings,
197197
largelists,

src/table.jl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -419,7 +419,7 @@ function Table(blobs::Vector{ArrowBlob}; convert::Bool=true)
419419
dictencoded = Dict{Int64,Meta.Field}() # dictionary id => field
420420
sync = OrderedSynchronizer()
421421
tsks = Channel{Any}(Inf)
422-
tsk = Threads.@spawn begin
422+
tsk = @wkspawn begin
423423
i = 1
424424
for cols in tsks
425425
if i == 1
@@ -522,7 +522,7 @@ function Table(blobs::Vector{ArrowBlob}; convert::Bool=true)
522522
elseif header isa Meta.RecordBatch
523523
anyrecordbatches = true
524524
@debugv 1 "parsing record batch message: compression = $(header.compression)"
525-
Threads.@spawn begin
525+
@wkspawn begin
526526
cols = collect(VectorIterator(sch, $batch, dictencodings, convert))
527527
put!(() -> put!(tsks, cols), sync, $(rbi))
528528
end

src/write.jl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ function Base.open(
167167
# start message writing from channel
168168
threaded = Threads.nthreads() > 1
169169
task =
170-
threaded ? (Threads.@spawn for msg in msgs
170+
threaded ? (@wkspawn for msg in msgs
171171
Base.write(io, msg, blocks, schema, alignment)
172172
end) : (@async for msg in msgs
173173
Base.write(io, msg, blocks, schema, alignment)
@@ -296,7 +296,7 @@ function write(writer::Writer, source)
296296
put!(writer.msgs, recbatchmsg)
297297
else
298298
if writer.threaded
299-
Threads.@spawn process_partition(
299+
@wkspawn process_partition(
300300
tblcols,
301301
writer.dictencodings,
302302
writer.largelists,

0 commit comments

Comments
 (0)