From a5234e0faf70c5c1b8c0f47d45df3a5619cce7cb Mon Sep 17 00:00:00 2001 From: Philippe Gras Date: Wed, 11 Oct 2023 19:18:41 +0200 Subject: [PATCH 1/4] Add a copy(::LazyBranch) method to prevent branch materialization on copy In absence of such method, the copy(::AbstractVector) method that resulted in the materialization into a Vector was called. --- src/iteration.jl | 42 ++++++++++++++++++++++++++---------------- test/runtests.jl | 5 +++++ 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/iteration.jl b/src/iteration.jl index e4bd0af8..788ae73f 100644 --- a/src/iteration.jl +++ b/src/iteration.jl @@ -112,29 +112,39 @@ mutable struct LazyBranch{T,J,B} <: AbstractVector{T} buffer::Vector{B} thread_locks::Vector{ReentrantLock} buffer_range::Vector{UnitRange{Int64}} +end - function LazyBranch(f::ROOTFile, b::Union{TBranch,TBranchElement}) - T, J = auto_T_JaggT(f, b; customstructs=f.customstructs) - T = (T === Vector{Bool} ? BitVector : T) - _buffer = T[] - if J != Nojagg - # if branch is jagged, fix the buffer and eltype according to what - # VectorOfVectors would return in `getindex` - _buffer = VectorOfVectors(T(), Int32[1]) - T = SubArray{eltype(T), 1, T, Tuple{UnitRange{Int64}}, true} - end - Nthreads = _maxthreadid() - return new{T,J,typeof(_buffer)}(f, b, length(b), - b.fBasketEntry, - [_buffer for _ in 1:Nthreads], - [ReentrantLock() for _ in 1:Nthreads], - [0:-1 for _ in 1:Nthreads]) +function LazyBranch(f::ROOTFile, b::Union{TBranch,TBranchElement}) + T, J = auto_T_JaggT(f, b; customstructs=f.customstructs) + T = (T === Vector{Bool} ? BitVector : T) + _buffer = T[] + if J != Nojagg + # if branch is jagged, fix the buffer and eltype according to what + # VectorOfVectors would return in `getindex` + _buffer = VectorOfVectors(T(), Int32[1]) + T = SubArray{eltype(T), 1, T, Tuple{UnitRange{Int64}}, true} + end + Nthreads = _maxthreadid() + return LazyBranch{T,J,typeof(_buffer)}(f, b, length(b), + b.fBasketEntry, + [_buffer for _ in 1:Nthreads], + [ReentrantLock() for _ in 1:Nthreads], + [0:-1 for _ in 1:Nthreads]) end end + LazyBranch(f::ROOTFile, s::AbstractString) = LazyBranch(f, f[s]) basketarray(lb::LazyBranch, ithbasket) = basketarray(lb.f, lb.b, ithbasket) basketarray_iter(lb::LazyBranch) = basketarray_iter(lb.f, lb.b) +""" + copy(x::LazyBranch) + + Makes a shallow copy of x. +""" +Base.copy(x::LazyBranch{T,J,B}) where {T,J,B} = LazyBranch{T,J,B}(x.f, x.b, x.L, x.fEntry, x.buffer, + x.thread_locks, x.buffer_range) + function Base.hash(lb::LazyBranch, h::UInt) h = hash(lb.f, h) h = hash(lb.b.fClassName, h) diff --git a/test/runtests.jl b/test/runtests.jl index 8149f6db..8c26ba90 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -252,6 +252,11 @@ end @test [row.int32_array for row in table[20:30]] == BA[20:30] @test sum(table.int32_array) == sum(row.int32_array for row in table) @test [row.int32_array for row in table] == BA + BA_copy = copy(BA) + #Check the copy is a LazyBranch: + @test typeof(BA_copy) == typeof(BA) + #Check the content: + @test all(BA .== BA_copy) # do some hardcoded value checks bunches = Float32[] From 23e577c759e7247b5b1999f77e4e846e2b60b881 Mon Sep 17 00:00:00 2001 From: Philippe Gras Date: Fri, 13 Oct 2023 11:19:59 +0200 Subject: [PATCH 2/4] missmerged fixed --- src/iteration.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/iteration.jl b/src/iteration.jl index 788ae73f..7fbf4efd 100644 --- a/src/iteration.jl +++ b/src/iteration.jl @@ -130,7 +130,6 @@ function LazyBranch(f::ROOTFile, b::Union{TBranch,TBranchElement}) [_buffer for _ in 1:Nthreads], [ReentrantLock() for _ in 1:Nthreads], [0:-1 for _ in 1:Nthreads]) - end end LazyBranch(f::ROOTFile, s::AbstractString) = LazyBranch(f, f[s]) From 54c42b8d88b4f740476db7d71704274791dafdd4 Mon Sep 17 00:00:00 2001 From: Jerry Ling Date: Tue, 16 Jan 2024 12:18:03 -0500 Subject: [PATCH 3/4] Update src/iteration.jl --- src/iteration.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/iteration.jl b/src/iteration.jl index 7fbf4efd..5c94d681 100644 --- a/src/iteration.jl +++ b/src/iteration.jl @@ -141,8 +141,8 @@ basketarray_iter(lb::LazyBranch) = basketarray_iter(lb.f, lb.b) Makes a shallow copy of x. """ -Base.copy(x::LazyBranch{T,J,B}) where {T,J,B} = LazyBranch{T,J,B}(x.f, x.b, x.L, x.fEntry, x.buffer, - x.thread_locks, x.buffer_range) +Base.copy(x::LazyBranch{T,J,B}) where {T,J,B} = LazyBranch{T,J,B}(x.f, x.b, x.L, x.fEntry, deepcopy(x.buffer), + x.thread_locks, copy(x.buffer_range)) function Base.hash(lb::LazyBranch, h::UInt) h = hash(lb.f, h) From de28e08e5edbc63672b5e229c413d396322d9ef5 Mon Sep 17 00:00:00 2001 From: Jerry Ling Date: Fri, 1 Mar 2024 16:11:53 -0500 Subject: [PATCH 4/4] Update src/iteration.jl --- src/iteration.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/iteration.jl b/src/iteration.jl index 5c94d681..d2ddf3eb 100644 --- a/src/iteration.jl +++ b/src/iteration.jl @@ -141,7 +141,7 @@ basketarray_iter(lb::LazyBranch) = basketarray_iter(lb.f, lb.b) Makes a shallow copy of x. """ -Base.copy(x::LazyBranch{T,J,B}) where {T,J,B} = LazyBranch{T,J,B}(x.f, x.b, x.L, x.fEntry, deepcopy(x.buffer), +Base.copy(x::LazyBranch{T,J,B}) where {T,J,B} = LazyBranch{T,J,B}(x.f, x.b, x.L, x.fEntry, copy.(x.buffer), x.thread_locks, copy(x.buffer_range)) function Base.hash(lb::LazyBranch, h::UInt)