Don't pool small inline string columns by default

Fixes #982. Since the size of `String1` and `String3` are <= the size of
the ref integer type we use for pooling (`UInt32`), let's avoid pooling
them by default. Users can still request specific columns be pooled like
always.

Author: quinnj

src/utils.jl

@@ -51,7 +51,7 @@ tupcat(::Type{Tuple{T, T2, T3}}, S) where {T, T2, T3} = Tuple{T, T2, T3, S}
 tupcat(::Type{Tuple{T, T2, T3, T4}}, S) where {T, T2, T3, T4} = Tuple{T, T2, T3, T4, S}
 tupcat(::Type{T}, S) where {T <: Tuple} = Tuple{Any[(fieldtype(T, i) for i = 1:fieldcount(T))..., S]...}
 
-const StringTypes = Union{Type{String}, Type{PosLenString}, Type{<:InlineString}}
+const StringTypes = Union{Type{String}, Type{PosLenString}, Type{InlineString}, Type{String7}, Type{String15}, Type{String31}, Type{String63}, Type{String127}, Type{String255}}
 pickstringtype(T, maxstringsize) = T === InlineString ? (maxstringsize < DEFAULT_MAX_INLINE_STRING_LENGTH ? InlineStringType(maxstringsize) : String) : T
 
 # we define our own bit flag on a Parsers.ReturnCode to signal if a column needs to promote to string

test/basics.jl

@@ -777,4 +777,15 @@ f = CSV.File(IOBuffer(join((rand(("a,$(rand())", "b,$(rand())")) for _ = 1:10^6)
 f = CSV.File(IOBuffer("a\nfalse\n"))
 @test eltype(f.a) == Bool
 
+# 982
+data = """a,b,c,d
+A,BB,CCC,DDDD
+A,BB,CCC,DDDD
+"""
+f = CSV.File(IOBuffer(data))
+@test !(f.a isa PooledArray)
+@test !(f.b isa PooledArray)
+@test !(f.c isa PooledArray)
+@test f.d isa PooledArray
+
 end

test/runtests.jl

@@ -24,27 +24,27 @@ include("write.jl")
 
 @testset "PooledArrays" begin
 
-    f = CSV.File(IOBuffer("X\nb\nc\na\nc"), pool=true)
-    @test typeof(f.X) == PooledArrays.PooledArray{InlineString1,UInt32,1,Array{UInt32,1}}
+    f = CSV.File(IOBuffer("X\nbbbb\ncccc\naaaa\ncccc"), pool=true)
+    @test typeof(f.X) == PooledArrays.PooledArray{String7,UInt32,1,Array{UInt32,1}}
     @test (length(f), length(f.names)) == (4, 1)
-    @test f.X == ["b", "c", "a", "c"]
+    @test f.X == ["bbbb", "cccc", "aaaa", "cccc"]
     @test f.X.refs[2] == f.X.refs[4]
 
-    f = CSV.File(IOBuffer("X\nb\nc\na\nc"), pool=0.75)
-    @test typeof(f.X) == PooledArrays.PooledArray{InlineString1,UInt32,1,Array{UInt32,1}}
+    f = CSV.File(IOBuffer("X\nbbbb\ncccc\naaaa\ncccc"), pool=0.75)
+    @test typeof(f.X) == PooledArrays.PooledArray{String7,UInt32,1,Array{UInt32,1}}
     @test (length(f), length(f.names)) == (4, 1)
-    @test f.X == ["b", "c", "a", "c"]
+    @test f.X == ["bbbb", "cccc", "aaaa", "cccc"]
     @test f.X.refs[2] == f.X.refs[4]
 
-    f = CSV.File(IOBuffer("X\nb\nc\n\nc"), pool=true, ignoreemptyrows=false)
-    @test typeof(f.X) == PooledArray{Union{Missing, InlineString1},UInt32,1,Array{UInt32,1}}
+    f = CSV.File(IOBuffer("X\nbbbb\ncccc\n\ncccc"), pool=true, ignoreemptyrows=false)
+    @test typeof(f.X) == PooledArray{Union{Missing, String7},UInt32,1,Array{UInt32,1}}
     @test (length(f), length(f.names)) == (4, 1)
     @test f.X[3] === missing
 
-    f = CSV.File(IOBuffer("X\nc\nc\n\nc\nc\nc\nc\nc\nc"), pool=0.25, ignoreemptyrows=false)
-    @test typeof(f.X) == PooledArray{Union{Missing, InlineString1},UInt32,1,Array{UInt32,1}}
+    f = CSV.File(IOBuffer("X\ncccc\ncccc\n\ncccc\ncccc\ncccc\ncccc\ncccc\ncccc"), pool=0.25, ignoreemptyrows=false)
+    @test typeof(f.X) == PooledArray{Union{Missing, String7},UInt32,1,Array{UInt32,1}}
     @test (length(f), length(f.names)) == (9, 1)
-    @test isequal(f.X, ["c", "c", missing, "c", "c", "c", "c", "c", "c"])
+    @test isequal(f.X, ["cccc", "cccc", missing, "cccc", "cccc", "cccc", "cccc", "cccc", "cccc"])
 
 end