generator support, index support, update XAM, disable VCF

Author: Jonathan Bieler

.github/workflows/CI.yml

@@ -2,22 +2,22 @@
 
 julia_version = "1.10.0"
 manifest_format = "2.0"
-project_hash = "88cdf5bf224b9670f958866657ca51fb8330c936"
+project_hash = "08c55c789910edb857eb1c8aac4f01c644757cad"
 
 [[deps.Artifacts]]
 uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
 
 [[deps.Automa]]
-deps = ["ScanByte", "TranscodingStreams"]
-git-tree-sha1 = "48e54446df62fdf9ef76959c32dc33f3cff659ee"
+deps = ["PrecompileTools", "TranscodingStreams"]
+git-tree-sha1 = "014bc22d6c400a7703c0f5dc1fdc302440cf88be"
 uuid = "67c07d97-cdcb-5c2c-af73-a7f9c32a568b"
-version = "0.8.3"
+version = "1.0.4"
 
 [[deps.BGZFStreams]]
 deps = ["CodecZlib"]
-git-tree-sha1 = "a9c80401403c068c02784cd53d417d3a82e2d2bd"
+git-tree-sha1 = "3aca54d25f8c30056577aa37ea68184da68df685"
 uuid = "28d598bf-9b8f-59f1-b38c-5a06b4a0f5e6"
-version = "0.3.1"
+version = "0.3.2"
 
 [[deps.Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -30,15 +30,15 @@ version = "3.1.0"
 
 [[deps.BioGenerics]]
 deps = ["TranscodingStreams"]
-git-tree-sha1 = "0b581906418b93231d391b5dd78831fdc2da0c82"
+git-tree-sha1 = "017562e86afcd2a6a2a9220606a40b54604887c9"
 uuid = "47718e42-2ac5-11e9-14af-e5595289c2ea"
-version = "0.1.2"
+version = "0.1.5"
 
 [[deps.BioSequences]]
 deps = ["BioSymbols", "PrecompileTools", "Random", "Twiddle"]
-git-tree-sha1 = "6cf406ea0d5bc901eabe55fca609007bce2db026"
+git-tree-sha1 = "6fdba8b4279460fef5674e9aa2dac7ef5be361d5"
 uuid = "7e6ae17a-c86d-528c-b3b9-7f778a29fe59"
-version = "3.1.5"
+version = "3.1.6"
 
 [[deps.BioSymbols]]
 deps = ["PrecompileTools"]
@@ -48,15 +48,15 @@ version = "5.1.3"
 
 [[deps.CodecZlib]]
 deps = ["TranscodingStreams", "Zlib_jll"]
-git-tree-sha1 = "9c209fb7536406834aa938fb149964b985de6c83"
+git-tree-sha1 = "b8fe8546d52ca154ac556809e10c75e6e7430ac8"
 uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
-version = "0.7.1"
+version = "0.7.5"
 
 [[deps.Compat]]
-deps = ["UUIDs"]
-git-tree-sha1 = "4e88377ae7ebeaf29a047aa1ee40826e0b708a5d"
+deps = ["TOML", "UUIDs"]
+git-tree-sha1 = "b1c55339b7c6c350ee89f2c1604299660525b248"
 uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
-version = "4.7.0"
+version = "4.15.0"
 weakdeps = ["Dates", "LinearAlgebra"]
 
     [deps.Compat.extensions]
@@ -69,9 +69,9 @@ version = "1.0.5+1"
 
 [[deps.DataStructures]]
 deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
-git-tree-sha1 = "cf25ccb972fec4e4817764d01c82386ae94f77b4"
+git-tree-sha1 = "1d0a14036acb104d9e89698bd408f63ab58cdc82"
 uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
-version = "0.18.14"
+version = "0.18.20"
 
 [[deps.Dates]]
 deps = ["Printf"]
@@ -123,21 +123,21 @@ uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
 version = "0.3.23+2"
 
 [[deps.OrderedCollections]]
-git-tree-sha1 = "d321bf2de576bf25ec4d3e4360faca399afca282"
+git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5"
 uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
-version = "1.6.0"
+version = "1.6.3"
 
 [[deps.PrecompileTools]]
 deps = ["Preferences"]
-git-tree-sha1 = "9673d39decc5feece56ef3940e5dafba15ba0f81"
+git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f"
 uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
-version = "1.1.2"
+version = "1.2.1"
 
 [[deps.Preferences]]
 deps = ["TOML"]
-git-tree-sha1 = "7eb1686b4f04b82f96ed7a4ea5890a4f0c7a09f1"
+git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6"
 uuid = "21216c6a-2e73-6563-6e65-726566657250"
-version = "1.4.0"
+version = "1.4.3"
 
 [[deps.Printf]]
 deps = ["Unicode"]
@@ -151,18 +151,6 @@ uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 version = "0.7.0"
 
-[[deps.SIMD]]
-deps = ["PrecompileTools"]
-git-tree-sha1 = "0e270732477b9e551d884e6b07e23bb2ec947790"
-uuid = "fdea26ae-647d-5447-a871-4b548cad5224"
-version = "3.4.5"
-
-[[deps.ScanByte]]
-deps = ["Libdl", "SIMD"]
-git-tree-sha1 = "d49e35f413186528f1d7cc675e67d0ed16fd7800"
-uuid = "7b38b023-a4d7-4c5e-8d43-3f3097f304eb"
-version = "0.4.0"
-
 [[deps.Serialization]]
 uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 
@@ -195,9 +183,9 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 
 [[deps.XAM]]
 deps = ["Automa", "BGZFStreams", "BioAlignments", "BioGenerics", "BioSequences", "GenomicFeatures", "Indexes", "Printf", "TranscodingStreams"]
-git-tree-sha1 = "634c3177fdf7d52498d4b947080bc73be833eeab"
+git-tree-sha1 = "e402710abf3f0bbed192896851bef8e483cf7952"
 uuid = "d759349c-bcba-11e9-07c2-5b90f8f05f7c"
-version = "0.3.1"
+version = "0.4.0"
 
 [[deps.Zlib_jll]]
 deps = ["Libdl"]

.github/workflows/CompatHelper.yml

@@ -1,7 +1,7 @@
 name = "BioRecordsProcessing"
 uuid = "321bc2d7-9525-42af-8b23-17b2621d5ea8"
 authors = ["Jonathan Bieler <[email protected]> and contributors"]
-version = "0.2.2"
+version = "0.2.3"
 
 [deps]
 BGZFStreams = "28d598bf-9b8f-59f1-b38c-5a06b4a0f5e6"
@@ -15,7 +15,7 @@ BGZFStreams = "0.3"
 CodecZlib = "0.7"
 Glob = "1.3"
 julia = "1.7"
-XAM = "0.3, 0.4"
+XAM = "0.4"
 GenomicFeatures = "2, 2.1"
 
 [extras]

.github/workflows/TagBot.yml

@@ -3,5 +3,4 @@ BioRecordsProcessing = "321bc2d7-9525-42af-8b23-17b2621d5ea8"
 BioSequences = "7e6ae17a-c86d-528c-b3b9-7f778a29fe59"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 FASTX = "c2308a5c-f048-11e8-3e8a-31650f418d12"
-VariantCallFormat = "28eba6e3-a997-4ad9-87c6-d933b8bca6c1"
 XAM = "d759349c-bcba-11e9-07c2-5b90f8f05f7c"

.github/workflows/documentation.yml

@@ -57,5 +57,5 @@ end
 BAMPairedReadGrouper() = RecordGrouper{BAM.Record, String}(
     r -> BAM.tempname(r),
     records -> length(records) >= 2,
-    r -> BAM.isprimary(r)
+    r -> BAM.isprimaryalignment(r)
 )

.gitignore

@@ -94,35 +94,43 @@ Reader(FASTX.FASTA, File("test.fa"))
 Reader(FASTX.FASTQ, Directory("data/", "*.fastq"))
 ```
 """
-struct Reader{F} <: AbstractSource where {F <: AbstractFileProvider}
+struct Reader{F,I} <: AbstractSource where {F <: AbstractFileProvider, I}
     record_module::Module
     file_provider::F
+    index::I
 
-    Reader(record_module::Module, file_provider::F) where {F <: AbstractFileProvider} = new{F}(record_module, file_provider)
+    Reader(record_module::Module, file_provider::F, index::I) where {F <: AbstractFileProvider, I} = new{F, I}(record_module, file_provider, index)
 end
-Reader(record_module::Module, filename::String) = Reader(record_module, File(filename))
+Reader(record_module::Module, file_provider::F; index=nothing) where {F <: AbstractFileProvider} =  Reader(record_module, file_provider, index)
+Reader(record_module::Module, filename::String; index=nothing) = Reader(record_module, File(filename); index=index)
 
 record_type(reader::Reader{F}) where {F} = reader.record_module
 _filename(reader::Reader) = _filename(reader.file_provider)
 is_paired(reader::Reader) = is_paired(reader.file_provider)
 
-function open_reader(s::Reader, filepath, filename, extension)
+function open_reader(source::Reader, filepath, filename, extension)
 
-    RecordType = record_type(s)
+    RecordType = record_type(source)
 
     if extension == ".gz"
         reader = RecordType.Reader(GzipDecompressorStream(open(filepath)))
 
     elseif extension == ".bam"
-        index_file = filepath * ".bai"
-        if !isfile(index_file)
-            @warn "Index file not found : $index_file"
-            reader = RecordType.Reader(open(filepath))
+        
+        if !isnothing(source.index)
+            reader = RecordType.Reader(open(filepath); index = source.index)
         else
-            reader = RecordType.Reader(open(filepath); index = index_file)
+            index_file = filepath * ".bai"
+            if !isfile(index_file)
+                @warn "Index file not found : $index_file"
+                reader = RecordType.Reader(open(filepath))
+            else
+                reader = RecordType.Reader(open(filepath); index = index_file)
+            end
         end
-        if !isnothing(interval(s.file_provider))
-            seek_region!(reader, interval(s.file_provider))
+
+        if !isnothing(interval(source.file_provider))
+            seek_region!(reader, interval(source.file_provider))
         end
 
     else
@@ -131,18 +139,24 @@ function open_reader(s::Reader, filepath, filename, extension)
     reader, RecordType
 end
 
+
+function Base.show(io::IO, source::Reader) 
+    print(io, "Reader($(source.record_module), $(source.file_provider))")
+end
+
+
 """
 ```julia
-Buffer(data::Vector{T}; filename = "")
+Buffer(data::T; filename = "")
 ```
 
-Use the array `data` as a source of records. An optional filename can be provided when a `Writer`
+Use the collection `data` as a source of records. An optional filename can be provided when a `Writer`
 is used as a sink.
 """
 struct Buffer{T} <: AbstractSource 
-    data::Vector{T}
+    data::T
     filename::String
-    Buffer(data::Vector{T}; filename = "") where T = new{T}(data, filename)
+    Buffer(data::T; filename = "") where T = new{T}(data, filename)
 end
 
 function Base.show(io::IO, source::Buffer{T}) where T
@@ -159,7 +173,7 @@ function seek_region!(reader::BAM.Reader, region)
     refindex = findfirst(isequal(region.seqname), reader.refseqnames)
     refindex == nothing && throw(ArgumentError("sequence name $(iter.refname) is not found in the header"))
 
-    chunks = XAM.BAM.Indexes.overlapchunks(reader.index.index, refindex, region.first:region.last )
+    chunks = XAM.BAM.Indexes.overlapchunks(reader.index.index, refindex, region.first:region.last)
     if !isempty(chunks)
         seek(reader, first(chunks).start)
     end

LICENSE

@@ -3,5 +3,4 @@ BioSequences = "7e6ae17a-c86d-528c-b3b9-7f778a29fe59"
 FASTX = "c2308a5c-f048-11e8-3e8a-31650f418d12"
 FormatSpecimens = "3372ea36-2a1a-11e9-3eb7-996970b6ffbd"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-VariantCallFormat = "28eba6e3-a997-4ad9-87c6-d933b8bca6c1"
 XAM = "d759349c-bcba-11e9-07c2-5b90f8f05f7c"

Manifest.toml

@@ -1,5 +1,6 @@
 using BioRecordsProcessing
-using Test, FASTX, XAM, VariantCallFormat, BioSequences, FormatSpecimens
+using Test, FASTX, XAM, BioSequences, FormatSpecimens
+#VariantCallFormat
 
 @testset "Internals" begin
     @test BioRecordsProcessing.insert_suffix("name", ".fastq", ".processed") == "name.processed.fastq"

Project.toml

@@ -88,6 +88,8 @@
         @test out[1] != out[2] # make sure Collect is copied
     end
 
+    #disable until https://github.com/rasmushenningsson/VariantCallFormat.jl/issues/5 is fixed
+    if false
     @testset "VCF" begin
         mktempdir() do dir
             filepath = joinpath(path_of_format("VCF"), "adeno_virus.vcf")
@@ -117,9 +119,9 @@
             @test_throws ErrorException run(p)    
         end
     end
+    end 
 
-    @testset "Buffer + Collect" begin
-        
+    @testset "Buffer + Collect" begin 
         input = rand(10)
         p = Pipeline(
             Buffer(input),
@@ -129,7 +131,18 @@
         @show p
         output = run(p)
         @test all(output .≈ 2*input)
-    
+    end
+
+    @testset "Buffer + Collect with generator" begin 
+        input = (i for i in 1:10)
+        p = Pipeline(
+            Buffer(input),
+            x -> 2x,
+            Collect(Float64),
+        )
+        @show p
+        output = run(p)
+        @test all(output .≈ 2 .* input)
     end
 
     @testset "Buffer + Writer" begin
@@ -281,6 +294,24 @@
         end
     end
 
+    @testset "BAM + Index" begin
+        mktempdir() do dir
+            spec = list_valid_specimens("BAM")
+            bam = joinpath(path_of_format("BAM"), "SRR7993829_1.100K.forward.bam")
+            index = BAM.BAI(bam * ".bai")
+
+            p = Pipeline(
+                Reader(BAM, File(bam); index = index),
+                Collect(BAM.Record)
+            )
+            @show p
+            out = run(p)
+            @test length(out) == 36405
+        end
+    end
+
+    
+
     @testset "BAM to paired FASTQ" begin
         mktempdir() do dir
             spec = list_valid_specimens("BAM")