From abe4a1491f14a25050ddabfba851e6cd3fc1fded Mon Sep 17 00:00:00 2001 From: Justin Willmert Date: Mon, 5 Oct 2020 14:00:28 -0500 Subject: [PATCH 1/8] Initialize runtime constants at runtime * Hide all underlying constants within private module --- src/HDF5.jl | 325 ++++++++++++++--------------- src/api_types.jl | 493 ++++++++++++++++++++++++++------------------ src/blosc_filter.jl | 8 +- src/deprecated.jl | 10 +- src/macros.jl | 100 +++++++++ test/compound.jl | 20 +- test/custom.jl | 2 +- test/extend_test.jl | 6 +- test/gc.jl | 4 +- test/mmap.jl | 2 +- test/mpio.jl | 2 +- test/plain.jl | 26 +-- test/properties.jl | 18 +- test/swmr.jl | 2 +- 14 files changed, 611 insertions(+), 407 deletions(-) create mode 100644 src/macros.jl diff --git a/src/HDF5.jl b/src/HDF5.jl index c9e8260fb..94d97ccde 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -10,6 +10,7 @@ import Mmap using Compat export +H5, H5D, H5E, H5F, H5FD, H5I, H5L, H5O, H5P, H5R, H5S, H5T, H5Z, a_create, a_delete, a_open, a_read, a_write, attrs, d_create, d_create_external, d_open, d_read, d_write, dataspace, datatype, file, filename, @@ -33,6 +34,7 @@ end include("datafile.jl") +include("macros.jl") # Core API ccall wrappers include("api_types.jl") include("api.jl") @@ -63,33 +65,33 @@ chartype(::Type{String}) = ASCIIChar stringtype(::Type{ASCIIChar}) = String stringtype(::Type{UTF8Char}) = String -cset(::Type{<:AbstractString}) = H5T_CSET_UTF8 -cset(::Type{UTF8Char}) = H5T_CSET_UTF8 -cset(::Type{ASCIIChar}) = H5T_CSET_ASCII +cset(::Type{<:AbstractString}) = H5T.CSET_UTF8 +cset(::Type{UTF8Char}) = H5T.CSET_UTF8 +cset(::Type{ASCIIChar}) = H5T.CSET_ASCII ## Conversion between Julia types and HDF5 atomic types -hdf5_type_id(::Type{Bool}) = H5T_NATIVE_B8 -hdf5_type_id(::Type{Int8}) = H5T_NATIVE_INT8 -hdf5_type_id(::Type{UInt8}) = H5T_NATIVE_UINT8 -hdf5_type_id(::Type{Int16}) = H5T_NATIVE_INT16 -hdf5_type_id(::Type{UInt16}) = H5T_NATIVE_UINT16 -hdf5_type_id(::Type{Int32}) = H5T_NATIVE_INT32 -hdf5_type_id(::Type{UInt32}) = H5T_NATIVE_UINT32 -hdf5_type_id(::Type{Int64}) = H5T_NATIVE_INT64 -hdf5_type_id(::Type{UInt64}) = H5T_NATIVE_UINT64 -hdf5_type_id(::Type{Float32}) = H5T_NATIVE_FLOAT -hdf5_type_id(::Type{Float64}) = H5T_NATIVE_DOUBLE -hdf5_type_id(::Type{Reference}) = H5T_STD_REF_OBJ - -hdf5_type_id(::Type{<:AbstractString}) = H5T_C_S1 +hdf5_type_id(::Type{Bool}) = H5T.NATIVE_B8 +hdf5_type_id(::Type{Int8}) = H5T.NATIVE_INT8 +hdf5_type_id(::Type{UInt8}) = H5T.NATIVE_UINT8 +hdf5_type_id(::Type{Int16}) = H5T.NATIVE_INT16 +hdf5_type_id(::Type{UInt16}) = H5T.NATIVE_UINT16 +hdf5_type_id(::Type{Int32}) = H5T.NATIVE_INT32 +hdf5_type_id(::Type{UInt32}) = H5T.NATIVE_UINT32 +hdf5_type_id(::Type{Int64}) = H5T.NATIVE_INT64 +hdf5_type_id(::Type{UInt64}) = H5T.NATIVE_UINT64 +hdf5_type_id(::Type{Float32}) = H5T.NATIVE_FLOAT +hdf5_type_id(::Type{Float64}) = H5T.NATIVE_DOUBLE +hdf5_type_id(::Type{Reference}) = H5T.STD_REF_OBJ + +hdf5_type_id(::Type{<:AbstractString}) = H5T.C_S1 const BitsType = Union{Bool,Int8,UInt8,Int16,UInt16,Int32,UInt32,Int64,UInt64,Float32,Float64} const ScalarType = Union{BitsType,Reference} # It's not safe to use particular id codes because these can change, so we use characteristics of the type. function _hdf5_type_map(class_id, is_signed, native_size) - if class_id == H5T_INTEGER - if is_signed == H5T_SGN_2 + if class_id == H5T.INTEGER + if is_signed == H5T.SGN_2 return native_size === Csize_t(1) ? Int8 : native_size === Csize_t(2) ? Int16 : native_size === Csize_t(4) ? Int32 : @@ -172,16 +174,16 @@ end mutable struct Properties id::hid_t class::hid_t - function Properties(id, class::hid_t = H5P_DEFAULT) + function Properties(id, class::hid_t = H5P.DEFAULT) p = new(id, class) finalizer(close, p) #Essential, otherwise we get a memory leak, since closing file with CLOSE_STRONG is not doing it for us p end end -Properties() = Properties(H5P_DEFAULT) +Properties() = Properties(H5P.DEFAULT) Base.convert(::Type{hid_t}, p::Properties) = p.id function Base.show(io::IO, prop::Properties) - if prop.class == H5P_DEFAULT + if prop.class == H5P.DEFAULT print(io, "HDF5 property: default class") elseif isvalid(prop) print(io, "HDF5 property: ", h5p_get_class_name(prop.class), " class") @@ -239,19 +241,19 @@ function Base.show(io::IO, dtype::Datatype) print(io, h5lt_dtype_to_text(dtype)) else # Note that h5i_is_valid returns `false` on the built-in datatypes (e.g. - # H5T_NATIVE_INT), apparently because they have refcounts of 0 yet are always + # H5T.NATIVE_INT), apparently because they have refcounts of 0 yet are always # valid. Just temporarily turn off error printing and try the call to probe if # dtype is valid since H5LTdtype_to_text special-cases all of the built-in types # internally. - old_func, old_client_data = h5e_get_auto(H5E_DEFAULT) - h5e_set_auto(H5E_DEFAULT, C_NULL, C_NULL) + old_func, old_client_data = h5e_get_auto(H5E.DEFAULT) + h5e_set_auto(H5E.DEFAULT, C_NULL, C_NULL) local text try text = h5lt_dtype_to_text(dtype) catch text = "(invalid)" finally - h5e_set_auto(H5E_DEFAULT, old_func, old_client_data) + h5e_set_auto(H5E.DEFAULT, old_func, old_client_data) end print(io, text) end @@ -292,7 +294,7 @@ attrs(p::Union{File,Group,Dataset}) = Attributes(p) # Methods for reference types function Reference(parent::Union{File,Group,Dataset}, name::AbstractString) ref = Ref{hobj_ref_t}() - h5r_create(ref, checkvalid(parent), name, H5R_OBJECT, -1) + h5r_create(ref, checkvalid(parent), name, H5R.OBJECT, -1) return Reference(ref[]) end Base.:(==)(a::Reference, b::Reference) = a.r == b.r @@ -325,7 +327,7 @@ Base.show(io::IO, ::MIME"text/plain", E::EmptyArray) = show(io, E) Base.cat_size(::EmptyArray) = error("concatenation of HDF5.EmptyArray is unsupported") Base.cat_size(::EmptyArray, d) = error("concatenation of HDF5.EmptyArray is unsupported") -# Stub types to encode fixed-size arrays for H5T_ARRAY +# Stub types to encode fixed-size arrays for H5T.ARRAY struct FixedArray{T,D,L} data::NTuple{L,T} end @@ -405,23 +407,23 @@ function h5open(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Bool error("HDF5 does not support appending without writing") end close_apl = false - if apl.id == H5P_DEFAULT - apl = p_create(H5P_FILE_ACCESS) + if apl.id == H5P.DEFAULT + apl = p_create(H5P.FILE_ACCESS) close_apl = true # With garbage collection, the other modes don't make sense - apl[:fclose_degree] = H5F_CLOSE_STRONG + apl[:fclose_degree] = H5F.CLOSE_STRONG end if cr && (tr || !isfile(filename)) - flag = swmr ? H5F_ACC_TRUNC|H5F_ACC_SWMR_WRITE : H5F_ACC_TRUNC + flag = swmr ? H5F.ACC_TRUNC|H5F.ACC_SWMR_WRITE : H5F.ACC_TRUNC fid = h5f_create(filename, flag, cpl, apl) else if !h5f_is_hdf5(filename) error("This does not appear to be an HDF5 file") end if wr - flag = swmr ? H5F_ACC_RDWR|H5F_ACC_SWMR_WRITE : H5F_ACC_RDWR + flag = swmr ? H5F.ACC_RDWR|H5F.ACC_SWMR_WRITE : H5F.ACC_RDWR else - flag = swmr ? H5F_ACC_RDONLY|H5F_ACC_SWMR_READ : H5F_ACC_RDONLY + flag = swmr ? H5F.ACC_RDONLY|H5F.ACC_SWMR_READ : H5F.ACC_RDONLY end fid = h5f_open(filename, flag, apl) end @@ -446,11 +448,11 @@ Pass `swmr=true` to enable (Single Writer Multiple Reader) SWMR write access for "r+", or SWMR read access for "r". """ function h5open(filename::AbstractString, mode::AbstractString="r"; swmr=false, pv...) - fapl = p_create(H5P_FILE_ACCESS; pv...) # file access property list + fapl = p_create(H5P.FILE_ACCESS; pv...) # file access property list # With garbage collection, the other modes don't make sense # (Set this first, so that the user-passed properties can overwrite this.) - fapl[:fclose_degree] = H5F_CLOSE_STRONG - fcpl = p_create(H5P_FILE_CREATE; pv...) # file create property list + fapl[:fclose_degree] = H5F.CLOSE_STRONG + fcpl = p_create(H5P.FILE_CREATE; pv...) # file create property list modes = mode == "r" ? (true, false, false, false, false) : mode == "r+" ? (true, true, false, false, true ) : @@ -656,7 +658,7 @@ file(o::Union{Object,Attribute}) = o.file fd(obj::Object) = h5i_get_file_id(checkvalid(obj).id) # Flush buffers -Base.flush(f::Union{Object,Attribute,Datatype,File}, scope = H5F_SCOPE_GLOBAL) = h5f_flush(checkvalid(f).id, scope) +Base.flush(f::Union{Object,Attribute,Datatype,File}, scope = H5F.SCOPE_GLOBAL) = h5f_flush(checkvalid(f).id, scope) # Open objects g_open(parent::Union{File,Group}, name::AbstractString, apl::Properties=DEFAULT_PROPERTIES) = Group(h5g_open(checkvalid(parent), name, apl), file(parent)) @@ -666,9 +668,9 @@ a_open(parent::Union{File,Object}, name::AbstractString, apl::Properties=DEFAULT # Object (group, named datatype, or dataset) open function h5object(obj_id::hid_t, parent) obj_type = h5i_get_type(obj_id) - obj_type == H5I_GROUP ? Group(obj_id, file(parent)) : - obj_type == H5I_DATATYPE ? Datatype(obj_id, file(parent)) : - obj_type == H5I_DATASET ? Dataset(obj_id, file(parent)) : + obj_type == H5I.GROUP ? Group(obj_id, file(parent)) : + obj_type == H5I.DATATYPE ? Datatype(obj_id, file(parent)) : + obj_type == H5I.DATASET ? Dataset(obj_id, file(parent)) : error("Invalid object type for path ", path) end o_open(parent, path::AbstractString) = h5object(h5o_open(checkvalid(parent), path), parent) @@ -687,15 +689,15 @@ Base.getindex(x::Attributes, name::AbstractString) = a_open(x.parent, name) function Base.getindex(parent::Union{File,Group}, path::AbstractString; pv...) objtype = gettype(parent, path) - if objtype == H5I_DATASET - dapl = p_create(H5P_DATASET_ACCESS; pv...) - dxpl = p_create(H5P_DATASET_XFER; pv...) + if objtype == H5I.DATASET + dapl = p_create(H5P.DATASET_ACCESS; pv...) + dxpl = p_create(H5P.DATASET_XFER; pv...) return d_open(parent, path, dapl, dxpl) - elseif objtype == H5I_GROUP - gapl = p_create(H5P_GROUP_ACCESS; pv...) + elseif objtype == H5I.GROUP + gapl = p_create(H5P.GROUP_ACCESS; pv...) return g_open(parent, path, gapl) - else#if objtype == H5I_DATATYPE # only remaining choice - tapl = p_create(H5P_DATATYPE_ACCESS; pv...) + else#if objtype == H5I.DATATYPE # only remaining choice + tapl = p_create(H5P.DATATYPE_ACCESS; pv...) return t_open(parent, path, tapl) end end @@ -734,9 +736,9 @@ end # Setting dset creation properties with name/value pairs function d_create(parent::Union{File,Group}, path::AbstractString, dtype::Datatype, dspace::Dataspace; pv...) - dcpl = isempty(pv) ? DEFAULT_PROPERTIES : p_create(H5P_DATASET_CREATE; pv...) - dxpl = isempty(pv) ? DEFAULT_PROPERTIES : p_create(H5P_DATASET_XFER; pv...) - dapl = isempty(pv) ? DEFAULT_PROPERTIES : p_create(H5P_DATASET_ACCESS; pv...) + dcpl = isempty(pv) ? DEFAULT_PROPERTIES : p_create(H5P.DATASET_CREATE; pv...) + dxpl = isempty(pv) ? DEFAULT_PROPERTIES : p_create(H5P.DATASET_XFER; pv...) + dapl = isempty(pv) ? DEFAULT_PROPERTIES : p_create(H5P.DATASET_ACCESS; pv...) Dataset(h5d_create(checkvalid(parent), path, dtype, dspace, _link_properties(path), dcpl, dapl), file(parent), dxpl) end d_create(parent::Union{File,Group}, path::AbstractString, dtype::Datatype, dspace_dims::Dims; pv...) = d_create(checkvalid(parent), path, dtype, dataspace(dspace_dims); pv...) @@ -746,7 +748,7 @@ d_create(parent::Union{File,Group}, path::AbstractString, dtype::Type, dspace_di # Note that H5Tcreate is very different; H5Tcommit is the analog of these others t_create(class_id, sz) = Datatype(h5t_create(class_id, sz)) function t_commit(parent::Union{File,Group}, path::AbstractString, dtype::Datatype, - lcpl::Properties=p_create(H5P_LINK_CREATE), tcpl::Properties=DEFAULT_PROPERTIES, tapl::Properties=DEFAULT_PROPERTIES) + lcpl::Properties=p_create(H5P.LINK_CREATE), tcpl::Properties=DEFAULT_PROPERTIES, tapl::Properties=DEFAULT_PROPERTIES) h5p_set_char_encoding(lcpl, cset(typeof(path))) h5t_commit(checkvalid(parent), path, dtype, lcpl, tcpl, tapl) dtype.file = file(parent) @@ -758,13 +760,13 @@ a_create(parent::Union{File,Object}, name::AbstractString, dtype::Datatype, dspa function _prop_get(p::Properties, name::Symbol) class = p.class - if class == H5P_FILE_CREATE + if class == H5P.FILE_CREATE return name === :userblock ? h5p_get_userblock(p) : - name === :track_times ? h5p_get_obj_track_times(p) : # H5P_OBJECT_CREATE + name === :track_times ? h5p_get_obj_track_times(p) : # H5P.OBJECT_CREATE error("unknown file create property ", name) end - if class == H5P_FILE_ACCESS + if class == H5P.FILE_ACCESS return name === :alignment ? h5p_get_alignment(p) : name === :driver ? h5p_get_driver(p) : name === :driver_info ? h5p_get_driver_info(p) : @@ -774,33 +776,33 @@ function _prop_get(p::Properties, name::Symbol) error("unknown file access property ", name) end - if class == H5P_GROUP_CREATE + if class == H5P.GROUP_CREATE return name === :local_heap_size_hint ? h5p_get_local_heap_size_hint(p) : - name === :track_times ? h5p_get_obj_track_times(p) : # H5P_OBJECT_CREATE + name === :track_times ? h5p_get_obj_track_times(p) : # H5P.OBJECT_CREATE error("unknown group create property ", name) end - if class == H5P_LINK_CREATE + if class == H5P.LINK_CREATE return name === :char_encoding ? h5p_get_char_encoding(p) : name === :create_intermediate_group ? h5p_get_create_intermediate_group(p) : error("unknown link create property ", name) end - if class == H5P_DATASET_CREATE + if class == H5P.DATASET_CREATE return name === :alloc_time ? h5p_get_alloc_time(p) : name === :chunk ? get_chunk(p) : #name === :external ? h5p_get_external(p) : name === :layout ? h5p_get_layout(p) : - name === :track_times ? h5p_get_obj_track_times(p) : # H5P_OBJECT_CREATE + name === :track_times ? h5p_get_obj_track_times(p) : # H5P.OBJECT_CREATE error("unknown dataset create property ", name) end - if class == H5P_DATASET_XFER + if class == H5P.DATASET_XFER return name === :dxpl_mpio ? h5p_get_dxpl_mpio(p) : error("unknown dataset transfer property ", name) end - if class == H5P_ATTRIBUTE_CREATE + if class == H5P.ATTRIBUTE_CREATE return name === :char_encoding ? h5p_get_char_encoding(p) : error("unknown attribute create property ", name) end @@ -811,13 +813,13 @@ end function _prop_set!(p::Properties, name::Symbol, val, check::Bool = true) class = p.class - if class == H5P_FILE_CREATE + if class == H5P.FILE_CREATE return name === :userblock ? h5p_set_userblock(p, val...) : - name === :track_times ? h5p_set_obj_track_times(p, val...) : # H5P_OBJECT_CREATE + name === :track_times ? h5p_set_obj_track_times(p, val...) : # H5P.OBJECT_CREATE check ? error("unknown file create property ", name) : nothing end - if class == H5P_FILE_ACCESS + if class == H5P.FILE_ACCESS return name === :alignment ? h5p_set_alignment(p, val...) : name === :fapl_mpio ? h5p_set_fapl_mpio(p, val...) : name === :fclose_degree ? h5p_set_fclose_degree(p, val...) : @@ -825,19 +827,19 @@ function _prop_set!(p::Properties, name::Symbol, val, check::Bool = true) check ? error("unknown file access property ", name) : nothing end - if class == H5P_GROUP_CREATE + if class == H5P.GROUP_CREATE return name === :local_heap_size_hint ? h5p_set_local_heap_size_hint(p, val...) : - name === :track_times ? h5p_set_obj_track_times(p, val...) : # H5P_OBJECT_CREATE + name === :track_times ? h5p_set_obj_track_times(p, val...) : # H5P.OBJECT_CREATE check ? error("unknown group create property ", name) : nothing end - if class == H5P_LINK_CREATE + if class == H5P.LINK_CREATE return name === :char_encoding ? h5p_set_char_encoding(p, val...) : name === :create_intermediate_group ? h5p_set_create_intermediate_group(p, val...) : check ? error("unknown link create property ", name) : nothing end - if class == H5P_DATASET_CREATE + if class == H5P.DATASET_CREATE return name === :alloc_time ? h5p_set_alloc_time(p, val...) : name === :blosc ? h5p_set_blosc(p, val...) : name === :chunk ? set_chunk(p, val...) : @@ -846,16 +848,16 @@ function _prop_set!(p::Properties, name::Symbol, val, check::Bool = true) name === :external ? h5p_set_external(p, val...) : name === :layout ? h5p_set_layout(p, val...) : name === :shuffle ? h5p_set_shuffle(p, val...) : - name === :track_times ? h5p_set_obj_track_times(p, val...) : # H5P_OBJECT_CREATE + name === :track_times ? h5p_set_obj_track_times(p, val...) : # H5P.OBJECT_CREATE check ? error("unknown dataset create property ", name) : nothing end - if class == H5P_DATASET_XFER + if class == H5P.DATASET_XFER return name === :dxpl_mpio ? h5p_set_dxpl_mpio(p, val...) : check ? error("unknown dataset transfer property ", name) : nothing end - if class == H5P_ATTRIBUTE_CREATE + if class == H5P.ATTRIBUTE_CREATE return name === :char_encoding ? h5p_set_char_encoding(p, val...) : check ? error("unknown attribute create property ", name) : nothing end @@ -877,8 +879,8 @@ o_delete(parent::Union{File,Group}, path::AbstractString, lapl::Properties=DEFAU o_delete(obj::Object) = o_delete(parent(obj), ascii(split(name(obj),"/")[end])) # FIXME: remove ascii? # Copy objects -o_copy(src_parent::Union{File,Group}, src_path::AbstractString, dst_parent::Union{File,Group}, dst_path::AbstractString) = h5o_copy(checkvalid(src_parent), src_path, checkvalid(dst_parent), dst_path, H5P_DEFAULT, _link_properties(dst_path)) -o_copy(src_obj::Object, dst_parent::Union{File,Group}, dst_path::AbstractString) = h5o_copy(checkvalid(src_obj), ".", checkvalid(dst_parent), dst_path, H5P_DEFAULT, _link_properties(dst_path)) +o_copy(src_parent::Union{File,Group}, src_path::AbstractString, dst_parent::Union{File,Group}, dst_path::AbstractString) = h5o_copy(checkvalid(src_parent), src_path, checkvalid(dst_parent), dst_path, H5P.DEFAULT, _link_properties(dst_path)) +o_copy(src_obj::Object, dst_parent::Union{File,Group}, dst_path::AbstractString) = h5o_copy(checkvalid(src_obj), ".", checkvalid(dst_parent), dst_path, H5P.DEFAULT, _link_properties(dst_path)) # Assign syntax: obj[path] = value # Creates a dataset unless obj is a dataset, in which case it creates an attribute @@ -957,7 +959,7 @@ function Base.eltype(dset::Union{Dataset,Attribute}) end function isnull(obj::Union{Dataset,Attribute}) dspace = dataspace(obj) - ret = h5s_get_simple_extent_type(dspace) == H5S_NULL + ret = h5s_get_simple_extent_type(dspace) == H5S.NULL close(dspace) ret end @@ -969,13 +971,13 @@ name(attr::Attribute) = h5a_get_name(attr) function Base.keys(x::Union{Group,File}) checkvalid(x) n = length(x) - return [h5l_get_name_by_idx(x, ".", H5_INDEX_NAME, H5_ITER_INC, i-1, H5P_DEFAULT) for i = 1:n] + return [h5l_get_name_by_idx(x, ".", H5.INDEX_NAME, H5.ITER_INC, i-1, H5P.DEFAULT) for i = 1:n] end function Base.keys(x::Attributes) checkvalid(x.parent) n = length(x) - return [h5a_get_name_by_idx(x.parent, ".", H5_INDEX_NAME, H5_ITER_INC, i-1, H5P_DEFAULT) for i = 1:n] + return [h5a_get_name_by_idx(x.parent, ".", H5.INDEX_NAME, H5.ITER_INC, i-1, H5P.DEFAULT) for i = 1:n] end # iteration by objects @@ -983,7 +985,7 @@ function Base.iterate(parent::Union{File,Group}, iter = (1,nothing)) n, prev_obj = iter prev_obj ≢ nothing && close(prev_obj) n > length(parent) && return nothing - obj = h5object(h5o_open_by_idx(checkvalid(parent), ".", H5_INDEX_NAME, H5_ITER_INC, n-1, H5P_DEFAULT), parent) + obj = h5object(h5o_open_by_idx(checkvalid(parent), ".", H5.INDEX_NAME, H5.ITER_INC, n-1, H5P.DEFAULT), parent) return (obj, (n+1,obj)) end @@ -1015,7 +1017,7 @@ datatype(::Type{T}) where {T<:ScalarType} = Datatype(hdf5_type_id(T), false) datatype(A::AbstractArray{T}) where {T<:ScalarType} = Datatype(hdf5_type_id(T), false) function datatype(::Type{Complex{T}}) where {T<:ScalarType} COMPLEX_SUPPORT[] || error("complex support disabled. call HDF5.enable_complex_support() to enable") - dtype = h5t_create(H5T_COMPOUND, 2*sizeof(T)) + dtype = h5t_create(H5T.COMPOUND, 2*sizeof(T)) h5t_insert(dtype, COMPLEX_FIELD_NAMES[][1], 0, hdf5_type_id(T)) h5t_insert(dtype, COMPLEX_FIELD_NAMES[][2], sizeof(T), hdf5_type_id(T)) return Datatype(dtype) @@ -1031,7 +1033,7 @@ function datatype(str::AbstractString) end function datatype(::Array{S}) where {S<:AbstractString} type_id = h5t_copy(hdf5_type_id(S)) - h5t_set_size(type_id, H5T_VARIABLE) + h5t_set_size(type_id, H5T.VARIABLE) h5t_set_cset(type_id, cset(S)) Datatype(type_id) end @@ -1051,8 +1053,8 @@ dataspace(dset::Dataset) = Dataspace(h5d_get_space(checkvalid(dset))) dataspace(attr::Attribute) = Dataspace(h5a_get_space(checkvalid(attr))) # Create a dataspace from in-memory types -dataspace(x::Union{T, Complex{T}}) where {T<:ScalarType} = Dataspace(h5s_create(H5S_SCALAR)) -dataspace(::AbstractString) = Dataspace(h5s_create(H5S_SCALAR)) +dataspace(x::Union{T, Complex{T}}) where {T<:ScalarType} = Dataspace(h5s_create(H5S.SCALAR)) +dataspace(::AbstractString) = Dataspace(h5s_create(H5S.SCALAR)) function _dataspace(sz::Dims{N}, max_dims::Union{Dims{N}, Tuple{}}=()) where N dims = hsize_t[sz[i] for i in N:-1:1] @@ -1068,8 +1070,8 @@ end dataspace(A::AbstractArray{T,N}; max_dims::Union{Dims{N},Tuple{}} = ()) where {T,N} = _dataspace(size(A), max_dims) # special array types dataspace(v::VLen; max_dims::Union{Dims,Tuple{}}=()) = _dataspace(size(v.data), max_dims) -dataspace(A::EmptyArray) = Dataspace(h5s_create(H5S_NULL)) -dataspace(n::Nothing) = Dataspace(h5s_create(H5S_NULL)) +dataspace(A::EmptyArray) = Dataspace(h5s_create(H5S.NULL)) +dataspace(n::Nothing) = Dataspace(h5s_create(H5S.NULL)) # for giving sizes explicitly dataspace(sz::Dims{N}; max_dims::Union{Dims{N},Tuple{}}=()) where {N} = _dataspace(sz, max_dims) dataspace(sz1::Int, sz2::Int, sz3::Int...; max_dims::Union{Dims,Tuple{}}=()) = _dataspace(tuple(sz1, sz2, sz3...), max_dims) @@ -1175,7 +1177,7 @@ function Base.read(obj::DatasetOrAttribute, ::Type{T}, I...) where T dspace = dataspace(obj) stype = h5s_get_simple_extent_type(dspace) - stype == H5S_NULL && return EmptyArray{T}() + stype == H5S.NULL && return EmptyArray{T}() if !isempty(I) indices = Base.to_indices(obj, I) @@ -1183,7 +1185,7 @@ function Base.read(obj::DatasetOrAttribute, ::Type{T}, I...) where T end scalar = false - if stype == H5S_SCALAR + if stype == H5S.SCALAR sz = (1,) scalar = true elseif isempty(I) @@ -1207,7 +1209,7 @@ function Base.read(obj::DatasetOrAttribute, ::Type{T}, I...) where T out = do_normalize(T) ? normalize_types.(buf) : buf - xfer_id = obj isa Dataset ? obj.xfer.id : H5P_DEFAULT + xfer_id = obj isa Dataset ? obj.xfer.id : H5P.DEFAULT do_reclaim(T) && h5d_vlen_reclaim(memtype.id, memspace.id, xfer_id, buf) close(memtype) @@ -1260,12 +1262,12 @@ Array(x::Dataset) = read(x) # Clean up string buffer according to padding mode function unpad(s::String, pad::Integer) - if pad == H5T_STR_NULLTERM + if pad == H5T.STR_NULLTERM v = findfirst(isequal('\0'), s) v === nothing ? s : s[1:v-1] - elseif pad == H5T_STR_NULLPAD + elseif pad == H5T.STR_NULLPAD rstrip(s, '\0') - elseif pad == H5T_STR_SPACEPAD + elseif pad == H5T.STR_SPACEPAD rstrip(s, ' ') else error("Unrecognized string padding mode $pad") @@ -1276,7 +1278,7 @@ unpad(s, pad::Integer) = unpad(String(s), pad) # Dereference function _deref(parent, r::Reference) r == Reference() && error("Reference is null") - obj_id = h5r_dereference(checkvalid(parent), H5P_DEFAULT, H5R_OBJECT, r) + obj_id = h5r_dereference(checkvalid(parent), H5P.DEFAULT, H5R.OBJECT, r) h5object(obj_id, parent) end Base.getindex(parent::Union{File,Group}, r::Reference) = _deref(parent, r) @@ -1304,7 +1306,7 @@ Base.read(attr::Attributes, name::AbstractString) = a_read(attr.parent, name) function iscontiguous(obj::Dataset) prop = h5d_get_create_plist(checkvalid(obj)) try - h5p_get_layout(prop) == H5D_CONTIGUOUS + h5p_get_layout(prop) == H5D.CONTIGUOUS finally h5p_close(prop) end @@ -1347,7 +1349,7 @@ function readmmap(obj::Dataset, ::Type{Array{T}}) where {T} # Check permissions intent = h5f_get_intent(obj.file.id) - flag = intent == H5F_ACC_RDONLY ? "r" : "r+" + flag = intent == H5F.ACC_RDONLY ? "r" : "r+" fd = open(obj.file.filename, flag) end @@ -1560,7 +1562,7 @@ function hyperslab(dspace::Dataspace, I::Union{AbstractRange{Int},Int}...) error("index out of range") end end - h5s_select_hyperslab(dsel_id, H5S_SELECT_SET, dsel_start, dsel_stride, dsel_count, C_NULL) + h5s_select_hyperslab(dsel_id, H5S.SELECT_SET, dsel_start, dsel_stride, dsel_count, C_NULL) finally close(dspace) end @@ -1576,15 +1578,15 @@ end # If you need to link to multiple segments, use low-level interface function d_create_external(parent::Union{File,Group}, name::AbstractString, filepath::AbstractString, t, sz::Dims, offset::Integer=0) checkvalid(parent) - dcpl = p_create(H5P_DATASET_CREATE) - h5p_set_external(dcpl , filepath, Int(offset), prod(sz)*sizeof(t)) # TODO: allow H5F_UNLIMITED + dcpl = p_create(H5P.DATASET_CREATE) + h5p_set_external(dcpl , filepath, Int(offset), prod(sz)*sizeof(t)) # TODO: allow H5F.UNLIMITED d_create(parent, name, datatype(t), dataspace(sz); dcpl=dcpl) end function do_write_chunk(dataset::Dataset, offset, chunk_bytes::Vector{UInt8}, filter_mask=0) checkvalid(dataset) offs = collect(hsize_t, reverse(offset)) .- 1 - h5do_write_chunk(dataset, H5P_DEFAULT, UInt32(filter_mask), offs, length(chunk_bytes), chunk_bytes) + h5do_write_chunk(dataset, H5P.DEFAULT, UInt32(filter_mask), offs, length(chunk_bytes), chunk_bytes) end struct ChunkStorage @@ -1623,9 +1625,9 @@ function hdf5_to_julia(obj::Union{Dataset, Attribute}) objspace = dataspace(obj) try stype = h5s_get_simple_extent_type(objspace) - if stype == H5S_SIMPLE + if stype == H5S.SIMPLE return Array{T} - elseif stype == H5S_NULL + elseif stype == H5S.NULL return EmptyArray{T} else return T @@ -1638,32 +1640,32 @@ end function hdf5_to_julia_eltype(objtype) local T class_id = h5t_get_class(objtype) - if class_id == H5T_STRING + if class_id == H5T.STRING cset = h5t_get_cset(objtype) n = h5t_get_size(objtype) - if cset == H5T_CSET_ASCII + if cset == H5T.CSET_ASCII T = (n == 1) ? ASCIIChar : String - elseif cset == H5T_CSET_UTF8 + elseif cset == H5T.CSET_UTF8 T = (n == 1) ? UTF8Char : String else error("character set ", cset, " not recognized") end - elseif class_id == H5T_INTEGER || class_id == H5T_FLOAT + elseif class_id == H5T.INTEGER || class_id == H5T.FLOAT T = get_mem_compatible_jl_type(objtype) - elseif class_id == H5T_BITFIELD + elseif class_id == H5T.BITFIELD T = get_mem_compatible_jl_type(objtype) - elseif class_id == H5T_ENUM + elseif class_id == H5T.ENUM T = get_mem_compatible_jl_type(objtype) - elseif class_id == H5T_REFERENCE + elseif class_id == H5T.REFERENCE T = get_mem_compatible_jl_type(objtype) - elseif class_id == H5T_OPAQUE + elseif class_id == H5T.OPAQUE T = Opaque - elseif class_id == H5T_VLEN + elseif class_id == H5T.VLEN super_id = h5t_get_super(objtype) T = VLen{hdf5_to_julia_eltype(Datatype(super_id))} - elseif class_id == H5T_COMPOUND + elseif class_id == H5T.COMPOUND T = get_mem_compatible_jl_type(objtype) - elseif class_id == H5T_ARRAY + elseif class_id == H5T.ARRAY T = get_mem_compatible_jl_type(objtype) else error("Class id ", class_id, " is not yet supported") @@ -1673,7 +1675,7 @@ end function get_jl_type(objtype::Datatype) class_id = h5t_get_class(objtype) - if class_id == H5T_OPAQUE + if class_id == H5T.OPAQUE return Opaque else return get_mem_compatible_jl_type(objtype) @@ -1691,7 +1693,7 @@ end function get_mem_compatible_jl_type(objtype::Datatype) class_id = h5t_get_class(objtype) - if class_id == H5T_STRING + if class_id == H5T.STRING if h5t_is_variable_str(objtype) return Cstring else @@ -1699,11 +1701,11 @@ function get_mem_compatible_jl_type(objtype::Datatype) pad = h5t_get_strpad(objtype) return FixedString{Int(n), pad} end - elseif class_id == H5T_INTEGER || class_id == H5T_FLOAT + elseif class_id == H5T.INTEGER || class_id == H5T.FLOAT native_type = h5t_get_native_type(objtype) try native_size = h5t_get_size(native_type) - if class_id == H5T_INTEGER + if class_id == H5T.INTEGER is_signed = h5t_get_sign(native_type) else is_signed = nothing @@ -1712,29 +1714,29 @@ function get_mem_compatible_jl_type(objtype::Datatype) finally h5t_close(native_type) end - elseif class_id == H5T_BITFIELD + elseif class_id == H5T.BITFIELD return Bool - elseif class_id == H5T_ENUM + elseif class_id == H5T.ENUM super_type = h5t_get_super(objtype) try native_type = h5t_get_native_type(super_type) try native_size = h5t_get_size(native_type) is_signed = h5t_get_sign(native_type) - return _hdf5_type_map(H5T_INTEGER, is_signed, native_size) + return _hdf5_type_map(H5T.INTEGER, is_signed, native_size) finally h5t_close(native_type) end finally h5t_close(super_type) end - elseif class_id == H5T_REFERENCE + elseif class_id == H5T.REFERENCE # TODO update to use version 1.12 reference functions/types return Reference - elseif class_id == H5T_VLEN + elseif class_id == H5T.VLEN superid = h5t_get_super(objtype) return VariableArray{get_mem_compatible_jl_type(Datatype(superid))} - elseif class_id == H5T_COMPOUND + elseif class_id == H5T.COMPOUND N = h5t_get_nmembers(objtype) membernames = ntuple(N) do i @@ -1758,7 +1760,7 @@ function get_mem_compatible_jl_type(objtype::Datatype) else return NamedTuple{Symbol.(membernames), Tuple{membertypes...}} end - elseif class_id == H5T_ARRAY + elseif class_id == H5T.ARRAY dims = h5t_get_array_dims(objtype) nd = length(dims) eltyp = Datatype(h5t_get_super(objtype)) @@ -1791,42 +1793,42 @@ function h5a_write(attr_id::hid_t, memtype_id::hid_t, v::VLen{T}) where {T<:Unio vp = vlenpack(v) h5a_write(attr_id, memtype_id, vp) end -h5a_create(loc_id, name, type_id, space_id) = h5a_create(loc_id, name, type_id, space_id, _attr_properties(name), H5P_DEFAULT) -h5a_open(obj_id, name) = h5a_open(obj_id, name, H5P_DEFAULT) -h5d_create(loc_id, name, type_id, space_id) = h5d_create(loc_id, name, type_id, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) -h5d_open(obj_id, name) = h5d_open(obj_id, name, H5P_DEFAULT) -function h5d_read(dataset_id::hid_t, memtype_id::hid_t, buf::AbstractArray, xfer::hid_t=H5P_DEFAULT) +h5a_create(loc_id, name, type_id, space_id) = h5a_create(loc_id, name, type_id, space_id, _attr_properties(name), H5P.DEFAULT) +h5a_open(obj_id, name) = h5a_open(obj_id, name, H5P.DEFAULT) +h5d_create(loc_id, name, type_id, space_id) = h5d_create(loc_id, name, type_id, space_id, H5P.DEFAULT, H5P.DEFAULT, H5P.DEFAULT) +h5d_open(obj_id, name) = h5d_open(obj_id, name, H5P.DEFAULT) +function h5d_read(dataset_id::hid_t, memtype_id::hid_t, buf::AbstractArray, xfer::hid_t=H5P.DEFAULT) stride(buf, 1) != 1 && throw(ArgumentError("Cannot read arrays with a different stride than `Array`")) - h5d_read(dataset_id, memtype_id, H5S_ALL, H5S_ALL, xfer, buf) + h5d_read(dataset_id, memtype_id, H5S.ALL, H5S.ALL, xfer, buf) end -function h5d_write(dataset_id::hid_t, memtype_id::hid_t, buf::AbstractArray, xfer::hid_t=H5P_DEFAULT) +function h5d_write(dataset_id::hid_t, memtype_id::hid_t, buf::AbstractArray, xfer::hid_t=H5P.DEFAULT) stride(buf, 1) != 1 && throw(ArgumentError("Cannot write arrays with a different stride than `Array`")) - h5d_write(dataset_id, memtype_id, H5S_ALL, H5S_ALL, xfer, buf) + h5d_write(dataset_id, memtype_id, H5S.ALL, H5S.ALL, xfer, buf) end -function h5d_write(dataset_id::hid_t, memtype_id::hid_t, str::AbstractString, xfer::hid_t=H5P_DEFAULT) - ccall((:H5Dwrite, libhdf5), herr_t, (hid_t, hid_t, hid_t, hid_t, hid_t, Cstring), dataset_id, memtype_id, H5S_ALL, H5S_ALL, xfer, str) +function h5d_write(dataset_id::hid_t, memtype_id::hid_t, str::AbstractString, xfer::hid_t=H5P.DEFAULT) + ccall((:H5Dwrite, libhdf5), herr_t, (hid_t, hid_t, hid_t, hid_t, hid_t, Cstring), dataset_id, memtype_id, H5S.ALL, H5S.ALL, xfer, str) end -function h5d_write(dataset_id::hid_t, memtype_id::hid_t, x::T, xfer::hid_t=H5P_DEFAULT) where {T<:Union{ScalarType, Complex{<:ScalarType}}} +function h5d_write(dataset_id::hid_t, memtype_id::hid_t, x::T, xfer::hid_t=H5P.DEFAULT) where {T<:Union{ScalarType, Complex{<:ScalarType}}} tmp = Ref{T}(x) - h5d_write(dataset_id, memtype_id, H5S_ALL, H5S_ALL, xfer, tmp) + h5d_write(dataset_id, memtype_id, H5S.ALL, H5S.ALL, xfer, tmp) end -function h5d_write(dataset_id::hid_t, memtype_id::hid_t, strs::Array{<:AbstractString}, xfer::hid_t=H5P_DEFAULT) +function h5d_write(dataset_id::hid_t, memtype_id::hid_t, strs::Array{<:AbstractString}, xfer::hid_t=H5P.DEFAULT) p = Ref{Cstring}(strs) - h5d_write(dataset_id, memtype_id, H5S_ALL, H5S_ALL, xfer, p) + h5d_write(dataset_id, memtype_id, H5S.ALL, H5S.ALL, xfer, p) end -function h5d_write(dataset_id::hid_t, memtype_id::hid_t, v::VLen{T}, xfer::hid_t=H5P_DEFAULT) where {T<:Union{ScalarType,CharType}} +function h5d_write(dataset_id::hid_t, memtype_id::hid_t, v::VLen{T}, xfer::hid_t=H5P.DEFAULT) where {T<:Union{ScalarType,CharType}} vp = vlenpack(v) - h5d_write(dataset_id, memtype_id, H5S_ALL, H5S_ALL, xfer, vp) -end -h5f_create(filename) = h5f_create(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT) -h5f_open(filename, mode) = h5f_open(filename, mode, H5P_DEFAULT) -h5g_create(obj_id, name) = h5g_create(obj_id, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) -h5g_create(obj_id, name, lcpl_id, gcpl_id) = h5g_create(obj_id, name, lcpl_id, gcpl_id, H5P_DEFAULT) -h5g_open(file_id, name) = h5g_open(file_id, name, H5P_DEFAULT) -h5l_exists(loc_id, name) = h5l_exists(loc_id, name, H5P_DEFAULT) -h5o_open(obj_id, name) = h5o_open(obj_id, name, H5P_DEFAULT) + h5d_write(dataset_id, memtype_id, H5S.ALL, H5S.ALL, xfer, vp) +end +h5f_create(filename) = h5f_create(filename, H5F.ACC_TRUNC, H5P.DEFAULT, H5P.DEFAULT) +h5f_open(filename, mode) = h5f_open(filename, mode, H5P.DEFAULT) +h5g_create(obj_id, name) = h5g_create(obj_id, name, H5P.DEFAULT, H5P.DEFAULT, H5P.DEFAULT) +h5g_create(obj_id, name, lcpl_id, gcpl_id) = h5g_create(obj_id, name, lcpl_id, gcpl_id, H5P.DEFAULT) +h5g_open(file_id, name) = h5g_open(file_id, name, H5P.DEFAULT) +h5l_exists(loc_id, name) = h5l_exists(loc_id, name, H5P.DEFAULT) +h5o_open(obj_id, name) = h5o_open(obj_id, name, H5P.DEFAULT) #h5s_get_simple_extent_ndims(space_id::hid_t) = h5s_get_simple_extent_ndims(space_id, C_NULL, C_NULL) -h5t_get_native_type(type_id) = h5t_get_native_type(type_id, H5T_DIR_ASCEND) +h5t_get_native_type(type_id) = h5t_get_native_type(type_id, H5T.DIR_ASCEND) # Functions that require special handling @@ -1836,12 +1838,12 @@ const libversion = h5_get_libversion() vlen_get_buf_size(dset::Dataset, dtype::Datatype, dspace::Dataspace) = h5d_vlen_get_buf_size(dset, dtype, dspace) ### Property manipulation ### -get_access_properties(d::Dataset) = Properties(h5d_get_access_plist(d), H5P_DATASET_ACCESS) -get_access_properties(f::File) = Properties(h5f_get_access_plist(f), H5P_FILE_ACCESS) -get_create_properties(d::Dataset) = Properties(h5d_get_create_plist(d), H5P_DATASET_CREATE) -get_create_properties(g::Group) = Properties(h5g_get_create_plist(g), H5P_GROUP_CREATE) -get_create_properties(f::File) = Properties(h5f_get_create_plist(f), H5P_FILE_CREATE) -get_create_properties(a::Attribute) = Properties(h5a_get_create_plist(a), H5P_ATTRIBUTE_CREATE) +get_access_properties(d::Dataset) = Properties(h5d_get_access_plist(d), H5P.DATASET_ACCESS) +get_access_properties(f::File) = Properties(h5f_get_access_plist(f), H5P.FILE_ACCESS) +get_create_properties(d::Dataset) = Properties(h5d_get_create_plist(d), H5P.DATASET_CREATE) +get_create_properties(g::Group) = Properties(h5g_get_create_plist(g), H5P.GROUP_CREATE) +get_create_properties(f::File) = Properties(h5f_get_create_plist(f), H5P.FILE_CREATE) +get_create_properties(a::Attribute) = Properties(h5a_get_create_plist(a), H5P.ATTRIBUTE_CREATE) get_chunk(p::Properties) = tuple(convert(Vector{Int}, reverse(h5p_get_chunk(p)))...) set_chunk(p::Properties, dims...) = h5p_set_chunk(p, length(dims), hsize_t[reverse(dims)...]) @@ -1888,19 +1890,19 @@ const chunked_props = (; compress=nothing, deflate=nothing, blosc=nothing, shuff """ create_external(source::Union{HDF5.File, HDF5.Group}, source_relpath, target_filename, target_path; - lcpl_id=HDF5.H5P_DEFAULT, lapl_id=HDF5.H5P.DEFAULT) + lcpl_id=HDF5.H5P.DEFAULT, lapl_id=HDF5.H5P.DEFAULT) Create an external link such that `source[source_relpath]` points to `target_path` within the file with path `target_filename`; Calls `[H5Lcreate_external](https://www.hdfgroup.org/HDF5/doc/RM/RM_H5L.html#Link-CreateExternal)`. """ -function create_external(source::Union{File,Group}, source_relpath, target_filename, target_path; lcpl_id=H5P_DEFAULT, lapl_id=H5P_DEFAULT) +function create_external(source::Union{File,Group}, source_relpath, target_filename, target_path; lcpl_id=H5P.DEFAULT, lapl_id=H5P.DEFAULT) h5l_create_external(target_filename, target_path, source, source_relpath, lcpl_id, lapl_id) nothing end # error handling function hiding_errors(f) - error_stack = H5E_DEFAULT + error_stack = H5E.DEFAULT # error_stack = ccall((:H5Eget_current_stack, libhdf5), hid_t, ()) old_func, old_client_data = h5e_get_auto(error_stack) h5e_set_auto(error_stack, C_NULL, C_NULL) @@ -1921,7 +1923,7 @@ _attr_properties(::AbstractString) = UTF8_ATTRIBUTE_PROPERTIES[] const ASCII_LINK_PROPERTIES = Ref{Properties}() const ASCII_ATTRIBUTE_PROPERTIES = Ref{Properties}() -const DEFAULT_PROPERTIES = Properties(H5P_DEFAULT, H5P_DEFAULT) +const DEFAULT_PROPERTIES = Properties(H5P.DEFAULT, H5P.DEFAULT) const HAS_PARALLEL = Ref(false) @@ -1943,17 +1945,18 @@ function __init__() ENV["HDF5_USE_FILE_LOCKING"] = "FALSE" end + __init_globals__() register_blosc() # Turn off automatic error printing - # h5e_set_auto(H5E_DEFAULT, C_NULL, C_NULL) + # h5e_set_auto(H5E.DEFAULT, C_NULL, C_NULL) - ASCII_LINK_PROPERTIES[] = p_create(H5P_LINK_CREATE; char_encoding = H5T_CSET_ASCII, + ASCII_LINK_PROPERTIES[] = p_create(H5P.LINK_CREATE; char_encoding = H5T.CSET_ASCII, create_intermediate_group = 1) - UTF8_LINK_PROPERTIES[] = p_create(H5P_LINK_CREATE; char_encoding = H5T_CSET_UTF8, + UTF8_LINK_PROPERTIES[] = p_create(H5P.LINK_CREATE; char_encoding = H5T.CSET_UTF8, create_intermediate_group = 1) - ASCII_ATTRIBUTE_PROPERTIES[] = p_create(H5P_ATTRIBUTE_CREATE; char_encoding = H5T_CSET_ASCII) - UTF8_ATTRIBUTE_PROPERTIES[] = p_create(H5P_ATTRIBUTE_CREATE; char_encoding = H5T_CSET_UTF8) + ASCII_ATTRIBUTE_PROPERTIES[] = p_create(H5P.ATTRIBUTE_CREATE; char_encoding = H5T.CSET_ASCII) + UTF8_ATTRIBUTE_PROPERTIES[] = p_create(H5P.ATTRIBUTE_CREATE; char_encoding = H5T.CSET_UTF8) @require MPI="da04e1cc-30fd-572f-bb4f-1f8673147195" @eval include("mpio.jl") diff --git a/src/api_types.jl b/src/api_types.jl index 28d71b970..930ed7fc2 100644 --- a/src/api_types.jl +++ b/src/api_types.jl @@ -104,222 +104,315 @@ abstract type Hmpih end primitive type Hmpih32 <: Hmpih 32 end # MPICH C/Fortran, OpenMPI Fortran: 32 bit handles primitive type Hmpih64 <: Hmpih 64 end # OpenMPI C: pointers (mostly 64 bit) -# Private function to extract exported global library constants. -# Need to call H5open to ensure library is initalized before reading these constants. -# Although these are runtime initalized constants, in practice their values are stable, so -# we can precompile for improved latency. -let libhdf5handle = Ref(Libdl.dlopen(libhdf5)) - ccall(Libdl.dlsym(libhdf5handle[], :H5open), herr_t, ()) - global _read_const(sym::Symbol) = unsafe_load(cglobal(Libdl.dlsym(libhdf5handle[], sym), hid_t)) +@defconstants H5 begin + # iteration order constants + ITER_UNKNOWN::Cint = -1 + ITER_INC::Cint = 0 + ITER_DEC::Cint = 1 + ITER_NATIVE::Cint = 2 + ITER_N::Cint = 3 + + # indexing type constants + INDEX_UNKNOWN::Cint = -1 + INDEX_NAME::Cint = 0 + INDEX_CRT_ORDER::Cint = 1 + INDEX_N::Cint = 2 end -# iteration order constants -const H5_ITER_UNKNOWN = -1 -const H5_ITER_INC = 0 -const H5_ITER_DEC = 1 -const H5_ITER_NATIVE = 2 -const H5_ITER_N = 3 - -# indexing type constants -const H5_INDEX_UNKNOWN = -1 -const H5_INDEX_NAME = 0 -const H5_INDEX_CRT_ORDER = 1 -const H5_INDEX_N = 2 - # dataset constants -const H5D_COMPACT = 0 -const H5D_CONTIGUOUS = 1 -const H5D_CHUNKED = 2 - -# allocation times (C enum H5D_alloc_time_t) -const H5D_ALLOC_TIME_ERROR = -1 -const H5D_ALLOC_TIME_DEFAULT = 0 -const H5D_ALLOC_TIME_EARLY = 1 -const H5D_ALLOC_TIME_LATE = 2 -const H5D_ALLOC_TIME_INCR = 3 - -# used to "unset" chunk cache configuration parameters -const H5D_CHUNK_CACHE_NSLOTS_DEFAULT = -1 % Csize_t -const H5D_CHUNK_CACHE_NBYTES_DEFAULT = -1 % Csize_t -const H5D_CHUNK_CACHE_W0_DEFAULT = Cdouble(-1) +@defconstants H5D begin + # layouts (C enum H5D_layout_t + COMPACT::Cint = 0 + CONTIGUOUS::Cint = 1 + CHUNKED::Cint = 2 + + # allocation times (C enum H5D_alloc_time_t) + ALLOC_TIME_ERROR::Cint = -1 + ALLOC_TIME_DEFAULT::Cint = 0 + ALLOC_TIME_EARLY::Cint = 1 + ALLOC_TIME_LATE::Cint = 2 + ALLOC_TIME_INCR::Cint = 3 + + # used to "unset" chunk cache configuration parameters + CHUNK_CACHE_NSLOTS_DEFAULT::Csize_t = -1 % Csize_t + CHUNK_CACHE_NBYTES_DEFAULT::Csize_t = -1 % Csize_t + CHUNK_CACHE_W0_DEFAULT::Cfloat = -1.0f0 +end # error-related constants -const H5E_DEFAULT = 0 - -# file access modes -const H5F_ACC_RDONLY = 0x0000 -const H5F_ACC_RDWR = 0x0001 -const H5F_ACC_TRUNC = 0x0002 -const H5F_ACC_EXCL = 0x0004 -const H5F_ACC_DEBUG = 0x0008 -const H5F_ACC_CREAT = 0x0010 -const H5F_ACC_SWMR_WRITE = 0x0020 -const H5F_ACC_SWMR_READ = 0x0040 - -# Library versions -const H5F_LIBVER_EARLIEST = 0 -const H5F_LIBVER_V18 = 1 -const H5F_LIBVER_V110 = 2 -const H5F_LIBVER_LATEST = H5F_LIBVER_V110 - -# object types -const H5F_OBJ_FILE = 0x0001 -const H5F_OBJ_DATASET = 0x0002 -const H5F_OBJ_GROUP = 0x0004 -const H5F_OBJ_DATATYPE = 0x0008 -const H5F_OBJ_ATTR = 0x0010 -const H5F_OBJ_ALL = (H5F_OBJ_FILE|H5F_OBJ_DATASET|H5F_OBJ_GROUP|H5F_OBJ_DATATYPE|H5F_OBJ_ATTR) -const H5F_OBJ_LOCAL = 0x0020 - -# other file constants -const H5F_SCOPE_LOCAL = 0 -const H5F_SCOPE_GLOBAL = 1 -const H5F_CLOSE_DEFAULT = 0 -const H5F_CLOSE_WEAK = 1 -const H5F_CLOSE_SEMI = 2 -const H5F_CLOSE_STRONG = 3 +@defconstants H5E begin + DEFAULT::hid_t = 0 +end + +@defconstants H5F begin + # file access modes + ACC_RDONLY::Cuint = 0x0000 + ACC_RDWR::Cuint = 0x0001 + ACC_TRUNC::Cuint = 0x0002 + ACC_EXCL::Cuint = 0x0004 + ACC_DEBUG::Cuint = 0x0008 + ACC_CREAT::Cuint = 0x0010 + ACC_SWMR_WRITE::Cuint = 0x0020 + ACC_SWMR_READ::Cuint = 0x0040 + + # Library versions + LIBVER_EARLIEST::Cint = 0 + LIBVER_V18::Cint = 1 + LIBVER_V110::Cint = 2 + LIBVER_LATEST::Cint = 2 # H5F_LIBVER_V110 + + # object types + OBJ_FILE::Cuint = 0x0001 + OBJ_DATASET::Cuint = 0x0002 + OBJ_GROUP::Cuint = 0x0004 + OBJ_DATATYPE::Cuint = 0x0008 + OBJ_ATTR::Cuint = 0x0010 + OBJ_ALL::Cuint = 0x001f # (H5F_OBJ_FILE|H5F_OBJ_DATASET|H5F_OBJ_GROUP|H5F_OBJ_DATATYPE|H5F_OBJ_ATTR) + OBJ_LOCAL::Cuint = 0x0020 + + # other file constants + SCOPE_LOCAL::Cint = 0 + SCOPE_GLOBAL::Cint = 1 + CLOSE_DEFAULT::Cint = 0 + CLOSE_WEAK::Cint = 1 + CLOSE_SEMI::Cint = 2 + CLOSE_STRONG::Cint = 3 +end # file driver constants -const H5FD_MPIO_INDEPENDENT = 0 -const H5FD_MPIO_COLLECTIVE = 1 -const H5FD_MPIO_CHUNK_DEFAULT = 0 -const H5FD_MPIO_CHUNK_ONE_IO = 1 -const H5FD_MPIO_CHUNK_MULTI_IO = 2 -const H5FD_MPIO_COLLECTIVE_IO = 0 -const H5FD_MPIO_INDIVIDUAL_IO = 1 - -# object types (C enum H5Itype_t) -const H5I_FILE = 1 -const H5I_GROUP = 2 -const H5I_DATATYPE = 3 -const H5I_DATASPACE = 4 -const H5I_DATASET = 5 -const H5I_ATTR = 6 -const H5I_REFERENCE = 7 -const H5I_VFL = 8 +@defconstants H5FD begin + # C enum H5FD_mpio_xfer_t + MPIO_INDEPENDENT::Cint = 0 + MPIO_COLLECTIVE::Cint = 1 + + # C enum H5FD_mpio_chunk_opt_t + MPIO_CHUNK_DEFAULT::Cint = 0 + MPIO_CHUNK_ONE_IO::Cint = 1 + MPIO_CHUNK_MULTI_IO::Cint = 2 + + # C enum H5FD_mpio_collective_opt_t + MPIO_COLLECTIVE_IO::Cint = 0 + MPIO_INDIVIDUAL_IO::Cint = 1 +end + +@defconstants H5I begin + # object types (C enum H5Itype_t) + FILE::Cint = 1 + GROUP::Cint = 2 + DATATYPE::Cint = 3 + DATASPACE::Cint = 4 + DATASET::Cint = 5 + ATTR::Cint = 6 + REFERENCE::Cint = 7 + VFL::Cint = 8 +end # Link constants -const H5L_TYPE_HARD = 0 -const H5L_TYPE_SOFT = 1 -const H5L_TYPE_EXTERNAL= 2 +@defconstants H5L begin + TYPE_HARD::Cint = 0 + TYPE_SOFT::Cint = 1 + TYPE_EXTERNAL::Cint = 2 +end # Object constants -const H5O_TYPE_GROUP = 0 -const H5O_TYPE_DATASET = 1 -const H5O_TYPE_NAMED_DATATYPE = 2 +@defconstants H5O begin + TYPE_GROUP::Cint = 0 + TYPE_DATASET::Cint = 1 + TYPE_NAMED_DATATYPE::Cint = 2 +end # Property constants -const H5P_DEFAULT = hid_t(0) -const H5P_OBJECT_CREATE = _read_const(:H5P_CLS_OBJECT_CREATE_ID_g) -const H5P_FILE_CREATE = _read_const(:H5P_CLS_FILE_CREATE_ID_g) -const H5P_FILE_ACCESS = _read_const(:H5P_CLS_FILE_ACCESS_ID_g) -const H5P_DATASET_CREATE = _read_const(:H5P_CLS_DATASET_CREATE_ID_g) -const H5P_DATASET_ACCESS = _read_const(:H5P_CLS_DATASET_ACCESS_ID_g) -const H5P_DATASET_XFER = _read_const(:H5P_CLS_DATASET_XFER_ID_g) -const H5P_FILE_MOUNT = _read_const(:H5P_CLS_FILE_MOUNT_ID_g) -const H5P_GROUP_CREATE = _read_const(:H5P_CLS_GROUP_CREATE_ID_g) -const H5P_GROUP_ACCESS = _read_const(:H5P_CLS_GROUP_ACCESS_ID_g) -const H5P_DATATYPE_CREATE = _read_const(:H5P_CLS_DATATYPE_CREATE_ID_g) -const H5P_DATATYPE_ACCESS = _read_const(:H5P_CLS_DATATYPE_ACCESS_ID_g) -const H5P_STRING_CREATE = _read_const(:H5P_CLS_STRING_CREATE_ID_g) -const H5P_ATTRIBUTE_CREATE = _read_const(:H5P_CLS_ATTRIBUTE_CREATE_ID_g) -const H5P_OBJECT_COPY = _read_const(:H5P_CLS_OBJECT_COPY_ID_g) -const H5P_LINK_CREATE = _read_const(:H5P_CLS_LINK_CREATE_ID_g) -const H5P_LINK_ACCESS = _read_const(:H5P_CLS_LINK_ACCESS_ID_g) +@defconstants H5P begin + DEFAULT::hid_t = 0 + OBJECT_CREATE::hid_t + FILE_CREATE::hid_t + FILE_ACCESS::hid_t + DATASET_CREATE::hid_t + DATASET_ACCESS::hid_t + DATASET_XFER::hid_t + FILE_MOUNT::hid_t + GROUP_CREATE::hid_t + GROUP_ACCESS::hid_t + DATATYPE_CREATE::hid_t + DATATYPE_ACCESS::hid_t + STRING_CREATE::hid_t + ATTRIBUTE_CREATE::hid_t + OBJECT_COPY::hid_t + LINK_CREATE::hid_t + LINK_ACCESS::hid_t +end # Reference constants -const H5R_OBJECT = 0 -const H5R_DATASET_REGION = 1 -const H5R_OBJ_REF_BUF_SIZE = 8 # == sizeof(hobj_ref_t) -const H5R_DSET_REG_REF_BUF_SIZE = 12 # == sizeof(hdset_reg_ref_t) +@defconstants H5R begin + OBJECT::Cint = 0 + DATASET_REGION::Cint = 1 + OBJ_REF_BUF_SIZE::Cint = 8 # == sizeof(hobj_ref_t) + DSET_REG_REF_BUF_SIZE::Cint = 12 # == sizeof(hdset_reg_ref_t) +end # Dataspace constants -const H5S_ALL = hid_t(0) -const H5S_SCALAR = hid_t(0) -const H5S_SIMPLE = hid_t(1) -const H5S_NULL = hid_t(2) -const H5S_UNLIMITED = typemax(hsize_t) - -# Dataspace selection constants -const H5S_SELECT_SET = 0 -const H5S_SELECT_OR = 1 -const H5S_SELECT_AND = 2 -const H5S_SELECT_XOR = 3 -const H5S_SELECT_NOTB = 4 -const H5S_SELECT_NOTA = 5 -const H5S_SELECT_APPEND = 6 -const H5S_SELECT_PREPEND = 7 - -# type classes (C enum H5T_class_t) -const H5T_INTEGER = hid_t(0) -const H5T_FLOAT = hid_t(1) -const H5T_TIME = hid_t(2) # not supported by HDF5 library -const H5T_STRING = hid_t(3) -const H5T_BITFIELD = hid_t(4) -const H5T_OPAQUE = hid_t(5) -const H5T_COMPOUND = hid_t(6) -const H5T_REFERENCE = hid_t(7) -const H5T_ENUM = hid_t(8) -const H5T_VLEN = hid_t(9) -const H5T_ARRAY = hid_t(10) - -# Character types -const H5T_CSET_ASCII = 0 -const H5T_CSET_UTF8 = 1 - -# Sign types (C enum H5T_sign_t) -const H5T_SGN_NONE = Cint(0) # unsigned -const H5T_SGN_2 = Cint(1) # 2's complement - -# Search directions -const H5T_DIR_ASCEND = 1 -const H5T_DIR_DESCEND = 2 - -# String padding modes -const H5T_STR_NULLTERM = 0 -const H5T_STR_NULLPAD = 1 -const H5T_STR_SPACEPAD = 2 - -# Type_id constants (LE = little endian, I16 = Int16, etc) -const H5T_STD_I8LE = _read_const(:H5T_STD_I8LE_g) -const H5T_STD_I8BE = _read_const(:H5T_STD_I8BE_g) -const H5T_STD_U8LE = _read_const(:H5T_STD_U8LE_g) -const H5T_STD_U8BE = _read_const(:H5T_STD_U8BE_g) -const H5T_STD_I16LE = _read_const(:H5T_STD_I16LE_g) -const H5T_STD_I16BE = _read_const(:H5T_STD_I16BE_g) -const H5T_STD_U16LE = _read_const(:H5T_STD_U16LE_g) -const H5T_STD_U16BE = _read_const(:H5T_STD_U16BE_g) -const H5T_STD_I32LE = _read_const(:H5T_STD_I32LE_g) -const H5T_STD_I32BE = _read_const(:H5T_STD_I32BE_g) -const H5T_STD_U32LE = _read_const(:H5T_STD_U32LE_g) -const H5T_STD_U32BE = _read_const(:H5T_STD_U32BE_g) -const H5T_STD_I64LE = _read_const(:H5T_STD_I64LE_g) -const H5T_STD_I64BE = _read_const(:H5T_STD_I64BE_g) -const H5T_STD_U64LE = _read_const(:H5T_STD_U64LE_g) -const H5T_STD_U64BE = _read_const(:H5T_STD_U64BE_g) -const H5T_IEEE_F32LE = _read_const(:H5T_IEEE_F32LE_g) -const H5T_IEEE_F32BE = _read_const(:H5T_IEEE_F32BE_g) -const H5T_IEEE_F64LE = _read_const(:H5T_IEEE_F64LE_g) -const H5T_IEEE_F64BE = _read_const(:H5T_IEEE_F64BE_g) -const H5T_C_S1 = _read_const(:H5T_C_S1_g) -const H5T_STD_REF_OBJ = _read_const(:H5T_STD_REF_OBJ_g) -const H5T_STD_REF_DSETREG = _read_const(:H5T_STD_REF_DSETREG_g) -# Native types -const H5T_NATIVE_B8 = _read_const(:H5T_NATIVE_B8_g) -const H5T_NATIVE_INT8 = _read_const(:H5T_NATIVE_INT8_g) -const H5T_NATIVE_UINT8 = _read_const(:H5T_NATIVE_UINT8_g) -const H5T_NATIVE_INT16 = _read_const(:H5T_NATIVE_INT16_g) -const H5T_NATIVE_UINT16 = _read_const(:H5T_NATIVE_UINT16_g) -const H5T_NATIVE_INT32 = _read_const(:H5T_NATIVE_INT32_g) -const H5T_NATIVE_UINT32 = _read_const(:H5T_NATIVE_UINT32_g) -const H5T_NATIVE_INT64 = _read_const(:H5T_NATIVE_INT64_g) -const H5T_NATIVE_UINT64 = _read_const(:H5T_NATIVE_UINT64_g) -const H5T_NATIVE_FLOAT = _read_const(:H5T_NATIVE_FLOAT_g) -const H5T_NATIVE_DOUBLE = _read_const(:H5T_NATIVE_DOUBLE_g) -# Other type constants -const H5T_VARIABLE = reinterpret(UInt, -1) +@defconstants H5S begin + # atomic data types + ALL::hid_t = 0 + UNLIMITED::hid_t = typemax(hsize_t) + + # Types of dataspaces (C enum H5S_class_t) + SCALAR::hid_t = hid_t(0) + SIMPLE::hid_t = hid_t(1) + NULL::hid_t = hid_t(2) + + # Dataspace selection constants (C enum H5S_seloper_t) + SELECT_SET::Cint = 0 + SELECT_OR::Cint = 1 + SELECT_AND::Cint = 2 + SELECT_XOR::Cint = 3 + SELECT_NOTB::Cint = 4 + SELECT_NOTA::Cint = 5 + SELECT_APPEND::Cint = 6 + SELECT_PREPEND::Cint = 7 +end + +@defconstants H5T begin + # type classes (C enum H5T_class_t) + INTEGER::hid_t = 0 + FLOAT::hid_t = 1 + TIME::hid_t = 2 # not supported by HDF5 library + STRING::hid_t = 3 + BITFIELD::hid_t = 4 + OPAQUE::hid_t = 5 + COMPOUND::hid_t = 6 + REFERENCE::hid_t = 7 + ENUM::hid_t = 8 + VLEN::hid_t = 9 + ARRAY::hid_t = 10 + + # Character types (C enum H5T_cset_t) + CSET_ASCII::Cint = 0 + CSET_UTF8::Cint = 1 + + # String padding modes (C enum H5T_str_t) + STR_NULLTERM::Cint = 0 + STR_NULLPAD::Cint = 1 + STR_SPACEPAD::Cint = 2 + + # Variable length string + VARIABLE::Csize_t = -1 % Csize_t + + # Sign types (C enum H5T_sign_t) + SGN_NONE::Cint = 0 # unsigned + SGN_2::Cint = 1 # 2's complement + + # Search directions (C enum H5T_direction_t) + DIR_ASCEND::Cint = 1 + DIR_DESCEND::Cint = 2 + + # Type "constants" (LE = little endian, I16 = Int16, etc) + STD_I8LE::hid_t + STD_I8BE::hid_t + STD_U8LE::hid_t + STD_U8BE::hid_t + STD_I16LE::hid_t + STD_I16BE::hid_t + STD_U16LE::hid_t + STD_U16BE::hid_t + STD_I32LE::hid_t + STD_I32BE::hid_t + STD_U32LE::hid_t + STD_U32BE::hid_t + STD_I64LE::hid_t + STD_I64BE::hid_t + STD_U64LE::hid_t + STD_U64BE::hid_t + IEEE_F32LE::hid_t + IEEE_F32BE::hid_t + IEEE_F64LE::hid_t + IEEE_F64BE::hid_t + C_S1::hid_t + STD_REF_OBJ::hid_t + STD_REF_DSETREG::hid_t + NATIVE_B8::hid_t + NATIVE_INT8::hid_t + NATIVE_UINT8::hid_t + NATIVE_INT16::hid_t + NATIVE_UINT16::hid_t + NATIVE_INT32::hid_t + NATIVE_UINT32::hid_t + NATIVE_INT64::hid_t + NATIVE_UINT64::hid_t + NATIVE_FLOAT::hid_t + NATIVE_DOUBLE::hid_t +end # Filter constants -const H5Z_FLAG_OPTIONAL = 0x0001 -const H5Z_FLAG_REVERSE = 0x0100 -const H5Z_CLASS_T_VERS = 1 +@defconstants H5Z begin + FLAG_OPTIONAL::Cuint = 0x0001 + FLAG_REVERSE::Cuint = 0x0100 + CLASS_T_VERS::Cint = 1 +end + +function __init_globals__() + libh = Libdl.dlopen(libhdf5) + # Ensure runtime is initialized + ccall(Libdl.dlsym(libh, :H5open), herr_t, ()) + # Note: dlsym must occur outside cglobal statement on Julia 1.5 and earlier or else + # segfaults. + function read_const(sym::Symbol) + symptr = Libdl.dlsym(libh, sym) + return unsafe_load(cglobal(symptr, hid_t)) + end + + _H5P.OBJECT_CREATE[] = read_const(:H5P_CLS_OBJECT_CREATE_ID_g) + _H5P.FILE_CREATE[] = read_const(:H5P_CLS_FILE_CREATE_ID_g) + _H5P.FILE_ACCESS[] = read_const(:H5P_CLS_FILE_ACCESS_ID_g) + _H5P.DATASET_CREATE[] = read_const(:H5P_CLS_DATASET_CREATE_ID_g) + _H5P.DATASET_ACCESS[] = read_const(:H5P_CLS_DATASET_ACCESS_ID_g) + _H5P.DATASET_XFER[] = read_const(:H5P_CLS_DATASET_XFER_ID_g) + _H5P.FILE_MOUNT[] = read_const(:H5P_CLS_FILE_MOUNT_ID_g) + _H5P.GROUP_CREATE[] = read_const(:H5P_CLS_GROUP_CREATE_ID_g) + _H5P.GROUP_ACCESS[] = read_const(:H5P_CLS_GROUP_ACCESS_ID_g) + _H5P.DATATYPE_CREATE[] = read_const(:H5P_CLS_DATATYPE_CREATE_ID_g) + _H5P.DATATYPE_ACCESS[] = read_const(:H5P_CLS_DATATYPE_ACCESS_ID_g) + _H5P.STRING_CREATE[] = read_const(:H5P_CLS_STRING_CREATE_ID_g) + _H5P.ATTRIBUTE_CREATE[] = read_const(:H5P_CLS_ATTRIBUTE_CREATE_ID_g) + _H5P.OBJECT_COPY[] = read_const(:H5P_CLS_OBJECT_COPY_ID_g) + _H5P.LINK_CREATE[] = read_const(:H5P_CLS_LINK_CREATE_ID_g) + _H5P.LINK_ACCESS[] = read_const(:H5P_CLS_LINK_ACCESS_ID_g) + + _H5T.STD_I8LE[] = read_const(:H5T_STD_I8LE_g) + _H5T.STD_I8BE[] = read_const(:H5T_STD_I8BE_g) + _H5T.STD_U8LE[] = read_const(:H5T_STD_U8LE_g) + _H5T.STD_U8BE[] = read_const(:H5T_STD_U8BE_g) + _H5T.STD_I16LE[] = read_const(:H5T_STD_I16LE_g) + _H5T.STD_I16BE[] = read_const(:H5T_STD_I16BE_g) + _H5T.STD_U16LE[] = read_const(:H5T_STD_U16LE_g) + _H5T.STD_U16BE[] = read_const(:H5T_STD_U16BE_g) + _H5T.STD_I32LE[] = read_const(:H5T_STD_I32LE_g) + _H5T.STD_I32BE[] = read_const(:H5T_STD_I32BE_g) + _H5T.STD_U32LE[] = read_const(:H5T_STD_U32LE_g) + _H5T.STD_U32BE[] = read_const(:H5T_STD_U32BE_g) + _H5T.STD_I64LE[] = read_const(:H5T_STD_I64LE_g) + _H5T.STD_I64BE[] = read_const(:H5T_STD_I64BE_g) + _H5T.STD_U64LE[] = read_const(:H5T_STD_U64LE_g) + _H5T.STD_U64BE[] = read_const(:H5T_STD_U64BE_g) + _H5T.IEEE_F32LE[] = read_const(:H5T_IEEE_F32LE_g) + _H5T.IEEE_F32BE[] = read_const(:H5T_IEEE_F32BE_g) + _H5T.IEEE_F64LE[] = read_const(:H5T_IEEE_F64LE_g) + _H5T.IEEE_F64BE[] = read_const(:H5T_IEEE_F64BE_g) + _H5T.C_S1[] = read_const(:H5T_C_S1_g) + _H5T.STD_REF_OBJ[] = read_const(:H5T_STD_REF_OBJ_g) + _H5T.STD_REF_DSETREG[] = read_const(:H5T_STD_REF_DSETREG_g) + _H5T.NATIVE_B8[] = read_const(:H5T_NATIVE_B8_g) + _H5T.NATIVE_INT8[] = read_const(:H5T_NATIVE_INT8_g) + _H5T.NATIVE_UINT8[] = read_const(:H5T_NATIVE_UINT8_g) + _H5T.NATIVE_INT16[] = read_const(:H5T_NATIVE_INT16_g) + _H5T.NATIVE_UINT16[] = read_const(:H5T_NATIVE_UINT16_g) + _H5T.NATIVE_INT32[] = read_const(:H5T_NATIVE_INT32_g) + _H5T.NATIVE_UINT32[] = read_const(:H5T_NATIVE_UINT32_g) + _H5T.NATIVE_INT64[] = read_const(:H5T_NATIVE_INT64_g) + _H5T.NATIVE_UINT64[] = read_const(:H5T_NATIVE_UINT64_g) + _H5T.NATIVE_FLOAT[] = read_const(:H5T_NATIVE_FLOAT_g) + _H5T.NATIVE_DOUBLE[] = read_const(:H5T_NATIVE_DOUBLE_g) + + nothing +end +__init_globals__() diff --git a/src/blosc_filter.jl b/src/blosc_filter.jl index 624c381c1..16993776a 100644 --- a/src/blosc_filter.jl +++ b/src/blosc_filter.jl @@ -28,7 +28,7 @@ function blosc_set_local(dcpl::hid_t, htype::hid_t, space::hid_t) end htypesize = h5t_get_size(htype) - if h5t_get_class(htype) == H5T_ARRAY + if h5t_get_class(htype) == H5T.ARRAY hsuper = h5t_get_super(htype) basetypesize = h5t_get_size(hsuper) h5t_close(hsuper) @@ -61,7 +61,7 @@ function blosc_filter(flags::Cuint, cd_nelmts::Csize_t, doshuffle = cd_nelmts >= 6 ? unsafe_load(cd_values, 6) != 0 : true # to do: set compressor based on compcode in unsafe_load(cd_values, 7)? - if (flags & H5Z_FLAG_REVERSE) == 0 # compressing + if (flags & H5Z.FLAG_REVERSE) == 0 # compressing # Allocate an output buffer exactly as long as the input data; if # the result is larger, we simply return 0. The filter is flagged # as optional, so HDF5 marks the chunk as uncompressed and proceeds. @@ -100,7 +100,7 @@ function register_blosc() c_blosc_filter = @cfunction(blosc_filter, Csize_t, (Cuint, Csize_t, Ptr{Cuint}, Csize_t, Ptr{Csize_t}, Ptr{Ptr{Cvoid}})) - h5z_register(H5Z_class_t(H5Z_CLASS_T_VERS, FILTER_BLOSC, 1, 1, pointer(blosc_name), C_NULL, c_blosc_set_local, c_blosc_filter)) + h5z_register(H5Z_class_t(H5Z.CLASS_T_VERS, FILTER_BLOSC, 1, 1, pointer(blosc_name), C_NULL, c_blosc_set_local, c_blosc_filter)) return nothing end @@ -109,7 +109,7 @@ const set_blosc_values = Cuint[0,0,0,0,5,1,0] function h5p_set_blosc(p::Properties, level::Integer=5) 0 <= level <= 9 || throw(ArgumentError("blosc compression $level not in [0,9]")) set_blosc_values[5] = level - h5p_set_filter(p.id, FILTER_BLOSC, H5Z_FLAG_OPTIONAL, length(set_blosc_values), set_blosc_values) + h5p_set_filter(p.id, FILTER_BLOSC, H5Z.FLAG_OPTIONAL, length(set_blosc_values), set_blosc_values) return nothing end diff --git a/src/deprecated.jl b/src/deprecated.jl index 2489cd2d8..fb684d9f0 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -188,7 +188,7 @@ end ### Changed in PR#689 # - switch from H5Rdereference1 to H5Rdereference2 -@deprecate h5r_dereference(obj_id, ref_type, ref) h5r_dereference(obj_id, H5P_DEFAULT, ref_type, ref) false +@deprecate h5r_dereference(obj_id, ref_type, ref) h5r_dereference(obj_id, H5P.DEFAULT, ref_type, ref) false ### Changed in PR#690, PR#695 # - rename exported bindings @@ -222,3 +222,11 @@ import Base: names ### Changed in PR#694 @deprecate has(parent::Union{File,Group,Dataset}, path::AbstractString) Base.haskey(parent, path) @deprecate exists(parent::Union{File,Group,Dataset,Datatype,Attributes}, path::AbstractString) Base.haskey(parent, path) false + +for pseudomod in (H5, H5D, H5E, H5F, H5FD, H5I, H5L, H5O, H5P, H5R, H5S, H5T, H5Z) + modsym = nameof(pseudomod) + for C in propertynames(pseudomod) + constname = Symbol(modsym, :_, C) + @eval @deprecate_binding($constname, $modsym.$C, false) + end +end diff --git a/src/macros.jl b/src/macros.jl new file mode 100644 index 000000000..3c2659095 --- /dev/null +++ b/src/macros.jl @@ -0,0 +1,100 @@ +import Base.Meta: isexpr + +""" +```julia +@defconstants H5PREFIX begin + STATIC_CONSTANT::hid_t = 1 + DYNAMIC_CONSTANT::hid_t + ... +end +``` + +Defines a collection of "constants" (`STATIC_CONSTANT`, `DYANMIC_CONSTANT`, ...) within +the namespace `H5PREFIX` wherein the distinction between runtime and static values is +abstracted away from the user's view. + +For truly constant values such as `STATIC_CONSTANT`, the syntax is given as in the first +line above where the value is assigned to a name with a type assertion. The value may also +be an expression which is evaluated the current module context. Note that the value +evaluation occurs before the constants are assigned, so the expression cannot refer to +any previous constants within the same block; e.g. the following will not work: +```julia +julia> @defconstants BITMASK begin + BIT_ONE::UInt = 0x1 + BIT_TWO::UInt = 0x2 + BIT_BOTH::Uint = BIT_ONE | BIT_TWO + end +ERROR: LoadError: UndefVarError: BIT_ONE not defined +... +``` + +The second permitted syntax is for runtime constants which require initialization. +In this form, only the name and value type are given, and an appropriately-typed, +uninitialized `Ref` container is assigned to the constant binding. These values must be +assigned at runtime from an `__init__()` function: +```julia +function __init__() + H5PREFIX.DYANMIC_CONSTANT = my_runtime_value() +end +``` +""" +macro defconstants(prefix::Symbol, expr::Expr) + isexpr(expr, :block) || error("Expected block expression") + stmts = expr.args + + symbols = Vector{Symbol}(undef, 0) + imports = Vector{Union{Symbol,Expr}}(undef, 0) + defbody = Vector{Expr}(undef, 0) + getbody = Vector{Expr}(undef, 0) + + innermod = Symbol(:_, prefix) + for line in stmts + line isa LineNumberNode && continue + + isruntime = isexpr(line, :(=)) ? false : + isexpr(line, :(::)) ? true : + error("Unexpected statement: ", repr(line)) + + # Get the name and type pair + nametype = isruntime ? line : line.args[1] + isexpr(nametype, :(::)) || error("Expected `name::type`, got ", repr(nametype)) + name = nametype.args[1]::Symbol + type = nametype.args[2] + # Save type for later use + push!(imports, type) + + value = isruntime ? esc(:(Ref{$type}())) : + QuoteNode(Core.eval(__module__, line.args[2])) + fullname = esc(:($innermod.$name)) + getexpr = isruntime ? :($(fullname)[]) : fullname + + push!(symbols, name) + push!(defbody, :(const $(esc(name)) = $value)) + push!(getbody, :(sym === $(QuoteNode(name)) && return $getexpr)) + end + + # Build expressions to import all necessary types from the parent module. + push!(imports, :Ref) # explicitly add `Ref` + imports = Expr[Expr(:(.), :(.), :(.), sym) for sym in unique!(imports)] + # Push the imports into the definition body + pushfirst!(defbody, Expr(:import, imports...)) + + prefix = esc(prefix) + innermod = esc(innermod) + block = quote + Base.@__doc__(struct $prefix end) + baremodule $innermod + $(defbody...) + end + function Base.propertynames(::Type{$prefix}) + return $((symbols...,)) + end + function Base.getproperty(::Type{$prefix}, sym::Symbol) + $(getbody...) + return getfield($prefix, sym) + end + end + Base.remove_linenums!(block) + block.head = :toplevel + return block +end diff --git a/test/compound.jl b/test/compound.jl index 4cac14cd0..516a6a1e6 100644 --- a/test/compound.jl +++ b/test/compound.jl @@ -30,18 +30,18 @@ function unsafe_convert(::Type{foo_hdf5}, x::foo) end function datatype(::Type{foo_hdf5}) - dtype = HDF5.h5t_create(HDF5.H5T_COMPOUND, sizeof(foo_hdf5)) + dtype = HDF5.h5t_create(HDF5.H5T.COMPOUND, sizeof(foo_hdf5)) HDF5.h5t_insert(dtype, "a", fieldoffset(foo_hdf5, 1), datatype(Float64)) - vlenstr_dtype = HDF5.h5t_copy(HDF5.H5T_C_S1) - HDF5.h5t_set_size(vlenstr_dtype, HDF5.H5T_VARIABLE) - HDF5.h5t_set_cset(vlenstr_dtype, HDF5.H5T_CSET_UTF8) + vlenstr_dtype = HDF5.h5t_copy(HDF5.H5T.C_S1) + HDF5.h5t_set_size(vlenstr_dtype, HDF5.H5T.VARIABLE) + HDF5.h5t_set_cset(vlenstr_dtype, HDF5.H5T.CSET_UTF8) HDF5.h5t_insert(dtype, "b", fieldoffset(foo_hdf5, 2), vlenstr_dtype) - fixedstr_dtype = HDF5.h5t_copy(HDF5.H5T_C_S1) + fixedstr_dtype = HDF5.h5t_copy(HDF5.H5T.C_S1) HDF5.h5t_set_size(fixedstr_dtype, 20 * sizeof(UInt8)) - HDF5.h5t_set_cset(fixedstr_dtype, HDF5.H5T_CSET_UTF8) - HDF5.h5t_set_strpad(fixedstr_dtype, HDF5.H5T_STR_NULLPAD) + HDF5.h5t_set_cset(fixedstr_dtype, HDF5.H5T.CSET_UTF8) + HDF5.h5t_set_strpad(fixedstr_dtype, HDF5.H5T.STR_NULLPAD) HDF5.h5t_insert(dtype, "c", fieldoffset(foo_hdf5, 3), fixedstr_dtype) hsz = HDF5.hsize_t[3,3] @@ -63,11 +63,11 @@ struct bar_hdf5 end function datatype(::Type{bar_hdf5}) - dtype = HDF5.h5t_create(HDF5.H5T_COMPOUND, sizeof(bar_hdf5)) + dtype = HDF5.h5t_create(HDF5.H5T.COMPOUND, sizeof(bar_hdf5)) - fixedstr_dtype = HDF5.h5t_copy(HDF5.H5T_C_S1) + fixedstr_dtype = HDF5.h5t_copy(HDF5.H5T.C_S1) HDF5.h5t_set_size(fixedstr_dtype, 20 * sizeof(UInt8)) - HDF5.h5t_set_cset(fixedstr_dtype, HDF5.H5T_CSET_UTF8) + HDF5.h5t_set_cset(fixedstr_dtype, HDF5.H5T.CSET_UTF8) hsz = HDF5.hsize_t[2] array_dtype = HDF5.h5t_array_create(fixedstr_dtype, 1, hsz) diff --git a/test/custom.jl b/test/custom.jl index f260a5405..f48cd72c2 100644 --- a/test/custom.jl +++ b/test/custom.jl @@ -8,7 +8,7 @@ struct Simple end function datatype(::Type{Simple}) - dtype = HDF5.h5t_create(HDF5.H5T_COMPOUND, sizeof(Simple)) + dtype = HDF5.h5t_create(H5T.COMPOUND, sizeof(Simple)) HDF5.h5t_insert(dtype, "a", fieldoffset(Simple, 1), datatype(Float64)) HDF5.h5t_insert(dtype, "b", fieldoffset(Simple, 2), datatype(Int)) HDF5.Datatype(dtype) diff --git a/test/extend_test.jl b/test/extend_test.jl index 6bac00864..97c75d42a 100644 --- a/test/extend_test.jl +++ b/test/extend_test.jl @@ -45,11 +45,11 @@ b = d_create(fid, "b", Int, ((1000,), (-1,)), chunk=(100,)) #-1 is equivalent to b[1:200] = ones(200) dims, max_dims = HDF5.get_dims(b) @test dims == (UInt64(1000),) -@test max_dims == (HDF5.H5S_UNLIMITED,) +@test max_dims == (H5S.UNLIMITED,) set_dims!(b, (10000,)) dims, max_dims = HDF5.get_dims(b) @test dims == (UInt64(10000),) -@test max_dims == (HDF5.H5S_UNLIMITED,) +@test max_dims == (H5S.UNLIMITED,) #println("b is size current $(map(int,HDF5.get_dims(b)[1])) max $(map(int,HDF5.get_dims(b)[2]))") # b[:] = [1:10000] # gave error no method lastindex(HDF5.Dataset{PlainHDF5File},), # so I defined lastindex(dset::HDF5.Dataset) = length(dset), and exported lastindex @@ -71,7 +71,7 @@ dims, max_dims = HDF5.get_dims(d_again) b_again = fid["b"] dims, max_dims = HDF5.get_dims(b_again) @test dims == (UInt64(10000),) -@test max_dims == (HDF5.H5S_UNLIMITED,) +@test max_dims == (H5S.UNLIMITED,) #println("b is size current $(map(int,HDF5.get_dims(b)[1])) max $(map(int,HDF5.get_dims(b)[2]))") close(fid) diff --git a/test/gc.jl b/test/gc.jl index 67ca24ddb..9a75c1470 100644 --- a/test/gc.jl +++ b/test/gc.jl @@ -27,7 +27,7 @@ GC.enable(false) fn = tempname() for i = 1:10 file = h5open(fn, "w") - memtype_id = HDF5.h5t_create(HDF5.H5T_COMPOUND, 2*sizeof(Float64)) + memtype_id = HDF5.h5t_create(HDF5.H5T.COMPOUND, 2*sizeof(Float64)) HDF5.h5t_insert(memtype_id, "real", 0, HDF5.hdf5_type_id(Float64)) HDF5.h5t_insert(memtype_id, "imag", sizeof(Float64), HDF5.hdf5_type_id(Float64)) dt = HDF5.Datatype(memtype_id) @@ -64,7 +64,7 @@ for i = 1:10 end GC.enable(true) -let plist = p_create(HDF5.H5P_FILE_ACCESS) # related to issue #620 +let plist = p_create(HDF5.H5P.FILE_ACCESS) # related to issue #620 HDF5.h5p_close(plist) @test_nowarn finalize(plist) end diff --git a/test/mmap.jl b/test/mmap.jl index e19e8d216..d323c65fe 100644 --- a/test/mmap.jl +++ b/test/mmap.jl @@ -12,7 +12,7 @@ f = h5open(fn, "w") # datasets) and the other with explicit early allocation. hdf5_A = d_create(f, "A", datatype(Int64), dataspace(3,3)) hdf5_B = d_create(f, "B", datatype(Float64), dataspace(3,3); - alloc_time = HDF5.H5D_ALLOC_TIME_EARLY) + alloc_time = H5D.ALLOC_TIME_EARLY) # The late case cannot be mapped yet. @test_throws ErrorException("Error getting offset") readmmap(f["A"]) # Then write and fill dataset A, making it mappable. B was filled with 0.0 at diff --git a/test/mpio.jl b/test/mpio.jl index 3e38b5e3d..45b8967d4 100644 --- a/test/mpio.jl +++ b/test/mpio.jl @@ -14,7 +14,7 @@ myrank = MPI.Comm_rank(comm) @test HDF5.has_parallel() -let fileprop = p_create(HDF5.H5P_FILE_ACCESS) +let fileprop = p_create(HDF5.H5P.FILE_ACCESS) HDF5.h5p_set_fapl_mpio(fileprop, comm, info) h5comm, h5info = HDF5.h5p_get_fapl_mpio(fileprop) diff --git a/test/plain.jl b/test/plain.jl index 1b621f8f8..9ed295dc6 100644 --- a/test/plain.jl +++ b/test/plain.jl @@ -41,13 +41,13 @@ write(f, "salut", salut) write(f, "ucode", ucode) # Manually write a variable-length string (issue #187) let - dtype = HDF5.Datatype(HDF5.h5t_copy(HDF5.H5T_C_S1)) - HDF5.h5t_set_size(dtype.id, HDF5.H5T_VARIABLE) + dtype = HDF5.Datatype(HDF5.h5t_copy(H5T.C_S1)) + HDF5.h5t_set_size(dtype.id, H5T.VARIABLE) HDF5.h5t_set_cset(dtype.id, HDF5.cset(typeof(salut))) dspace = HDF5.dataspace(salut) dset = HDF5.d_create(f, "salut-vlen", dtype, dspace) GC.@preserve salut begin - HDF5.h5d_write(dset, dtype, HDF5.H5S_ALL, HDF5.H5S_ALL, HDF5.H5P_DEFAULT, [pointer(salut)]) + HDF5.h5d_write(dset, dtype, H5S.ALL, H5S.ALL, H5P.DEFAULT, [pointer(salut)]) end end # Arrays of strings @@ -355,14 +355,14 @@ close(fd) rm(fn) # File creation and access property lists -cpl = p_create(HDF5.H5P_FILE_CREATE) +cpl = p_create(H5P.FILE_CREATE) cpl[:userblock] = 1024 -apl = p_create(HDF5.H5P_FILE_ACCESS) -apl[:libver_bounds] = (HDF5.H5F_LIBVER_EARLIEST, HDF5.H5F_LIBVER_LATEST) +apl = p_create(H5P.FILE_ACCESS) +apl[:libver_bounds] = (H5F.LIBVER_EARLIEST, H5F.LIBVER_LATEST) h5open(fn, false, true, true, true, false, cpl, apl) do fid write(fid, "intarray", [1, 2, 3]) end -h5open(fn, "r", libver_bounds=(HDF5.H5F_LIBVER_EARLIEST, HDF5.H5F_LIBVER_LATEST)) do fid +h5open(fn, "r", libver_bounds=(H5F.LIBVER_EARLIEST, H5F.LIBVER_LATEST)) do fid intarray = read(fid, "intarray") @test intarray == [1, 2, 3] end @@ -626,8 +626,8 @@ end # empty and 0-size arrays fn = tempname() hfile = h5open(fn, "w") -dtype_varstring = HDF5.Datatype(HDF5.h5t_copy(HDF5.H5T_C_S1)) -HDF5.h5t_set_size(dtype_varstring, HDF5.H5T_VARIABLE) +dtype_varstring = HDF5.Datatype(HDF5.h5t_copy(H5T.C_S1)) +HDF5.h5t_set_size(dtype_varstring, H5T.VARIABLE) write(hfile, "uint8_array", UInt8[(1:8)...]) write(hfile, "bool_scalar", true) @@ -637,7 +637,7 @@ varstring = "var" write(hfile, "fixed_string", fixstring) vardset = d_create(hfile, "variable_string", dtype_varstring, dataspace(varstring)) GC.@preserve varstring begin - HDF5.h5d_write(vardset, dtype_varstring, HDF5.H5S_ALL, HDF5.H5S_ALL, HDF5.H5P_DEFAULT, [pointer(varstring)]) + HDF5.h5d_write(vardset, dtype_varstring, H5S.ALL, H5S.ALL, H5P.DEFAULT, [pointer(varstring)]) end flush(hfile) close(dtype_varstring) @@ -699,10 +699,10 @@ dset = d_create(group, "dset", datatype(Int), dataspace((1,))) meta = a_create(dset, "meta", datatype(Bool), dataspace((1,))) @test sprint(show, meta) == "HDF5 attribute: meta" -prop = p_create(HDF5.H5P_DATASET_CREATE) +prop = p_create(H5P.DATASET_CREATE) @test sprint(show, prop) == "HDF5 property: dataset create class" -dtype = HDF5.Datatype(HDF5.h5t_copy(HDF5.H5T_IEEE_F64LE)) +dtype = HDF5.Datatype(HDF5.h5t_copy(H5T.IEEE_F64LE)) @test sprint(show, dtype) == "HDF5 datatype: H5T_IEEE_F64LE" dspace = dataspace((1,)) @@ -839,7 +839,7 @@ hfile[GenericString("test")] = 17.2 @test_nowarn a_delete(dset1, GenericString("meta1")) # transient types -memtype_id = HDF5.h5t_copy(HDF5.H5T_NATIVE_DOUBLE) +memtype_id = HDF5.h5t_copy(H5T.NATIVE_DOUBLE) dt = HDF5.Datatype(memtype_id) @test !HDF5.h5t_committed(dt) t_commit(hfile, GenericString("dt"), dt) diff --git a/test/properties.jl b/test/properties.jl index 7c0c83b24..147a38e38 100644 --- a/test/properties.jl +++ b/test/properties.jl @@ -7,12 +7,12 @@ fn = tempname() h5open(fn, "w"; userblock = 1024, alignment = (0, sizeof(Int)), - libver_bounds = (HDF5.H5F_LIBVER_EARLIEST, HDF5.H5F_LIBVER_LATEST), + libver_bounds = (H5F.LIBVER_EARLIEST, H5F.LIBVER_LATEST), ) do hfile # generic g = g_create(hfile, "group") d = d_create(g, "dataset", datatype(Int), dataspace((500,500)), - alloc_time = HDF5.H5D_ALLOC_TIME_EARLY, + alloc_time = H5D.ALLOC_TIME_EARLY, chunk = (5, 10), track_times = false) attrs(d)["metadata"] = "test" @@ -38,27 +38,27 @@ h5open(fn, "w"; @test fapl[:alignment] == (0, sizeof(Int)) # value is H5FD_SEC2, but "constant" is runtime value not loadable by _read_const() - @test HDF5.h5i_get_type(fapl[:driver]) == HDF5.H5I_VFL + @test HDF5.h5i_get_type(fapl[:driver]) == H5I.VFL # Docs say h5p_get_driver_info() doesn't error, but it does print an error message... # https://portal.hdfgroup.org/display/HDF5/H5P_GET_DRIVER_INFO HDF5.hiding_errors() do @test fapl[:driver_info] == C_NULL end - @test fapl[:fclose_degree] == HDF5.H5F_CLOSE_STRONG - @test fapl[:libver_bounds] == (HDF5.H5F_LIBVER_EARLIEST, HDF5.H5F_LIBVER_LATEST) + @test fapl[:fclose_degree] == H5F.CLOSE_STRONG + @test fapl[:libver_bounds] == (H5F.LIBVER_EARLIEST, H5F.LIBVER_LATEST) @test gcpl[:local_heap_size_hint] == 0 @test gcpl[:track_times] == true - @test HDF5.UTF8_LINK_PROPERTIES[][:char_encoding] == HDF5.H5T_CSET_UTF8 + @test HDF5.UTF8_LINK_PROPERTIES[][:char_encoding] == H5T.CSET_UTF8 @test HDF5.UTF8_LINK_PROPERTIES[][:create_intermediate_group] == 1 - @test dcpl[:alloc_time] == HDF5.H5D_ALLOC_TIME_EARLY + @test dcpl[:alloc_time] == H5D.ALLOC_TIME_EARLY @test dcpl[:chunk] == (5, 10) - @test dcpl[:layout] == HDF5.H5D_CHUNKED + @test dcpl[:layout] == H5D.CHUNKED @test dcpl[:track_times] == false - @test acpl[:char_encoding] == HDF5.H5T_CSET_UTF8 + @test acpl[:char_encoding] == H5T.CSET_UTF8 nothing end diff --git a/test/swmr.jl b/test/swmr.jl index f13497e03..9fc6d3839 100644 --- a/test/swmr.jl +++ b/test/swmr.jl @@ -96,7 +96,7 @@ end @testset "create by libver, then start_swmr_write" begin #test this h5open method with keyword arg - h5open(fname, "w", libver_bounds=(HDF5.H5F_LIBVER_LATEST, HDF5.H5F_LIBVER_LATEST), swmr=false) do h5 + h5open(fname, "w", libver_bounds=(H5F.LIBVER_LATEST, H5F.LIBVER_LATEST), swmr=false) do h5 prep_h5_file(h5) HDF5.start_swmr_write(h5) # after creating datasets remote_test(h5) From af8c6a7a9c56a84e0920b40895722836977a621c Mon Sep 17 00:00:00 2001 From: Justin Willmert Date: Thu, 8 Oct 2020 09:18:42 -0500 Subject: [PATCH 2/8] Overload setproperty! as well for easier initialization --- src/api_types.jl | 102 +++++++++++++++++++++++------------------------ src/macros.jl | 20 ++++++++++ 2 files changed, 71 insertions(+), 51 deletions(-) diff --git a/src/api_types.jl b/src/api_types.jl index 930ed7fc2..548b5c4e3 100644 --- a/src/api_types.jl +++ b/src/api_types.jl @@ -361,57 +361,57 @@ function __init_globals__() return unsafe_load(cglobal(symptr, hid_t)) end - _H5P.OBJECT_CREATE[] = read_const(:H5P_CLS_OBJECT_CREATE_ID_g) - _H5P.FILE_CREATE[] = read_const(:H5P_CLS_FILE_CREATE_ID_g) - _H5P.FILE_ACCESS[] = read_const(:H5P_CLS_FILE_ACCESS_ID_g) - _H5P.DATASET_CREATE[] = read_const(:H5P_CLS_DATASET_CREATE_ID_g) - _H5P.DATASET_ACCESS[] = read_const(:H5P_CLS_DATASET_ACCESS_ID_g) - _H5P.DATASET_XFER[] = read_const(:H5P_CLS_DATASET_XFER_ID_g) - _H5P.FILE_MOUNT[] = read_const(:H5P_CLS_FILE_MOUNT_ID_g) - _H5P.GROUP_CREATE[] = read_const(:H5P_CLS_GROUP_CREATE_ID_g) - _H5P.GROUP_ACCESS[] = read_const(:H5P_CLS_GROUP_ACCESS_ID_g) - _H5P.DATATYPE_CREATE[] = read_const(:H5P_CLS_DATATYPE_CREATE_ID_g) - _H5P.DATATYPE_ACCESS[] = read_const(:H5P_CLS_DATATYPE_ACCESS_ID_g) - _H5P.STRING_CREATE[] = read_const(:H5P_CLS_STRING_CREATE_ID_g) - _H5P.ATTRIBUTE_CREATE[] = read_const(:H5P_CLS_ATTRIBUTE_CREATE_ID_g) - _H5P.OBJECT_COPY[] = read_const(:H5P_CLS_OBJECT_COPY_ID_g) - _H5P.LINK_CREATE[] = read_const(:H5P_CLS_LINK_CREATE_ID_g) - _H5P.LINK_ACCESS[] = read_const(:H5P_CLS_LINK_ACCESS_ID_g) - - _H5T.STD_I8LE[] = read_const(:H5T_STD_I8LE_g) - _H5T.STD_I8BE[] = read_const(:H5T_STD_I8BE_g) - _H5T.STD_U8LE[] = read_const(:H5T_STD_U8LE_g) - _H5T.STD_U8BE[] = read_const(:H5T_STD_U8BE_g) - _H5T.STD_I16LE[] = read_const(:H5T_STD_I16LE_g) - _H5T.STD_I16BE[] = read_const(:H5T_STD_I16BE_g) - _H5T.STD_U16LE[] = read_const(:H5T_STD_U16LE_g) - _H5T.STD_U16BE[] = read_const(:H5T_STD_U16BE_g) - _H5T.STD_I32LE[] = read_const(:H5T_STD_I32LE_g) - _H5T.STD_I32BE[] = read_const(:H5T_STD_I32BE_g) - _H5T.STD_U32LE[] = read_const(:H5T_STD_U32LE_g) - _H5T.STD_U32BE[] = read_const(:H5T_STD_U32BE_g) - _H5T.STD_I64LE[] = read_const(:H5T_STD_I64LE_g) - _H5T.STD_I64BE[] = read_const(:H5T_STD_I64BE_g) - _H5T.STD_U64LE[] = read_const(:H5T_STD_U64LE_g) - _H5T.STD_U64BE[] = read_const(:H5T_STD_U64BE_g) - _H5T.IEEE_F32LE[] = read_const(:H5T_IEEE_F32LE_g) - _H5T.IEEE_F32BE[] = read_const(:H5T_IEEE_F32BE_g) - _H5T.IEEE_F64LE[] = read_const(:H5T_IEEE_F64LE_g) - _H5T.IEEE_F64BE[] = read_const(:H5T_IEEE_F64BE_g) - _H5T.C_S1[] = read_const(:H5T_C_S1_g) - _H5T.STD_REF_OBJ[] = read_const(:H5T_STD_REF_OBJ_g) - _H5T.STD_REF_DSETREG[] = read_const(:H5T_STD_REF_DSETREG_g) - _H5T.NATIVE_B8[] = read_const(:H5T_NATIVE_B8_g) - _H5T.NATIVE_INT8[] = read_const(:H5T_NATIVE_INT8_g) - _H5T.NATIVE_UINT8[] = read_const(:H5T_NATIVE_UINT8_g) - _H5T.NATIVE_INT16[] = read_const(:H5T_NATIVE_INT16_g) - _H5T.NATIVE_UINT16[] = read_const(:H5T_NATIVE_UINT16_g) - _H5T.NATIVE_INT32[] = read_const(:H5T_NATIVE_INT32_g) - _H5T.NATIVE_UINT32[] = read_const(:H5T_NATIVE_UINT32_g) - _H5T.NATIVE_INT64[] = read_const(:H5T_NATIVE_INT64_g) - _H5T.NATIVE_UINT64[] = read_const(:H5T_NATIVE_UINT64_g) - _H5T.NATIVE_FLOAT[] = read_const(:H5T_NATIVE_FLOAT_g) - _H5T.NATIVE_DOUBLE[] = read_const(:H5T_NATIVE_DOUBLE_g) + H5P.OBJECT_CREATE = read_const(:H5P_CLS_OBJECT_CREATE_ID_g) + H5P.FILE_CREATE = read_const(:H5P_CLS_FILE_CREATE_ID_g) + H5P.FILE_ACCESS = read_const(:H5P_CLS_FILE_ACCESS_ID_g) + H5P.DATASET_CREATE = read_const(:H5P_CLS_DATASET_CREATE_ID_g) + H5P.DATASET_ACCESS = read_const(:H5P_CLS_DATASET_ACCESS_ID_g) + H5P.DATASET_XFER = read_const(:H5P_CLS_DATASET_XFER_ID_g) + H5P.FILE_MOUNT = read_const(:H5P_CLS_FILE_MOUNT_ID_g) + H5P.GROUP_CREATE = read_const(:H5P_CLS_GROUP_CREATE_ID_g) + H5P.GROUP_ACCESS = read_const(:H5P_CLS_GROUP_ACCESS_ID_g) + H5P.DATATYPE_CREATE = read_const(:H5P_CLS_DATATYPE_CREATE_ID_g) + H5P.DATATYPE_ACCESS = read_const(:H5P_CLS_DATATYPE_ACCESS_ID_g) + H5P.STRING_CREATE = read_const(:H5P_CLS_STRING_CREATE_ID_g) + H5P.ATTRIBUTE_CREATE = read_const(:H5P_CLS_ATTRIBUTE_CREATE_ID_g) + H5P.OBJECT_COPY = read_const(:H5P_CLS_OBJECT_COPY_ID_g) + H5P.LINK_CREATE = read_const(:H5P_CLS_LINK_CREATE_ID_g) + H5P.LINK_ACCESS = read_const(:H5P_CLS_LINK_ACCESS_ID_g) + + H5T.STD_I8LE = read_const(:H5T_STD_I8LE_g) + H5T.STD_I8BE = read_const(:H5T_STD_I8BE_g) + H5T.STD_U8LE = read_const(:H5T_STD_U8LE_g) + H5T.STD_U8BE = read_const(:H5T_STD_U8BE_g) + H5T.STD_I16LE = read_const(:H5T_STD_I16LE_g) + H5T.STD_I16BE = read_const(:H5T_STD_I16BE_g) + H5T.STD_U16LE = read_const(:H5T_STD_U16LE_g) + H5T.STD_U16BE = read_const(:H5T_STD_U16BE_g) + H5T.STD_I32LE = read_const(:H5T_STD_I32LE_g) + H5T.STD_I32BE = read_const(:H5T_STD_I32BE_g) + H5T.STD_U32LE = read_const(:H5T_STD_U32LE_g) + H5T.STD_U32BE = read_const(:H5T_STD_U32BE_g) + H5T.STD_I64LE = read_const(:H5T_STD_I64LE_g) + H5T.STD_I64BE = read_const(:H5T_STD_I64BE_g) + H5T.STD_U64LE = read_const(:H5T_STD_U64LE_g) + H5T.STD_U64BE = read_const(:H5T_STD_U64BE_g) + H5T.IEEE_F32LE = read_const(:H5T_IEEE_F32LE_g) + H5T.IEEE_F32BE = read_const(:H5T_IEEE_F32BE_g) + H5T.IEEE_F64LE = read_const(:H5T_IEEE_F64LE_g) + H5T.IEEE_F64BE = read_const(:H5T_IEEE_F64BE_g) + H5T.C_S1 = read_const(:H5T_C_S1_g) + H5T.STD_REF_OBJ = read_const(:H5T_STD_REF_OBJ_g) + H5T.STD_REF_DSETREG = read_const(:H5T_STD_REF_DSETREG_g) + H5T.NATIVE_B8 = read_const(:H5T_NATIVE_B8_g) + H5T.NATIVE_INT8 = read_const(:H5T_NATIVE_INT8_g) + H5T.NATIVE_UINT8 = read_const(:H5T_NATIVE_UINT8_g) + H5T.NATIVE_INT16 = read_const(:H5T_NATIVE_INT16_g) + H5T.NATIVE_UINT16 = read_const(:H5T_NATIVE_UINT16_g) + H5T.NATIVE_INT32 = read_const(:H5T_NATIVE_INT32_g) + H5T.NATIVE_UINT32 = read_const(:H5T_NATIVE_UINT32_g) + H5T.NATIVE_INT64 = read_const(:H5T_NATIVE_INT64_g) + H5T.NATIVE_UINT64 = read_const(:H5T_NATIVE_UINT64_g) + H5T.NATIVE_FLOAT = read_const(:H5T_NATIVE_FLOAT_g) + H5T.NATIVE_DOUBLE = read_const(:H5T_NATIVE_DOUBLE_g) nothing end diff --git a/src/macros.jl b/src/macros.jl index 3c2659095..baf248022 100644 --- a/src/macros.jl +++ b/src/macros.jl @@ -46,6 +46,7 @@ macro defconstants(prefix::Symbol, expr::Expr) imports = Vector{Union{Symbol,Expr}}(undef, 0) defbody = Vector{Expr}(undef, 0) getbody = Vector{Expr}(undef, 0) + setbody = Vector{Expr}(undef, 0) innermod = Symbol(:_, prefix) for line in stmts @@ -71,6 +72,15 @@ macro defconstants(prefix::Symbol, expr::Expr) push!(symbols, name) push!(defbody, :(const $(esc(name)) = $value)) push!(getbody, :(sym === $(QuoteNode(name)) && return $getexpr)) + if isruntime + setexpr = quote + sym === $(QuoteNode(name)) && begin + $fullname[] = value + return value + end + end + append!(setbody, Base.remove_linenums!(setexpr).args) + end end # Build expressions to import all necessary types from the parent module. @@ -94,6 +104,16 @@ macro defconstants(prefix::Symbol, expr::Expr) return getfield($prefix, sym) end end + if !isempty(setbody) + setfn = quote + function Base.setproperty!(::Type{$prefix}, sym::Symbol, value) + $(setbody...) + return setfield!($prefix, sym, value) + end + end + append!(block.args, Base.remove_linenums!(setfn).args) + end + Base.remove_linenums!(block) block.head = :toplevel return block From 31488046bc289d959a59414f9b4d32f67a58623d Mon Sep 17 00:00:00 2001 From: Justin Willmert Date: Thu, 8 Oct 2020 11:06:07 -0500 Subject: [PATCH 3/8] Use new runtime ability to define Float16 type --- src/HDF5.jl | 14 ++++++++++++-- src/api_types.jl | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/HDF5.jl b/src/HDF5.jl index 94d97ccde..2ac8a3475 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -79,13 +79,14 @@ hdf5_type_id(::Type{Int32}) = H5T.NATIVE_INT32 hdf5_type_id(::Type{UInt32}) = H5T.NATIVE_UINT32 hdf5_type_id(::Type{Int64}) = H5T.NATIVE_INT64 hdf5_type_id(::Type{UInt64}) = H5T.NATIVE_UINT64 +hdf5_type_id(::Type{Float16}) = H5T.NATIVE_FLOAT16 hdf5_type_id(::Type{Float32}) = H5T.NATIVE_FLOAT hdf5_type_id(::Type{Float64}) = H5T.NATIVE_DOUBLE hdf5_type_id(::Type{Reference}) = H5T.STD_REF_OBJ hdf5_type_id(::Type{<:AbstractString}) = H5T.C_S1 -const BitsType = Union{Bool,Int8,UInt8,Int16,UInt16,Int32,UInt32,Int64,UInt64,Float32,Float64} +const BitsType = Union{Bool,Int8,UInt8,Int16,UInt16,Int32,UInt32,Int64,UInt64,Float16,Float32,Float64} const ScalarType = Union{BitsType,Reference} # It's not safe to use particular id codes because these can change, so we use characteristics of the type. @@ -105,7 +106,8 @@ function _hdf5_type_map(class_id, is_signed, native_size) throw(KeyError(class_id, is_signed, native_size)) end else - return native_size === Csize_t(4) ? Float32 : + return native_size === Csize_t(2) ? Float16 : + native_size === Csize_t(4) ? Float32 : native_size === Csize_t(8) ? Float64 : throw(KeyError(class_id, is_signed, native_size)) end @@ -1948,6 +1950,14 @@ function __init__() __init_globals__() register_blosc() + # Generate the Float16 datatype: + float16 = h5t_copy(H5T.NATIVE_FLOAT) + h5t_set_fields(float16, 15, 10, 5, 0, 10) + h5t_set_size(float16, 2) + h5t_set_ebias(float16, 15) + h5t_lock(float16) + H5T.NATIVE_FLOAT16 = float16 + # Turn off automatic error printing # h5e_set_auto(H5E.DEFAULT, C_NULL, C_NULL) diff --git a/src/api_types.jl b/src/api_types.jl index 548b5c4e3..5d006aca1 100644 --- a/src/api_types.jl +++ b/src/api_types.jl @@ -341,6 +341,7 @@ end NATIVE_UINT64::hid_t NATIVE_FLOAT::hid_t NATIVE_DOUBLE::hid_t + NATIVE_FLOAT16::hid_t end # Filter constants From de9460cfe519da98989a85774b3090d6c03ec0e6 Mon Sep 17 00:00:00 2001 From: jmert <2965436+jmert@users.noreply.github.com> Date: Tue, 13 Oct 2020 10:51:23 -0500 Subject: [PATCH 4/8] =?UTF-8?q?Address=20review=20=E2=80=94=20remove=20res?= =?UTF-8?q?idual=20forced=20initialization?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mustafa M --- src/api_types.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/api_types.jl b/src/api_types.jl index 5d006aca1..f9fa06230 100644 --- a/src/api_types.jl +++ b/src/api_types.jl @@ -416,4 +416,3 @@ function __init_globals__() nothing end -__init_globals__() From 64c2f2d7ceb32f843deed7a17da58798f4e5ede0 Mon Sep 17 00:00:00 2001 From: Justin Willmert Date: Tue, 13 Oct 2020 11:36:57 -0500 Subject: [PATCH 5/8] Use singleton instances, not just singleton type. This is crucial to avoid creating a huge number of invalidations in Base code. Using SnoopCompile (on Julia master ~v1.6): ```julia-repl julia> using SnoopCompileCore julia> invs = @snoopr using HDF5; julia> using SnoopCompile ``` **Before:** ```julia-repl julia> trees = invalidation_trees(invs) 1-element Vector{SnoopCompile.MethodInvalidations}: inserting getproperty(::Type{H5E}, sym::Symbol) in HDF5 invalidated: backedges: 1: superseding getproperty(x::Type, f::Symbol) in Base at Base.jl:28 with MethodInstance for getproperty(::DataType, ::Symbol) (1220 children) 3 mt_cache ``` **After:** ```julia-repl julia> trees = invalidation_trees(invs) SnoopCompile.MethodInvalidations[] ``` --- src/deprecated.jl | 2 +- src/macros.jl | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/deprecated.jl b/src/deprecated.jl index fb684d9f0..57624f945 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -224,7 +224,7 @@ import Base: names @deprecate exists(parent::Union{File,Group,Dataset,Datatype,Attributes}, path::AbstractString) Base.haskey(parent, path) false for pseudomod in (H5, H5D, H5E, H5F, H5FD, H5I, H5L, H5O, H5P, H5R, H5S, H5T, H5Z) - modsym = nameof(pseudomod) + modsym = nameof(typeof(pseudomod)) for C in propertynames(pseudomod) constname = Symbol(modsym, :_, C) @eval @deprecate_binding($constname, $modsym.$C, false) diff --git a/src/macros.jl b/src/macros.jl index baf248022..1535f07ae 100644 --- a/src/macros.jl +++ b/src/macros.jl @@ -89,26 +89,25 @@ macro defconstants(prefix::Symbol, expr::Expr) # Push the imports into the definition body pushfirst!(defbody, Expr(:import, imports...)) - prefix = esc(prefix) - innermod = esc(innermod) + eprefix = esc(prefix) + einnermod = esc(innermod) block = quote - Base.@__doc__(struct $prefix end) - baremodule $innermod + baremodule $einnermod + struct $eprefix end $(defbody...) end - function Base.propertynames(::Type{$prefix}) + const $eprefix = $einnermod.$prefix() + function Base.propertynames(::$einnermod.$prefix) return $((symbols...,)) end - function Base.getproperty(::Type{$prefix}, sym::Symbol) + function Base.getproperty(::$einnermod.$prefix, sym::Symbol) $(getbody...) - return getfield($prefix, sym) end end if !isempty(setbody) setfn = quote - function Base.setproperty!(::Type{$prefix}, sym::Symbol, value) + function Base.setproperty!(::$einnermod.$prefix, sym::Symbol, value) $(setbody...) - return setfield!($prefix, sym, value) end end append!(block.args, Base.remove_linenums!(setfn).args) From 54227dd48a20f2cc455cdbbe5b121fd71d089188 Mon Sep 17 00:00:00 2001 From: Justin Willmert Date: Tue, 13 Oct 2020 11:59:15 -0500 Subject: [PATCH 6/8] Ensure QuoteNodes contain requested data type --- src/api_types.jl | 4 ++-- src/macros.jl | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/api_types.jl b/src/api_types.jl index f9fa06230..1cdb81639 100644 --- a/src/api_types.jl +++ b/src/api_types.jl @@ -253,8 +253,8 @@ end # Dataspace constants @defconstants H5S begin # atomic data types - ALL::hid_t = 0 - UNLIMITED::hid_t = typemax(hsize_t) + ALL::hsize_t = 0 + UNLIMITED::hsize_t = typemax(hsize_t) # Types of dataspaces (C enum H5S_class_t) SCALAR::hid_t = hid_t(0) diff --git a/src/macros.jl b/src/macros.jl index 1535f07ae..a8f029724 100644 --- a/src/macros.jl +++ b/src/macros.jl @@ -64,8 +64,12 @@ macro defconstants(prefix::Symbol, expr::Expr) # Save type for later use push!(imports, type) - value = isruntime ? esc(:(Ref{$type}())) : - QuoteNode(Core.eval(__module__, line.args[2])) + if isruntime + value = esc(:(Ref{$type}())) + else + valexpr = :(convert($type, $(line.args[2]))) + value = QuoteNode(Core.eval(__module__, valexpr)) + end fullname = esc(:($innermod.$name)) getexpr = isruntime ? :($(fullname)[]) : fullname From 2055b283c37eee9b455fd06bd0693359a935dd74 Mon Sep 17 00:00:00 2001 From: Justin Willmert Date: Tue, 13 Oct 2020 17:23:12 -0500 Subject: [PATCH 7/8] Address review comments --- src/HDF5.jl | 8 -------- src/api_types.jl | 10 +++++++++- src/macros.jl | 8 +++++--- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/HDF5.jl b/src/HDF5.jl index 2ac8a3475..647da6ed5 100644 --- a/src/HDF5.jl +++ b/src/HDF5.jl @@ -1950,14 +1950,6 @@ function __init__() __init_globals__() register_blosc() - # Generate the Float16 datatype: - float16 = h5t_copy(H5T.NATIVE_FLOAT) - h5t_set_fields(float16, 15, 10, 5, 0, 10) - h5t_set_size(float16, 2) - h5t_set_ebias(float16, 15) - h5t_lock(float16) - H5T.NATIVE_FLOAT16 = float16 - # Turn off automatic error printing # h5e_set_auto(H5E.DEFAULT, C_NULL, C_NULL) diff --git a/src/api_types.jl b/src/api_types.jl index 1cdb81639..4b78179c2 100644 --- a/src/api_types.jl +++ b/src/api_types.jl @@ -253,7 +253,7 @@ end # Dataspace constants @defconstants H5S begin # atomic data types - ALL::hsize_t = 0 + ALL::hid_t = 0 UNLIMITED::hsize_t = typemax(hsize_t) # Types of dataspaces (C enum H5S_class_t) @@ -414,5 +414,13 @@ function __init_globals__() H5T.NATIVE_FLOAT = read_const(:H5T_NATIVE_FLOAT_g) H5T.NATIVE_DOUBLE = read_const(:H5T_NATIVE_DOUBLE_g) + # Defines a type corresponding to Julia's IEEE Float16 type. + float16 = h5t_copy(H5T.NATIVE_FLOAT) + h5t_set_fields(float16, 15, 10, 5, 0, 10) + h5t_set_size(float16, 2) + h5t_set_ebias(float16, 15) + h5t_lock(float16) + H5T.NATIVE_FLOAT16 = float16 + nothing end diff --git a/src/macros.jl b/src/macros.jl index a8f029724..8a0856aeb 100644 --- a/src/macros.jl +++ b/src/macros.jl @@ -54,13 +54,13 @@ macro defconstants(prefix::Symbol, expr::Expr) isruntime = isexpr(line, :(=)) ? false : isexpr(line, :(::)) ? true : - error("Unexpected statement: ", repr(line)) + error("Unexpected statement: ", line) # Get the name and type pair nametype = isruntime ? line : line.args[1] - isexpr(nametype, :(::)) || error("Expected `name::type`, got ", repr(nametype)) + isexpr(nametype, :(::)) || error("Expected `name::type`, got: ", nametype) name = nametype.args[1]::Symbol - type = nametype.args[2] + type = nametype.args[2]::Union{Symbol,Expr} # Save type for later use push!(imports, type) @@ -106,12 +106,14 @@ macro defconstants(prefix::Symbol, expr::Expr) end function Base.getproperty(::$einnermod.$prefix, sym::Symbol) $(getbody...) + error($(string(prefix) * " has no constant "), sym) end end if !isempty(setbody) setfn = quote function Base.setproperty!(::$einnermod.$prefix, sym::Symbol, value) $(setbody...) + error($(string(prefix) * "."), sym, " cannot be set") end end append!(block.args, Base.remove_linenums!(setfn).args) From 42060a00fcc63b5705510dedf366eea58f806465 Mon Sep 17 00:00:00 2001 From: Justin Willmert Date: Thu, 29 Oct 2020 11:02:34 -0500 Subject: [PATCH 8/8] Remove unnecessary import of Core.Ref --- src/macros.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/macros.jl b/src/macros.jl index 8a0856aeb..1663de3ac 100644 --- a/src/macros.jl +++ b/src/macros.jl @@ -88,7 +88,6 @@ macro defconstants(prefix::Symbol, expr::Expr) end # Build expressions to import all necessary types from the parent module. - push!(imports, :Ref) # explicitly add `Ref` imports = Expr[Expr(:(.), :(.), :(.), sym) for sym in unique!(imports)] # Push the imports into the definition body pushfirst!(defbody, Expr(:import, imports...))