4
4
n5py: The N5 file format implemented in Python
5
5
"""
6
6
7
-
8
7
from __future__ import annotations
9
8
10
9
from ._version import version as __version__
18
17
import os
19
18
import struct
20
19
import sys
21
- from typing import Any , Dict , Optional , cast
22
20
import warnings
21
+ from typing import Any , Dict , Optional , cast
23
22
24
23
import numpy as np
25
24
from numcodecs .abc import Codec
26
25
from numcodecs .compat import ndarray_copy
27
26
from numcodecs .registry import get_codec , register_codec
28
-
29
27
from zarr .meta import ZARR_FORMAT , json_dumps , json_loads
30
- from zarr .storage import FSStore , NestedDirectoryStore , _prog_ckey , _prog_number , normalize_storage_path
28
+ from zarr .storage import (
29
+ FSStore ,
30
+ NestedDirectoryStore ,
31
+ _prog_ckey ,
32
+ _prog_number ,
33
+ normalize_storage_path ,
34
+ )
31
35
from zarr .storage import array_meta_key as zarr_array_meta_key
32
36
from zarr .storage import attrs_key as zarr_attrs_key
33
37
from zarr .storage import group_meta_key as zarr_group_meta_key
@@ -94,7 +98,9 @@ def __getitem__(self, key: str) -> bytes:
94
98
elif key .endswith (zarr_array_meta_key ):
95
99
key_new = key .replace (zarr_array_meta_key , n5_attrs_key )
96
100
top_level = key == zarr_array_meta_key
97
- value = array_metadata_to_zarr (self ._load_n5_attrs (key_new ), top_level = top_level )
101
+ value = array_metadata_to_zarr (
102
+ self ._load_n5_attrs (key_new ), top_level = top_level
103
+ )
98
104
return json_dumps (value )
99
105
100
106
elif key .endswith (zarr_attrs_key ):
@@ -127,7 +133,9 @@ def __setitem__(self, key: str, value: Any):
127
133
key_new = key .replace (zarr_array_meta_key , n5_attrs_key )
128
134
top_level = key == zarr_array_meta_key
129
135
n5_attrs = self ._load_n5_attrs (key_new )
130
- n5_attrs .update (** array_metadata_to_n5 (json_loads (value ), top_level = top_level ))
136
+ n5_attrs .update (
137
+ ** array_metadata_to_n5 (json_loads (value ), top_level = top_level )
138
+ )
131
139
value = json_dumps (n5_attrs )
132
140
133
141
elif key .endswith (zarr_attrs_key ):
@@ -138,7 +146,9 @@ def __setitem__(self, key: str, value: Any):
138
146
139
147
for k in n5_keywords :
140
148
if k in zarr_attrs :
141
- warnings .warn (f"Attribute { k } is a reserved N5 keyword" , UserWarning )
149
+ warnings .warn (
150
+ f"Attribute { k } is a reserved N5 keyword" , UserWarning
151
+ )
142
152
143
153
# remove previous user attributes
144
154
for k in list (n5_attrs .keys ()):
@@ -272,11 +282,10 @@ def _is_array(self, path: str):
272
282
def _contains_attrs (self , path : str ):
273
283
if path is None :
274
284
attrs_key = n5_attrs_key
285
+ elif not path .endswith (n5_attrs_key ):
286
+ attrs_key = os .path .join (path , n5_attrs_key )
275
287
else :
276
- if not path .endswith (n5_attrs_key ):
277
- attrs_key = os .path .join (path , n5_attrs_key )
278
- else :
279
- attrs_key = path
288
+ attrs_key = path
280
289
281
290
attrs = attrs_to_zarr (self ._load_n5_attrs (attrs_key ))
282
291
return len (attrs ) > 0
@@ -379,7 +388,9 @@ def __getitem__(self, key: str) -> bytes:
379
388
elif key .endswith (zarr_array_meta_key ):
380
389
key_new = key .replace (zarr_array_meta_key , self ._array_meta_key )
381
390
top_level = key == zarr_array_meta_key
382
- value = array_metadata_to_zarr (self ._load_n5_attrs (key_new ), top_level = top_level )
391
+ value = array_metadata_to_zarr (
392
+ self ._load_n5_attrs (key_new ), top_level = top_level
393
+ )
383
394
return json_dumps (value )
384
395
385
396
elif key .endswith (zarr_attrs_key ):
@@ -412,7 +423,9 @@ def __setitem__(self, key: str, value: Any):
412
423
key_new = key .replace (zarr_array_meta_key , self ._array_meta_key )
413
424
top_level = key == zarr_array_meta_key
414
425
n5_attrs = self ._load_n5_attrs (key_new )
415
- n5_attrs .update (** array_metadata_to_n5 (json_loads (value ), top_level = top_level ))
426
+ n5_attrs .update (
427
+ ** array_metadata_to_n5 (json_loads (value ), top_level = top_level )
428
+ )
416
429
417
430
value = json_dumps (n5_attrs )
418
431
@@ -424,7 +437,9 @@ def __setitem__(self, key: str, value: Any):
424
437
425
438
for k in n5_keywords :
426
439
if k in zarr_attrs .keys ():
427
- warnings .warn (f"Attribute { k } is a reserved N5 keyword" , UserWarning )
440
+ warnings .warn (
441
+ f"Attribute { k } is a reserved N5 keyword" , UserWarning
442
+ )
428
443
429
444
# replace previous user attributes
430
445
for k in list (n5_attrs .keys ()):
@@ -552,11 +567,10 @@ def _is_array(self, path: Optional[str]):
552
567
def _contains_attrs (self , path : Optional [str ]):
553
568
if path is None :
554
569
attrs_key = self ._attrs_key
570
+ elif not path .endswith (self ._attrs_key ):
571
+ attrs_key = os .path .join (path , self ._attrs_key )
555
572
else :
556
- if not path .endswith (self ._attrs_key ):
557
- attrs_key = os .path .join (path , self ._attrs_key )
558
- else :
559
- attrs_key = path
573
+ attrs_key = path
560
574
561
575
attrs = attrs_to_zarr (self ._load_n5_attrs (attrs_key ))
562
576
return len (attrs ) > 0
@@ -599,7 +613,9 @@ def group_metadata_to_zarr(group_metadata: Dict[str, Any]) -> Dict[str, Any]:
599
613
return group_metadata
600
614
601
615
602
- def array_metadata_to_n5 (array_metadata : Dict [str , Any ], top_level = False ) -> Dict [str , Any ]:
616
+ def array_metadata_to_n5 (
617
+ array_metadata : Dict [str , Any ], top_level = False
618
+ ) -> Dict [str , Any ]:
603
619
"""Convert array metadata from zarr to N5 format. If the `top_level` keyword argument is True,
604
620
then the `N5` : N5_FORMAT key : value pair will be inserted into the metadata."""
605
621
@@ -611,14 +627,19 @@ def array_metadata_to_n5(array_metadata: Dict[str, Any], top_level=False) -> Dic
611
627
try :
612
628
dtype = np .dtype (array_metadata ["dataType" ])
613
629
except TypeError :
614
- raise TypeError (f"Data type { array_metadata ['dataType' ]} is not supported by N5" )
630
+ raise TypeError (
631
+ f"Data type { array_metadata ['dataType' ]} is not supported by N5"
632
+ )
615
633
616
634
array_metadata ["dataType" ] = dtype .name
617
635
array_metadata ["dimensions" ] = array_metadata ["dimensions" ][::- 1 ]
618
636
array_metadata ["blockSize" ] = array_metadata ["blockSize" ][::- 1 ]
619
637
620
638
if "fill_value" in array_metadata :
621
- if array_metadata ["fill_value" ] != 0 and array_metadata ["fill_value" ] is not None :
639
+ if (
640
+ array_metadata ["fill_value" ] != 0
641
+ and array_metadata ["fill_value" ] is not None
642
+ ):
622
643
raise ValueError (
623
644
f"""Received fill_value = { array_metadata ['fill_value' ]} ,
624
645
but N5 only supports fill_value = 0"""
@@ -634,7 +655,9 @@ def array_metadata_to_n5(array_metadata: Dict[str, Any], top_level=False) -> Dic
634
655
635
656
if "filters" in array_metadata :
636
657
if array_metadata ["filters" ] != [] and array_metadata ["filters" ] is not None :
637
- raise ValueError ("Received filters, but N5 storage does not support zarr filters" )
658
+ raise ValueError (
659
+ "Received filters, but N5 storage does not support zarr filters"
660
+ )
638
661
del array_metadata ["filters" ]
639
662
640
663
assert "compression" in array_metadata
@@ -691,7 +714,9 @@ def attrs_to_zarr(attrs: Dict[str, Any]) -> Dict[str, Any]:
691
714
return attrs
692
715
693
716
694
- def compressor_config_to_n5 (compressor_config : Optional [Dict [str , Any ]]) -> Dict [str , Any ]:
717
+ def compressor_config_to_n5 (
718
+ compressor_config : Optional [Dict [str , Any ]],
719
+ ) -> Dict [str , Any ]:
695
720
if compressor_config is None :
696
721
return {"type" : "raw" }
697
722
else :
@@ -751,7 +776,9 @@ def compressor_config_to_n5(compressor_config: Optional[Dict[str, Any]]) -> Dict
751
776
return n5_config
752
777
753
778
754
- def compressor_config_to_zarr (compressor_config : Dict [str , Any ]) -> Optional [Dict [str , Any ]]:
779
+ def compressor_config_to_zarr (
780
+ compressor_config : Dict [str , Any ],
781
+ ) -> Optional [Dict [str , Any ]]:
755
782
codec_id = compressor_config ["type" ]
756
783
zarr_config = {"id" : codec_id }
757
784
@@ -779,7 +806,7 @@ def compressor_config_to_zarr(compressor_config: Dict[str, Any]) -> Optional[Dic
779
806
zarr_config ["filters" ] = None
780
807
781
808
elif codec_id == "gzip" :
782
- if "useZlib" in compressor_config and compressor_config [ "useZlib" ] :
809
+ if compressor_config . get ( "useZlib" ) :
783
810
zarr_config ["id" ] = "zlib"
784
811
zarr_config ["level" ] = compressor_config ["level" ]
785
812
else :
@@ -808,10 +835,16 @@ def __init__(self, dtype, chunk_shape, compressor_config=None, compressor=None):
808
835
809
836
if compressor :
810
837
if compressor_config is not None :
811
- raise ValueError ("Only one of compressor_config or compressor should be given." )
838
+ raise ValueError (
839
+ "Only one of compressor_config or compressor should be given."
840
+ )
812
841
compressor_config = compressor .get_config ()
813
842
814
- if compressor_config is None and compressor is None or compressor_config ["id" ] == "raw" :
843
+ if (
844
+ compressor_config is None
845
+ and compressor is None
846
+ or compressor_config ["id" ] == "raw"
847
+ ):
815
848
self .compressor_config = None
816
849
self ._compressor = None
817
850
else :
@@ -884,7 +917,8 @@ def _create_header(chunk):
884
917
def _read_header (chunk ):
885
918
num_dims = struct .unpack (">H" , chunk [2 :4 ])[0 ]
886
919
shape = tuple (
887
- struct .unpack (">I" , chunk [i : i + 4 ])[0 ] for i in range (4 , num_dims * 4 + 4 , 4 )
920
+ struct .unpack (">I" , chunk [i : i + 4 ])[0 ]
921
+ for i in range (4 , num_dims * 4 + 4 , 4 )
888
922
)[::- 1 ]
889
923
890
924
len_header = 4 + num_dims * 4
@@ -908,4 +942,4 @@ def _from_big_endian(self, data):
908
942
return a .astype (self .dtype )
909
943
910
944
911
- register_codec (N5ChunkWrapper , N5ChunkWrapper .codec_id )
945
+ register_codec (N5ChunkWrapper , N5ChunkWrapper .codec_id )
0 commit comments