Represent '-1' indices as empty strings in StringArrayEncoding

src/biotite/database/afdb/download.py

@@ -16,8 +16,11 @@
 _METADATA_URL = "https://alphafold.com/api/prediction"
 _BINARY_FORMATS = ["bcif"]
 # Adopted from https://www.uniprot.org/help/accession_numbers
+# adding the optional 'AF-' prefix and '-F1' suffix used by RCSB
 _UNIPROT_PATTERN = (
-    "[OPQ][0-9][A-Z0-9]{3}[0-9]|[A-NR-Z][0-9]([A-Z][A-Z0-9]{2}[0-9]){1,2}"
+    r"^(?P<prefix>AF-)?"
+    r"(?P<id>[OPQ][0-9][A-Z0-9]{3}[0-9]|[A-NR-Z][0-9]([A-Z][A-Z0-9]{2}[0-9]){1,2})"
+    r"(?P<suffix>-?F1)?$"
 )
 
 
@@ -167,10 +170,10 @@ def _extract_id(id):
     uniprot_id : str
         The UniProt ID.
     """
-    match = re.search(_UNIPROT_PATTERN, id)
+    match = re.match(_UNIPROT_PATTERN, id)
     if match is None:
         raise ValueError(f"Cannot extract AFDB identifier from '{id}'")
-    return match.group()
+    return match.group("id")
 
 
 def _assert_valid_file(response, id):

src/biotite/structure/io/pdbx/bcif.py

@@ -292,7 +292,7 @@ def as_array(self, dtype=None, masked_value=None):
         else:
             # Array needs to be converted, but masked values are
             # not necessarily convertible
-            # (e.g. '' cannot be converted to int)
+            # (e.g. '.' cannot be converted to int)
             if masked_value is None:
                 array = np.zeros(len(self._data), dtype=dtype)
             else:

src/biotite/structure/io/pdbx/cif.py

@@ -243,7 +243,7 @@ def as_array(self, dtype=str, masked_value=None):
         else:
             # Array needs to be converted, but masked values are
             # not necessarily convertible
-            # (e.g. '' cannot be converted to int)
+            # (e.g. '.' cannot be converted to int)
             if masked_value is None:
                 array = np.zeros(len(self._data), dtype=dtype)
             else:

src/biotite/structure/io/pdbx/compress.py

@@ -140,8 +140,8 @@ def _compress_data(bcif_data, rtol, atol):
         # Run encode to initialize the data and offset arrays
         indices = encoding.encode(array)
         offsets = np.cumsum([0] + [len(s) for s in encoding.strings])
-        encoding.data_encoding, _ = _find_best_integer_compression(indices)
-        encoding.offset_encoding, _ = _find_best_integer_compression(offsets)
+        encoding.data_encoding = _find_best_integer_compression(indices)
+        encoding.offset_encoding = _find_best_integer_compression(offsets)
         return bcif.BinaryCIFData(array, [encoding])
 
     elif np.issubdtype(array.dtype, np.floating):
@@ -159,18 +159,22 @@ def _compress_data(bcif_data, rtol, atol):
             # -> do not use integer encoding
             return bcif.BinaryCIFData(array, [ByteArrayEncoding()])
         else:
-            best_encoding, size_compressed = _find_best_integer_compression(
-                integer_array
+            best_encoding = _find_best_integer_compression(integer_array)
+            compressed_data = bcif.BinaryCIFData(
+                array, [to_integer_encoding] + best_encoding
             )
-            if size_compressed < _data_size_in_file(bcif.BinaryCIFData(array)):
-                return bcif.BinaryCIFData(array, [to_integer_encoding] + best_encoding)
+            uncompressed_data = bcif.BinaryCIFData(array, [ByteArrayEncoding()])
+            if _data_size_in_file(compressed_data) < _data_size_in_file(
+                uncompressed_data
+            ):
+                return compressed_data
             else:
                 # The float array is smaller -> encode it directly as bytes
-                return bcif.BinaryCIFData(array, [ByteArrayEncoding()])
+                return uncompressed_data
 
     elif np.issubdtype(array.dtype, np.integer):
         array = _to_smallest_integer_type(array)
-        encodings, _ = _find_best_integer_compression(array)
+        encodings = _find_best_integer_compression(array)
         return bcif.BinaryCIFData(array, encodings)
 
     else:
@@ -233,7 +237,7 @@ def _find_best_integer_compression(array):
                 if size < smallest_size:
                     best_encoding_sequence = encodings
                     smallest_size = size
-    return best_encoding_sequence, smallest_size
+    return best_encoding_sequence
 
 
 def _estimate_packed_length(array, packed_byte_count):

src/biotite/structure/io/pdbx/encoding.pyx

@@ -225,9 +225,13 @@ class Encoding(_Component, metaclass=ABCMeta):
         -------
         decoded_data : ndarray
             The decoded data.
+
+        Warnings
+        --------
+        When overriding this method, do not omit bound checks with
+        ``@cython.boundscheck(False)`` or ``@cython.wraparound(False)``,
+        since the file content may be invalid/malicious.
         """
-        # Important: Do not omit bound checks for decoding,
-        # since the file content may be invalid/malicious.
         raise NotImplementedError()
 
     def __str__(self):
@@ -883,17 +887,39 @@ class StringArrayEncoding(Encoding):
         else:
             check_present = True
 
-        string_order = _safe_cast(np.argsort(self.strings), np.int32)
-        sorted_strings = self.strings[string_order]
-        sorted_indices = np.searchsorted(sorted_strings, data)
-        indices = string_order[sorted_indices]
-        if check_present and not np.all(self.strings[indices] == data):
+        if len(self.strings) > 0:
+            string_order = _safe_cast(np.argsort(self.strings), np.int32)
+            sorted_strings = self.strings[string_order]
+            sorted_indices = np.searchsorted(sorted_strings, data)
+            indices = string_order[sorted_indices]
+            # `"" not in self.strings` can be quite costly and is only necessary,
+            # if the the `strings` were given by the user, as otherwise we always
+            # include an empty string explicitly when we compute them in this function
+            # -> Only run if `check_present` is True
+            if check_present and "" not in self.strings:
+                # Represent empty strings as -1
+                indices[data == ""] = -1
+        else:
+            # There are no strings -> The indices can only ever be -1 to indicate
+            # missing values
+            # The check if this is correct is done below
+            indices = np.full(data.shape[0], -1, dtype=np.int32)
+
+        valid_indices_mask = indices != -1
+        if check_present and not np.all(
+            self.strings[indices[valid_indices_mask]] == data[valid_indices_mask]
+        ):
             raise ValueError("Data contains strings not present in 'strings'")
         return encode_stepwise(indices, self.data_encoding)
 
     def decode(self, data):
         indices = decode_stepwise(data, self.data_encoding)
-        return self.strings[indices]
+        # Initialize with empty strings
+        strings = np.zeros(indices.shape[0], dtype=self.strings.dtype)
+        # `-1`` indices indicate missing values
+        valid_indices_mask = indices != -1
+        strings[valid_indices_mask] = self.strings[indices[valid_indices_mask]]
+        return strings
 
     def __eq__(self, other):
         if not isinstance(other, type(self)):
@@ -1009,6 +1035,11 @@ def decode_stepwise(data, encoding):
     """
     for enc in reversed(encoding):
         data = enc.decode(data)
+    # ByteEncoding may decode in a non-writable array,
+    # as it creates the ndarray cheaply from buffer
+    if not data.flags.writeable:
+        # Make the resulting ndarray writable, by copying the underlying buffer
+        data = data.copy()
     return data
 
 

tests/database/test_afdb.py

@@ -15,11 +15,12 @@
 
 @pytest.mark.skipif(cannot_connect_to(AFDB_URL), reason="AlphaFold DB is not available")
 @pytest.mark.parametrize("as_file_like", [False, True])
-@pytest.mark.parametrize("entry_id", ["P12345", "AF-P12345-F1"])
+@pytest.mark.parametrize("entry_id", ["P12345", "AF-P12345-F1", "AF-P12345F1"])
 @pytest.mark.parametrize("format", ["pdb", "cif", "bcif"])
 def test_fetch(as_file_like, entry_id, format):
     """
     Check if files in different formats can be downloaded by being able to parse them.
+    Also ensure that the downloaded file refers to the given input ID
     """
     path = None if as_file_like else tempfile.gettempdir()
     file_path_or_obj = afdb.fetch(entry_id, format, path, overwrite=True)
@@ -29,9 +30,11 @@ def test_fetch(as_file_like, entry_id, format):
     elif format == "cif":
         file = pdbx.CIFFile.read(file_path_or_obj)
         pdbx.get_structure(file)
+        assert file.block["struct_ref"]["pdbx_db_accession"].as_item() == "P12345"
     elif format == "bcif":
         file = pdbx.BinaryCIFFile.read(file_path_or_obj)
         pdbx.get_structure(file)
+        assert file.block["struct_ref"]["pdbx_db_accession"].as_item() == "P12345"
 
 
 @pytest.mark.skipif(cannot_connect_to(AFDB_URL), reason="AlphaFold DB is not available")

tests/structure/io/test_pdbx.py

@@ -764,13 +764,13 @@ def test_bcif_encoding():
         encoding: False
         for encoding in [
             pdbx.ByteArrayEncoding,
-            pdbx.FixedPointEncoding,
-            # This encoding is not used in the test file
-            # pdbx.IntervalQuantizationEncoding,
             pdbx.RunLengthEncoding,
             pdbx.DeltaEncoding,
             pdbx.IntegerPackingEncoding,
             pdbx.StringArrayEncoding,
+            # These encodings are not used in the test file
+            # pdbx.IntervalQuantizationEncoding,
+            # pdbx.FixedPointEncoding,
         ]
     }
 
@@ -860,16 +860,28 @@ def test_bcif_cif_consistency():
                 if cif_column.mask is None:
                     assert bcif_column.mask is None
                 else:
+                    # Currently the reference written by py-mmcif always writes masks as
+                    # `MISSING`, even if `INAPPLICABLE` would be correct
+                    # -> Check only the presence of mask values
                     assert (
-                        cif_column.mask.array.tolist()
-                        == bcif_column.mask.array.tolist()
-                    )
+                        cif_column.mask.array == pdbx.MaskValue.PRESENT
+                    ).tolist() == (
+                        bcif_column.mask.array == pdbx.MaskValue.PRESENT
+                    ).tolist()
                 # In CIF format, all vales are strings
                 # -> ensure consistency
                 dtype = bcif_column.data.array.dtype
-                assert cif_column.as_array(dtype).tolist() == pytest.approx(
-                    bcif_column.as_array(dtype).tolist()
+                cif_array = cif_column.as_array(dtype)
+                bcif_array = bcif_column.as_array(dtype)
+                mask = (
+                    cif_column.mask.array == pdbx.MaskValue.PRESENT
+                    if cif_column.mask is not None
+                    else np.full(len(cif_column.data), True)
                 )
+                assert cif_array[mask].tolist() == pytest.approx(
+                    bcif_array[mask].tolist()
+                )
+
             except Exception:
                 raise Exception(
                     f"Comparison failed for '{category_name}.{column_name}'"
@@ -965,8 +977,8 @@ def test_compress_data():
     bcif_file = pdbx.BinaryCIFFile.read(path)
     for category_name, category in bcif_file.block.items():
         for column_name, column in category.items():
-            try:
-                for attr_name, data in [("data", column.data), ("mask", column.mask)]:
+            for attr_name, data in [("data", column.data), ("mask", column.mask)]:
+                try:
                     if data is None:
                         continue
                     ref_size = len(
@@ -993,8 +1005,10 @@ def test_compress_data():
                         serialized_compressed_data
                     )
                     assert restored_data.array.tolist() == data.array.tolist()
-            except AssertionError:
-                raise AssertionError(f"{category_name}.{column_name} {attr_name}")
+                except AssertionError:
+                    print(data.encoding)
+                    print(compressed_data.encoding)
+                    raise AssertionError(f"{category_name}.{column_name} {attr_name}")
 
 
 @pytest.mark.parametrize(