diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a06aefa0..ed62e43cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,9 +19,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * `FileVersion` has a mandatory `api` parameter * `B2Folder` holds a handle to B2Api * `Bucket` unit tests for v1 and v2 are now common +* `download_file_*` methods refactored to allow for inspecting FileVersion before downloading the whole file ### Fixed * Fix call to incorrect internal api in `B2Api.get_download_url_for_file_name` +* Fix EncryptionSetting.from_response_headers +* Fix FileVersion.size and FileVersion.mod_time_millis type ambiguity ## [1.8.0] - 2021-05-21 diff --git a/b2sdk/_v2/__init__.py b/b2sdk/_v2/__init__.py index b1180f630..b5472b1d8 100644 --- a/b2sdk/_v2/__init__.py +++ b/b2sdk/_v2/__init__.py @@ -116,11 +116,7 @@ # source / destination -from b2sdk.download_dest import AbstractDownloadDestination -from b2sdk.download_dest import DownloadDestBytes -from b2sdk.download_dest import DownloadDestLocalFile -from b2sdk.download_dest import DownloadDestProgressWrapper -from b2sdk.download_dest import PreSeekedDownloadDest +from b2sdk.transfer.inbound.downloaded_file import DownloadedFile from b2sdk.transfer.outbound.outbound_source import OutboundTransferSource from b2sdk.transfer.outbound.copy_source import CopySource @@ -136,7 +132,6 @@ # trasfer from b2sdk.transfer.inbound.downloader.abstract import AbstractDownloader -from b2sdk.transfer.inbound.file_metadata import FileMetadata from b2sdk.transfer.outbound.large_file_upload_state import LargeFileUploadState from b2sdk.transfer.inbound.downloader.parallel import AbstractDownloaderThread from b2sdk.transfer.inbound.downloader.parallel import FirstPartDownloaderThread diff --git a/b2sdk/api.py b/b2sdk/api.py index 2630d9665..d32e2ee31 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -8,7 +8,7 @@ # ###################################################################### -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Tuple from .bucket import Bucket, BucketFactory from .encryption.setting import EncryptionSetting @@ -17,6 +17,7 @@ from .file_version import FileIdAndName from .large_file.services import LargeFileServices from .raw_api import API_VERSION +from .progress import AbstractProgressListener from .session import B2Session from .transfer import ( CopyManager, @@ -24,6 +25,7 @@ Emerger, UploadManager, ) +from .transfer.inbound.downloaded_file import DownloadedFile from .utils import B2TraceMeta, b2_url_encode, limit_trace_arguments @@ -223,42 +225,30 @@ def create_bucket( def download_file_by_id( self, - file_id, - download_dest, - progress_listener=None, - range_=None, + file_id: str, + progress_listener: Optional[AbstractProgressListener] = None, + range_: Optional[Tuple[int, int]] = None, encryption: Optional[EncryptionSetting] = None, - ): + allow_seeking: bool = True, + ) -> DownloadedFile: """ Download a file with the given ID. :param str file_id: a file ID - :param download_dest: an instance of the one of the following classes: \ - :class:`~b2sdk.v1.DownloadDestLocalFile`,\ - :class:`~b2sdk.v1.DownloadDestBytes`,\ - :class:`~b2sdk.v1.DownloadDestProgressWrapper`,\ - :class:`~b2sdk.v1.PreSeekedDownloadDest`,\ - or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination` - :param progress_listener: an instance of the one of the following classes: \ - :class:`~b2sdk.v1.PartProgressReporter`,\ - :class:`~b2sdk.v1.TqdmProgressListener`,\ - :class:`~b2sdk.v1.SimpleProgressListener`,\ - :class:`~b2sdk.v1.DoNothingProgressListener`,\ - :class:`~b2sdk.v1.ProgressListenerForTest`,\ - :class:`~b2sdk.v1.SyncFileReporter`,\ - or any sub class of :class:`~b2sdk.v1.AbstractProgressListener` - :param list range_: a list of two integers, the first one is a start\ + :param progress_listener: a progress listener object to use, or ``None`` to not track progress + :param range_: a list of two integers, the first one is a start\ position, and the second one is the end position in the file - :param b2sdk.v1.EncryptionSetting encryption: encryption settings (``None`` if unknown) - :return: context manager that returns an object that supports iter_content() + :param encryption: encryption settings (``None`` if unknown) + :param allow_seeking: if true, download strategies requiring seeking on the download destination will be + taken into account """ url = self.session.get_download_url_by_id(file_id) return self.services.download_manager.download_file_from_url( url, - download_dest, progress_listener, range_, encryption, + allow_seeking, ) def update_file_retention( diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index bf97996a7..5f74ae423 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -9,7 +9,7 @@ ###################################################################### import logging -from typing import Optional +from typing import Optional, Tuple from .encryption.setting import EncryptionSetting, EncryptionSettingFactory from .encryption.types import EncryptionMode @@ -22,9 +22,10 @@ LegalHold, ) from .file_version import FileVersion, FileVersionFactory -from .progress import DoNothingProgressListener +from .progress import AbstractProgressListener, DoNothingProgressListener from .transfer.emerge.executor import AUTO_CONTENT_TYPE from .transfer.emerge.write_intent import WriteIntent +from .transfer.inbound.downloaded_file import DownloadedFile from .transfer.outbound.copy_source import CopySource from .transfer.outbound.upload_source import UploadSourceBytes, UploadSourceLocalFile from .utils import B2TraceMeta, disable_trace, limit_trace_arguments @@ -158,45 +159,41 @@ def cancel_large_file(self, file_id): def download_file_by_id( self, - file_id, - download_dest, - progress_listener=None, - range_=None, + file_id: str, + progress_listener: Optional[AbstractProgressListener] = None, + range_: Optional[Tuple[int, int]] = None, encryption: Optional[EncryptionSetting] = None, - ): + allow_seeking: bool = True, + ) -> DownloadedFile: """ Download a file by ID. .. note:: download_file_by_id actually belongs in :py:class:`b2sdk.v1.B2Api`, not in :py:class:`b2sdk.v1.Bucket`; we just provide a convenient redirect here - :param str file_id: a file ID - :param download_dest: an instance of the one of the following classes: \ - :class:`~b2sdk.v1.DownloadDestLocalFile`,\ - :class:`~b2sdk.v1.DownloadDestBytes`,\ - :class:`~b2sdk.v1.DownloadDestProgressWrapper`,\ - :class:`~b2sdk.v1.PreSeekedDownloadDest`,\ - or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination` - :param b2sdk.v1.AbstractProgressListener, None progress_listener: a progress listener object to use, or ``None`` to not report progress - :param tuple[int, int] range_: two integer values, start and end offsets - :param b2sdk.v1.EncryptionSetting encryption: encryption settings (``None`` if unknown) + :param file_id: a file ID + :param progress_listener: a progress listener object to use, or ``None`` to not track progress + :param range_: two integer values, start and end offsets + :param encryption: encryption settings (``None`` if unknown) + :param allow_seeking: if true, download strategies requiring seeking on the download destination will be + taken into account """ return self.api.download_file_by_id( file_id, - download_dest, progress_listener, range_=range_, encryption=encryption, + allow_seeking=allow_seeking, ) def download_file_by_name( self, - file_name, - download_dest, - progress_listener=None, - range_=None, + file_name: str, + progress_listener: Optional[AbstractProgressListener] = None, + range_: Optional[Tuple[int, int]] = None, encryption: Optional[EncryptionSetting] = None, - ): + allow_seeking: bool = True, + ) -> DownloadedFile: """ Download a file by name. @@ -204,24 +201,20 @@ def download_file_by_name( :ref:`Synchronizer `, a *high-performance* utility that synchronizes a local folder with a Bucket. - :param str file_name: a file name - :param download_dest: an instance of the one of the following classes: \ - :class:`~b2sdk.v1.DownloadDestLocalFile`,\ - :class:`~b2sdk.v1.DownloadDestBytes`,\ - :class:`~b2sdk.v1.DownloadDestProgressWrapper`,\ - :class:`~b2sdk.v1.PreSeekedDownloadDest`,\ - or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination` - :param b2sdk.v1.AbstractProgressListener, None progress_listener: a progress listener object to use, or ``None`` to not track progress - :param tuple[int, int] range_: two integer values, start and end offsets - :param b2sdk.v1.EncryptionSetting encryption: encryption settings (``None`` if unknown) + :param file_name: a file name + :param progress_listener: a progress listener object to use, or ``None`` to not track progress + :param range_: two integer values, start and end offsets + :param encryption: encryption settings (``None`` if unknown) + :param allow_seeking: if true, download strategies requiring seeking on the download destination will be + taken into account """ url = self.api.session.get_download_url_by_name(self.name, file_name) return self.api.services.download_manager.download_file_from_url( url, - download_dest, progress_listener, range_, encryption=encryption, + allow_seeking=allow_seeking, ) def get_file_info_by_id(self, file_id: str) -> FileVersion: diff --git a/b2sdk/encryption/setting.py b/b2sdk/encryption/setting.py index 95d0bae1b..5ddecee49 100644 --- a/b2sdk/encryption/setting.py +++ b/b2sdk/encryption/setting.py @@ -16,8 +16,10 @@ from ..utils import b64_of_bytes, md5_of_bytes from .types import ENCRYPTION_MODES_WITH_MANDATORY_ALGORITHM, ENCRYPTION_MODES_WITH_MANDATORY_KEY from .types import EncryptionAlgorithm, EncryptionMode +from ..utils import FILE_INFO_HEADER_PREFIX SSE_C_KEY_ID_FILE_INFO_KEY_NAME = 'sse_c_key_id' +SSE_C_KEY_ID_HEADER = FILE_INFO_HEADER_PREFIX + SSE_C_KEY_ID_FILE_INFO_KEY_NAME logger = logging.getLogger(__name__) @@ -147,7 +149,7 @@ def add_to_upload_headers(self, headers): elif self.mode == EncryptionMode.SSE_C: self._add_sse_c_headers(headers) if self.key.key_id is not None: - header = 'X-Bz-Info-%s' % (SSE_C_KEY_ID_FILE_INFO_KEY_NAME,) + header = SSE_C_KEY_ID_HEADER if headers.get(header) is not None and headers[header] != self.key.key_id: raise ValueError( 'Ambiguous key id set: "%s" in headers and "%s" in %s' % @@ -309,18 +311,22 @@ def _from_value_dict(cls, value_dict, key_id=None): @classmethod def from_response_headers(cls, headers): - - mode = EncryptionMode(headers.get('X-Bz-Server-Side-Encryption', 'none')) - kwargs = { - 'mode': mode, - } - if mode == EncryptionMode.SSE_C: - kwargs['key'] = EncryptionKey(secret=None, key_id=None) - algorithm = headers.get('X-Bz-Server-Side-Encryption-Customer-Algorithm') - if algorithm is not None: - kwargs['algorithm'] = EncryptionAlgorithm(algorithm) - - return EncryptionSetting(**kwargs) + if 'X-Bz-Server-Side-Encryption' in headers: + mode = EncryptionMode.SSE_B2 + algorithm = EncryptionAlgorithm(headers['X-Bz-Server-Side-Encryption']) + return EncryptionSetting(mode, algorithm) + if 'X-Bz-Server-Side-Encryption-Customer-Algorithm' in headers: + mode = EncryptionMode.SSE_C + algorithm = EncryptionAlgorithm( + headers['X-Bz-Server-Side-Encryption-Customer-Algorithm'] + ) + if SSE_C_KEY_ID_HEADER in headers: + key_id = headers[SSE_C_KEY_ID_HEADER] + else: + key_id = None + key = EncryptionKey(secret=None, key_id=key_id) + return EncryptionSetting(mode, algorithm, key) + return EncryptionSetting(EncryptionMode.NONE) SSE_NONE = EncryptionSetting(mode=EncryptionMode.NONE,) diff --git a/b2sdk/file_version.py b/b2sdk/file_version.py index 20a30cc3e..797f95456 100644 --- a/b2sdk/file_version.py +++ b/b2sdk/file_version.py @@ -11,6 +11,7 @@ from .encryption.setting import EncryptionSetting, EncryptionSettingFactory from .file_lock import FileRetentionSetting, LegalHold, NO_RETENTION_FILE_SETTING from .raw_api import SRC_LAST_MODIFIED_MILLIS +from .utils import FILE_INFO_HEADER_PREFIX_LOWER class FileVersion: @@ -68,7 +69,7 @@ def __init__( self.api = api self.id_ = id_ self.file_name = file_name - self.size = size + self.size = size and int(size) self.content_type = content_type self.content_sha1 = content_sha1 self.content_md5 = content_md5 @@ -82,7 +83,7 @@ def __init__( if SRC_LAST_MODIFIED_MILLIS in self.file_info: self.mod_time_millis = int(self.file_info[SRC_LAST_MODIFIED_MILLIS]) else: - self.mod_time_millis = self.upload_timestamp + self.mod_time_millis = self.upload_timestamp and int(self.upload_timestamp) def as_dict(self): """ represents the object as a dict which looks almost exactly like the raw api output for upload/list """ @@ -201,6 +202,12 @@ def from_api_response(cls, api, file_version_dict, force_action=None): @classmethod def from_response_headers(cls, api, headers): + file_info = {} + prefix_len = len(FILE_INFO_HEADER_PREFIX_LOWER) + for header_name, header_value in headers.items(): + if header_name[:prefix_len].lower() == FILE_INFO_HEADER_PREFIX_LOWER: + file_info_key = header_name[prefix_len:] + file_info[file_info_key] = header_value return FileVersion( api=api, id_=headers.get('x-bz-file-id'), @@ -208,9 +215,9 @@ def from_response_headers(cls, api, headers): size=headers.get('content-length'), content_type=headers.get('content-type'), content_sha1=headers.get('x-bz-content-sha1'), - file_info=None, + file_info=file_info, upload_timestamp=headers.get('x-bz-upload-timestamp'), - action=None, + action='upload', content_md5=None, server_side_encryption=EncryptionSettingFactory.from_response_headers(headers), file_retention=FileRetentionSetting.from_response_headers(headers), diff --git a/b2sdk/raw_api.py b/b2sdk/raw_api.py index 813b271bc..68b6d7539 100644 --- a/b2sdk/raw_api.py +++ b/b2sdk/raw_api.py @@ -25,7 +25,7 @@ from .exception import FileOrBucketNotFound, ResourceNotFound, UnusableFileName, InvalidMetadataDirective, WrongEncryptionModeForBucketDefault, AccessDenied, SSECKeyError, RetentionWriteError from .encryption.setting import EncryptionAlgorithm, EncryptionMode, EncryptionSetting from .file_lock import BucketRetentionSetting, FileRetentionSetting, NO_RETENTION_FILE_SETTING, RetentionMode, RetentionPeriod, LegalHold -from .utils import b2_url_encode, hex_sha1_of_stream +from .utils import b2_url_encode, hex_sha1_of_stream, FILE_INFO_HEADER_PREFIX # All supported realms REALM_URLS = { @@ -863,7 +863,7 @@ def upload_file( 'X-Bz-Content-Sha1': content_sha1, } for k, v in file_infos.items(): - headers['X-Bz-Info-' + k] = b2_url_encode(v) + headers[FILE_INFO_HEADER_PREFIX + k] = b2_url_encode(v) if server_side_encryption is not None: assert server_side_encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index 42fefb62a..b6cc7980c 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -18,6 +18,8 @@ import threading from contextlib import contextmanager +from requests.structures import CaseInsensitiveDict + from .b2http import ResponseContextManager from .encryption.setting import EncryptionMode, EncryptionSetting from .exception import ( @@ -44,6 +46,7 @@ b2_url_encode, ConcurrentUsedAuthTokenGuard, hex_sha1_of_bytes, + FILE_INFO_HEADER_PREFIX, ) from .stream.hashing import StreamWithHash @@ -206,14 +209,18 @@ def as_download_headers(self, account_auth_token_or_none, range_=None): content_length = range_[1] - range_[0] + 1 else: content_length = len(self.data_bytes) - headers = { - 'content-length': content_length, - 'content-type': self.content_type, - 'x-bz-content-sha1': self.content_sha1, - 'x-bz-upload-timestamp': self.upload_timestamp, - 'x-bz-file-id': self.file_id, - 'x-bz-file-name': self.name, - } + headers = CaseInsensitiveDict( + { + 'content-length': content_length, + 'content-type': self.content_type, + 'x-bz-content-sha1': self.content_sha1, + 'x-bz-upload-timestamp': self.upload_timestamp, + 'x-bz-file-id': self.file_id, + 'x-bz-file-name': self.name, + } + ) + for key, value in self.file_info.items(): + headers[FILE_INFO_HEADER_PREFIX + key] = value if account_auth_token_or_none is not None and self.bucket.is_file_lock_enabled: not_permitted = [] @@ -233,12 +240,27 @@ def as_download_headers(self, account_auth_token_or_none, range_=None): if not_permitted: headers['X-Bz-Client-Unauthorized-To-Read'] = ','.join(not_permitted) + if self.server_side_encryption is not None: + if self.server_side_encryption.mode == EncryptionMode.SSE_B2: + headers['X-Bz-Server-Side-Encryption'] = self.server_side_encryption.algorithm.value + elif self.server_side_encryption.mode == EncryptionMode.SSE_C: + headers['X-Bz-Server-Side-Encryption-Customer-Algorithm' + ] = self.server_side_encryption.algorithm.value + headers['X-Bz-Server-Side-Encryption-Customer-Key-Md5' + ] = self.server_side_encryption.key.key_md5() + elif self.server_side_encryption.mode in (EncryptionMode.NONE, EncryptionMode.UNKNOWN): + pass + else: + raise ValueError( + 'Unsupported encryption mode: %s' % (self.server_side_encryption.mode,) + ) + if range_ is not None: headers['Content-Range'] = 'bytes %d-%d/%d' % ( range_[0], range_[0] + content_length, len(self.data_bytes) ) # yapf: disable for key, value in self.file_info.items(): - headers['x-bz-info-' + key] = value + headers[FILE_INFO_HEADER_PREFIX + key] = value return headers def as_upload_result(self, account_auth_token): @@ -419,7 +441,7 @@ def iter_content(self, chunk_size=1): @property def request(self): - headers = {} + headers = CaseInsensitiveDict() if self.range_ is not None: headers['Range'] = '%s-%s' % self.range_ return FakeRequest(self.url, headers) diff --git a/b2sdk/stream/range.py b/b2sdk/stream/range.py index facb4778e..6cdc66bd5 100644 --- a/b2sdk/stream/range.py +++ b/b2sdk/stream/range.py @@ -70,6 +70,12 @@ def read(self, size=None): self.relative_pos += len(data) return data + def close(self): + super().close() + # TODO: change the use cases of this class to close the file objects passed to it, instead of having + # RangeOfInputStream close it's members upon garbage collection + self.stream.close() + def wrap_with_range(stream, stream_length, range_offset, range_length): if range_offset == 0 and range_length == stream_length: diff --git a/b2sdk/stream/wrapper.py b/b2sdk/stream/wrapper.py index 96dc5deb0..883504fb5 100644 --- a/b2sdk/stream/wrapper.py +++ b/b2sdk/stream/wrapper.py @@ -79,10 +79,6 @@ def write(self, data): """ return self.stream.write(data) - def close(self): - super(StreamWrapper, self).close() - self.stream.close() - class StreamWithLengthWrapper(StreamWrapper): """ diff --git a/b2sdk/sync/action.py b/b2sdk/sync/action.py index ec1e826ae..9fab316e2 100644 --- a/b2sdk/sync/action.py +++ b/b2sdk/sync/action.py @@ -12,7 +12,6 @@ import logging import os -from ..download_dest import DownloadDestLocalFile from .encryption_provider import AbstractSyncEncryptionSettingsProvider from ..bucket import Bucket @@ -260,19 +259,18 @@ def do_action(self, bucket, reporter): # Download the file to a .tmp file download_path = self.local_full_path + '.b2.sync.tmp' - download_dest = DownloadDestLocalFile(download_path) encryption = self.encryption_settings_provider.get_setting_for_download( bucket=bucket, file_version=self.source_path.selected_version, ) - bucket.download_file_by_id( + downloaded_file = bucket.download_file_by_id( self.source_path.selected_version.id_, - download_dest, - progress_listener, + progress_listener=progress_listener, encryption=encryption, ) + downloaded_file.save_to(download_path) # Move the file into place try: diff --git a/b2sdk/transfer/emerge/planner/upload_subpart.py b/b2sdk/transfer/emerge/planner/upload_subpart.py index 40e242c5a..88fddb018 100644 --- a/b2sdk/transfer/emerge/planner/upload_subpart.py +++ b/b2sdk/transfer/emerge/planner/upload_subpart.py @@ -13,7 +13,6 @@ from abc import ABCMeta, abstractmethod from functools import partial -from b2sdk.download_dest import DownloadDestBytes from b2sdk.stream.chained import StreamOpener from b2sdk.stream.range import wrap_with_range from b2sdk.utils import hex_sha1_of_unlimited_stream @@ -64,12 +63,13 @@ def get_stream_opener(self, emerge_execution=None): def _download(self, emerge_execution): url = emerge_execution.services.session.get_download_url_by_id(self.outbound_source.file_id) absolute_offset = self.outbound_source.offset + self.relative_offset - download_dest = DownloadDestBytes() range_ = (absolute_offset, absolute_offset + self.length - 1) - emerge_execution.services.download_manager.download_file_from_url( - url, download_dest, range_=range_, encryption=self.outbound_source.encryption - ) - return download_dest.get_bytes_written() + with io.BytesIO() as bytes_io: + downloaded_file = emerge_execution.services.download_manager.download_file_from_url( + url, range_=range_, encryption=self.outbound_source.encryption + ) + downloaded_file.save(bytes_io) + return bytes_io.getvalue() class LocalSourceUploadSubpart(BaseUploadSubpart): diff --git a/b2sdk/transfer/inbound/download_manager.py b/b2sdk/transfer/inbound/download_manager.py index c0900618d..b7824b86e 100644 --- a/b2sdk/transfer/inbound/download_manager.py +++ b/b2sdk/transfer/inbound/download_manager.py @@ -11,21 +11,18 @@ import logging from typing import Optional -from b2sdk.download_dest import DownloadDestProgressWrapper from b2sdk.encryption.setting import EncryptionSetting from b2sdk.progress import DoNothingProgressListener from b2sdk.exception import ( - ChecksumMismatch, InvalidRange, - TruncatedOutput, ) -from b2sdk.raw_api import SRC_LAST_MODIFIED_MILLIS from b2sdk.utils import B2TraceMetaAbstract +from b2sdk.file_version import FileVersionFactory +from .downloaded_file import DownloadedFile from .downloader.parallel import ParallelDownloader from .downloader.simple import SimpleDownloader -from .file_metadata import FileMetadata logger = logging.getLogger(__name__) @@ -61,7 +58,6 @@ def __init__(self, services): min_chunk_size=self.MIN_CHUNK_SIZE, max_chunk_size=self.MAX_CHUNK_SIZE, ), - # IOTDownloader(), # TODO: curl -s httpbin.org/get | tee /dev/stderr 2>ble | sha1sum | cut -c -40 SimpleDownloader( min_chunk_size=self.MIN_CHUNK_SIZE, max_chunk_size=self.MAX_CHUNK_SIZE, @@ -71,80 +67,39 @@ def __init__(self, services): def download_file_from_url( self, url, - download_dest, progress_listener=None, range_=None, encryption: Optional[EncryptionSetting] = None, - ): + allow_seeking=True, + ) -> DownloadedFile: """ :param url: url from which the file should be downloaded - :param download_dest: where to put the file when it is downloaded - :param progress_listener: where to notify about progress downloading + :param progress_listener: where to notify about downloading progress :param range_: 2-element tuple containing data of http Range header :param b2sdk.v1.EncryptionSetting encryption: encryption setting (``None`` if unknown) + :param bool allow_seeking: if False, download strategies that rely on seeking to write data + (parallel strategies) will be discarded. """ progress_listener = progress_listener or DoNothingProgressListener() - download_dest = DownloadDestProgressWrapper(download_dest, progress_listener) with self.services.session.download_file_from_url( url, range_=range_, encryption=encryption, ) as response: - metadata = FileMetadata.from_response(response) + file_version = FileVersionFactory.from_response_headers( + self.services.api, response.headers + ) if range_ is not None: # 2021-05-20: unfortunately for a read of a complete object server does not return the 'Content-Range' header - if (range_[1] - range_[0] + 1) != metadata.content_length: - raise InvalidRange(metadata.content_length, range_) - - mod_time_millis = int( - metadata.file_info.get( - SRC_LAST_MODIFIED_MILLIS, - response.headers['x-bz-upload-timestamp'], - ) - ) - - with download_dest.make_file_context( - metadata.file_id, - metadata.file_name, - metadata.content_length, - metadata.content_type, - metadata.content_sha1, - metadata.file_info, - mod_time_millis, - range_=range_, - ) as file: - - for strategy in self.strategies: - if strategy.is_suitable(metadata, progress_listener): - bytes_read, actual_sha1 = strategy.download( - file, - response, - metadata, - self.services.session, - encryption=encryption, - ) - break - else: - assert False, 'no strategy suitable for download was found!' - - self._validate_download( - range_, bytes_read, actual_sha1, metadata - ) # raises exceptions - return metadata.as_info_dict() - - @classmethod - def _validate_download(cls, range_, bytes_read, actual_sha1, metadata): - if range_ is None: - if bytes_read != metadata.content_length: - raise TruncatedOutput(bytes_read, metadata.content_length) - - if metadata.content_sha1 != 'none' and actual_sha1 != metadata.content_sha1: - raise ChecksumMismatch( - checksum_type='sha1', - expected=metadata.content_sha1, - actual=actual_sha1, - ) - else: - desired_length = range_[1] - range_[0] + 1 - if bytes_read != desired_length: - raise TruncatedOutput(bytes_read, desired_length) + if (range_[1] - range_[0] + 1) != file_version.size: + raise InvalidRange(file_version.size, range_) + + for strategy in self.strategies: + + if strategy.is_suitable(file_version, allow_seeking): + return DownloadedFile( + file_version, strategy, range_, response, encryption, self.services.session, + progress_listener + ) + else: + raise ValueError('no strategy suitable for download was found!') diff --git a/b2sdk/transfer/inbound/downloaded_file.py b/b2sdk/transfer/inbound/downloaded_file.py new file mode 100644 index 000000000..380602edc --- /dev/null +++ b/b2sdk/transfer/inbound/downloaded_file.py @@ -0,0 +1,149 @@ +###################################################################### +# +# File: b2sdk/transfer/inbound/downloaded_file.py +# +# Copyright 2021 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +import io +import logging +from typing import Optional, Tuple + +from requests.models import Response + +from .downloader.abstract import AbstractDownloader +from ...encryption.setting import EncryptionSetting +from ...file_version import FileVersion +from ...progress import AbstractProgressListener +from ...session import B2Session +from ...stream.progress import WritingStreamWithProgress + +from b2sdk.exception import ( + ChecksumMismatch, + TruncatedOutput, +) +from b2sdk.utils import set_file_mtime + +logger = logging.getLogger(__name__) + + +class MtimeUpdatedFile(io.IOBase): + """ + Helper class that facilitates updating a files mod_time after closing. + Usage: + + .. code-block: python + + downloaded_file = bucket.download_file_by_id('b2_file_id') + with MtimeUpdatedFile('some_local_path') as file: + downloaded_file.save(file, file.set_mod_time) + # 'some_local_path' has the mod_time set according to metadata in B2 + """ + + def __init__(self, path_, mode='wb+'): + self.path_ = path_ + self.mode = mode + self.mod_time_to_set = None + self.file = None + + def set_mod_time(self, mod_time): + self.mod_time_to_set = mod_time + + def write(self, value): + """ + This method is overwritten (monkey-patched) in __enter__ for performance reasons + """ + raise NotImplementedError + + def read(self, *a): + """ + This method is overwritten (monkey-patched) in __enter__ for performance reasons + """ + raise NotImplementedError + + def seek(self, offset, whence=0): + return self.file.seek(offset, whence) + + def tell(self): + return self.file.tell() + + def __enter__(self): + self.file = open(self.path_, self.mode) + self.write = self.file.write + self.read = self.file.read + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.file.close() + if self.mod_time_to_set is not None: + set_file_mtime(self.path_, self.mod_time_to_set) + + +class DownloadedFile: + def __init__( + self, + file_version: FileVersion, + strategy: AbstractDownloader, + range_: Optional[Tuple[int, int]], + response: Response, + encryption: Optional[EncryptionSetting], + session: B2Session, + progress_listener: AbstractProgressListener, + ): + self.file_version = file_version + self.strategy = strategy + self.range_ = range_ + self.progress_listener = progress_listener + self.response = response + self.encryption = encryption + self.session = session + + def _validate_download(self, bytes_read, actual_sha1): + if self.range_ is None: + if bytes_read != self.file_version.size: + raise TruncatedOutput(bytes_read, self.file_version.size) + + if self.file_version.content_sha1 != 'none' and actual_sha1 != self.file_version.content_sha1: + raise ChecksumMismatch( + checksum_type='sha1', + expected=self.file_version.content_sha1, + actual=actual_sha1, + ) + else: + desired_length = self.range_[1] - self.range_[0] + 1 + if bytes_read != desired_length: + raise TruncatedOutput(bytes_read, desired_length) + + def save(self, file, mod_time_callback=None): + """ + Read data from B2 cloud and write it to a file-like object + :param file: a file-like object + :param mod_time_callback: a callable accepting a single argument: the mod time of the downloaded file in milliseconds + """ + if self.progress_listener: + file = WritingStreamWithProgress(file, self.progress_listener) + if self.range_ is not None: + total_bytes = self.range_[1] - self.range_[0] + 1 + else: + total_bytes = self.file_version.size + self.progress_listener.set_total_bytes(total_bytes) + if mod_time_callback is not None: + mod_time_callback(self.file_version.mod_time_millis) + bytes_read, actual_sha1 = self.strategy.download( + file, + self.response, + self.file_version, + self.session, + encryption=self.encryption, + ) + self._validate_download(bytes_read, actual_sha1) + + def save_to(self, path_, mode='wb+'): + """ + Open a local file and write data from B2 cloud to it, also update the mod_time. + """ + with MtimeUpdatedFile(path_, mode) as file: + self.save(file, file.set_mod_time) diff --git a/b2sdk/transfer/inbound/downloader/abstract.py b/b2sdk/transfer/inbound/downloader/abstract.py index f01674c06..3251c5705 100644 --- a/b2sdk/transfer/inbound/downloader/abstract.py +++ b/b2sdk/transfer/inbound/downloader/abstract.py @@ -18,6 +18,9 @@ class AbstractDownloader(metaclass=B2TraceMetaAbstract): + + REQUIRES_SEEKING = True + def __init__( self, force_chunk_size=None, @@ -41,33 +44,34 @@ def _get_chunk_size(self, content_length): return aligned @classmethod - def _get_remote_range(cls, response, metadata): + def _get_remote_range(cls, response, file_version): """ Get a range from response or original request (as appropriate). :param response: requests.Response of initial request - :param metadata: metadata dict of the target file + :param file_version: b2sdk.v1.FileVersionInfo :return: a range object """ raw_range_header = response.request.headers.get('Range') # 'bytes 0-11' if raw_range_header is None: - return Range(0, 0 if metadata.content_length == 0 else metadata.content_length - 1) + return Range(0, 0 if file_version.size == 0 else file_version.size - 1) return Range.from_header(raw_range_header) - @abstractmethod - def is_suitable(self, metadata, progress_listener): + def is_suitable(self, file_version, allow_seeking): """ - Analyze metadata (possibly against options passed earlier to constructor + Analyze file_version (possibly against options passed earlier to constructor to find out whether the given download request should be handled by this downloader). """ - pass + if self.REQUIRES_SEEKING and not allow_seeking: + return False + return True @abstractmethod def download( self, file, response, - metadata, + file_version, session, encryption: Optional[EncryptionSetting] = None, ): diff --git a/b2sdk/transfer/inbound/downloader/parallel.py b/b2sdk/transfer/inbound/downloader/parallel.py index 0f8590171..0d7a194c9 100644 --- a/b2sdk/transfer/inbound/downloader/parallel.py +++ b/b2sdk/transfer/inbound/downloader/parallel.py @@ -53,16 +53,18 @@ def __init__(self, max_streams, min_part_size, *args, **kwargs): self.min_part_size = min_part_size super(ParallelDownloader, self).__init__(*args, **kwargs) - def is_suitable(self, metadata, progress_listener): + def is_suitable(self, file_version, allow_seeking): + if not super().is_suitable(file_version, allow_seeking): + return False return self._get_number_of_streams( - metadata.content_length - ) >= 2 and metadata.content_length >= 2 * self.min_part_size + file_version.size + ) >= 2 and file_version.size >= 2 * self.min_part_size def _get_number_of_streams(self, content_length): return min(self.max_streams, content_length // self.min_part_size) or 1 def download( - self, file, response, metadata, session, encryption: Optional[EncryptionSetting] = None + self, file, response, file_version, session, encryption: Optional[EncryptionSetting] = None ): """ Download a file from given url using parallel download sessions and stores it in the given download_destination. @@ -71,14 +73,14 @@ def download( :param response: the response of the first request made to the cloud service with download intent :return: """ - remote_range = self._get_remote_range(response, metadata) + remote_range = self._get_remote_range(response, file_version) actual_size = remote_range.size() start_file_position = file.tell() parts_to_download = list( gen_parts( remote_range, Range(start_file_position, start_file_position + actual_size - 1), - part_count=self._get_number_of_streams(metadata.content_length), + part_count=self._get_number_of_streams(file_version.size), ) ) @@ -101,7 +103,7 @@ def download( # At this point the hasher already consumed the data until the end of first stream. # Consume the rest of the file to complete the hashing process - self._finish_hashing(first_part, file, hasher, metadata.content_length) + self._finish_hashing(first_part, file, hasher, file_version.size) return bytes_written, hasher.hexdigest() diff --git a/b2sdk/transfer/inbound/downloader/simple.py b/b2sdk/transfer/inbound/downloader/simple.py index 61cb86cd5..d686ee38d 100644 --- a/b2sdk/transfer/inbound/downloader/simple.py +++ b/b2sdk/transfer/inbound/downloader/simple.py @@ -19,16 +19,16 @@ class SimpleDownloader(AbstractDownloader): + + REQUIRES_SEEKING = False + def __init__(self, *args, **kwargs): super(SimpleDownloader, self).__init__(*args, **kwargs) - def is_suitable(self, metadata, progress_listener): - return True - def download( - self, file, response, metadata, session, encryption: Optional[EncryptionSetting] = None + self, file, response, file_version, session, encryption: Optional[EncryptionSetting] = None ): - actual_size = self._get_remote_range(response, metadata).size() + actual_size = self._get_remote_range(response, file_version).size() chunk_size = self._get_chunk_size(actual_size) digest = hashlib.sha1() @@ -40,16 +40,16 @@ def download( assert actual_size >= 1 # code below does `actual_size - 1`, but it should never reach that part with an empty file - # now, normally bytes_read == metadata.content_length, but sometimes there is a timeout + # now, normally bytes_read == file_version.size, but sometimes there is a timeout # or something and the server closes connection, while neither tcp or http have a problem # with the truncated output, so we detect it here and try to continue num_tries = 5 # this is hardcoded because we are going to replace the entire retry interface soon, so we'll avoid deprecation here and keep it private retries_left = num_tries - 1 - while retries_left and bytes_read < metadata.content_length: + while retries_left and bytes_read < file_version.size: new_range = self._get_remote_range( response, - metadata, + file_version, ).subrange(bytes_read, actual_size - 1) # original response is not closed at this point yet, as another layer is responsible for closing it, so a new socket might be allocated, # but this is a very rare case and so it is not worth the optimization diff --git a/b2sdk/utils.py b/b2sdk/utils.py index d0e88a6e1..e09bb22b5 100644 --- a/b2sdk/utils.py +++ b/b2sdk/utils.py @@ -21,6 +21,10 @@ from logfury.v0_1 import DefaultTraceAbstractMeta, DefaultTraceMeta, limit_trace_arguments, disable_trace, trace_call +# File info header prefix +FILE_INFO_HEADER_PREFIX = 'X-Bz-Info-' +FILE_INFO_HEADER_PREFIX_LOWER = FILE_INFO_HEADER_PREFIX.lower() + def interruptible_get_result(future): """ diff --git a/b2sdk/v1/__init__.py b/b2sdk/v1/__init__.py index c0756f347..4ee508c04 100644 --- a/b2sdk/v1/__init__.py +++ b/b2sdk/v1/__init__.py @@ -15,7 +15,12 @@ from b2sdk.v1.api import B2Api from b2sdk.v1.bucket import Bucket, BucketFactory from b2sdk.v1.cache import AbstractCache +from b2sdk.v1.download_dest import ( + AbstractDownloadDestination, DownloadDestLocalFile, PreSeekedDownloadDest, DownloadDestBytes, + DownloadDestProgressWrapper +) from b2sdk.v1.exception import CommandError, DestFileNewer +from b2sdk.v1.file_metadata import FileMetadata from b2sdk.v1.file_version import FileVersionInfo from b2sdk.v1.session import B2Session from b2sdk.v1.sync import ( diff --git a/b2sdk/v1/api.py b/b2sdk/v1/api.py index 8b62f6025..70dbc45f8 100644 --- a/b2sdk/v1/api.py +++ b/b2sdk/v1/api.py @@ -8,8 +8,11 @@ # ###################################################################### +from typing import Optional, overload, Tuple + +from .download_dest import AbstractDownloadDestination from b2sdk import _v2 as v2 -from .bucket import Bucket, BucketFactory +from .bucket import Bucket, BucketFactory, download_file_and_return_info_dict from .file_version import FileVersionInfo, file_version_info_from_id_and_name from .session import B2Session @@ -18,6 +21,7 @@ # public API method # and to use v1.Bucket # and to retain cancel_large_file return type +# and to retain old style download_file_by_id signature (allowing for the new one as well) class B2Api(v2.B2Api): SESSION_CLASS = staticmethod(B2Session) BUCKET_FACTORY_CLASS = staticmethod(BucketFactory) @@ -48,3 +52,70 @@ def check_bucket_restrictions(self, bucket_name): def cancel_large_file(self, file_id: str) -> FileVersionInfo: file_id_and_name = super().cancel_large_file(file_id) return file_version_info_from_id_and_name(file_id_and_name, self) + + @overload + def download_file_by_id( + self, + file_id: str, + download_dest: AbstractDownloadDestination, + progress_listener: Optional[v2.AbstractProgressListener] = None, + range_: Optional[Tuple[int, int]] = None, + encryption: Optional[v2.EncryptionSetting] = None, + allow_seeking: bool = True, + ) -> dict: + ... + + @overload + def download_file_by_id( + self, + file_id: str, + progress_listener: Optional[v2.AbstractProgressListener] = None, + range_: Optional[Tuple[int, int]] = None, + encryption: Optional[v2.EncryptionSetting] = None, + allow_seeking: bool = True, + ) -> v2.DownloadedFile: + ... + + def download_file_by_id( + self, + file_id: str, + download_dest: Optional[AbstractDownloadDestination] = None, + progress_listener: Optional[v2.AbstractProgressListener] = None, + range_: Optional[Tuple[int, int]] = None, + encryption: Optional[v2.EncryptionSetting] = None, + allow_seeking: bool = True, + ): + """ + Download a file with the given ID. + + :param file_id: a file ID + :param download_dest: an instance of the one of the following classes: \ + :class:`~b2sdk.v1.DownloadDestLocalFile`,\ + :class:`~b2sdk.v1.DownloadDestBytes`,\ + :class:`~b2sdk.v1.PreSeekedDownloadDest`,\ + or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination` + :param progress_listener: an instance of the one of the following classes: \ + :class:`~b2sdk.v1.PartProgressReporter`,\ + :class:`~b2sdk.v1.TqdmProgressListener`,\ + :class:`~b2sdk.v1.SimpleProgressListener`,\ + :class:`~b2sdk.v1.DoNothingProgressListener`,\ + :class:`~b2sdk.v1.ProgressListenerForTest`,\ + :class:`~b2sdk.v1.SyncFileReporter`,\ + or any sub class of :class:`~b2sdk.v1.AbstractProgressListener` + :param range_: a list of two integers, the first one is a start\ + position, and the second one is the end position in the file + :param encryption: encryption settings (``None`` if unknown) + :param allow_seeking: if true, download strategies requiring seeking on the download destination will be + taken into account + """ + downloaded_file = super().download_file_by_id( + file_id=file_id, + progress_listener=progress_listener, + range_=range_, + encryption=encryption, + allow_seeking=allow_seeking, + ) + if download_dest is not None: + return download_file_and_return_info_dict(downloaded_file, download_dest, range_) + else: + return downloaded_file diff --git a/b2sdk/v1/bucket.py b/b2sdk/v1/bucket.py index 60bc51aa1..cb888307e 100644 --- a/b2sdk/v1/bucket.py +++ b/b2sdk/v1/bucket.py @@ -8,14 +8,18 @@ # ###################################################################### +from .download_dest import AbstractDownloadDestination +from .file_metadata import FileMetadata from .file_version import FileVersionInfoFactory -from typing import Optional +from typing import Optional, overload, Tuple from b2sdk import _v2 as v2 from b2sdk.utils import validate_b2_file_name # Overridden to retain the obsolete copy_file and start_large_file methods # and to return old style FILE_VERSION_FACTORY +# and to retain old style download_file_by_name signature +# and to retain old style download_file_by_id signature (allowing for the new one as well) class Bucket(v2.Bucket): FILE_VERSION_FACTORY = staticmethod(FileVersionInfoFactory) @@ -89,6 +93,118 @@ def start_large_file( legal_hold=legal_hold, ) + def download_file_by_name( + self, + file_name: str, + download_dest: AbstractDownloadDestination, + progress_listener: Optional[v2.AbstractProgressListener] = None, + range_: Optional[Tuple[int, int]] = None, + encryption: Optional[v2.EncryptionSetting] = None, + allow_seeking: bool = True, + ): + """ + Download a file by name. + + .. seealso:: + + :ref:`Synchronizer `, a *high-performance* utility that synchronizes a local folder with a Bucket. + + :param str file_name: a file name + :param download_dest: an instance of the one of the following classes: \ + :class:`~b2sdk.v1.DownloadDestLocalFile`,\ + :class:`~b2sdk.v1.DownloadDestBytes`,\ + :class:`~b2sdk.v1.PreSeekedDownloadDest`,\ + or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination` + :param progress_listener: a progress listener object to use, or ``None`` to not track progress + :param range_: two integer values, start and end offsets + :param encryption: encryption settings (``None`` if unknown) + """ + downloaded_file = super().download_file_by_name( + file_name=file_name, + progress_listener=progress_listener, + range_=range_, + encryption=encryption, + allow_seeking=allow_seeking + ) + return download_file_and_return_info_dict(downloaded_file, download_dest, range_) + + @overload + def download_file_by_id( + self, + file_id: str, + download_dest: AbstractDownloadDestination = None, + progress_listener: Optional[v2.AbstractProgressListener] = None, + range_: Optional[Tuple[int, int]] = None, + encryption: Optional[v2.EncryptionSetting] = None, + allow_seeking: bool = True, + ) -> dict: + ... + + @overload + def download_file_by_id( + self, + file_id: str, + progress_listener: Optional[v2.AbstractProgressListener] = None, + range_: Optional[Tuple[int, int]] = None, + encryption: Optional[v2.EncryptionSetting] = None, + allow_seeking: bool = True, + ) -> v2.DownloadedFile: + ... + + def download_file_by_id( + self, + file_id: str, + download_dest: Optional[AbstractDownloadDestination] = None, + progress_listener: Optional[v2.AbstractProgressListener] = None, + range_: Optional[Tuple[int, int]] = None, + encryption: Optional[v2.EncryptionSetting] = None, + allow_seeking: bool = True, + ): + """ + Download a file by ID. + + .. note:: + download_file_by_id actually belongs in :py:class:`b2sdk.v1.B2Api`, not in :py:class:`b2sdk.v1.Bucket`; we just provide a convenient redirect here + + :param file_id: a file ID + :param download_dest: an instance of the one of the following classes: \ + :class:`~b2sdk.v1.DownloadDestLocalFile`,\ + :class:`~b2sdk.v1.DownloadDestBytes`,\ + :class:`~b2sdk.v1.PreSeekedDownloadDest`,\ + or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination` + :param progress_listener: a progress listener object to use, or ``None`` to not report progress + :param range_: two integer values, start and end offsets + :param encryption: encryption settings (``None`` if unknown) + :param allow_seeking: if true, download strategies requiring seeking on the download destination will be + taken into account + """ + return self.api.download_file_by_id( + file_id, + download_dest, + progress_listener, + range_=range_, + encryption=encryption, + allow_seeking=allow_seeking, + ) + + +def download_file_and_return_info_dict( + downloaded_file: v2.DownloadedFile, download_dest: AbstractDownloadDestination, + range_: Optional[Tuple[int, int]] +): + with download_dest.make_file_context( + file_id=downloaded_file.file_version.id_, + file_name=downloaded_file.file_version.file_name, + content_length=downloaded_file.file_version.size, + content_type=downloaded_file.file_version.content_type, + content_sha1=downloaded_file.file_version.content_sha1, + file_info=downloaded_file.file_version.file_info, + mod_time_millis=downloaded_file.file_version.mod_time_millis, + range_=range_, + ) as file: + downloaded_file.save(file) + return FileMetadata.from_file_version(downloaded_file.file_version).as_info_dict() + class BucketFactory(v2.BucketFactory): BUCKET_CLASS = staticmethod(Bucket) diff --git a/b2sdk/download_dest.py b/b2sdk/v1/download_dest.py similarity index 98% rename from b2sdk/download_dest.py rename to b2sdk/v1/download_dest.py index 5cdfdf375..d9941274f 100644 --- a/b2sdk/download_dest.py +++ b/b2sdk/v1/download_dest.py @@ -1,6 +1,6 @@ ###################################################################### # -# File: b2sdk/download_dest.py +# File: b2sdk/v1/download_dest.py # # Copyright 2019 Backblaze Inc. All Rights Reserved. # @@ -15,7 +15,7 @@ from b2sdk.stream.progress import WritingStreamWithProgress -from .utils import B2TraceMetaAbstract, limit_trace_arguments, set_file_mtime +from ..utils import B2TraceMetaAbstract, limit_trace_arguments, set_file_mtime class AbstractDownloadDestination(metaclass=B2TraceMetaAbstract): diff --git a/b2sdk/transfer/inbound/file_metadata.py b/b2sdk/v1/file_metadata.py similarity index 70% rename from b2sdk/transfer/inbound/file_metadata.py rename to b2sdk/v1/file_metadata.py index 1bac90eaf..8a71da6cf 100644 --- a/b2sdk/transfer/inbound/file_metadata.py +++ b/b2sdk/v1/file_metadata.py @@ -1,13 +1,15 @@ ###################################################################### # -# File: b2sdk/transfer/inbound/file_metadata.py +# File: b2sdk/v1/file_metadata.py # -# Copyright 2020 Backblaze Inc. All Rights Reserved. +# Copyright 2021 Backblaze Inc. All Rights Reserved. # # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +from ..file_version import FileVersion + class FileMetadata(object): """ @@ -15,16 +17,6 @@ class FileMetadata(object): """ UNVERIFIED_CHECKSUM_PREFIX = 'unverified:' - __slots__ = ( - 'file_id', - 'file_name', - 'content_type', - 'content_length', - 'content_sha1', - 'content_sha1_verified', - 'file_info', - ) - def __init__( self, file_id, @@ -41,18 +33,6 @@ def __init__( self.content_sha1, self.content_sha1_verified = self._decode_content_sha1(content_sha1) self.file_info = file_info - @classmethod - def from_response(cls, response): - info = response.headers - return cls( - file_id=info['x-bz-file-id'], - file_name=info['x-bz-file-name'], - content_type=info['content-type'], - content_length=int(info['content-length']), - content_sha1=info['x-bz-content-sha1'], - file_info=dict((k[10:], info[k]) for k in info if k.startswith('x-bz-info-')), - ) - def as_info_dict(self): return { 'fileId': self.file_id, @@ -74,3 +54,14 @@ def _encode_content_sha1(cls, content_sha1, content_sha1_verified): if not content_sha1_verified: return '%s%s' % (cls.UNVERIFIED_CHECKSUM_PREFIX, content_sha1) return content_sha1 + + @classmethod + def from_file_version(cls, file_version: FileVersion): + return cls( + file_id=file_version.id_, + file_name=file_version.file_name, + content_type=file_version.content_type, + content_length=file_version.size, + content_sha1=file_version.content_sha1, + file_info=file_version.file_info, + ) diff --git a/b2sdk/v1/file_version.py b/b2sdk/v1/file_version.py index 27d2b54a3..962f653d7 100644 --- a/b2sdk/v1/file_version.py +++ b/b2sdk/v1/file_version.py @@ -55,7 +55,7 @@ def __init__( ): self.id_ = id_ self.file_name = file_name - self.size = size + self.size = size and int(size) self.content_type = content_type self.content_sha1 = content_sha1 self.content_md5 = content_md5 @@ -70,7 +70,7 @@ def __init__( if SRC_LAST_MODIFIED_MILLIS in self.file_info: self.mod_time_millis = int(self.file_info[SRC_LAST_MODIFIED_MILLIS]) else: - self.mod_time_millis = self.upload_timestamp + self.mod_time_millis = self.upload_timestamp and int(self.upload_timestamp) @property def api(self): diff --git a/doc/source/api/download_dest.rst b/doc/source/api/download_dest.rst deleted file mode 100644 index 9e4d37f90..000000000 --- a/doc/source/api/download_dest.rst +++ /dev/null @@ -1,24 +0,0 @@ -Download destination -================================================== - -.. note:: - Concrete classes described in this chapter implement methods defined in ``AbstractDownloadDestination`` - -.. autoclass:: b2sdk.v1.AbstractDownloadDestination - :members: - -.. autoclass:: b2sdk.v1.DownloadDestLocalFile() - :no-members: - :special-members: __init__ - -.. autoclass:: b2sdk.v1.DownloadDestBytes - :no-members: - :special-members: __init__ - -.. autoclass:: b2sdk.v1.DownloadDestProgressWrapper() - :no-members: - :special-members: __init__ - -.. autoclass:: b2sdk.v1.PreSeekedDownloadDest() - :no-members: - :special-members: __init__ diff --git a/doc/source/api/downloaded_file.rst b/doc/source/api/downloaded_file.rst new file mode 100644 index 000000000..7bb6ca7b0 --- /dev/null +++ b/doc/source/api/downloaded_file.rst @@ -0,0 +1,4 @@ +Downloaded File +=============== + +.. automodule:: b2sdk.transfer.inbound.downloaded_file diff --git a/doc/source/api/internal/download_dest.rst b/doc/source/api/internal/download_dest.rst deleted file mode 100644 index 7c9f663a0..000000000 --- a/doc/source/api/internal/download_dest.rst +++ /dev/null @@ -1,8 +0,0 @@ -:mod:`b2sdk.download_dest` -- Download destination -================================================== - -.. automodule:: b2sdk.download_dest - :members: - :undoc-members: - :show-inheritance: - :special-members: __init__ diff --git a/doc/source/api/internal/transfer/inbound/file_metadata.rst b/doc/source/api/internal/transfer/inbound/file_metadata.rst index fcd64b9d5..56eed0f4e 100644 --- a/doc/source/api/internal/transfer/inbound/file_metadata.rst +++ b/doc/source/api/internal/transfer/inbound/file_metadata.rst @@ -1,7 +1,7 @@ -:mod:`b2sdk.transfer.inbound.file_metadata` -=========================================== +:mod:`b2sdk.v1.file_metadata` +============================= -.. automodule:: b2sdk.transfer.inbound.file_metadata +.. automodule:: b2sdk.v1.file_metadata :members: :undoc-members: :show-inheritance: diff --git a/doc/source/api_reference.rst b/doc/source/api_reference.rst index 70c4cfae1..bc9af9695 100644 --- a/doc/source/api_reference.rst +++ b/doc/source/api_reference.rst @@ -27,13 +27,13 @@ Public API api/bucket api/file_lock api/data_classes + api/downloaded_file api/enums api/progress api/sync api/utils api/transfer/emerge/write_intent api/transfer/outbound/outbound_source - api/download_dest api/encryption/setting api/encryption/types @@ -50,7 +50,6 @@ Internal API api/internal/b2http api/internal/utils api/internal/cache - api/internal/download_dest api/internal/stream/chained api/internal/stream/hashing api/internal/stream/progress diff --git a/doc/source/quick_start.rst b/doc/source/quick_start.rst index 77d10983c..f262b417a 100644 --- a/doc/source/quick_start.rst +++ b/doc/source/quick_start.rst @@ -210,33 +210,28 @@ By id .. code-block:: python - >>> from b2sdk.v1 import DownloadDestLocalFile >>> from b2sdk.v1 import DoNothingProgressListener >>> local_file_path = '/home/user1/b2_example/new2.pdf' >>> file_id = '4_z5485a1682662eb3e60980d10_f1195145f42952533_d20190403_m130258_c002_v0001111_t0002' - >>> download_dest = DownloadDestLocalFile(local_file_path) >>> progress_listener = DoNothingProgressListener() - >>> b2_api.download_file_by_id(file_id, download_dest, progress_listener) - {'fileId': '4_z5485a1682662eb3e60980d10_f1195145f42952533_d20190403_m130258_c002_v0001111_t0002', - 'fileName': 'som2.pdf', - 'contentType': 'application/pdf', - 'contentLength': 1870579, - 'contentSha1': 'd821849a70922e87c2b0786c0be7266b89d87df0', - 'fileInfo': {'src_last_modified_millis': '1550988084299'}} + >>> downloaded_file = b2_api.download_file_by_id(file_id, progress_listener) # only the headers + # and the beginning of the file is downloaded at this stage - >>> print('File name: ', download_dest.file_name) + >>> print('File name: ', downloaded_file.file_version.file_name) File name: som2.pdf - >>> print('File id: ', download_dest.file_id) + >>> print('File id: ', downloaded_file.file_version.id_) File id: 4_z5485a1682662eb3e60980d10_f1195145f42952533_d20190403_m130258_c002_v0001111_t0002 - >>> print('File size: ', download_dest.content_length) + >>> print('File size: ', downloaded_file.file_version.size) File size: 1870579 - >>> print('Content type:', download_dest.content_type) + >>> print('Content type:', downloaded_file.file_version.content_type) Content type: application/pdf - >>> print('Content sha1:', download_dest.content_sha1) + >>> print('Content sha1:', downloaded_file.file_version.content_sha1) Content sha1: d821849a70922e87c2b0786c0be7266b89d87df0 + >>> downloaded_file.save_to(local_file_path) # this downloads the whole file + By name ------- @@ -245,14 +240,8 @@ By name >>> bucket = b2_api.get_bucket_by_name(bucket_name) >>> b2_file_name = 'dummy_new.pdf' >>> local_file_name = '/home/user1/b2_example/new3.pdf' - >>> download_dest = DownloadDestLocalFile(local_file_name) - >>> bucket.download_file_by_name(b2_file_name, download_dest) - {'fileId': '4_z5485a1682662eb3e60980d10_f113f963288e711a6_d20190404_m065910_c002_v0001095_t0044', - 'fileName': 'dummy_new.pdf', - 'contentType': 'application/pdf', - 'contentLength': 1870579, - 'contentSha1': 'd821849a70922e87c2b0786c0be7266b89d87df0', - 'fileInfo': {'how': 'good-file'}} + >>> downloaded_file = bucket.download_file_by_name(b2_file_name) + >>> downloaded_file.save_to(local_file_path) Downloading encrypted files diff --git a/test/unit/bucket/test_bucket.py b/test/unit/bucket/test_bucket.py index d91b83750..48ae2825d 100644 --- a/test/unit/bucket/test_bucket.py +++ b/test/unit/bucket/test_bucket.py @@ -7,7 +7,7 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### - +import io from io import BytesIO import os import platform @@ -17,6 +17,7 @@ from ..test_base import TestBase +import apiver_deps from apiver_deps_exception import ( AlreadyFailed, B2Error, @@ -30,9 +31,13 @@ FileSha1Mismatch, SSECKeyError, ) +if apiver_deps.V <= 1: + from apiver_deps import DownloadDestBytes, PreSeekedDownloadDest +else: + DownloadDestBytes, PreSeekedDownloadDest = None, None # these classes are not present, thus not needed, in v2 from apiver_deps import B2Api +from apiver_deps import DownloadedFile from apiver_deps import LargeFileUploadState -from apiver_deps import DownloadDestBytes, PreSeekedDownloadDest from apiver_deps import MetadataDirectiveMode from apiver_deps import Part from apiver_deps import AbstractProgressListener @@ -44,7 +49,6 @@ from apiver_deps import EncryptionAlgorithm, EncryptionSetting, EncryptionMode, EncryptionKey, SSE_NONE, SSE_B2_AES from apiver_deps import CopySource, UploadSourceLocalFile, WriteIntent from apiver_deps import FileRetentionSetting, LegalHold, RetentionMode, NO_RETENTION_FILE_SETTING -import apiver_deps if apiver_deps.V <= 1: from apiver_deps import FileVersionInfo as VFileVersionInfo else: @@ -308,7 +312,7 @@ def test_version_by_name(self): self.assertIsInstance(info, VFileVersionInfo) expected = ( - a_id, 'a', 11, None, 'b2/x-auto', 'none', NO_RETENTION_FILE_SETTING, LegalHold.UNSET + a_id, 'a', 11, 'upload', 'b2/x-auto', 'none', NO_RETENTION_FILE_SETTING, LegalHold.UNSET ) actual = ( info.id_, @@ -546,18 +550,26 @@ def test_encryption(self): actual = [info.server_side_encryption for info in self.bucket.list_file_versions('a')][0] self.assertEqual(SSE_NONE, actual) # bucket default + actual = self.bucket.get_file_info_by_name('a').server_side_encryption + self.assertEqual(SSE_NONE, actual) # bucket default actual = [info.server_side_encryption for info in self.bucket.list_file_versions('b')][0] self.assertEqual(SSE_B2_AES, actual) # explicitly requested sse-b2 + actual = self.bucket.get_file_info_by_name('b').server_side_encryption + self.assertEqual(SSE_B2_AES, actual) # explicitly requested sse-b2 # actual = [info.server_side_encryption for info in self.bucket.list_file_versions('c')][0] # self.assertEqual(SSE_NONE, actual) # explicitly requested none actual = [info.server_side_encryption for info in self.bucket.list_file_versions('d')][0] self.assertEqual(SSE_B2_AES, actual) # explicitly requested sse-b2 + actual = self.bucket.get_file_info_by_name('d').server_side_encryption + self.assertEqual(SSE_B2_AES, actual) # explicitly requested sse-b2 actual = [info.server_side_encryption for info in self.bucket.list_file_versions('e')][0] self.assertEqual(SSE_C_AES_NO_SECRET, actual) # explicitly requested sse-c + actual = self.bucket.get_file_info_by_name('e').server_side_encryption + self.assertEqual(SSE_C_AES_NO_SECRET, actual) # explicitly requested sse-c class TestCopyFile(TestCaseWithBucket): @@ -1075,10 +1087,20 @@ def _upload_part(self, large_file_id, part_number, part_data): ) def _check_file_contents(self, file_name, expected_contents): - download = DownloadDestBytes() + contents = self._download_file(file_name) + self.assertEqual(expected_contents, contents) + + def _download_file(self, file_name): with FileSimulator.dont_check_encryption(): - self.bucket.download_file_by_name(file_name, download) - self.assertEqual(expected_contents, download.get_bytes_written()) + if apiver_deps.V <= 1: + download = DownloadDestBytes() + self.bucket.download_file_by_name(file_name, download) + return download.get_bytes_written() + else: + with io.BytesIO() as bytes_io: + downloaded_file = self.bucket.download_file_by_name(file_name) + downloaded_file.save(bytes_io) + return bytes_io.getvalue() class TestConcatenate(TestCaseWithBucket): @@ -1164,23 +1186,25 @@ def _create_remote(self, sources, file_name, encryption=None): ) -# Downloads - - class DownloadTestsBase(object): DATA = NotImplemented def setUp(self): super(DownloadTestsBase, self).setUp() - self.file_info = self.bucket.upload_bytes(self.DATA.encode(), 'file1') - self.encrypted_file_info = self.bucket.upload_bytes( + self.file_version = self.bucket.upload_bytes(self.DATA.encode(), 'file1') + self.encrypted_file_version = self.bucket.upload_bytes( self.DATA.encode(), 'enc_file1', encryption=SSE_C_AES ) - self.download_dest = DownloadDestBytes() + if apiver_deps.V <= 1: + self.download_dest = DownloadDestBytes() + self.bytes_io = None + else: + self.download_dest = None + self.bytes_io = io.BytesIO() self.progress_listener = StubProgressListener() def _verify(self, expected_result, check_progress_listener=True): - assert self.download_dest.get_bytes_written() == expected_result.encode() + self._assert_downloaded_data(expected_result) if check_progress_listener: valid, reason = self.progress_listener.is_valid_reason( check_closed=False, @@ -1189,45 +1213,96 @@ def _verify(self, expected_result, check_progress_listener=True): ) assert valid, reason + def _assert_downloaded_data(self, expected_result): + if apiver_deps.V <= 1: + assert self.download_dest.get_bytes_written() == expected_result.encode() + else: + assert self.bytes_io.getvalue() == expected_result.encode() + + def download_file_by_id(self, file_id, v1_download_dest=None, v2_file=None, **kwargs): + if apiver_deps.V <= 1: + self.bucket.download_file_by_id( + file_id, v1_download_dest or self.download_dest, **kwargs + ) + else: + self.bucket.download_file_by_id(file_id, **kwargs).save(v2_file or self.bytes_io) + + def download_file_by_name(self, file_id, download_dest=None, **kwargs): + if apiver_deps.V <= 1: + self.bucket.download_file_by_name( + file_id, download_dest or self.download_dest, **kwargs + ) + else: + self.bucket.download_file_by_name(file_id, **kwargs).save(self.bytes_io) + class DownloadTests(DownloadTestsBase): DATA = 'abcdefghijklmnopqrs' + @pytest.mark.apiver(from_ver=2) + def test_v2_return_types(self): + download_kwargs = { + 'range_': (7, 18), + 'encryption': SSE_C_AES, + 'progress_listener': self.progress_listener, + } + file_version = self.bucket.upload_bytes( + self.DATA.encode(), 'enc_file2', encryption=SSE_C_AES + ) + file_version.size = 12 # we're only downloading a part of the file + other_properties = { + 'file_version': file_version, + } + ret = self.bucket.download_file_by_id(file_version.id_, **download_kwargs) + assert isinstance(ret, DownloadedFile), type(ret) + for attr_name, expected_value in {**download_kwargs, **other_properties}.items(): + assert getattr(ret, attr_name) == expected_value, attr_name + + ret = self.bucket.download_file_by_name(file_version.file_name, **download_kwargs) + assert isinstance(ret, DownloadedFile), type(ret) + for attr_name, expected_value in {**download_kwargs, **other_properties}.items(): + assert getattr(ret, attr_name) == expected_value, attr_name + + @pytest.mark.apiver(to_ver=1) + def test_v1_return_types(self): + expected = { + 'contentLength': 19, + 'contentSha1': '893e69ff0109f3459c4243013b3de8b12b41a30e', + 'contentType': 'b2/x-auto', + 'fileId': '9999', + 'fileInfo': {}, + 'fileName': 'file1' + } + ret = self.bucket.download_file_by_id(self.file_version.id_, self.download_dest) + assert ret == expected + ret = self.bucket.download_file_by_name(self.file_version.file_name, self.download_dest) + assert ret == expected + def test_download_by_id_no_progress(self): - self.bucket.download_file_by_id(self.file_info.id_, self.download_dest) + self.download_file_by_id(self.file_version.id_) self._verify(self.DATA, check_progress_listener=False) def test_download_by_name_no_progress(self): - self.bucket.download_file_by_name('file1', self.download_dest) + self.download_file_by_name('file1') self._verify(self.DATA, check_progress_listener=False) def test_download_by_name_progress(self): - self.bucket.download_file_by_name( - 'file1', self.download_dest, progress_listener=self.progress_listener - ) + self.download_file_by_name('file1', progress_listener=self.progress_listener) self._verify(self.DATA) def test_download_by_id_progress(self): - self.bucket.download_file_by_id( - self.file_info.id_, self.download_dest, progress_listener=self.progress_listener - ) + self.download_file_by_id(self.file_version.id_, progress_listener=self.progress_listener) self._verify(self.DATA) def test_download_by_id_progress_partial(self): - self.bucket.download_file_by_id( - self.file_info.id_, - self.download_dest, - progress_listener=self.progress_listener, - range_=(3, 9) + self.download_file_by_id( + self.file_version.id_, progress_listener=self.progress_listener, range_=(3, 9) ) self._verify('defghij') def test_download_by_id_progress_exact_range(self): - self.bucket.download_file_by_id( - self.file_info.id_, - self.download_dest, - progress_listener=self.progress_listener, - range_=(0, 18) + self.download_file_by_id( + self.file_version.id_, progress_listener=self.progress_listener, range_=(0, 18) ) self._verify(self.DATA) @@ -1236,14 +1311,14 @@ def test_download_by_id_progress_range_one_off(self): InvalidRange, msg='A range of 0-19 was requested (size of 20), but cloud could only serve 19 of that', ): - self.bucket.download_file_by_id( - self.file_info.id_, - self.download_dest, - self.progress_listener, + self.download_file_by_id( + self.file_version.id_, + progress_listener=self.progress_listener, range_=(0, 19), ) - def test_download_by_id_progress_partial_inplace_overwrite(self): + @pytest.mark.apiver(to_ver=1) + def test_download_by_id_progress_partial_inplace_overwrite_v1(self): # LOCAL is # 12345678901234567890 # @@ -1261,15 +1336,54 @@ def test_download_by_id_progress_partial_inplace_overwrite(self): download_dest = PreSeekedDownloadDest(seek_target=3, local_file_path=path) data = b'12345678901234567890' write_file(path, data) - self.bucket.download_file_by_id( - self.file_info.id_, + self.download_file_by_id( + self.file_version.id_, download_dest, progress_listener=self.progress_listener, range_=(3, 9), ) self._check_local_file_contents(path, b'123defghij1234567890') - def test_download_by_id_progress_partial_shifted_overwrite(self): + @pytest.mark.apiver(from_ver=2) + def test_download_by_id_progress_partial_inplace_overwrite_v2(self): + # LOCAL is + # 12345678901234567890 + # + # and then: + # + # abcdefghijklmnopqrs + # ||||||| + # ||||||| + # vvvvvvv + # + # 123defghij1234567890 + + with TempDir() as d: + path = os.path.join(d, 'file2') + data = b'12345678901234567890' + write_file(path, data) + with io.open(path, 'rb+') as file: + file.seek(3) + self.download_file_by_id( + self.file_version.id_, + v2_file=file, + progress_listener=self.progress_listener, + range_=(3, 9), + ) + self._check_local_file_contents(path, b'123defghij1234567890') + + @pytest.mark.apiver(from_ver=2) + def test_download_update_mtime_v2(self): + with TempDir() as d: + file_version = self.bucket.upload_bytes( + self.DATA.encode(), 'file1', file_infos={'src_last_modified_millis': '1000'} + ) + path = os.path.join(d, 'file2') + self.bucket.download_file_by_id(file_version.id_).save_to(path) + assert pytest.approx(1, rel=0.001) == os.path.getmtime(path) + + @pytest.mark.apiver(to_ver=1) + def test_download_by_id_progress_partial_shifted_overwrite_v1(self): # LOCAL is # 12345678901234567890 # @@ -1292,25 +1406,54 @@ def test_download_by_id_progress_partial_shifted_overwrite(self): download_dest = PreSeekedDownloadDest(seek_target=7, local_file_path=path) data = b'12345678901234567890' write_file(path, data) - self.bucket.download_file_by_id( - self.file_info.id_, + self.download_file_by_id( + self.file_version.id_, download_dest, progress_listener=self.progress_listener, range_=(3, 9), ) self._check_local_file_contents(path, b'1234567defghij567890') + @pytest.mark.apiver(from_ver=2) + def test_download_by_id_progress_partial_shifted_overwrite_v2(self): + # LOCAL is + # 12345678901234567890 + # + # and then: + # + # abcdefghijklmnopqrs + # ||||||| + # \\\\\\\ + # \\\\\\\ + # \\\\\\\ + # \\\\\\\ + # \\\\\\\ + # ||||||| + # vvvvvvv + # + # 1234567defghij567890 + + with TempDir() as d: + path = os.path.join(d, 'file2') + data = b'12345678901234567890' + write_file(path, data) + with io.open(path, 'rb+') as file: + file.seek(7) + self.download_file_by_id( + self.file_version.id_, + v2_file=file, + progress_listener=self.progress_listener, + range_=(3, 9), + ) + self._check_local_file_contents(path, b'1234567defghij567890') + def test_download_by_id_no_progress_encryption(self): - self.bucket.download_file_by_id( - self.encrypted_file_info.id_, self.download_dest, encryption=SSE_C_AES - ) + self.download_file_by_id(self.encrypted_file_version.id_, encryption=SSE_C_AES) self._verify(self.DATA, check_progress_listener=False) def test_download_by_id_no_progress_wrong_encryption(self): with self.assertRaises(SSECKeyError): - self.bucket.download_file_by_id( - self.encrypted_file_info.id_, self.download_dest, encryption=SSE_C_AES_2 - ) + self.download_file_by_id(self.encrypted_file_version.id_, encryption=SSE_C_AES_2) def _check_local_file_contents(self, path, expected_contents): with open(path, 'rb') as f: @@ -1325,13 +1468,49 @@ class EmptyFileDownloadScenarioMixin(object): """ use with DownloadTests, but not for TestDownloadParallel as it does not like empty files """ def test_download_by_name_empty_file(self): - self.file_info = self.bucket.upload_bytes(b'', 'empty') - self.bucket.download_file_by_name('empty', self.download_dest, self.progress_listener) + self.file_version = self.bucket.upload_bytes(b'', 'empty') + self.download_file_by_name('empty', progress_listener=self.progress_listener) self._verify('') # actual tests +# test choosing strategy + + +@pytest.mark.apiver(from_ver=2) +class TestChooseStrategy(TestCaseWithBucket): + def test_choose_strategy(self): + file_version = self.bucket.upload_bytes(b'hello world' * 8, 'file1') + parallel_downloader = ParallelDownloader( + force_chunk_size=1, + max_streams=32, + min_part_size=16, + ) + simple_downloader = self.bucket.api.services.download_manager.strategies[1] + self.bucket.api.services.download_manager.strategies = [ + parallel_downloader, + simple_downloader, + ] + downloaded_file = self.bucket.download_file_by_id(file_version.id_, allow_seeking=True) + assert downloaded_file.strategy == parallel_downloader + + downloaded_file = self.bucket.download_file_by_id(file_version.id_, allow_seeking=False) + assert downloaded_file.strategy == simple_downloader + + downloaded_file = self.bucket.download_file_by_name( + file_version.file_name, allow_seeking=True + ) + assert downloaded_file.strategy == parallel_downloader + + downloaded_file = self.bucket.download_file_by_name( + file_version.file_name, allow_seeking=False + ) + assert downloaded_file.strategy == simple_downloader + + +# Default tests + class TestDownloadDefault(DownloadTests, EmptyFileDownloadScenarioMixin, TestCaseWithBucket): pass @@ -1374,9 +1553,7 @@ def setUp(self): ] def test_download_by_id_progress_monotonic(self): - self.bucket.download_file_by_id( - self.file_info.id_, self.download_dest, self.progress_listener - ) + self.download_file_by_id(self.file_version.id_, progress_listener=self.progress_listener) self._verify(self.DATA) diff --git a/test/unit/sync/test_sync.py b/test/unit/sync/test_sync.py index cdae583bd..6af856675 100644 --- a/test/unit/sync/test_sync.py +++ b/test/unit/sync/test_sync.py @@ -723,9 +723,9 @@ def test_encryption_b2_to_local(self, synchronizer_factory, apiver): except: pass - assert bucket.mock_calls == [ - mock.call.download_file_by_id('id_d_100', mock.ANY, mock.ANY, encryption=encryption), - ] + assert bucket.mock_calls[0] == mock.call.download_file_by_id( + 'id_d_100', progress_listener=mock.ANY, encryption=encryption + ) if apiver in ['v0', 'v1']: file_version_kwarg = 'file_version_info' diff --git a/test/unit/v0/test_bucket.py b/test/unit/v0/test_bucket.py index 3321b1e3c..f972b07f6 100644 --- a/test/unit/v0/test_bucket.py +++ b/test/unit/v0/test_bucket.py @@ -283,7 +283,7 @@ def test_version_by_name(self): self.assertIsInstance(info, FileVersionInfo) expected = ( - a_id, 'a', 11, None, 'b2/x-auto', 'none', NO_RETENTION_FILE_SETTING, LegalHold.UNSET + a_id, 'a', 11, 'upload', 'b2/x-auto', 'none', NO_RETENTION_FILE_SETTING, LegalHold.UNSET ) actual = ( info.id_,