Apiver2docs

CHANGELOG.md

@@ -19,9 +19,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * `FileVersion` has a mandatory `api` parameter 
 * `B2Folder` holds a handle to B2Api 
 * `Bucket` unit tests for v1 and v2 are now common
+* `download_file_*` methods refactored to allow for inspecting FileVersion before downloading the whole file
 
 ### Fixed
 * Fix call to incorrect internal api in `B2Api.get_download_url_for_file_name`
+* Fix EncryptionSetting.from_response_headers
+* Fix FileVersion.size and FileVersion.mod_time_millis type ambiguity
 
 ## [1.8.0] - 2021-05-21
 

b2sdk/_v2/__init__.py

@@ -116,11 +116,7 @@
 
 # source / destination
 
-from b2sdk.download_dest import AbstractDownloadDestination
-from b2sdk.download_dest import DownloadDestBytes
-from b2sdk.download_dest import DownloadDestLocalFile
-from b2sdk.download_dest import DownloadDestProgressWrapper
-from b2sdk.download_dest import PreSeekedDownloadDest
+from b2sdk.transfer.inbound.downloaded_file import DownloadedFile
 
 from b2sdk.transfer.outbound.outbound_source import OutboundTransferSource
 from b2sdk.transfer.outbound.copy_source import CopySource
@@ -136,7 +132,6 @@
 # trasfer
 
 from b2sdk.transfer.inbound.downloader.abstract import AbstractDownloader
-from b2sdk.transfer.inbound.file_metadata import FileMetadata
 from b2sdk.transfer.outbound.large_file_upload_state import LargeFileUploadState
 from b2sdk.transfer.inbound.downloader.parallel import AbstractDownloaderThread
 from b2sdk.transfer.inbound.downloader.parallel import FirstPartDownloaderThread

b2sdk/api.py

@@ -8,7 +8,7 @@
 #
 ######################################################################
 
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, Tuple
 
 from .bucket import Bucket, BucketFactory
 from .encryption.setting import EncryptionSetting
@@ -17,13 +17,15 @@
 from .file_version import FileIdAndName
 from .large_file.services import LargeFileServices
 from .raw_api import API_VERSION
+from .progress import AbstractProgressListener
 from .session import B2Session
 from .transfer import (
     CopyManager,
     DownloadManager,
     Emerger,
     UploadManager,
 )
+from .transfer.inbound.downloaded_file import DownloadedFile
 from .utils import B2TraceMeta, b2_url_encode, limit_trace_arguments
 
 
@@ -223,42 +225,30 @@ def create_bucket(
 
     def download_file_by_id(
         self,
-        file_id,
-        download_dest,
-        progress_listener=None,
-        range_=None,
+        file_id: str,
+        progress_listener: Optional[AbstractProgressListener] = None,
+        range_: Optional[Tuple[int, int]] = None,
         encryption: Optional[EncryptionSetting] = None,
-    ):
+        allow_seeking: bool = True,
+    ) -> DownloadedFile:
         """
         Download a file with the given ID.
 
         :param str file_id: a file ID
-        :param download_dest: an instance of the one of the following classes: \
-        :class:`~b2sdk.v1.DownloadDestLocalFile`,\
-        :class:`~b2sdk.v1.DownloadDestBytes`,\
-        :class:`~b2sdk.v1.DownloadDestProgressWrapper`,\
-        :class:`~b2sdk.v1.PreSeekedDownloadDest`,\
-        or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination`
-        :param progress_listener: an instance of the one of the following classes: \
-        :class:`~b2sdk.v1.PartProgressReporter`,\
-        :class:`~b2sdk.v1.TqdmProgressListener`,\
-        :class:`~b2sdk.v1.SimpleProgressListener`,\
-        :class:`~b2sdk.v1.DoNothingProgressListener`,\
-        :class:`~b2sdk.v1.ProgressListenerForTest`,\
-        :class:`~b2sdk.v1.SyncFileReporter`,\
-        or any sub class of :class:`~b2sdk.v1.AbstractProgressListener`
-        :param list range_: a list of two integers, the first one is a start\
+        :param progress_listener: a progress listener object to use, or ``None`` to not track progress
+        :param range_: a list of two integers, the first one is a start\
         position, and the second one is the end position in the file
-        :param b2sdk.v1.EncryptionSetting encryption: encryption settings (``None`` if unknown)
-        :return: context manager that returns an object that supports iter_content()
+        :param encryption: encryption settings (``None`` if unknown)
+        :param allow_seeking: if true, download strategies requiring seeking on the download destination will be
+                              taken into account
         """
         url = self.session.get_download_url_by_id(file_id)
         return self.services.download_manager.download_file_from_url(
             url,
-            download_dest,
             progress_listener,
             range_,
             encryption,
+            allow_seeking,
         )
 
     def update_file_retention(

b2sdk/bucket.py

@@ -9,7 +9,7 @@
 ######################################################################
 
 import logging
-from typing import Optional
+from typing import Optional, Tuple
 
 from .encryption.setting import EncryptionSetting, EncryptionSettingFactory
 from .encryption.types import EncryptionMode
@@ -22,9 +22,10 @@
     LegalHold,
 )
 from .file_version import FileVersion, FileVersionFactory
-from .progress import DoNothingProgressListener
+from .progress import AbstractProgressListener, DoNothingProgressListener
 from .transfer.emerge.executor import AUTO_CONTENT_TYPE
 from .transfer.emerge.write_intent import WriteIntent
+from .transfer.inbound.downloaded_file import DownloadedFile
 from .transfer.outbound.copy_source import CopySource
 from .transfer.outbound.upload_source import UploadSourceBytes, UploadSourceLocalFile
 from .utils import B2TraceMeta, disable_trace, limit_trace_arguments
@@ -158,70 +159,62 @@ def cancel_large_file(self, file_id):
 
     def download_file_by_id(
         self,
-        file_id,
-        download_dest,
-        progress_listener=None,
-        range_=None,
+        file_id: str,
+        progress_listener: Optional[AbstractProgressListener] = None,
+        range_: Optional[Tuple[int, int]] = None,
         encryption: Optional[EncryptionSetting] = None,
-    ):
+        allow_seeking: bool = True,
+    ) -> DownloadedFile:
         """
         Download a file by ID.
 
         .. note::
           download_file_by_id actually belongs in :py:class:`b2sdk.v1.B2Api`, not in :py:class:`b2sdk.v1.Bucket`; we just provide a convenient redirect here
 
-        :param str file_id: a file ID
-        :param download_dest: an instance of the one of the following classes: \
-        :class:`~b2sdk.v1.DownloadDestLocalFile`,\
-        :class:`~b2sdk.v1.DownloadDestBytes`,\
-        :class:`~b2sdk.v1.DownloadDestProgressWrapper`,\
-        :class:`~b2sdk.v1.PreSeekedDownloadDest`,\
-        or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination`
-        :param b2sdk.v1.AbstractProgressListener, None progress_listener: a progress listener object to use, or ``None`` to not report progress
-        :param tuple[int, int] range_: two integer values, start and end offsets
-        :param b2sdk.v1.EncryptionSetting encryption: encryption settings (``None`` if unknown)
+        :param file_id: a file ID
+        :param progress_listener: a progress listener object to use, or ``None`` to not track progress
+        :param range_: two integer values, start and end offsets
+        :param encryption: encryption settings (``None`` if unknown)
+        :param allow_seeking: if true, download strategies requiring seeking on the download destination will be
+                              taken into account
         """
         return self.api.download_file_by_id(
             file_id,
-            download_dest,
             progress_listener,
             range_=range_,
             encryption=encryption,
+            allow_seeking=allow_seeking,
         )
 
     def download_file_by_name(
         self,
-        file_name,
-        download_dest,
-        progress_listener=None,
-        range_=None,
+        file_name: str,
+        progress_listener: Optional[AbstractProgressListener] = None,
+        range_: Optional[Tuple[int, int]] = None,
         encryption: Optional[EncryptionSetting] = None,
-    ):
+        allow_seeking: bool = True,
+    ) -> DownloadedFile:
         """
         Download a file by name.
 
         .. seealso::
 
             :ref:`Synchronizer <sync>`, a *high-performance* utility that synchronizes a local folder with a Bucket.
 
-        :param str file_name: a file name
-        :param download_dest: an instance of the one of the following classes: \
-        :class:`~b2sdk.v1.DownloadDestLocalFile`,\
-        :class:`~b2sdk.v1.DownloadDestBytes`,\
-        :class:`~b2sdk.v1.DownloadDestProgressWrapper`,\
-        :class:`~b2sdk.v1.PreSeekedDownloadDest`,\
-        or any sub class of :class:`~b2sdk.v1.AbstractDownloadDestination`
-        :param b2sdk.v1.AbstractProgressListener, None progress_listener: a progress listener object to use, or ``None`` to not track progress
-        :param tuple[int, int] range_: two integer values, start and end offsets
-        :param b2sdk.v1.EncryptionSetting encryption: encryption settings (``None`` if unknown)
+        :param file_name: a file name
+        :param progress_listener: a progress listener object to use, or ``None`` to not track progress
+        :param range_: two integer values, start and end offsets
+        :param encryption: encryption settings (``None`` if unknown)
+        :param allow_seeking: if true, download strategies requiring seeking on the download destination will be
+                              taken into account
         """
         url = self.api.session.get_download_url_by_name(self.name, file_name)
         return self.api.services.download_manager.download_file_from_url(
             url,
-            download_dest,
             progress_listener,
             range_,
             encryption=encryption,
+            allow_seeking=allow_seeking,
         )
 
     def get_file_info_by_id(self, file_id: str) -> FileVersion:

b2sdk/encryption/setting.py

@@ -16,8 +16,10 @@
 from ..utils import b64_of_bytes, md5_of_bytes
 from .types import ENCRYPTION_MODES_WITH_MANDATORY_ALGORITHM, ENCRYPTION_MODES_WITH_MANDATORY_KEY
 from .types import EncryptionAlgorithm, EncryptionMode
+from ..utils import FILE_INFO_HEADER_PREFIX
 
 SSE_C_KEY_ID_FILE_INFO_KEY_NAME = 'sse_c_key_id'
+SSE_C_KEY_ID_HEADER = FILE_INFO_HEADER_PREFIX + SSE_C_KEY_ID_FILE_INFO_KEY_NAME
 
 logger = logging.getLogger(__name__)
 
@@ -147,7 +149,7 @@ def add_to_upload_headers(self, headers):
         elif self.mode == EncryptionMode.SSE_C:
             self._add_sse_c_headers(headers)
             if self.key.key_id is not None:
-                header = 'X-Bz-Info-%s' % (SSE_C_KEY_ID_FILE_INFO_KEY_NAME,)
+                header = SSE_C_KEY_ID_HEADER
                 if headers.get(header) is not None and headers[header] != self.key.key_id:
                     raise ValueError(
                         'Ambiguous key id set: "%s" in headers and "%s" in %s' %
@@ -309,18 +311,22 @@ def _from_value_dict(cls, value_dict, key_id=None):
 
     @classmethod
     def from_response_headers(cls, headers):
-
-        mode = EncryptionMode(headers.get('X-Bz-Server-Side-Encryption', 'none'))
-        kwargs = {
-            'mode': mode,
-        }
-        if mode == EncryptionMode.SSE_C:
-            kwargs['key'] = EncryptionKey(secret=None, key_id=None)
-        algorithm = headers.get('X-Bz-Server-Side-Encryption-Customer-Algorithm')
-        if algorithm is not None:
-            kwargs['algorithm'] = EncryptionAlgorithm(algorithm)
-
-        return EncryptionSetting(**kwargs)
+        if 'X-Bz-Server-Side-Encryption' in headers:
+            mode = EncryptionMode.SSE_B2
+            algorithm = EncryptionAlgorithm(headers['X-Bz-Server-Side-Encryption'])
+            return EncryptionSetting(mode, algorithm)
+        if 'X-Bz-Server-Side-Encryption-Customer-Algorithm' in headers:
+            mode = EncryptionMode.SSE_C
+            algorithm = EncryptionAlgorithm(
+                headers['X-Bz-Server-Side-Encryption-Customer-Algorithm']
+            )
+            if SSE_C_KEY_ID_HEADER in headers:
+                key_id = headers[SSE_C_KEY_ID_HEADER]
+            else:
+                key_id = None
+            key = EncryptionKey(secret=None, key_id=key_id)
+            return EncryptionSetting(mode, algorithm, key)
+        return EncryptionSetting(EncryptionMode.NONE)
 
 
 SSE_NONE = EncryptionSetting(mode=EncryptionMode.NONE,)

b2sdk/file_version.py

@@ -11,6 +11,7 @@
 from .encryption.setting import EncryptionSetting, EncryptionSettingFactory
 from .file_lock import FileRetentionSetting, LegalHold, NO_RETENTION_FILE_SETTING
 from .raw_api import SRC_LAST_MODIFIED_MILLIS
+from .utils import FILE_INFO_HEADER_PREFIX_LOWER
 
 
 class FileVersion:
@@ -68,7 +69,7 @@ def __init__(
         self.api = api
         self.id_ = id_
         self.file_name = file_name
-        self.size = size
+        self.size = size and int(size)
         self.content_type = content_type
         self.content_sha1 = content_sha1
         self.content_md5 = content_md5
@@ -82,7 +83,7 @@ def __init__(
         if SRC_LAST_MODIFIED_MILLIS in self.file_info:
             self.mod_time_millis = int(self.file_info[SRC_LAST_MODIFIED_MILLIS])
         else:
-            self.mod_time_millis = self.upload_timestamp
+            self.mod_time_millis = self.upload_timestamp and int(self.upload_timestamp)
 
     def as_dict(self):
         """ represents the object as a dict which looks almost exactly like the raw api output for upload/list """
@@ -201,16 +202,22 @@ def from_api_response(cls, api, file_version_dict, force_action=None):
 
     @classmethod
     def from_response_headers(cls, api, headers):
+        file_info = {}
+        prefix_len = len(FILE_INFO_HEADER_PREFIX_LOWER)
+        for header_name, header_value in headers.items():
+            if header_name[:prefix_len].lower() == FILE_INFO_HEADER_PREFIX_LOWER:
+                file_info_key = header_name[prefix_len:]
+                file_info[file_info_key] = header_value
         return FileVersion(
             api=api,
             id_=headers.get('x-bz-file-id'),
             file_name=headers.get('x-bz-file-name'),
             size=headers.get('content-length'),
             content_type=headers.get('content-type'),
             content_sha1=headers.get('x-bz-content-sha1'),
-            file_info=None,
+            file_info=file_info,
             upload_timestamp=headers.get('x-bz-upload-timestamp'),
-            action=None,
+            action='upload',
             content_md5=None,
             server_side_encryption=EncryptionSettingFactory.from_response_headers(headers),
             file_retention=FileRetentionSetting.from_response_headers(headers),

b2sdk/raw_api.py

@@ -25,7 +25,7 @@
 from .exception import FileOrBucketNotFound, ResourceNotFound, UnusableFileName, InvalidMetadataDirective, WrongEncryptionModeForBucketDefault, AccessDenied, SSECKeyError, RetentionWriteError
 from .encryption.setting import EncryptionAlgorithm, EncryptionMode, EncryptionSetting
 from .file_lock import BucketRetentionSetting, FileRetentionSetting, NO_RETENTION_FILE_SETTING, RetentionMode, RetentionPeriod, LegalHold
-from .utils import b2_url_encode, hex_sha1_of_stream
+from .utils import b2_url_encode, hex_sha1_of_stream, FILE_INFO_HEADER_PREFIX
 
 # All supported realms
 REALM_URLS = {
@@ -863,7 +863,7 @@ def upload_file(
             'X-Bz-Content-Sha1': content_sha1,
         }
         for k, v in file_infos.items():
-            headers['X-Bz-Info-' + k] = b2_url_encode(v)
+            headers[FILE_INFO_HEADER_PREFIX + k] = b2_url_encode(v)
         if server_side_encryption is not None:
             assert server_side_encryption.mode in (
                 EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C