diff --git a/b2sdk/api.py b/b2sdk/api.py index 62a2eb563..b97989206 100644 --- a/b2sdk/api.py +++ b/b2sdk/api.py @@ -13,6 +13,7 @@ from .bucket import Bucket, BucketFactory from .encryption.setting import EncryptionSetting from .exception import NonExistentBucket, RestrictedBucket +from .file_lock import BucketRetentionSetting from .file_version import FileIdAndName from .large_file.services import LargeFileServices from .raw_api import API_VERSION @@ -177,6 +178,8 @@ def create_bucket( cors_rules=None, lifecycle_rules=None, default_server_side_encryption: Optional[EncryptionSetting] = None, + default_retention: Optional[BucketRetentionSetting] = None, + is_file_lock_enabled: Optional[bool] = None, ): """ Create a bucket. @@ -187,6 +190,8 @@ def create_bucket( :param dict cors_rules: bucket CORS rules to store with the bucket :param dict lifecycle_rules: bucket lifecycle rules to store with the bucket :param b2sdk.v1.EncryptionSetting default_server_side_encryption: default server side encryption settings (``None`` if unknown) + :param b2sdk.v1.BucketRetentionSetting default_retention: default retention setting + :param bool is_file_lock_enabled: boolean value specifies whether bucket is File Lock-enabled :return: a Bucket object :rtype: b2sdk.v1.Bucket """ @@ -200,6 +205,7 @@ def create_bucket( cors_rules=cors_rules, lifecycle_rules=lifecycle_rules, default_server_side_encryption=default_server_side_encryption, + is_file_lock_enabled=is_file_lock_enabled, ) bucket = self.BUCKET_FACTORY_CLASS.from_api_bucket_dict(self, response) assert name == bucket.name, 'API created a bucket with different name\ @@ -208,6 +214,12 @@ def create_bucket( than requested: %s != %s' % ( bucket_type, bucket.type_ ) + if default_retention is not None: + # server does not support setting default retention on create + # so we provide convinient helper for it + bucket = self.BUCKET_FACTORY_CLASS.from_api_bucket_dict( + self, bucket.update(default_retention=default_retention) + ) self.cache.save_bucket(bucket) return bucket diff --git a/b2sdk/bucket.py b/b2sdk/bucket.py index 101bbd2bd..9584737bc 100644 --- a/b2sdk/bucket.py +++ b/b2sdk/bucket.py @@ -14,6 +14,12 @@ from .encryption.setting import EncryptionSetting, EncryptionSettingFactory from .encryption.types import EncryptionMode from .exception import FileNotPresent, FileOrBucketNotFound, UnexpectedCloudBehaviour, UnrecognizedBucketType +from .file_lock import ( + BucketRetentionSetting, + FileLockConfiguration, + FileRetentionSetting, + UNKNOWN_BUCKET_RETENTION, +) from .file_version import FileVersionInfo, FileVersionInfoFactory from .progress import DoNothingProgressListener from .transfer.emerge.executor import AUTO_CONTENT_TYPE @@ -48,6 +54,8 @@ def __init__( default_server_side_encryption: EncryptionSetting = EncryptionSetting( EncryptionMode.UNKNOWN ), + default_retention: BucketRetentionSetting = UNKNOWN_BUCKET_RETENTION, + is_file_lock_enabled: Optional[bool] = None, ): """ :param b2sdk.v1.B2Api api: an API object @@ -61,6 +69,8 @@ def __init__( :param dict bucket_dict: a dictionary which contains bucket parameters :param set options_set: set of bucket options strings :param b2sdk.v1.EncryptionSetting default_server_side_encryption: default server side encryption settings + :param b2sdk.v1.BucketRetentionSetting default_retention: default retention setting + :param bool is_file_lock_enabled: whether file locking is enabled or not """ self.api = api self.id_ = id_ @@ -73,6 +83,8 @@ def __init__( self.bucket_dict = bucket_dict or {} self.options_set = options_set or set() self.default_server_side_encryption = default_server_side_encryption + self.default_retention = default_retention + self.is_file_lock_enabled = is_file_lock_enabled def get_id(self): """ @@ -107,6 +119,7 @@ def update( lifecycle_rules=None, if_revision_is=None, default_server_side_encryption: Optional[EncryptionSetting] = None, + default_retention: Optional[BucketRetentionSetting] = None, ): """ Update various bucket parameters. @@ -118,6 +131,7 @@ def update( :param dict lifecycle_rules: lifecycle rules to store with a bucket :param int if_revision_is: revision number, update the info **only if** *revision* equals to *if_revision_is* :param b2sdk.v1.EncryptionSetting default_server_side_encryption: default server side encryption settings (``None`` if unknown) + :param b2sdk.v1.BucketRetentionSetting default_retention: bucket default retention setting """ account_id = self.api.account_info.get_account_id() return self.api.session.update_bucket( @@ -129,6 +143,7 @@ def update( lifecycle_rules=lifecycle_rules, if_revision_is=if_revision_is, default_server_side_encryption=default_server_side_encryption, + default_retention=default_retention, ) def cancel_large_file(self, file_id): @@ -400,6 +415,8 @@ def upload_bytes( file_infos=None, progress_listener=None, encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): """ Upload bytes in memory to a B2 file. @@ -410,6 +427,8 @@ def upload_bytes( :param dict,None file_infos: a file info to store with the file or ``None`` to not store anything :param b2sdk.v1.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not track progress :param b2sdk.v1.EncryptionSetting encryption: encryption settings (``None`` if unknown) + :param b2sdk.v1.FileRetentionSetting file_retention: file retention setting + :param bool legal_hold: legal hold setting :rtype: generator[b2sdk.v1.FileVersion] """ upload_source = UploadSourceBytes(data_bytes) @@ -420,6 +439,8 @@ def upload_bytes( file_info=file_infos, progress_listener=progress_listener, encryption=encryption, + file_retention=file_retention, + legal_hold=legal_hold, ) def upload_local_file( @@ -432,6 +453,8 @@ def upload_local_file( min_part_size=None, progress_listener=None, encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): """ Upload a file on local disk to a B2 file. @@ -448,6 +471,8 @@ def upload_local_file( :param int min_part_size: a minimum size of a part :param b2sdk.v1.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not report progress :param b2sdk.v1.EncryptionSetting encryption: encryption settings (``None`` if unknown) + :param b2sdk.v1.FileRetentionSetting file_retention: file retention setting + :param bool legal_hold: legal hold setting :rtype: b2sdk.v1.FileVersionInfo """ upload_source = UploadSourceLocalFile(local_path=local_file, content_sha1=sha1_sum) @@ -459,6 +484,8 @@ def upload_local_file( min_part_size=min_part_size, progress_listener=progress_listener, encryption=encryption, + file_retention=file_retention, + legal_hold=legal_hold, ) def upload( @@ -470,6 +497,8 @@ def upload( min_part_size=None, progress_listener=None, encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): """ Upload a file to B2, retrying as needed. @@ -489,6 +518,8 @@ def upload( :param int,None min_part_size: the smallest part size to use or ``None`` to determine automatically :param b2sdk.v1.AbstractProgressListener,None progress_listener: a progress listener object to use, or ``None`` to not report progress :param b2sdk.v1.EncryptionSetting encryption: encryption settings (``None`` if unknown) + :param b2sdk.v1.FileRetentionSetting file_retention: file retention setting + :param bool legal_hold: legal hold setting :rtype: b2sdk.v1.FileVersionInfo """ return self.create_file( @@ -500,6 +531,8 @@ def upload( # FIXME: Bucket.upload documents wrong logic recommended_upload_part_size=min_part_size, encryption=encryption, + file_retention=file_retention, + legal_hold=legal_hold, ) def create_file( @@ -512,6 +545,8 @@ def create_file( recommended_upload_part_size=None, continue_large_file_id=None, encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): """ Creates a new file in this bucket using an iterable (list, tuple etc) of remote or local sources. @@ -534,6 +569,8 @@ def create_file( :param str,None continue_large_file_id: large file id that should be selected to resume file creation for multipart upload/copy, ``None`` for automatic search for this id :param b2sdk.v1.EncryptionSetting encryption: encryption settings (``None`` if unknown) + :param b2sdk.v1.FileRetentionSetting file_retention: file retention setting + :param bool legal_hold: legal hold setting """ return self._create_file( self.api.services.emerger.emerge, @@ -545,6 +582,8 @@ def create_file( continue_large_file_id=continue_large_file_id, recommended_upload_part_size=recommended_upload_part_size, encryption=encryption, + file_retention=file_retention, + legal_hold=legal_hold, ) def create_file_stream( @@ -557,6 +596,8 @@ def create_file_stream( recommended_upload_part_size=None, continue_large_file_id=None, encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): """ Creates a new file in this bucket using a stream of multiple remote or local sources. @@ -581,6 +622,8 @@ def create_file_stream( for multipart upload/copy, if ``None`` in multipart case it would always start a new large file :param b2sdk.v1.EncryptionSetting encryption: encryption settings (``None`` if unknown) + :param b2sdk.v1.FileRetentionSetting file_retention: file retention setting + :param bool legal_hold: legal hold setting """ return self._create_file( self.api.services.emerger.emerge_stream, @@ -592,6 +635,8 @@ def create_file_stream( continue_large_file_id=continue_large_file_id, recommended_upload_part_size=recommended_upload_part_size, encryption=encryption, + file_retention=file_retention, + legal_hold=legal_hold, ) def _create_file( @@ -605,6 +650,8 @@ def _create_file( recommended_upload_part_size=None, continue_large_file_id=None, encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): validate_b2_file_name(file_name) progress_listener = progress_listener or DoNothingProgressListener() @@ -619,6 +666,8 @@ def _create_file( recommended_upload_part_size=recommended_upload_part_size, continue_large_file_id=continue_large_file_id, encryption=encryption, + file_retention=file_retention, + legal_hold=legal_hold, ) def concatenate( @@ -631,6 +680,8 @@ def concatenate( recommended_upload_part_size=None, continue_large_file_id=None, encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): """ Creates a new file in this bucket by concatenating multiple remote or local sources. @@ -650,6 +701,8 @@ def concatenate( :param str,None continue_large_file_id: large file id that should be selected to resume file creation for multipart upload/copy, ``None`` for automatic search for this id :param b2sdk.v1.EncryptionSetting encryption: encryption settings (``None`` if unknown) + :param b2sdk.v1.FileRetentionSetting file_retention: file retention setting + :param bool legal_hold: legal hold setting """ return self.create_file( WriteIntent.wrap_sources_iterator(outbound_sources), @@ -660,6 +713,8 @@ def concatenate( recommended_upload_part_size=recommended_upload_part_size, continue_large_file_id=continue_large_file_id, encryption=encryption, + file_retention=file_retention, + legal_hold=legal_hold, ) def concatenate_stream( @@ -672,6 +727,8 @@ def concatenate_stream( recommended_upload_part_size=None, continue_large_file_id=None, encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): """ Creates a new file in this bucket by concatenating stream of multiple remote or local sources. @@ -692,6 +749,8 @@ def concatenate_stream( for multipart upload/copy, if ``None`` in multipart case it would always start a new large file :param b2sdk.v1.EncryptionSetting encryption: encryption setting (``None`` if unknown) + :param b2sdk.v1.FileRetentionSetting file_retention: file retention setting + :param bool legal_hold: legal hold setting """ return self.create_file_stream( WriteIntent.wrap_sources_iterator(outbound_sources_iterator), @@ -702,6 +761,8 @@ def concatenate_stream( recommended_upload_part_size=recommended_upload_part_size, continue_large_file_id=continue_large_file_id, encryption=encryption, + file_retention=file_retention, + legal_hold=legal_hold, ) def get_download_url(self, filename): @@ -740,6 +801,8 @@ def copy( source_encryption: Optional[EncryptionSetting] = None, source_file_info: Optional[dict] = None, source_content_type: Optional[str] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): """ Creates a new file in this bucket by (server-side) copying from an existing file. @@ -763,6 +826,8 @@ def copy( (``None`` if unknown) :param dict,None source_file_info: source file's file_info dict, useful when copying files with SSE-C :param str,None source_content_type: source file's content type, useful when copying files with SSE-C + :param b2sdk.v1.FileRetentionSetting file_retention: file retention setting for the new file. + :param bool legal_hold: legal hold setting for the new file. """ copy_source = CopySource( @@ -786,6 +851,8 @@ def copy( progress_listener=progress_listener, destination_encryption=destination_encryption, source_encryption=source_encryption, + file_retention=file_retention, + legal_hold=legal_hold, ).result() else: return self.create_file( @@ -795,6 +862,8 @@ def copy( file_info=file_info, progress_listener=progress_listener, encryption=destination_encryption, + file_retention=file_retention, + legal_hold=legal_hold, ) def delete_file_version(self, file_id, file_name): @@ -828,6 +897,9 @@ def as_dict(self): result['revision'] = self.revision result['options'] = self.options_set result['defaultServerSideEncryption'] = self.default_server_side_encryption.as_dict() + result['isFileLockEnabled'] = self.is_file_lock_enabled + result['defaultRetention'] = self.default_retention.as_dict() + return result def __repr__(self): @@ -872,7 +944,17 @@ def from_api_bucket_dict(cls, api, bucket_dict): "algorithm" : "AES256", "mode" : "SSE-B2" } - } + }, + "fileLockConfiguration": { + "isClientAuthorizedToRead": true, + "value": { + "defaultRetention": { + "mode": null, + "period": null + }, + "isFileLockEnabled": false + } + } } into a Bucket object. @@ -896,6 +978,7 @@ def from_api_bucket_dict(cls, api, bucket_dict): if 'defaultServerSideEncryption' not in bucket_dict: raise UnexpectedCloudBehaviour('server did not provide `defaultServerSideEncryption`') default_server_side_encryption = EncryptionSettingFactory.from_bucket_dict(bucket_dict) + file_lock_configuration = FileLockConfiguration.from_bucket_dict(bucket_dict) return cls.BUCKET_CLASS( api, bucket_id, @@ -908,4 +991,6 @@ def from_api_bucket_dict(cls, api, bucket_dict): bucket_dict, options, default_server_side_encryption, + file_lock_configuration.default_retention, + file_lock_configuration.is_file_lock_enabled, ) diff --git a/b2sdk/file_lock.py b/b2sdk/file_lock.py new file mode 100644 index 000000000..5cf6eecb9 --- /dev/null +++ b/b2sdk/file_lock.py @@ -0,0 +1,270 @@ +###################################################################### +# +# File: b2sdk/file_lock.py +# +# Copyright 2021 Backblaze Inc. All Rights Reserved. +# +# License https://www.backblaze.com/using_b2_code.html +# +###################################################################### + +from typing import Optional +import enum + +from .exception import UnexpectedCloudBehaviour +# TODO: write __repr__ and __eq__ methods for the classes below + +ACTIONS_WITHOUT_LOCK_SETTINGS = frozenset(['hide', 'folder']) + + +@enum.unique +class RetentionMode(enum.Enum): + COMPLIANCE = "compliance" # TODO: docs + GOVERNANCE = "governance" # TODO: docs + NONE = None + UNKNOWN = "unknown" + + +RETENTION_MODES_REQUIRING_PERIODS = frozenset({RetentionMode.COMPLIANCE, RetentionMode.GOVERNANCE}) + + +class RetentionPeriod: + def __init__(self, years: Optional[int] = None, days: Optional[int] = None): + assert (years is None) != (days is None) + if years is not None: + self.duration = years + self.unit = 'years' + else: + self.duration = days + self.unit = 'days' + + @classmethod + def from_period_dict(cls, period_dict): + """ + Build a RetentionPeriod from an object returned by the server, such as: + + .. code-block :: + + { + "duration": 2, + "unit": "years" + } + """ + return cls(**{period_dict['unit']: period_dict['duration']}) + + def as_dict(self): + return { + "duration": self.duration, + "unit": self.unit, + } + + +class FileRetentionSetting: + def __init__(self, mode: RetentionMode, retain_until: Optional[int] = None): + if mode in RETENTION_MODES_REQUIRING_PERIODS and retain_until is None: + raise ValueError('must specify retain_until for retention mode %s' % (mode,)) + self.mode = mode + self.retain_until = retain_until + + @classmethod + def from_file_version_dict(cls, file_version_dict: dict): + """ + Returns FileRetentionSetting for the given file_version_dict retrieved from the api. E.g. + + .. code-block :: + + { + "action": "upload", + "fileRetention": { + "isClientAuthorizedToRead": false, + "value": null + }, + ... + } + + { + "action": "upload", + "fileRetention": { + "isClientAuthorizedToRead": true, + "value": { + "mode": "governance", + "retainUntilTimestamp": 1628942493000 + } + }, + ... + } + """ + if 'fileRetention' not in file_version_dict: + if file_version_dict['action'] not in ACTIONS_WITHOUT_LOCK_SETTINGS: + raise UnexpectedCloudBehaviour( + 'No fileRetention provided for file version with action=%s' % + (file_version_dict['action']) + ) + return NO_RETENTION_FILE_SETTING + + retention_dict = file_version_dict['fileRetention'] + + if not retention_dict['isClientAuthorizedToRead']: + return cls(RetentionMode.UNKNOWN, None) + + mode = retention_dict['value']['mode'] + if mode is None: + return NO_RETENTION_FILE_SETTING + + return cls( + RetentionMode(mode), + retention_dict['value']['retainUntilTimestamp'], + ) + + def serialize_to_json_for_request(self): + if self.mode is RetentionMode.UNKNOWN: + raise ValueError('cannot use an unknown file retention setting in requests') + return self.as_dict() + + def as_dict(self): + return { + "mode": self.mode.value, + "retainUntilTimestamp": self.retain_until, + } + + def add_to_to_upload_headers(self, headers): + if self.mode is RetentionMode.UNKNOWN: + raise ValueError('cannot use an unknown file retention setting in requests') + + headers['X-Bz-File-Retention-Mode'] = str( + self.mode.value + ) # mode = NONE is not supported by the server at the + # moment, but it should be + headers['X-Bz-File-Retention-Retain-Until-Timestamp'] = self.retain_until + + +class LegalHoldSerializer: + @classmethod + def from_server(cls, file_version_dict) -> Optional[bool]: + if 'legalHold' not in file_version_dict: + if file_version_dict['action'] not in ACTIONS_WITHOUT_LOCK_SETTINGS: + raise UnexpectedCloudBehaviour( + 'legalHold not provided for file version with action=%s' % + (file_version_dict['action']) + ) + return None + legal_hold_dict = file_version_dict['legalHold'] + if legal_hold_dict['value'] is None: + return None + if legal_hold_dict['value'] == 'on': + return True + elif legal_hold_dict['value'] == 'off': + return False + raise ValueError('Unknown legal hold value: %s' % (legal_hold_dict['value'],)) + + @classmethod + def to_server(cls, bool_value: Optional[bool]) -> str: + if bool_value is None: + raise ValueError('Cannot use unknown legal hold in requests') + if bool_value: + return 'on' + return 'off' + + @classmethod + def add_to_upload_headers(cls, bool_value: Optional[bool], headers): + headers['X-Bz-File-Legal-Hold'] = cls.to_server(bool_value) + + +class BucketRetentionSetting: + def __init__(self, mode: RetentionMode, period: Optional[RetentionPeriod] = None): + if mode in RETENTION_MODES_REQUIRING_PERIODS and period is None: + raise ValueError('must specify period for retention mode %s' % (mode,)) + self.mode = mode + self.period = period + + @classmethod + def from_bucket_retention_dict(cls, retention_dict: dict): + """ + Build a BucketRetentionSetting from an object returned by the server, such as: + + .. code-block:: + + { + "mode": "compliance", + "period": { + "duration": 7, + "unit": "days" + } + } + + """ + period = retention_dict['period'] + if period is not None: + period = RetentionPeriod.from_period_dict(period) + return cls(RetentionMode(retention_dict['mode']), period) + + def as_dict(self): + result = { + 'mode': self.mode.value, + } + if self.period is not None: + result['period'] = self.period.as_dict() + return result + + def serialize_to_json_for_request(self): + if self.mode == RetentionMode.UNKNOWN: + raise ValueError('cannot use an unknown file lock configuration in requests') + return self.as_dict() + + +class FileLockConfiguration: + def __init__( + self, + default_retention: BucketRetentionSetting, + is_file_lock_enabled: Optional[bool], + ): + self.default_retention = default_retention + self.is_file_lock_enabled = is_file_lock_enabled + + @classmethod + def from_bucket_dict(cls, bucket_dict): + """ + Build a FileLockConfiguration from an object returned by server, such as: + + .. code-block:: + { + "isClientAuthorizedToRead": true, + "value": { + "defaultRetention": { + "mode": "governance", + "period": { + "duration": 2, + "unit": "years" + } + }, + "isFileLockEnabled": true + } + } + + or + + { + "isClientAuthorizedToRead": false, + "value": null + } + """ + + if not bucket_dict['fileLockConfiguration']['isClientAuthorizedToRead']: + return cls(UNKNOWN_BUCKET_RETENTION, None) + retention = BucketRetentionSetting.from_bucket_retention_dict( + bucket_dict['fileLockConfiguration']['value']['defaultRetention'] + ) + is_file_lock_enabled = bucket_dict['fileLockConfiguration']['value']['isFileLockEnabled'] + return cls(retention, is_file_lock_enabled) + + def as_dict(self): + return { + "defaultRetention": self.default_retention.as_dict(), + "isFileLockEnabled": self.is_file_lock_enabled, + } + + +UNKNOWN_BUCKET_RETENTION = BucketRetentionSetting(RetentionMode.UNKNOWN) +UNKNOWN_FILE_LOCK_CONFIGURATION = FileLockConfiguration(UNKNOWN_BUCKET_RETENTION, None) +NO_RETENTION_BUCKET_SETTING = BucketRetentionSetting(RetentionMode.NONE) +NO_RETENTION_FILE_SETTING = FileRetentionSetting(RetentionMode.NONE) diff --git a/b2sdk/file_version.py b/b2sdk/file_version.py index 0887ac315..49dbbc1f9 100644 --- a/b2sdk/file_version.py +++ b/b2sdk/file_version.py @@ -12,6 +12,7 @@ import datetime from .encryption.setting import EncryptionSetting, EncryptionSettingFactory +from .file_lock import FileRetentionSetting, LegalHoldSerializer class FileVersionInfo(object): @@ -35,8 +36,18 @@ class FileVersionInfo(object): LS_ENTRY_TEMPLATE = '%83s %6s %10s %8s %9d %s' # order is file_id, action, date, time, size, name __slots__ = [ - 'id_', 'file_name', 'size', 'content_type', 'content_sha1', 'content_md5', 'file_info', - 'upload_timestamp', 'action', 'server_side_encryption' + 'id_', + 'file_name', + 'size', + 'content_type', + 'content_sha1', + 'content_md5', + 'file_info', + 'upload_timestamp', + 'action', + 'server_side_encryption', + 'legal_hold', + 'file_retention', ] def __init__( @@ -51,6 +62,8 @@ def __init__( action, content_md5=None, server_side_encryption: Optional[EncryptionSetting] = None, # TODO: make it mandatory in v2 + legal_hold: Optional[bool] = None, + file_retention: Optional[FileRetentionSetting] = None, ): self.id_ = id_ self.file_name = file_name @@ -62,6 +75,8 @@ def __init__( self.upload_timestamp = upload_timestamp self.action = action self.server_side_encryption = server_side_encryption + self.legal_hold = legal_hold + self.file_retention = file_retention def as_dict(self): """ represents the object as a dict which looks almost exactly like the raw api output for upload/list """ @@ -69,6 +84,7 @@ def as_dict(self): 'fileId': self.id_, 'fileName': self.file_name, 'fileInfo': self.file_info, + 'legalHold': self.legal_hold, } if self.size is not None: result['size'] = self.size @@ -84,6 +100,8 @@ def as_dict(self): result['contentMd5'] = self.content_md5 if self.server_side_encryption is not None: # this is for backward compatibility of interface only, b2sdk always sets it result['serverSideEncryption'] = self.server_side_encryption.as_dict() + if self.file_retention is not None: # this is for backward compatibility of interface only, b2sdk always sets it + result['fileRetention'] = self.file_retention.as_dict() return result def format_ls_entry(self): @@ -170,6 +188,9 @@ def from_api_response(cls, file_info_dict, force_action=None): content_md5 = file_info_dict.get('contentMd5') file_info = file_info_dict.get('fileInfo') server_side_encryption = EncryptionSettingFactory.from_file_version_dict(file_info_dict) + file_retention = FileRetentionSetting.from_file_version_dict(file_info_dict) + + legal_hold = LegalHoldSerializer.from_server(file_info_dict) return FileVersionInfo( id_, @@ -182,6 +203,8 @@ def from_api_response(cls, file_info_dict, force_action=None): action, content_md5, server_side_encryption, + legal_hold, + file_retention, ) @classmethod diff --git a/b2sdk/large_file/services.py b/b2sdk/large_file/services.py index f7b4180cc..6985e5d58 100644 --- a/b2sdk/large_file/services.py +++ b/b2sdk/large_file/services.py @@ -11,6 +11,7 @@ from typing import Optional from b2sdk.encryption.setting import EncryptionSetting +from b2sdk.file_lock import FileRetentionSetting from b2sdk.file_version import FileVersionInfoFactory from b2sdk.large_file.part import PartFactory from b2sdk.large_file.unfinished_large_file import UnfinishedLargeFile @@ -84,6 +85,8 @@ def start_large_file( content_type=None, file_info=None, encryption: Optional[EncryptionSetting] = None, + legal_hold: Optional[bool] = None, + file_retention: Optional[FileRetentionSetting] = None, ): """ Start a large file transfer. @@ -92,6 +95,8 @@ def start_large_file( :param str,None content_type: the MIME type, or ``None`` to accept the default based on file extension of the B2 file name :param dict,None file_info: a file info to store with the file or ``None`` to not store anything :param b2sdk.v1.EncryptionSetting encryption: encryption settings (``None`` if unknown) + :param bool legal_hold: legal hold setting + :param b2sdk.v1.FileRetentionSetting file_retention: file retention setting """ return UnfinishedLargeFile( self.services.session.start_large_file( @@ -100,6 +105,8 @@ def start_large_file( content_type, file_info, server_side_encryption=encryption, + legal_hold=legal_hold, + file_retention=file_retention, ) ) diff --git a/b2sdk/large_file/unfinished_large_file.py b/b2sdk/large_file/unfinished_large_file.py index 9d2c2d6bf..0ecdef235 100644 --- a/b2sdk/large_file/unfinished_large_file.py +++ b/b2sdk/large_file/unfinished_large_file.py @@ -9,6 +9,7 @@ ###################################################################### from b2sdk.encryption.setting import EncryptionSettingFactory +from b2sdk.file_lock import FileRetentionSetting, LegalHoldSerializer class UnfinishedLargeFile(object): @@ -34,6 +35,8 @@ def __init__(self, file_dict): self.content_type = file_dict['contentType'] self.file_info = file_dict['fileInfo'] self.encryption = EncryptionSettingFactory.from_file_version_dict(file_dict) + self.file_retention = FileRetentionSetting.from_file_version_dict(file_dict) + self.legal_hold = LegalHoldSerializer.from_server(file_dict) def __repr__(self): return '<%s %s %s>' % (self.__class__.__name__, self.bucket_id, self.file_name) diff --git a/b2sdk/raw_api.py b/b2sdk/raw_api.py index ae6db091e..c1b8ba55c 100644 --- a/b2sdk/raw_api.py +++ b/b2sdk/raw_api.py @@ -24,6 +24,7 @@ from .b2http import B2Http from .exception import FileOrBucketNotFound, ResourceNotFound, UnusableFileName, InvalidMetadataDirective, WrongEncryptionModeForBucketDefault from .encryption.setting import EncryptionAlgorithm, EncryptionMode, EncryptionSetting +from .file_lock import BucketRetentionSetting, FileRetentionSetting, NO_RETENTION_FILE_SETTING, RetentionMode, RetentionPeriod from .utils import b2_url_encode, hex_sha1_of_stream # All possible capabilities @@ -36,6 +37,12 @@ 'deleteBuckets', 'readBucketEncryption', 'writeBucketEncryption', + 'readBucketRetentions', + 'writeBucketRetentions', + 'writeFileRetentions', + 'writeFileLegalHolds', + 'readFileRetentions', + 'readFileLegalHolds', 'listFiles', 'readFiles', 'shareFiles', @@ -89,6 +96,8 @@ def copy_file( destination_bucket_id=None, destination_server_side_encryption: Optional[EncryptionSetting] = None, source_server_side_encryption: Optional[EncryptionSetting] = None, + legal_hold: Optional[bool] = None, + file_retention: Optional[FileRetentionSetting] = None, ): pass @@ -118,6 +127,7 @@ def create_bucket( cors_rules=None, lifecycle_rules=None, default_server_side_encryption: Optional[EncryptionSetting] = None, + is_file_lock_enabled: Optional[bool] = None, ): pass @@ -256,6 +266,8 @@ def start_large_file( content_type, file_info, server_side_encryption: Optional[EncryptionSetting] = None, + legal_hold: Optional[bool] = None, + file_retention: Optional[FileRetentionSetting] = None, ): pass @@ -272,6 +284,19 @@ def update_bucket( lifecycle_rules=None, if_revision_is=None, default_server_side_encryption: Optional[EncryptionSetting] = None, + default_retention: Optional[BucketRetentionSetting] = None, + ): + pass + + @abstractmethod + def update_file_retention( + self, + api_url, + account_auth_token, + file_id, + file_name, + file_retention: FileRetentionSetting, + bypass_governance: bool = False, ): pass @@ -287,6 +312,8 @@ def upload_file( file_infos, data_stream, server_side_encryption: Optional[EncryptionSetting] = None, + legal_hold: Optional[bool] = None, + file_retention: Optional[FileRetentionSetting] = None, ): pass @@ -364,7 +391,8 @@ def create_bucket( bucket_info=None, cors_rules=None, lifecycle_rules=None, - default_server_side_encryption=None, + default_server_side_encryption: Optional[EncryptionSetting] = None, + is_file_lock_enabled: Optional[bool] = None, ): kwargs = dict( accountId=account_id, @@ -382,6 +410,8 @@ def create_bucket( raise WrongEncryptionModeForBucketDefault(default_server_side_encryption.mode) kwargs['defaultServerSideEncryption' ] = default_server_side_encryption.serialize_to_json_for_request() + if is_file_lock_enabled is not None: + kwargs['fileLockEnabled'] = is_file_lock_enabled return self._post_json( api_url, 'b2_create_bucket', @@ -624,6 +654,8 @@ def start_large_file( content_type, file_info, server_side_encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): kwargs = {} if server_side_encryption is not None: @@ -631,6 +663,9 @@ def start_large_file( EncryptionMode.NONE, EncryptionMode.SSE_B2, EncryptionMode.SSE_C ) kwargs['serverSideEncryption'] = server_side_encryption.serialize_to_json_for_request() + + # FIXME: implement `legal_hold` and `file_retention` + return self._post_json( api_url, 'b2_start_large_file', @@ -653,7 +688,8 @@ def update_bucket( cors_rules=None, lifecycle_rules=None, if_revision_is=None, - default_server_side_encryption=None, + default_server_side_encryption: Optional[EncryptionSetting] = None, + default_retention: Optional[BucketRetentionSetting] = None, ): assert bucket_info is not None or bucket_type is not None @@ -669,11 +705,12 @@ def update_bucket( if lifecycle_rules is not None: kwargs['lifecycleRules'] = lifecycle_rules if default_server_side_encryption is not None: - if default_server_side_encryption is not None: - if not default_server_side_encryption.mode.can_be_set_as_bucket_default(): - raise WrongEncryptionModeForBucketDefault(default_server_side_encryption.mode) - kwargs['defaultServerSideEncryption' - ] = default_server_side_encryption.serialize_to_json_for_request() + if not default_server_side_encryption.mode.can_be_set_as_bucket_default(): + raise WrongEncryptionModeForBucketDefault(default_server_side_encryption.mode) + kwargs['defaultServerSideEncryption' + ] = default_server_side_encryption.serialize_to_json_for_request() + if default_retention is not None: + kwargs['defaultRetention'] = default_retention.serialize_to_json_for_request() return self._post_json( api_url, @@ -684,6 +721,27 @@ def update_bucket( **kwargs ) + def update_file_retention( + self, + api_url, + account_auth_token, + file_id, + file_name, + file_retention: FileRetentionSetting, + bypass_governance: bool = False, + ): + kwargs = {} + kwargs['fileRetention'] = file_retention.serialize_to_json_for_request() + return self._post_json( + api_url, + 'b2_update_file_retention', + account_auth_token, + fileId=file_id, + fileName=file_name, + bypassGovernance=bypass_governance, + **kwargs + ) + def unprintable_to_hex(self, string): """ Replace unprintable chars in string with a hex representation. @@ -742,6 +800,8 @@ def upload_file( file_infos, data_stream, server_side_encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): """ Upload one, small file to b2. @@ -773,6 +833,8 @@ def upload_file( ) server_side_encryption.add_to_upload_headers(headers) + # FIXME: implement `legal_hold` and `file_retention` + return self.b2_http.post_content_return_json(upload_url, headers, data_stream) def upload_part( @@ -812,6 +874,8 @@ def copy_file( destination_bucket_id=None, destination_server_side_encryption: Optional[EncryptionSetting] = None, source_server_side_encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): kwargs = {} if bytes_range is not None: @@ -850,6 +914,8 @@ def copy_file( kwargs['sourceServerSideEncryption' ] = source_server_side_encryption.serialize_to_json_for_request() + # FIXME: implement `legal_hold` and `file_retention` + return self._post_json( api_url, 'b2_copy_file', @@ -979,8 +1045,17 @@ def test_raw_api_helper(raw_api): bucket_name = 'test-raw-api-%s-%d-%d' % ( account_id, int(time.time()), random.randint(1000, 9999) ) + + # very verbose http debug + #import http.client; http.client.HTTPConnection.debuglevel = 1 + bucket_dict = raw_api.create_bucket( - api_url, account_auth_token, account_id, bucket_name, 'allPublic' + api_url, + account_auth_token, + account_id, + bucket_name, + 'allPublic', + is_file_lock_enabled=True, ) bucket_id = bucket_dict['bucketId'] first_bucket_revision = bucket_dict['revision'] @@ -992,9 +1067,13 @@ def test_raw_api_helper(raw_api): algorithm=EncryptionAlgorithm.AES256, ) sse_none = EncryptionSetting(mode=EncryptionMode.NONE) - for encryption_setting in [ - sse_none, - sse_b2_aes, + for encryption_setting, default_retention in [ + ( + sse_none, + BucketRetentionSetting(mode=RetentionMode.GOVERNANCE, period=RetentionPeriod(days=1)) + ), + (sse_b2_aes, None), + (sse_b2_aes, BucketRetentionSetting(RetentionMode.NONE)), ]: bucket_dict = raw_api.update_bucket( api_url, @@ -1003,11 +1082,13 @@ def test_raw_api_helper(raw_api): bucket_id, 'allPublic', default_server_side_encryption=encryption_setting, + default_retention=default_retention, ) # b2_list_buckets print('b2_list_buckets') bucket_list_dict = raw_api.list_buckets(api_url, account_auth_token, account_id) + #print(bucket_list_dict) # b2_get_upload_url print('b2_get_upload_url') @@ -1186,7 +1267,10 @@ def test_raw_api_helper(raw_api): account_id, bucket_id, 'allPrivate', - bucket_info={'color': 'blue'} + bucket_info={'color': 'blue'}, + default_retention=BucketRetentionSetting( + mode=RetentionMode.GOVERNANCE, period=RetentionPeriod(days=1) + ), ) assert first_bucket_revision < updated_bucket['revision'] @@ -1212,6 +1296,16 @@ def _clean_and_delete_bucket(raw_api, api_url, account_auth_token, account_id, b action = version_dict['action'] if action in ['hide', 'upload']: print('b2_delete_file', file_name, action) + if action == 'upload' and version_dict[ + 'fileRetention'] and version_dict['fileRetention']['value']['mode'] is not None: + raw_api.update_file_retention( + api_url, + account_auth_token, + file_id, + file_name, + NO_RETENTION_FILE_SETTING, + bypass_governance=True + ) raw_api.delete_file_version(api_url, account_auth_token, file_id, file_name) else: print('b2_cancel_large_file', file_name) diff --git a/b2sdk/raw_simulator.py b/b2sdk/raw_simulator.py index ed99cb8aa..3377501bc 100644 --- a/b2sdk/raw_simulator.py +++ b/b2sdk/raw_simulator.py @@ -37,6 +37,7 @@ UnsatisfiableRange, SSECKeyError, ) +from .file_lock import BucketRetentionSetting, FileRetentionSetting, NO_RETENTION_BUCKET_SETTING, UNKNOWN_FILE_LOCK_CONFIGURATION from .raw_api import AbstractRawApi, HEX_DIGITS_AT_END, MetadataDirectiveMode, ALL_CAPABILITIES from .utils import ( b2_url_decode, @@ -141,7 +142,7 @@ class FileSimulator(object): def __init__( self, account_id, - bucket_id, + bucket, file_id, action, name, @@ -152,14 +153,15 @@ def __init__( upload_timestamp, range_=None, server_side_encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): - logger.debug('FileSimulator called with sse=%s', server_side_encryption) if action == 'hide': assert server_side_encryption is None else: assert server_side_encryption is not None self.account_id = account_id - self.bucket_id = bucket_id + self.bucket = bucket self.file_id = file_id self.action = action self.name = name @@ -174,6 +176,8 @@ def __init__( self.upload_timestamp = upload_timestamp self.range_ = range_ self.server_side_encryption = server_side_encryption + self.file_retention = file_retention + self.legal_hold = legal_hold if action == 'start': self.parts = [] @@ -192,7 +196,7 @@ def sort_key(self): """ return (self.name, self.file_id) - def as_download_headers(self, range_=None): + def as_download_headers(self, account_auth_token_or_none, range_=None): if self.data_bytes is None: content_length = 0 elif range_ is not None: @@ -210,6 +214,25 @@ def as_download_headers(self, range_=None): 'x-bz-file-id': self.file_id, 'x-bz-file-name': self.name, } + + if account_auth_token_or_none is not None and self.bucket.is_file_lock_enabled: + not_permitted = [] + + if not self.is_allowed_to_read_file_retention(account_auth_token_or_none): + not_permitted.append('X-Bz-File-Retention-Mode') + not_permitted.append('X-Bz-File-Retain-Until-Timestamp') + else: + if self.file_retention is not None: + self.file_retention.add_to_to_upload_headers(headers) + + if not self.is_allowed_to_read_file_legal_hold(account_auth_token_or_none): + not_permitted.append('X-Bz-File-Legal-Hold') + else: + headers['X-Bz-File-Legal-Hold'] = self.legal_hold and 'on' or 'off' + + if not_permitted: + headers['X-Bz-Client-Unauthorized-To-Read'] = ','.join(not_permitted) + if range_ is not None: headers['Content-Range'] = 'bytes %d-%d/%d' % ( range_[0], range_[0] + content_length, len(self.data_bytes) @@ -218,12 +241,12 @@ def as_download_headers(self, range_=None): headers['x-bz-info-' + key] = value return headers - def as_upload_result(self): + def as_upload_result(self, account_auth_token): result = dict( fileId=self.file_id, fileName=self.name, accountId=self.account_id, - bucketId=self.bucket_id, + bucketId=self.bucket.bucket_id, contentLength=len(self.data_bytes) if self.data_bytes is not None else 0, contentType=self.content_type, contentSha1=self.content_sha1, @@ -234,9 +257,11 @@ def as_upload_result(self): if self.server_side_encryption is not None: result['serverSideEncryption' ] = self.server_side_encryption.serialize_to_json_for_request() + result['fileRetention'] = self._file_retention_dict(account_auth_token) + result['legalHold'] = self._legal_hold_dict(account_auth_token) return result - def as_list_files_dict(self): + def as_list_files_dict(self, account_auth_token): result = dict( fileId=self.file_id, fileName=self.name, @@ -250,14 +275,22 @@ def as_list_files_dict(self): if self.server_side_encryption is not None: result['serverSideEncryption' ] = self.server_side_encryption.serialize_to_json_for_request() + result['fileRetention'] = self._file_retention_dict(account_auth_token) + result['legalHold'] = self._legal_hold_dict(account_auth_token) return result - def as_start_large_file_result(self): + def is_allowed_to_read_file_retention(self, account_auth_token): + return self.bucket._check_capability(account_auth_token, 'readFileRetentions') + + def is_allowed_to_read_file_legal_hold(self, account_auth_token): + return self.bucket._check_capability(account_auth_token, 'readFileLegalHolds') + + def as_start_large_file_result(self, account_auth_token): result = dict( fileId=self.file_id, fileName=self.name, accountId=self.account_id, - bucketId=self.bucket_id, + bucketId=self.bucket.bucket_id, contentType=self.content_type, fileInfo=self.file_info, uploadTimestamp=self.upload_timestamp, @@ -265,8 +298,31 @@ def as_start_large_file_result(self): if self.server_side_encryption is not None: result['serverSideEncryption' ] = self.server_side_encryption.serialize_to_json_for_request() + result['fileRetention'] = self._file_retention_dict(account_auth_token) + result['legalHold'] = self._legal_hold_dict(account_auth_token) return result + def _file_retention_dict(self, account_auth_token): + if not self.is_allowed_to_read_file_retention(account_auth_token): + return UNKNOWN_FILE_LOCK_CONFIGURATION + + file_lock_configuration = {'isClientAuthorizedToRead': True} + if self.file_retention is None: + file_lock_configuration['value'] = {'mode': None} + else: + file_lock_configuration['value'] = {'mode': self.file_retention.mode.value} + if self.file_retention.period is not None: + file_lock_configuration['value']['period'] = self.file_retention.period + return file_lock_configuration + + def _legal_hold_dict(self, account_auth_token): + if not self.is_allowed_to_read_file_legal_hold(account_auth_token): + return UNKNOWN_FILE_LOCK_CONFIGURATION + return { + 'isClientAuthorizedToRead': True, + 'value': self.legal_hold and 'on' or 'off', + } + def add_part(self, part_number, part): while len(self.parts) < part_number + 1: self.parts.append(None) @@ -341,9 +397,9 @@ def _get_encryption_mode_and_secret(self, encryption: Optional[EncryptionSetting class FakeResponse(object): - def __init__(self, file_sim, url, range_=None): + def __init__(self, account_auth_token_or_none, file_sim, url, range_=None): self.data_bytes = file_sim.data_bytes - self.headers = file_sim.as_download_headers(range_) + self.headers = file_sim.as_download_headers(account_auth_token_or_none, range_) self.url = url self.range_ = range_ if range_ is not None: @@ -391,6 +447,7 @@ def __init__( lifecycle_rules=None, options_set=None, default_server_side_encryption=None, + is_file_lock_enabled: Optional[bool] = None, ): assert bucket_type in ['allPrivate', 'allPublic'] self.api = api @@ -413,29 +470,51 @@ def __init__( if default_server_side_encryption is None: default_server_side_encryption = EncryptionSetting(mode=EncryptionMode.NONE) self.default_server_side_encryption = default_server_side_encryption + self.is_file_lock_enabled = is_file_lock_enabled + self.default_retention = NO_RETENTION_BUCKET_SETTING + + def is_allowed_to_read_bucket_encryption_setting(self, account_auth_token): + return self._check_capability(account_auth_token, 'readBucketEncryption') + + def is_allowed_to_read_bucket_retention(self, account_auth_token): + return self._check_capability(account_auth_token, 'readBucketRetentions') + + def _check_capability(self, account_auth_token, capability): + try: + key = self.api.auth_token_to_key[account_auth_token] + except KeyError: + # looks like it's an upload token + # fortunately BucketSimulator makes it easy to retrieve the true account_auth_token + # from an upload url + real_auth_token = account_auth_token.split('/')[-1] + key = self.api.auth_token_to_key[real_auth_token] + capabilities = key.get_allowed()['capabilities'] + return capability in capabilities def bucket_dict(self, account_auth_token): default_sse = {'isClientAuthorizedToRead': False} - is_allowed_to_read_bucket_encryption_setting = 'readBucketEncryption' in self.api.auth_token_to_key[ - account_auth_token].get_allowed()['capabilities'] - logger.debug( - 'authtoken %s is %sallowed to read encryption setting of %s' % ( - account_auth_token, - not is_allowed_to_read_bucket_encryption_setting and 'not ' or '', - self, - ) - ) - if is_allowed_to_read_bucket_encryption_setting: - default_sse = { - 'isClientAuthorizedToRead': True, - } + if self.is_allowed_to_read_bucket_encryption_setting(account_auth_token): + default_sse['isClientAuthorizedToRead'] = True default_sse['value'] = {'mode': self.default_server_side_encryption.mode.value} if self.default_server_side_encryption.algorithm is not None: default_sse['value']['algorithm' ] = self.default_server_side_encryption.algorithm.value else: default_sse['value'] = {'mode': EncryptionMode.UNKNOWN} - logger.debug('default sse returned is: %s', default_sse) + + if self.is_allowed_to_read_bucket_retention(account_auth_token): + file_lock_configuration = { + 'isClientAuthorizedToRead': True, + 'value': { + 'defaultRetention': { + 'mode': self.default_retention.mode.value, + 'period': self.default_retention.period, + }, + 'isFileLockEnabled': self.is_file_lock_enabled, + }, + } # yapf: disable + else: + file_lock_configuration = {'isClientAuthorizedToRead': False, 'value': None} return dict( accountId=self.account_id, bucketName=self.bucket_name, @@ -447,6 +526,7 @@ def bucket_dict(self, account_auth_token): options=self.options_set, revision=self.revision, defaultServerSideEncryption=default_sse, + fileLockConfiguration=file_lock_configuration, ) def cancel_large_file(self, file_id): @@ -469,16 +549,27 @@ def delete_file_version(self, file_id, file_name): return dict(fileId=file_id, fileName=file_name, uploadTimestamp=file_sim.upload_timestamp) def download_file_by_id( - self, file_id, url, range_=None, encryption: Optional[EncryptionSetting] = None + self, + account_auth_token_or_none, + file_id, + url, + range_=None, + encryption: Optional[EncryptionSetting] = None, ): file_sim = self.file_id_to_file[file_id] file_sim.check_encryption(encryption) - return self._download_file_sim(file_sim, url, range_=range_) + return self._download_file_sim(account_auth_token_or_none, file_sim, url, range_=range_) def download_file_by_name( - self, file_name, url, range_=None, encryption: Optional[EncryptionSetting] = None + self, + account_auth_token_or_none, + file_name, + url, + range_=None, + encryption: Optional[EncryptionSetting] = None, ): - files = self.list_file_names(file_name, 1)['files'] + files = self.list_file_names(self.api.current_token, file_name, + 1)['files'] # token is not important here if len(files) == 0: raise FileNotPresent(file_id_or_name=file_name) file_dict = files[0] @@ -486,43 +577,68 @@ def download_file_by_name( raise FileNotPresent(file_id_or_name=file_name) file_sim = self.file_name_and_id_to_file[(file_name, file_dict['fileId'])] file_sim.check_encryption(encryption) - return self._download_file_sim(file_sim, url, range_=range_) - - def _download_file_sim(self, file_sim, url, range_=None): - return ResponseContextManager(self.RESPONSE_CLASS(file_sim, url, range_)) + return self._download_file_sim(account_auth_token_or_none, file_sim, url, range_=range_) + + def _download_file_sim(self, account_auth_token_or_none, file_sim, url, range_=None): + return ResponseContextManager( + self.RESPONSE_CLASS( + account_auth_token_or_none, + file_sim, + url, + range_, + ) + ) - def finish_large_file(self, file_id, part_sha1_array): + def finish_large_file(self, account_auth_token, file_id, part_sha1_array): file_sim = self.file_id_to_file[file_id] file_sim.finish(part_sha1_array) - return file_sim.as_upload_result() + return file_sim.as_upload_result(account_auth_token) - def get_file_info_by_id(self, file_id): - return self.file_id_to_file[file_id].as_upload_result() + def get_file_info_by_id(self, account_auth_token, file_id): + return self.file_id_to_file[file_id].as_upload_result(account_auth_token) - def get_file_info_by_name(self, file_name): + def get_file_info_by_name(self, account_auth_token, file_name): for ((name, id), file) in self.file_name_and_id_to_file.items(): if file_name == name: - return file.as_download_headers() + return file.as_download_headers(account_auth_token_or_none=account_auth_token) raise FileNotPresent(file_id_or_name=file_name, bucket_name=self.bucket_name) - def get_upload_url(self): + def get_upload_url(self, account_auth_token): upload_id = next(self.upload_url_counter) - upload_url = 'https://upload.example.com/%s/%s' % (self.bucket_id, upload_id) + upload_url = 'https://upload.example.com/%s/%d/%s' % ( + self.bucket_id, upload_id, account_auth_token + ) return dict(bucketId=self.bucket_id, uploadUrl=upload_url, authorizationToken=upload_url) - def get_upload_part_url(self, file_id): - upload_url = 'https://upload.example.com/part/%s/%d' % (file_id, random.randint(1, 10**9)) + def get_upload_part_url(self, account_auth_token, file_id): + upload_url = 'https://upload.example.com/part/%s/%d/%s' % ( + file_id, random.randint(1, 10**9), account_auth_token + ) return dict(bucketId=self.bucket_id, uploadUrl=upload_url, authorizationToken=upload_url) - def hide_file(self, file_name): + def hide_file(self, account_auth_token, file_name): file_id = self._next_file_id() file_sim = self.FILE_SIMULATOR_CLASS( - self.account_id, self.bucket_id, file_id, 'hide', file_name, None, "none", {}, b'', + self.account_id, self, file_id, 'hide', file_name, None, "none", {}, b'', next(self.upload_timestamp_counter) ) self.file_id_to_file[file_id] = file_sim self.file_name_and_id_to_file[file_sim.sort_key()] = file_sim - return file_sim.as_list_files_dict() + return file_sim.as_list_files_dict(account_auth_token) + + def update_file_retention( + self, + account_auth_token, + file_id, + file_name, + file_retention: FileRetentionSetting, + bypass_governance: bool = False, + ): + file_sim = self.file_id_to_file[file_id] + assert self.is_file_lock_enabled + assert file_sim.name == file_name + # TODO: check bypass etc + file_sim.file_retention = file_retention def copy_file( self, @@ -555,12 +671,11 @@ def copy_file( data_bytes = get_bytes_range(file_sim.data_bytes, bytes_range) - destination_bucket_id = destination_bucket_id or self.bucket_id + destination_bucket = self.api.bucket_id_to_bucket.get(destination_bucket_id, self) sse = destination_server_side_encryption or self.default_server_side_encryption - logger.debug('setting encryption to %s', sse) copy_file_sim = self.FILE_SIMULATOR_CLASS( self.account_id, - destination_bucket_id, + destination_bucket, new_file_id, 'copy', new_file_name, @@ -578,7 +693,13 @@ def copy_file( return copy_file_sim - def list_file_names(self, start_file_name=None, max_file_count=None, prefix=None): + def list_file_names( + self, + account_auth_token, + start_file_name=None, + max_file_count=None, + prefix=None, + ): assert prefix is None or start_file_name is None or start_file_name.startswith(prefix ), locals() start_file_name = start_file_name or '' @@ -595,7 +716,7 @@ def list_file_names(self, start_file_name=None, max_file_count=None, prefix=None prev_file_name = file_name file_sim = self.file_name_and_id_to_file[key] if file_sim.is_visible(): - result_files.append(file_sim.as_list_files_dict()) + result_files.append(file_sim.as_list_files_dict(account_auth_token)) if len(result_files) == max_file_count: next_file_name = file_sim.name + ' ' break @@ -603,6 +724,7 @@ def list_file_names(self, start_file_name=None, max_file_count=None, prefix=None def list_file_versions( self, + account_auth_token, start_file_name=None, start_file_id=None, max_file_count=None, @@ -625,7 +747,7 @@ def list_file_versions( file_sim = self.file_name_and_id_to_file[key] if prefix is not None and not file_name.startswith(prefix): break - result_files.append(file_sim.as_list_files_dict()) + result_files.append(file_sim.as_list_files_dict(account_auth_token)) if len(result_files) == max_file_count: next_file_name = file_sim.name next_file_id = str(int(file_id) + 1) @@ -636,7 +758,7 @@ def list_parts(self, file_id, start_part_number, max_part_count): file_sim = self.file_id_to_file[file_id] return file_sim.list_parts(start_part_number, max_part_count) - def list_unfinished_large_files(self, start_file_id=None, max_file_count=None, prefix=None): + def list_unfinished_large_files(self, account_auth_token, start_file_id=None, max_file_count=None, prefix=None): start_file_id = start_file_id or self.FIRST_FILE_ID max_file_count = max_file_count or 100 all_unfinished_ids = set( @@ -645,15 +767,9 @@ def list_unfinished_large_files(self, start_file_id=None, max_file_count=None, p (prefix is None or v.name.startswith(prefix)) ) ids_in_order = sorted(all_unfinished_ids, reverse=True) + file_dict_list = [ - dict( - fileId=file_sim.file_id, - fileName=file_sim.name, - accountId=file_sim.account_id, - bucketId=file_sim.bucket_id, - contentType=file_sim.content_type, - fileInfo=file_sim.file_info - ) + file_sim.as_start_large_file_result(account_auth_token) for file_sim in ( self.file_id_to_file[file_id] for file_id in ids_in_order[:max_file_count] ) @@ -665,6 +781,7 @@ def list_unfinished_large_files(self, start_file_id=None, max_file_count=None, p def start_large_file( self, + account_auth_token, file_name, content_type, file_info, @@ -674,14 +791,13 @@ def start_large_file( sse = server_side_encryption or self.default_server_side_encryption if sse: # FIXME: remove this part when RawApi<->Encryption adapters are implemented properly file_info = sse.add_key_id_to_file_info(file_info) - logger.debug('setting encryption to %s', sse) file_sim = self.FILE_SIMULATOR_CLASS( - self.account_id, self.bucket_id, file_id, 'start', file_name, content_type, 'none', + self.account_id, self, file_id, 'start', file_name, content_type, 'none', file_info, None, next(self.upload_timestamp_counter), server_side_encryption=sse, ) # yapf: disable self.file_id_to_file[file_id] = file_sim self.file_name_and_id_to_file[file_sim.sort_key()] = file_sim - return file_sim.as_start_large_file_result() + return file_sim.as_start_large_file_result(account_auth_token) def _update_bucket( self, @@ -689,8 +805,9 @@ def _update_bucket( bucket_info=None, cors_rules=None, lifecycle_rules=None, - if_revision_is=None, - default_server_side_encryption=None, + if_revision_is: Optional[int] = None, + default_server_side_encryption: Optional[EncryptionSetting] = None, + default_retention: Optional[BucketRetentionSetting] = None, ): if if_revision_is is not None and self.revision != if_revision_is: raise Conflict() @@ -705,6 +822,8 @@ def _update_bucket( self.lifecycle_rules = lifecycle_rules if default_server_side_encryption is not None: self.default_server_side_encryption = default_server_side_encryption + if default_retention: + self.default_retention = default_retention self.revision += 1 return self.bucket_dict(self.api.current_token) @@ -736,11 +855,10 @@ def upload_file( encryption = server_side_encryption or self.default_server_side_encryption if encryption: # FIXME: remove this part when RawApi<->Encryption adapters are implemented properly file_infos = encryption.add_key_id_to_file_info(file_infos) - logger.debug('setting encryption to %s', encryption) file_sim = self.FILE_SIMULATOR_CLASS( self.account_id, - self.bucket_id, + self, file_id, 'upload', file_name, @@ -753,7 +871,7 @@ def upload_file( ) self.file_id_to_file[file_id] = file_sim self.file_name_and_id_to_file[file_sim.sort_key()] = file_sim - return file_sim.as_upload_result() + return file_sim.as_upload_result(upload_auth_token) def upload_part( self, @@ -968,6 +1086,7 @@ def create_bucket( cors_rules=None, lifecycle_rules=None, default_server_side_encryption: Optional[EncryptionSetting] = None, + is_file_lock_enabled: Optional[bool] = None, ): if not re.match(r'^[-a-zA-Z0-9]*$', bucket_name): raise BadJson('illegal bucket name: ' + bucket_name) @@ -986,6 +1105,7 @@ def create_bucket( lifecycle_rules, # watch out for options! default_server_side_encryption=default_server_side_encryption, + is_file_lock_enabled=is_file_lock_enabled, ) self.bucket_name_to_bucket[bucket_name] = bucket self.bucket_id_to_bucket[bucket_id] = bucket @@ -1045,6 +1165,21 @@ def delete_file_version(self, api_url, account_auth_token, file_id, file_name): self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'deleteFiles') return bucket.delete_file_version(file_id, file_name) + def update_file_retention( + self, + api_url, + account_auth_token, + file_id, + file_name, + file_retention: FileRetentionSetting, + bypass_governance: bool = False, + ): + bucket_id = self.file_id_to_bucket_id[file_id] + bucket = self._get_bucket_by_id(bucket_id) + return bucket.update_file_retention( + account_auth_token, file_id, file_name, file_retention, bypass_governance + ) + def delete_bucket(self, api_url, account_auth_token, account_id, bucket_id): self._assert_account_auth(api_url, account_auth_token, account_id, 'deleteBuckets') bucket = self._get_bucket_by_id(bucket_id) @@ -1070,12 +1205,20 @@ def download_file_from_url( bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) return bucket.download_file_by_id( - file_id, range_=range_, url=url, encryption=encryption + account_auth_token_or_none, + file_id, + range_=range_, + url=url, + encryption=encryption, ) elif bucket_name is not None and file_name is not None: bucket = self._get_bucket_by_name(bucket_name) return bucket.download_file_by_name( - b2_url_decode(file_name), range_=range_, url=url, encryption=encryption + account_auth_token_or_none, + b2_url_decode(file_name), + range_=range_, + url=url, + encryption=encryption, ) else: assert False @@ -1093,7 +1236,7 @@ def finish_large_file(self, api_url, account_auth_token, file_id, part_sha1_arra bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') - return bucket.finish_large_file(file_id, part_sha1_array) + return bucket.finish_large_file(account_auth_token, file_id, part_sha1_array) def get_download_authorization( self, api_url, account_auth_token, bucket_id, file_name_prefix, valid_duration_in_seconds @@ -1116,28 +1259,28 @@ def get_download_authorization( def get_file_info_by_id(self, api_url, account_auth_token, file_id): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) - return bucket.get_file_info_by_id(file_id) + return bucket.get_file_info_by_id(account_auth_token, file_id) def get_file_info_by_name(self, api_url, account_auth_token, bucket_name, file_name): bucket = self._get_bucket_by_name(bucket_name) - info = bucket.get_file_info_by_name(file_name) + info = bucket.get_file_info_by_name(account_auth_token, file_name) return info def get_upload_url(self, api_url, account_auth_token, bucket_id): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') - return self._get_bucket_by_id(bucket_id).get_upload_url() + return self._get_bucket_by_id(bucket_id).get_upload_url(account_auth_token) def get_upload_part_url(self, api_url, account_auth_token, file_id): bucket_id = self.file_id_to_bucket_id[file_id] bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') - return self._get_bucket_by_id(bucket_id).get_upload_part_url(file_id) + return self._get_bucket_by_id(bucket_id).get_upload_part_url(account_auth_token, file_id) def hide_file(self, api_url, account_auth_token, bucket_id, file_name): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') - response = bucket.hide_file(file_name) + response = bucket.hide_file(account_auth_token, file_name) self.file_id_to_bucket_id[response['fileId']] = bucket_id return response @@ -1154,6 +1297,8 @@ def copy_file( destination_bucket_id=None, destination_server_side_encryption=None, source_server_side_encryption=None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): bucket_id = self.file_id_to_bucket_id[source_file_id] bucket = self._get_bucket_by_id(bucket_id) @@ -1179,7 +1324,10 @@ def copy_file( dest_bucket.file_id_to_file[copy_file_sim.file_id] = copy_file_sim dest_bucket.file_name_and_id_to_file[copy_file_sim.sort_key()] = copy_file_sim - return copy_file_sim.as_upload_result() + + # FIXME: implement `legal_hold` and `file_retention` + + return copy_file_sim.as_upload_result(account_auth_token) def copy_part( self, @@ -1271,7 +1419,7 @@ def list_file_names( bucket_id=bucket_id, file_name=prefix, ) - return bucket.list_file_names(start_file_name, max_file_count, prefix) + return bucket.list_file_names(account_auth_token, start_file_name, max_file_count, prefix) def list_file_versions( self, @@ -1292,7 +1440,13 @@ def list_file_versions( bucket_id=bucket_id, file_name=prefix, ) - return bucket.list_file_versions(start_file_name, start_file_id, max_file_count, prefix) + return bucket.list_file_versions( + account_auth_token, + start_file_name, + start_file_id, + max_file_count, + prefix, + ) def list_keys( self, @@ -1327,7 +1481,7 @@ def list_unfinished_large_files( ) start_file_id = start_file_id or '' max_file_count = max_file_count or 100 - return bucket.list_unfinished_large_files(start_file_id, max_file_count, prefix) + return bucket.list_unfinished_large_files(account_auth_token, start_file_id, max_file_count, prefix) def start_large_file( self, @@ -1338,16 +1492,22 @@ def start_large_file( content_type, file_info, server_side_encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): bucket = self._get_bucket_by_id(bucket_id) self._assert_account_auth(api_url, account_auth_token, bucket.account_id, 'writeFiles') result = bucket.start_large_file( + account_auth_token, file_name, content_type, file_info, server_side_encryption, ) self.file_id_to_bucket_id[result['fileId']] = bucket_id + + # FIXME: implement `legal_hold` and `file_retention` + return result def update_bucket( @@ -1361,7 +1521,8 @@ def update_bucket( cors_rules=None, lifecycle_rules=None, if_revision_is=None, - default_server_side_encryption=None, + default_server_side_encryption: Optional[EncryptionSetting] = None, + default_retention: Optional[BucketRetentionSetting] = None, ): assert bucket_type or bucket_info or cors_rules or lifecycle_rules or default_server_side_encryption bucket = self._get_bucket_by_id(bucket_id) @@ -1373,6 +1534,7 @@ def update_bucket( lifecycle_rules=lifecycle_rules, if_revision_is=if_revision_is, default_server_side_encryption=default_server_side_encryption, + default_retention=default_retention, ) def upload_file( @@ -1386,6 +1548,8 @@ def upload_file( file_infos, data_stream, server_side_encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): with ConcurrentUsedAuthTokenGuard( self.currently_used_auth_tokens[upload_auth_token], upload_auth_token @@ -1416,6 +1580,9 @@ def upload_file( ) file_id = response['fileId'] self.file_id_to_bucket_id[file_id] = bucket_id + + # FIXME: implement `legal_hold` and `file_retention` + return response def upload_part( diff --git a/b2sdk/session.py b/b2sdk/session.py index 7f57a93d1..b7375d42a 100644 --- a/b2sdk/session.py +++ b/b2sdk/session.py @@ -19,6 +19,7 @@ from b2sdk.cache import AuthInfoCache, DummyCache from b2sdk.encryption.setting import EncryptionSetting from b2sdk.exception import (InvalidAuthToken, Unauthorized) +from b2sdk.file_lock import BucketRetentionSetting, FileRetentionSetting from b2sdk.raw_api import ALL_CAPABILITIES, B2RawApi logger = logging.getLogger(__name__) @@ -141,6 +142,7 @@ def create_bucket( cors_rules=None, lifecycle_rules=None, default_server_side_encryption=None, + is_file_lock_enabled: Optional[bool] = None, ): return self._wrap_default_token( self.raw_api.create_bucket, @@ -151,6 +153,7 @@ def create_bucket( cors_rules=cors_rules, lifecycle_rules=lifecycle_rules, default_server_side_encryption=default_server_side_encryption, + is_file_lock_enabled=is_file_lock_enabled, ) def create_key( @@ -285,6 +288,8 @@ def start_large_file( content_type, file_info, server_side_encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): return self._wrap_default_token( self.raw_api.start_large_file, @@ -293,6 +298,8 @@ def start_large_file( content_type, file_info, server_side_encryption, + file_retention=file_retention, + legal_hold=legal_hold, ) def update_bucket( @@ -305,6 +312,7 @@ def update_bucket( lifecycle_rules=None, if_revision_is=None, default_server_side_encryption: Optional[EncryptionSetting] = None, + default_retention: Optional[BucketRetentionSetting] = None, ): return self._wrap_default_token( self.raw_api.update_bucket, @@ -316,6 +324,7 @@ def update_bucket( lifecycle_rules=lifecycle_rules, if_revision_is=if_revision_is, default_server_side_encryption=default_server_side_encryption, + default_retention=default_retention, ) def upload_file( @@ -328,6 +337,8 @@ def upload_file( file_infos, data_stream, server_side_encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): return self._wrap_token( self.raw_api.upload_file, @@ -340,6 +351,8 @@ def upload_file( file_infos, data_stream, server_side_encryption, + file_retention=file_retention, + legal_hold=legal_hold, ) def upload_part( @@ -381,6 +394,8 @@ def copy_file( destination_bucket_id=None, destination_server_side_encryption: Optional[EncryptionSetting] = None, source_server_side_encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): return self._wrap_default_token( self.raw_api.copy_file, @@ -393,6 +408,8 @@ def copy_file( destination_bucket_id=destination_bucket_id, destination_server_side_encryption=destination_server_side_encryption, source_server_side_encryption=source_server_side_encryption, + file_retention=file_retention, + legal_hold=legal_hold, ) def copy_part( diff --git a/b2sdk/transfer/emerge/emerger.py b/b2sdk/transfer/emerge/emerger.py index 93af5b3c1..e598ccc67 100644 --- a/b2sdk/transfer/emerge/emerger.py +++ b/b2sdk/transfer/emerge/emerger.py @@ -12,6 +12,7 @@ from typing import Optional from b2sdk.encryption.setting import EncryptionSetting +from b2sdk.file_lock import FileRetentionSetting from b2sdk.utils import B2TraceMetaAbstract from b2sdk.transfer.emerge.executor import EmergeExecutor from b2sdk.transfer.emerge.planner.planner import EmergePlanner @@ -50,6 +51,8 @@ def emerge( recommended_upload_part_size=None, continue_large_file_id=None, encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): """ Create a new file (object in the cloud, really) from an iterable (list, tuple etc) of write intents. @@ -74,6 +77,8 @@ def emerge( progress_listener, continue_large_file_id=continue_large_file_id, encryption=encryption, + file_retention=file_retention, + legal_hold=legal_hold, ) def emerge_stream( @@ -88,6 +93,8 @@ def emerge_stream( continue_large_file_id=None, max_queue_size=DEFAULT_STREAMING_MAX_QUEUE_SIZE, encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): """ Create a new file (object in the cloud, really) from a stream of write intents. @@ -112,6 +119,8 @@ def emerge_stream( continue_large_file_id=continue_large_file_id, max_queue_size=max_queue_size, encryption=encryption, + file_retention=file_retention, + legal_hold=legal_hold, ) def get_emerge_planner(self, recommended_upload_part_size=None): diff --git a/b2sdk/transfer/emerge/executor.py b/b2sdk/transfer/emerge/executor.py index edec968a0..b613d62f2 100644 --- a/b2sdk/transfer/emerge/executor.py +++ b/b2sdk/transfer/emerge/executor.py @@ -15,6 +15,7 @@ from b2sdk.encryption.setting import EncryptionSetting from b2sdk.exception import MaxFileSizeExceeded +from b2sdk.file_lock import FileRetentionSetting, NO_RETENTION_FILE_SETTING from b2sdk.file_version import FileVersionInfoFactory from b2sdk.transfer.outbound.large_file_upload_state import LargeFileUploadState from b2sdk.transfer.outbound.upload_source import UploadSourceStream @@ -38,6 +39,8 @@ def execute_emerge_plan( continue_large_file_id=None, max_queue_size=None, encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): if emerge_plan.is_large_file(): execution = LargeFileEmergeExecution( @@ -47,7 +50,9 @@ def execute_emerge_plan( content_type, file_info, progress_listener, - encryption, + encryption=encryption, + file_retention=file_retention, + legal_hold=legal_hold, continue_large_file_id=continue_large_file_id, max_queue_size=max_queue_size, ) @@ -61,7 +66,9 @@ def execute_emerge_plan( content_type, file_info, progress_listener, - encryption, + encryption=encryption, + file_retention=file_retention, + legal_hold=legal_hold, ) return execution.execute_plan(emerge_plan) @@ -78,6 +85,8 @@ def __init__( file_info, progress_listener, encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): self.services = services self.bucket_id = bucket_id @@ -86,6 +95,8 @@ def __init__( self.file_info = file_info self.progress_listener = progress_listener self.encryption = encryption + self.file_retention = file_retention + self.legal_hold = legal_hold @abstractmethod def execute_plan(self, emerge_plan): @@ -115,6 +126,8 @@ def __init__( file_info, progress_listener, encryption: Optional[EncryptionSetting] = None, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, continue_large_file_id=None, max_queue_size=None, ): @@ -125,7 +138,9 @@ def __init__( content_type, file_info, progress_listener, - encryption, + encryption=encryption, + file_retention=file_retention, + legal_hold=legal_hold, ) self.continue_large_file_id = continue_large_file_id self.max_queue_size = max_queue_size @@ -136,6 +151,7 @@ def __init__( def execute_plan(self, emerge_plan): total_length = emerge_plan.get_total_length() encryption = self.encryption + encryption = self.encryption if total_length is not None and total_length > self.MAX_LARGE_FILE_SIZE: raise MaxFileSizeExceeded(total_length, self.MAX_LARGE_FILE_SIZE) @@ -158,6 +174,8 @@ def execute_plan(self, emerge_plan): file_info, self.continue_large_file_id, encryption=encryption, + file_retention=self.file_retention, + legal_hold=self.legal_hold, emerge_parts_dict=emerge_parts_dict, ) @@ -171,7 +189,9 @@ def execute_plan(self, emerge_plan): self.file_name, content_type, file_info, - encryption, + encryption=encryption, + file_retention=self.file_retention, + legal_hold=self.legal_hold, ) file_id = unfinished_file.file_id @@ -224,6 +244,8 @@ def _get_unfinished_file_and_parts( file_info, continue_large_file_id, encryption: EncryptionSetting, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, emerge_parts_dict=None, ): if 'listFiles' not in self.services.session.account_info.get_allowed()['capabilities']: @@ -237,7 +259,6 @@ def _get_unfinished_file_and_parts( bucket_id, continue_large_file_id, prefix=file_name, - encryption=encryption, ) if unfinished_file.file_info != file_info: raise ValueError( @@ -255,6 +276,8 @@ def _get_unfinished_file_and_parts( file_info, emerge_parts_dict, encryption, + file_retention, + legal_hold, ) elif emerge_parts_dict is not None: unfinished_file, finished_parts = self._match_unfinished_file_if_possible( @@ -263,12 +286,22 @@ def _get_unfinished_file_and_parts( file_info, emerge_parts_dict, encryption, + file_retention, + legal_hold, ) return unfinished_file, finished_parts def _find_unfinished_file_by_plan_id( - self, bucket_id, file_name, file_info, emerge_parts_dict, encryption: EncryptionSetting + self, + bucket_id, + file_name, + file_info, + emerge_parts_dict, + encryption: EncryptionSetting, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): + file_retention = file_retention or NO_RETENTION_FILE_SETTING assert 'plan_id' in file_info best_match_file = None best_match_parts = {} @@ -278,8 +311,17 @@ def _find_unfinished_file_by_plan_id( ): if file_.file_info != file_info: continue + # FIXME: encryption is None ??? if encryption is None or file_.encryption != encryption: continue + if bool(legal_hold) != file_.legal_hold: + # when `file_.legal_hold is None` it means that `legal_hold` is unknown and we skip + continue + if file_retention != file_.file_retention: + # if `file_.file_retention` is UNKNOWN then we skip - lib user can still + # pass UKNOWN file_retention here - but raw_api/server won't allow it + # and we don't check it here + continue finished_parts = {} for part in self.services.large_file.list_parts(file_.file_id): emerge_part = emerge_parts_dict.get(part.part_number) @@ -307,6 +349,8 @@ def _match_unfinished_file_if_possible( file_info, emerge_parts_dict, encryption: EncryptionSetting, + file_retention: Optional[FileRetentionSetting] = None, + legal_hold: Optional[bool] = None, ): """ Find an unfinished file that may be used to resume a large file upload. The @@ -315,6 +359,7 @@ def _match_unfinished_file_if_possible( This is only possible if the application key being used allows ``listFiles`` access. """ + file_retention = file_retention or NO_RETENTION_FILE_SETTING for file_ in self.services.large_file.list_unfinished_large_files( bucket_id, prefix=file_name ): @@ -322,8 +367,17 @@ def _match_unfinished_file_if_possible( continue if file_.file_info != file_info: continue + # FIXME: what if `encryption is None` - match ANY encryption? :) if encryption is not None and encryption != file_.encryption: continue + if bool(legal_hold) != file_.legal_hold: + # when `file_.legal_hold is None` it means that `legal_hold` is unknown and we skip + continue + if file_retention != file_.file_retention: + # if `file_.file_retention` is UNKNOWN then we skip - lib user can still + # pass UKNOWN file_retention here - but raw_api/server won't allow it + # and we don't check it here + continue files_match = True finished_parts = {} for part in self.services.large_file.list_parts(file_.file_id): @@ -456,6 +510,8 @@ def execute(self): progress_listener=execution.progress_listener, destination_encryption=execution.encryption, source_encryption=self.copy_source_range.encryption, + file_retention=execution.file_retention, + legal_hold=execution.legal_hold, ) @@ -509,7 +565,9 @@ def execute(self): execution.content_type or execution.DEFAULT_CONTENT_TYPE, execution.file_info or {}, execution.progress_listener, - execution.encryption, + encryption=execution.encryption, + file_retention=execution.file_retention, + legal_hold=execution.legal_hold, ) diff --git a/b2sdk/transfer/outbound/copy_manager.py b/b2sdk/transfer/outbound/copy_manager.py index e797aea5b..0b3ebd263 100644 --- a/b2sdk/transfer/outbound/copy_manager.py +++ b/b2sdk/transfer/outbound/copy_manager.py @@ -14,6 +14,7 @@ from b2sdk.encryption.setting import EncryptionMode, EncryptionSetting, SSE_C_KEY_ID_FILE_INFO_KEY_NAME from b2sdk.exception import AlreadyFailed, SSECKeyIdMismatchInCopy +from b2sdk.file_lock import FileRetentionSetting from b2sdk.file_version import FileVersionInfoFactory from b2sdk.raw_api import MetadataDirectiveMode from b2sdk.utils import B2TraceMetaAbstract @@ -73,6 +74,8 @@ def copy_file( progress_listener, destination_encryption: Optional[EncryptionSetting] = None, source_encryption: Optional[EncryptionSetting] = None, + legal_hold: Optional[bool] = None, + file_retention: Optional[FileRetentionSetting] = None, ): # Run small copies in the same thread pool as large file copies, # so that they share resources during a sync. @@ -86,6 +89,8 @@ def copy_file( progress_listener=progress_listener, destination_encryption=destination_encryption, source_encryption=source_encryption, + legal_hold=legal_hold, + file_retention=file_retention, ) def copy_part( @@ -173,6 +178,8 @@ def _copy_small_file( progress_listener, destination_encryption: Optional[EncryptionSetting], source_encryption: Optional[EncryptionSetting], + legal_hold: Optional[bool] = None, + file_retention: Optional[FileRetentionSetting] = None, ): with progress_listener: progress_listener.set_total_bytes(copy_source.get_content_length() or 0) @@ -206,6 +213,8 @@ def _copy_small_file( destination_bucket_id=destination_bucket_id, destination_server_side_encryption=destination_encryption, source_server_side_encryption=source_encryption, + legal_hold=legal_hold, + file_retention=file_retention, ) file_info = FileVersionInfoFactory.from_api_response(response) if progress_listener is not None: diff --git a/b2sdk/transfer/outbound/upload_manager.py b/b2sdk/transfer/outbound/upload_manager.py index fab793497..a65ae2811 100644 --- a/b2sdk/transfer/outbound/upload_manager.py +++ b/b2sdk/transfer/outbound/upload_manager.py @@ -11,12 +11,15 @@ import logging import concurrent.futures as futures +from typing import Optional + from b2sdk.encryption.setting import EncryptionMode, EncryptionSetting from b2sdk.exception import ( AlreadyFailed, B2Error, MaxRetriesExceeded, ) +from b2sdk.file_lock import FileRetentionSetting from b2sdk.file_version import FileVersionInfoFactory from b2sdk.stream.progress import ReadingStreamWithProgress from b2sdk.stream.hashing import StreamWithHash @@ -78,7 +81,9 @@ def upload_file( content_type, file_info, progress_listener, - encryption: EncryptionSetting = None, + encryption: Optional[EncryptionSetting] = None, + legal_hold: Optional[bool] = None, + file_retention: Optional[FileRetentionSetting] = None, ): f = self.get_thread_pool().submit( self._upload_small_file, @@ -203,7 +208,9 @@ def _upload_small_file( content_type, file_info, progress_listener, - encryption: EncryptionSetting, + encryption: Optional[EncryptionSetting] = None, + legal_hold: Optional[bool] = None, + file_retention: Optional[FileRetentionSetting] = None, ): content_length = upload_source.get_content_length() exception_info_list = [] @@ -232,6 +239,8 @@ def _upload_small_file( file_info, input_stream, server_side_encryption=encryption, # todo: client side encryption + legal_hold=legal_hold, + file_retention=file_retention, ) if content_sha1 == HEX_DIGITS_AT_END: content_sha1 = input_stream.hash