bump to 7.14.0 (#450)

* feat: add retrier

* chore(utils): deprecate etag

* refactor: update circle import solution between region and conf

Make `qiniu.config` more independent and move test cases to `tests/cases/test_zone` from `test_qiniu.py`

BREAK CHANGE: default value of protected filed `qiniu.config._config['default_zone']` changes to `None` from `qiniu.region.Region()`, which should be never used directly in user code.

* feat: add new region and endpoint for source/accelerate uploading with retrying

* fix: default_client will not work with config.set_default by call once

* chore: remove iter self for endpoint

* chore: improve error text

* fix: some features are not compatible with old version and improve error text

- change preferred scheme to http from https
- change UploaderBase._get_regions return type to list[LegacyRegion] from list[LegacyRegion or Region]
- change all inner legacy region type name from Region to LegacyRegion
- LegacyRegion.get_bucket_hosts homecache argument not work

* fix: TokenExpiredRetryPolicy not work

* fix: AccUnavailableRetryPolicy modify origin region service

* fix: argument name typo on put_data

* feat: bucket support regions and endpoints retry

* test: add and improve test cases

* feat: add field persistentType to strict policy fields

* style: fix flake8 code styles

* fix: enum on python2

* test: fix compatibility of test cases on python 2.7

* test: change test region to na0 from z0

* chore: ci add no accelerate bucket

* test: fix test error with python2

* fix: LegacyRegion.get_bucket_host not working in python2

* doc: add more type info to functions

* chore: change default hosts for querying regions

* fix: CachedRegionsProvider shrink not working

* feat: add uc backup hosts

* feat: update version and changelog

* add Qiniu auth verify callback

* chore: remove `Region.from_region_id` backup domain qbox and s3

* feat: add idle-time fop support and get fop status

* docs: fix authorization token link

* fix: form retry not working by no resume recorder

* chore: fix flake8 lint on python >= 3.8

* Update CHANGELOG.md

* fix: legacy region get_xxx_host and add more cases

Author: lihsai0

.github/workflows/ci-test.yml

@@ -55,6 +55,7 @@ jobs:
           QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }}
           QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }}
           QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }}
+          QINIU_TEST_NO_ACC_BUCKET: ${{ secrets.QINIU_TEST_NO_ACC_BUCKET }}
           QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }}
           QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}}
           QINIU_TEST_ENV: "travis"

CHANGELOG.md

@@ -1,4 +1,11 @@
 # Changelog
+## 7.14.0
+* 对象存储，空间管理、上传文件新增备用域名重试逻辑
+* 对象存储，调整查询区域主备域名
+* 对象存储，支持空间级别加速域名开关
+* 对象存储，回调签名验证函数新增兼容 Qiniu 签名
+* 对象存储，持久化处理支持闲时任务
+
 ## 7.13.2（2024-05-28）
 * 对象存储，修复上传回调设置自定义变量失效（v7.12.0 引入）
 

examples/upload.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 # flake8: noqa
+# import hashlib
 
-from qiniu import Auth, put_file, etag, urlsafe_base64_encode
+from qiniu import Auth, put_file, urlsafe_base64_encode
 import qiniu.config
 from qiniu.compat import is_py2, is_py3
 
@@ -24,13 +25,28 @@
 # 要上传文件的本地路径
 localfile = '/Users/jemy/Documents/qiniu.png'
 
-ret, info = put_file(token, key, localfile)
+# 上传时，sdk 会自动计算文件 hash 作为参数传递给服务端确保上传完整性
+# （若不一致，服务端会拒绝完成上传）
+# 但在访问文件时，服务端可能不会提供 MD5 或者编码格式不是期望的
+# 因此若有需有，请通过元数据功能自定义 MD5 或其他 hash 字段
+# hasher = hashlib.md5()
+# with open(localfile, 'rb') as f:
+#     for d in f:
+#         hasher.update(d)
+# object_metadata = {
+#     'x-qn-meta-md5': hasher.hexdigest()
+# }
+
+ret, info = put_file(
+    token,
+    key,
+    localfile
+    # metadata=object_metadata
+)
 print(ret)
 print(info)
 
 if is_py2:
     assert ret['key'].encode('utf-8') == key
 elif is_py3:
     assert ret['key'] == key
-
-assert ret['hash'] == etag(localfile)

examples/upload_callback.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # flake8: noqa
 
-from qiniu import Auth, put_file, etag
+from qiniu import Auth, put_file
 
 access_key = '...'
 secret_key = '...'
@@ -25,4 +25,3 @@
 ret, info = put_file(token, key, localfile)
 print(info)
 assert ret['key'] == key
-assert ret['hash'] == etag(localfile)

examples/upload_pfops.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # flake8: noqa
-from qiniu import Auth, put_file, etag, urlsafe_base64_encode
+from qiniu import Auth, put_file, urlsafe_base64_encode
 
 access_key = '...'
 secret_key = '...'
@@ -36,4 +36,3 @@
 ret, info = put_file(token, key, localfile)
 print(info)
 assert ret['key'] == key
-assert ret['hash'] == etag(localfile)

examples/upload_with_qvmzone.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # flake8: noqa
 
-from qiniu import Auth, put_file, etag, urlsafe_base64_encode
+from qiniu import Auth, put_file, urlsafe_base64_encode
 import qiniu.config
 from qiniu import Zone, set_default
 
@@ -37,4 +37,3 @@
 ret, info = put_file(token, key, localfile)
 print(info)
 assert ret['key'] == key
-assert ret['hash'] == etag(localfile)

qiniu/__init__.py

@@ -9,16 +9,16 @@
 
 # flake8: noqa
 
-__version__ = '7.13.2'
+__version__ = '7.14.0'
 
 from .auth import Auth, QiniuMacAuth
 
 from .config import set_default
 from .zone import Zone
-from .region import Region
+from .region import LegacyRegion as Region
 
 from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, \
-    build_batch_stat, build_batch_delete, build_batch_restoreAr
+    build_batch_stat, build_batch_delete, build_batch_restoreAr, build_batch_restore_ar
 from .services.storage.uploader import put_data, put_file, put_stream
 from .services.storage.upload_progress_recorder import UploadProgressRecorder
 from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url, DomainManager

qiniu/auth.py

@@ -37,6 +37,8 @@
     str('persistentOps'),  # 持久化处理操作
     str('persistentNotifyUrl'),  # 持久化处理结果通知URL
     str('persistentPipeline'),  # 持久化处理独享队列
+    str('persistentType'),  # 为 `1` 时，开启闲时任务，必须是 int 类型
+
     str('deleteAfterDays'),  # 文件多少天后自动删除
     str('fileType'),  # 文件的存储类型，0为标准存储，1为低频存储，2为归档存储，3为深度归档存储，4为归档直读存储
     str('isPrefixalScope'),  # 指定上传文件必须使用的前缀
@@ -194,22 +196,56 @@ def __upload_token(self, policy):
         return self.token_with_data(data)
 
     def verify_callback(
-            self,
-            origin_authorization,
-            url,
-            body,
-            content_type='application/x-www-form-urlencoded'):
-        """回调验证
-
-        Args:
-            origin_authorization: 回调时请求Header中的Authorization字段
-            url:                  回调请求的url
-            body:                 回调请求的body
-            content_type:         回调请求body的Content-Type
-
-        Returns:
-            返回true表示验证成功，返回false表示验证失败
+        self,
+        origin_authorization,
+        url,
+        body,
+        content_type='application/x-www-form-urlencoded',
+        method='GET',
+        headers=None
+    ):
+        """
+        Qbox 回调验证
+
+        Parameters
+        ----------
+        origin_authorization: str
+            回调时请求 Header 中的 Authorization 字段
+        url: str
+            回调请求的 url
+        body: str
+            回调请求的 body
+        content_type: str
+            回调请求的 Content-Type
+        method: str
+            回调请求的 method，Qiniu 签名必须传入，默认 GET
+        headers: dict
+            回调请求的 headers，Qiniu 签名必须传入，默认为空字典
+
+        Returns
+        -------
+        bool
+            返回 True 表示验证成功，返回 False 表示验证失败
         """
+        if headers is None:
+            headers = {}
+
+        # 兼容 Qiniu 签名
+        if origin_authorization.startswith("Qiniu"):
+            qn_auth = QiniuMacAuth(
+                access_key=self.__access_key,
+                secret_key=self.__secret_key,
+                disable_qiniu_timestamp_signature=True
+            )
+            return qn_auth.verify_callback(
+                origin_authorization,
+                url=url,
+                body=body,
+                content_type=content_type,
+                method=method,
+                headers=headers
+            )
+
         token = self.token_of_request(url, body, content_type)
         authorization = 'QBox {0}'.format(token)
         return origin_authorization == authorization
@@ -243,7 +279,7 @@ class QiniuMacAuth(object):
         __access_key
         __secret_key
 
-    http://kirk-docs.qiniu.com/apidocs/#TOC_325b437b89e8465e62e958cccc25c63f
+    https://developer.qiniu.com/kodo/1201/access-token
     """
 
     def __init__(self, access_key, secret_key, disable_qiniu_timestamp_signature=None):
@@ -326,6 +362,50 @@ def qiniu_headers(self, headers):
             '%s: %s' % (canonical_mime_header_key(key), headers.get(key)) for key in sorted(qiniu_fields)
         ])
 
+    def verify_callback(
+        self,
+        origin_authorization,
+        url,
+        body,
+        content_type='application/x-www-form-urlencoded',
+        method='GET',
+        headers=None
+    ):
+        """
+        Qiniu 回调验证
+
+        Parameters
+        ----------
+        origin_authorization: str
+            回调时请求 Header 中的 Authorization 字段
+        url: str
+            回调请求的 url
+        body: str
+            回调请求的 body
+        content_type: str
+            回调请求的 Content-Type
+        method: str
+            回调请求的 Method
+        headers: dict
+            回调请求的 headers
+
+        Returns
+        -------
+
+        """
+        if headers is None:
+            headers = {}
+        token = self.token_of_request(
+            method=method,
+            host=headers.get('Host', None),
+            url=url,
+            qheaders=self.qiniu_headers(headers),
+            content_type=content_type,
+            body=body
+        )
+        authorization = 'Qiniu {0}'.format(token)
+        return origin_authorization == authorization
+
     @staticmethod
     def __checkKey(access_key, secret_key):
         if not (access_key and secret_key):

qiniu/config.py

@@ -1,45 +1,36 @@
 # -*- coding: utf-8 -*-
-from qiniu import region
-
 RS_HOST = 'http://rs.qiniu.com'  # 管理操作Host
 RSF_HOST = 'http://rsf.qbox.me'  # 列举操作Host
 API_HOST = 'http://api.qiniuapi.com'  # 数据处理操作Host
-UC_HOST = region.UC_HOST  # 获取空间信息Host
-QUERY_REGION_HOST = 'https://kodo-config.qiniuapi.com'
+QUERY_REGION_HOST = 'https://uc.qiniuapi.com'
+QUERY_REGION_BACKUP_HOSTS = [
+    'kodo-config.qiniuapi.com',
+    'uc.qbox.me'
+]
+UC_HOST = QUERY_REGION_HOST  # 获取空间信息Host
+UC_BACKUP_HOSTS = QUERY_REGION_BACKUP_HOSTS
 
 _BLOCK_SIZE = 1024 * 1024 * 4  # 断点续传分块大小，该参数为接口规格，暂不支持修改
 
 _config = {
-    'default_zone': region.Region(),
+    'default_zone': None,
     'default_rs_host': RS_HOST,
     'default_rsf_host': RSF_HOST,
     'default_api_host': API_HOST,
     'default_uc_host': UC_HOST,
+    'default_uc_backup_hosts': UC_BACKUP_HOSTS,
     'default_query_region_host': QUERY_REGION_HOST,
-    'default_query_region_backup_hosts': [
-        'uc.qbox.me',
-        'api.qiniu.com'
-    ],
-    'default_backup_hosts_retry_times': 2,
+    'default_query_region_backup_hosts': QUERY_REGION_BACKUP_HOSTS,
+    'default_backup_hosts_retry_times': 3,  # 仅控制旧区域 LegacyRegion 查询 Hosts 的重试次数
     'connection_timeout': 30,  # 链接超时为时间为30s
     'connection_retries': 3,  # 链接重试次数为3次
     'connection_pool': 10,  # 链接池个数为10
     'default_upload_threshold': 2 * _BLOCK_SIZE  # put_file上传方式的临界默认值
 }
 
 _is_customized_default = {
-    'default_zone': False,
-    'default_rs_host': False,
-    'default_rsf_host': False,
-    'default_api_host': False,
-    'default_uc_host': False,
-    'default_query_region_host': False,
-    'default_query_region_backup_hosts': False,
-    'default_backup_hosts_retry_times': False,
-    'connection_timeout': False,
-    'connection_retries': False,
-    'connection_pool': False,
-    'default_upload_threshold': False
+    k: False
+    for k in _config.keys()
 }
 
 
@@ -48,6 +39,10 @@ def is_customized_default(key):
 
 
 def get_default(key):
+    if key == 'default_zone' and not _is_customized_default[key]:
+        # prevent circle import
+        from .region import LegacyRegion
+        return LegacyRegion()
     return _config[key]
 
 
@@ -56,7 +51,7 @@ def set_default(
         connection_timeout=None, default_rs_host=None, default_uc_host=None,
         default_rsf_host=None, default_api_host=None, default_upload_threshold=None,
         default_query_region_host=None, default_query_region_backup_hosts=None,
-        default_backup_hosts_retry_times=None):
+        default_backup_hosts_retry_times=None, default_uc_backup_hosts=None):
     if default_zone:
         _config['default_zone'] = default_zone
         _is_customized_default['default_zone'] = True
@@ -72,16 +67,23 @@ def set_default(
     if default_uc_host:
         _config['default_uc_host'] = default_uc_host
         _is_customized_default['default_uc_host'] = True
+        _config['default_uc_backup_hosts'] = []
+        _is_customized_default['default_uc_backup_hosts'] = True
         _config['default_query_region_host'] = default_uc_host
         _is_customized_default['default_query_region_host'] = True
         _config['default_query_region_backup_hosts'] = []
         _is_customized_default['default_query_region_backup_hosts'] = True
+    if default_uc_backup_hosts is not None:
+        _config['default_uc_backup_hosts'] = default_uc_backup_hosts
+        _is_customized_default['default_uc_backup_hosts'] = True
+        _config['default_query_region_backup_hosts'] = default_uc_backup_hosts
+        _is_customized_default['default_query_region_backup_hosts'] = True
     if default_query_region_host:
         _config['default_query_region_host'] = default_query_region_host
         _is_customized_default['default_query_region_host'] = True
         _config['default_query_region_backup_hosts'] = []
         _is_customized_default['default_query_region_backup_hosts'] = True
-    if default_query_region_backup_hosts:
+    if default_query_region_backup_hosts is not None:
         _config['default_query_region_backup_hosts'] = default_query_region_backup_hosts
         _is_customized_default['default_query_region_backup_hosts'] = True
     if default_backup_hosts_retry_times: