Merge pull request #36 from Yulegu-bj/master

新增"获取已上传成功的分片列表"

Author: ufo2243

examples/example_get_multi_upload_part.py

@@ -0,0 +1,26 @@
+# !/usr/bin/env python
+# -*- encoding: utf-8 -*-
+
+from ufile import multipartuploadufile, filemanager
+
+public_key = ''  # 账户公钥
+private_key = ''  # 账户私钥
+
+bucket = ''  # 空间名称
+upload_id = ''  # 上传分片ID
+max_parts = None  # 最大分片数目
+part_number_marker = None  # 起始分片编号
+
+# 设置上传host后缀,外网可用后缀形如 .cn-bj.ufileos.com（cn-bj为北京地区，其他地区具体后缀可见控制台：对象存储-单地域空间管理-存储空间域名）
+# 默认值为'.cn-bj.ufileos.com'，如果上传文件的bucket所在地域不在北京，请务必设置此项
+upload_suffix = '.cn-bj.ufileos.com'
+
+multi_part_upload_ufile_handler = multipartuploadufile.MultipartUploadUFile(public_key, private_key, upload_suffix)
+
+ret, resp = multi_part_upload_ufile_handler.get_multi_upload_part(bucket, upload_id, max_parts, part_number_marker,
+                                                                  upload_suffix=upload_suffix)
+assert resp.status_code == 200, print(
+    "status: %d error: %s" % (resp.status_code, resp.error if resp.error else resp.content))
+
+print("list parts success")
+print(resp.content)

ufile/httprequest.py

@@ -327,6 +327,22 @@ def _bucket_request(url, param, header):
     return __return_wraper(response)
 
 
+def _get_multi_upload_part(url, header):
+    """
+    UCloud UFile 获取分片上传列表信息请求
+
+    :param url: String类型，获取分片上传列表信息请求的url
+    :param header: dict类型，键值对类型分别为string类型，HTTP请求头信息
+    :return: ret: 如果http状态码不为[200, 204, 206]之一则返回None，否则如果服务器返回json信息则返回dict类型，键值对类型分别为string, unicode string类型，否则返回空的dict
+    :return:  ResponseInfo: 响应的具体信息，UCloud UFile 服务器返回信息或者网络链接异常
+    """
+    try:
+        response = requests.get(url, headers=header)
+    except RequestException as e:
+        return None, ResponseInfo(None, e)
+    return __return_wraper(response)
+
+
 class ResponseInfo(object):
     """
     UCloud UFile 服务器返回信息,解析UCloud UFile服务器返回信息以及网络连接问题

ufile/multipartuploadufile.py

@@ -9,10 +9,10 @@
 from . import config
 from .baseufile import BaseUFile
 from .compact import s
-from .httprequest import ResponseInfo, _initialsharding, _finishsharding, _shardingupload
+from .httprequest import ResponseInfo, _initialsharding, _finishsharding, _shardingupload, _get_multi_upload_part
 from .logger import logger
 from .util import _check_dict, initialsharding_url, finishsharding_url, shardingupload_url, _file_iter, \
-    mimetype_from_file, mimetype_from_buffer, deprecated
+    mimetype_from_file, mimetype_from_buffer, deprecated, ufile_listparts_url
 
 
 class MultipartUploadUFile(BaseUFile):
@@ -115,8 +115,9 @@ def uploadstream(self, bucket, key, stream, maxthread=4, retrycount=3, retryinte
                 break
             self.etaglist.append("")
             thread1 = threading.Thread(target=self.__partthread, args=(
-            sem, self.__bucket, self.__key, self.uploadid, partnumber, self.__header, data, retrycount, retryinterval,
-            self.etaglist, self.__upload_suffix))
+                sem, self.__bucket, self.__key, self.uploadid, partnumber, self.__header, data, retrycount,
+                retryinterval,
+                self.etaglist, self.__upload_suffix))
             self.__threaddict[partnumber] = thread1
             thread1.start()
             partnumber += 1
@@ -397,6 +398,8 @@ def init_multipart_upload(self, bucket, key, header=None, upload_suffix=None):
         self.__bucket = bucket
         self.__key = key
         self.__header = header if header else dict()
+        if 'User-Agent' not in self.__header:
+            self.__header['User-Agent'] = config.get_default('user_agent')
         self.__upload_suffix = upload_suffix if upload_suffix else config.get_default('upload_suffix')
         ret, resp = self.__initialsharding()
         if resp.ok:
@@ -431,7 +434,8 @@ def upload_part_copy(self, source_bucket, source_key, mime_type, offset, size, p
         self.__header.update({"X-Ufile-Copy-Source": "/%s/%s" % (source_bucket, source_key)})
         self.__header.update({"Content-Type": self.__mimetype})
         self.__header.update({"X-Ufile-Copy-Source-Range": "bytes=%d-%d" % (offset, size - 1)})
-
+        if 'User-Agent' not in self.__header:
+            self.__header['User-Agent'] = config.get_default('user_agent')
         authorization = self.authorization('put', self.__bucket, self.__key, self.__header)
         self.__header.update({"Authorization": authorization})
 
@@ -441,3 +445,25 @@ def upload_part_copy(self, source_bucket, source_key, mime_type, offset, size, p
         if resp.ok:
             self.etaglist.append(resp.etag)
         return ret, resp
+
+    def get_multi_upload_part(self, bucket, upload_id, max_parts=None, part_number_marker=None, header=None, upload_suffix=None):
+        """
+        获取未完成分片上传的对象的已上传成功的分片列表。
+
+        :param bucket: string类型，空间名称
+        :param upload_id: string类型，初始化分片上传时返回的uploadid
+        :param max_parts: int类型，返回的最大条目数，默认为1000
+        :param part_number_marker: int类型，返回的起始条目，默认为0
+        :param header: dict类型，HTTP请求头部
+        :param upload_suffix: string类型, 如果传入此参数, 则会忽略 config 中配置的 upload_suffix 字段
+        :return: (dict, ResponseInfo) tuple类型，dict为返回的json信息，ResponseInfo为请求的返回信息
+        """
+        self.__header = header if header else dict()
+        if 'User-Agent' not in self.__header:
+            self.__header['User-Agent'] = config.get_default('user_agent')
+        if upload_suffix is not None:
+            self.__upload_suffix = upload_suffix
+        authorization = self.authorization('get', bucket, "", self.__header, action='muploadpart')
+        self.__header.update({"Authorization": authorization})
+        url = ufile_listparts_url(bucket, self.__upload_suffix, upload_id, max_parts, part_number_marker)
+        return _get_multi_upload_part(url, self.__header)

ufile/util.py

@@ -1,17 +1,16 @@
 # -*- coding: utf-8 -*-
 
-from .compact import *
-from . import config
-
 import base64
 import hashlib
+import mimetypes
 import os
 import struct
-
-import mimetypes
-from os import path
 import warnings
-from .config import BLOCKSIZE 
+from os import path
+
+from . import config
+from .compact import *
+from .config import BLOCKSIZE
 
 _EXTRA_TYPES_MAP = {
     ".js": "application/javascript",
@@ -428,7 +427,10 @@ def shardingupload_url(bucket, key, uploadid, part_number, upload_suffix=None):
     :param part_number: integer类型, 分片上传的编号,从0开始
     :return: string类型, 结束分片上传UFile的url
     """
-    return 'http://{0}{1}/{2}?uploadId={3}&partNumber={4}'.format(bucket, upload_suffix or config.get_default('upload_suffix'), key, uploadid, s(str(part_number)))
+    return 'http://{0}{1}/{2}?uploadId={3}&partNumber={4}'.format(bucket,
+                                                                  upload_suffix or config.get_default('upload_suffix'),
+                                                                  key, uploadid, s(str(part_number)))
+
 
 def ufile_getfilelist_url(bucket, upload_suffix=None):
     """
@@ -439,6 +441,7 @@ def ufile_getfilelist_url(bucket, upload_suffix=None):
     """
     return 'http://{0}{1}/?list'.format(bucket, upload_suffix or config.get_default('upload_suffix'))
 
+
 def mimetype_from_file(file):
     """
     获取文件的mimetype
@@ -454,6 +457,7 @@ def mimetype_from_file(file):
 
     return 'application/unknowntype'
 
+
 def mimetype_from_buffer(stream):
     """
     获取流对象的mimetype
@@ -463,6 +467,7 @@ def mimetype_from_buffer(stream):
     """
     return 'application/octet-stream'
 
+
 def ufile_restore_url(bucket, key, upload_suffix=None):
     """
     解冻冷存文件的url
@@ -473,6 +478,7 @@ def ufile_restore_url(bucket, key, upload_suffix=None):
     """
     return 'http://{0}{1}/{2}?restore'.format(bucket, upload_suffix or config.get_default('upload_suffix'), key)
 
+
 def ufile_classswitch_url(bucket, key, upload_suffix=None):
     """
     文件存储类型转换的url
@@ -483,6 +489,7 @@ def ufile_classswitch_url(bucket, key, upload_suffix=None):
     """
     return 'http://{0}{1}/{2}'.format(bucket, upload_suffix or config.get_default('upload_suffix'), key)
 
+
 def ufile_copy_url(bucket, key, upload_suffix=None):
     """
     拷贝文件的url
@@ -493,6 +500,7 @@ def ufile_copy_url(bucket, key, upload_suffix=None):
     """
     return 'http://{0}{1}/{2}'.format(bucket, upload_suffix or config.get_default('upload_suffix'), key)
 
+
 def ufile_rename_url(bucket, key, upload_suffix=None):
     """
     重命名文件的url
@@ -504,6 +512,7 @@ def ufile_rename_url(bucket, key, upload_suffix=None):
     """
     return 'http://{0}{1}/{2}'.format(bucket, upload_suffix or config.get_default('upload_suffix'), key)
 
+
 def ufile_listobjects_url(bucket, upload_suffix=None):
     """
     获取目录文件列表的url
@@ -513,13 +522,35 @@ def ufile_listobjects_url(bucket, upload_suffix=None):
     """
     return 'http://{0}{1}/?listobjects'.format(bucket, upload_suffix or config.get_default('upload_suffix'))
 
+
+def ufile_listparts_url(bucket, upload_suffix, upload_id, max_parts=None, part_number_marker=None):
+    """
+    获取未完成分片上传的对象的已上传成功的分片列表。
+
+    :param bucket: string类型, 空间名称
+    :param upload_suffix: string类型, 域名后缀
+    :param upload_id: string类型, 初始化分片上传获得的uploadid字符串
+    :param max_parts: integer类型, 规定在US3响应中的最大Part数目。
+    :param part_number_marker: integer类型, 指定List的起始位置，只有Part Number数目大于该参数的Part会被列出
+    """
+    url = 'http://{0}{1}/?muploadpart&uploadId={2}'.format(bucket, upload_suffix or config.get_default('upload_suffix'),
+                                                           upload_id)
+    if max_parts is not None:
+        url += '&max-parts={}'.format(max_parts)
+    if part_number_marker is not None:
+        url += '&part-number-marker={}'.format(part_number_marker)
+    return url
+
+
 def deprecated(message):
-  def deprecated_decorator(func):
-      def deprecated_func(*args, **kwargs):
-          warnings.warn("Call to deprecated function {} . -- {}".format(func.__name__, message),
-                        category=DeprecationWarning,
-                        stacklevel=2)
-          warnings.simplefilter('default', DeprecationWarning)
-          return func(*args, **kwargs)
-      return deprecated_func
-  return deprecated_decorator
+    def deprecated_decorator(func):
+        def deprecated_func(*args, **kwargs):
+            warnings.warn("Call to deprecated function {} . -- {}".format(func.__name__, message),
+                          category=DeprecationWarning,
+                          stacklevel=2)
+            warnings.simplefilter('default', DeprecationWarning)
+            return func(*args, **kwargs)
+
+        return deprecated_func
+
+    return deprecated_decorator