From 3e96af0d3cc89f26f25a85e72d3cc87951d10fca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=94=A1=E5=8A=A0=E5=BC=BA?= Date: Fri, 22 Jun 2018 15:24:51 +0800 Subject: [PATCH 1/6] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=A7=86=E9=A2=91?= =?UTF-8?q?=E4=B8=89=E9=89=B4=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/__init__.py | 1 + qiniu/services/appraisal/__init__.py | 0 qiniu/services/appraisal/pulp.py | 100 +++++++++++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 qiniu/services/appraisal/__init__.py create mode 100644 qiniu/services/appraisal/pulp.py diff --git a/qiniu/__init__.py b/qiniu/__init__.py index f229397a..98dad9ca 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -24,6 +24,7 @@ from .services.processing.cmd import build_op, pipe_cmd, op_save from .services.compute.app import AccountClient from .services.compute.qcos_api import QcosClient +from .services.appraisal.pulp import video_pulp, video_terror, video_politician, video_appraisal, AppraisalOperation from .services.pili.rtc_server_manager import RtcServer, get_room_token diff --git a/qiniu/services/appraisal/__init__.py b/qiniu/services/appraisal/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qiniu/services/appraisal/pulp.py b/qiniu/services/appraisal/pulp.py new file mode 100644 index 00000000..c358158d --- /dev/null +++ b/qiniu/services/appraisal/pulp.py @@ -0,0 +1,100 @@ +import json +from qiniu.http import _post_with_qiniu_mac +from qiniu.auth import QiniuMacAuth + + +class AppraisalOperation(object): + '''视频审核的动作（鉴黄，鉴恐， 鉴政治人物)''' + ops = ("pulp", "terror", "politician") + + def __init__(self, op, hook_url="", params=None): + if op not in self.ops: + raise ValueError("op must be in %s" % self.ops) + if params is not None and not isinstance(params, dict): + raise TypeError("params must be dict: %s" % params) + self.op = op + self.hook_url = hook_url + self.params = params + + def __str__(self): + return 'op: %s\nhook_url: %s\nparams: %s\n' % (self.op, self.hook_url, self.params) + + +def video_appraisal(auth, vid, url, ops, params=None): + """ + @vid 视频唯一的ID + @url 视频鉴黄的地址 + @params 字典，视频处理的参数 + @ops 视频检测命令 [AppraisalOperation...] + 具体参数格式参考: + https://developer.qiniu.com/dora/manual/4258/video-pulp''' + """ + if params is not None: + if not isinstance(params, dict): + raise TypeError("params must be instance of dict, invalid params: %s" % params) + if not isinstance(ops, list): + raise TypeError("ops must be instance of list, invalid ops: %s" % ops) + if len(ops) <= 0: + raise ValueError("length of ops must greater than zero") + if not isinstance(auth, QiniuMacAuth): + raise TypeError("auth must be instance of QiniuMacAuth") + + def getop(operation): + d = { + "op": operation.op, + } + if operation.hook_url: + d["hookURL"] = operation.hook_url + if operation.params: + d["params"] = operation.params + return d + + ops = [getop(op) for op in ops] + + if params: + data = { + "data": { + "uri": url, + }, + "params": params + } + else: + data = { + "data": { + "uri": url, + }, + } + data['ops'] = ops + + return _post_with_qiniu_mac("http://argus.atlab.ai/v1/video/%s" % vid, data, auth) + + +def video_pulp(auth, vid, url, op=None, params=None): + if op is None: + op = AppraisalOperation("pulp") + else: + if not isinstance(op, AppraisalOperation): + raise TypeError("op must be instance of AppraisalOperation: %s" % op) + if op.op != "pulp": + raise ValueError("pulp appraisal operation must be pulp: %s" % op.op) + return video_appraisal(auth, vid, url, [op], params) + +def video_terror(auth, vid, url, op=None, params=None): + if op is None: + op = AppraisalOperation("terror") + else: + if not isinstance(op, AppraisalOperation): + raise TypeError("op must be instance of AppraisalOperation: %s" % op) + if op.op != "politician": + raise ValueError("terror appraisal operation must be terror: %s" % op.op) + return video_appraisal(auth, vid, url, [op], params) + +def video_politician(auth, vid, url, op=None, params=None): + if op is None: + op = AppraisalOperation("politician") + else: + if not isinstance(op, AppraisalOperation): + raise TypeError("op must be instance of AppraisalOperation: %s" % op) + if op.op != "politician": + raise ValueError("politician appraisal operation must be politician: %s" % op.op) + return video_appraisal(auth, vid, url, [op], params) From 381c79d81a5ad36bf3e737877596fdc7dc5d4f94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=94=A1=E5=8A=A0=E5=BC=BA?= Date: Fri, 22 Jun 2018 15:41:41 +0800 Subject: [PATCH 2/6] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E9=89=B4=E9=BB=84example?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/video_pulp.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 examples/video_pulp.py diff --git a/examples/video_pulp.py b/examples/video_pulp.py new file mode 100644 index 00000000..6fca86a3 --- /dev/null +++ b/examples/video_pulp.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from qiniu import QiniuMacAuth, video_pulp + +# 对已经上传到七牛的视频发起异步转码操作 +access_key = 'Access_Key' +secret_key = 'Secret_Key' +q = QiniuMacAuth(access_key, secret_key) + + +url = '' # 要鉴别的视频地址 +video_id = '' # 视频的唯一ID + + +ret, info = video_pulp(q, video_id, url) + +print(info) +assert 'pulp' in ret From 4d677d54a8d1e516ae749cd46e422e8fca1856d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=94=A1=E5=8A=A0=E5=BC=BA?= Date: Fri, 22 Jun 2018 16:02:02 +0800 Subject: [PATCH 3/6] =?UTF-8?q?=E8=A7=84=E8=8C=83=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/appraisal/pulp.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qiniu/services/appraisal/pulp.py b/qiniu/services/appraisal/pulp.py index c358158d..ce6402e0 100644 --- a/qiniu/services/appraisal/pulp.py +++ b/qiniu/services/appraisal/pulp.py @@ -1,4 +1,4 @@ -import json +# -*- coding: utf-8 -*- from qiniu.http import _post_with_qiniu_mac from qiniu.auth import QiniuMacAuth @@ -79,6 +79,7 @@ def video_pulp(auth, vid, url, op=None, params=None): raise ValueError("pulp appraisal operation must be pulp: %s" % op.op) return video_appraisal(auth, vid, url, [op], params) + def video_terror(auth, vid, url, op=None, params=None): if op is None: op = AppraisalOperation("terror") @@ -89,6 +90,7 @@ def video_terror(auth, vid, url, op=None, params=None): raise ValueError("terror appraisal operation must be terror: %s" % op.op) return video_appraisal(auth, vid, url, [op], params) + def video_politician(auth, vid, url, op=None, params=None): if op is None: op = AppraisalOperation("politician") From 4b5ffd024f786671bfc61f6348fc5766d37f0a88 Mon Sep 17 00:00:00 2001 From: Tony Cai Date: Mon, 16 Jul 2018 19:50:17 +0800 Subject: [PATCH 4/6] =?UTF-8?q?=E4=BF=AE=E6=94=B9DomainManager=20=E4=B8=AD?= =?UTF-8?q?=E7=9A=84get=5Fdomain=E6=8E=A5=E5=8F=A3=EF=BC=8C=E5=BA=94?= =?UTF-8?q?=E8=AF=A5=E4=BD=BF=E7=94=A8GET=20=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/cdn/manager.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index 313736ef..3cb8e3e3 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -212,7 +212,7 @@ def get_domain(self, name): - ResponseInfo 请求的Response信息 """ url = '{0}/domain/{1}'.format(self.server, name) - return self.__post(url) + return self.__get(url) def put_httpsconf(self, name, certid, forceHttps): """ @@ -264,6 +264,10 @@ def __post(self, url, data=None): headers = {'Content-Type': 'application/json'} return http._post_with_auth_and_headers(url, data, self.auth, headers) + def __get(self, url, params=None): + headers = {'Content-Type': 'application/json'} + return http._get(url, params, self.auth) + def __put(self, url, data=None): headers = {'Content-Type': 'application/json'} return http._put_with_auth_and_headers(url, data, self.auth, headers) From 61ee92e3e3290cb0644d4eabd1f38f856caeb2a1 Mon Sep 17 00:00:00 2001 From: Tony Cai Date: Mon, 16 Jul 2018 20:09:56 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E5=9F=9F=E5=90=8D=E5=88=97=E8=A1=A8=E7=9A=84=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 12 ++++++++++++ qiniu/services/cdn/manager.py | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 main.py diff --git a/main.py b/main.py new file mode 100644 index 00000000..70e6ebc6 --- /dev/null +++ b/main.py @@ -0,0 +1,12 @@ +from qiniu import Auth, DomainManager + +access_key = '' +secret_key = '' + +at = Auth(access_key, secret_key) + +dm = DomainManager(at) + +for ret, resp_info in dm.get_domain_list(limit=100): + for domain in ret['domains']: + print(domain['name']) diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index 3cb8e3e3..395309ba 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -214,6 +214,25 @@ def get_domain(self, name): url = '{0}/domain/{1}'.format(self.server, name) return self.__get(url) + def get_domain_list(self, marker="", limit=100): + """ + 获取域名信息，文档 https://developer.qiniu.com/fusion/api/4246/the-domain-name + + Args: + name: 域名, 如果是泛域名，必须以点号 . 开头 + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/domain'.format(self.server) + ret, respInfo = self.__get(url, params={"marker": marker, "limit": limit}) + yield ret, respInfo + + if ret.get("marker", ""): + for result in self.get_domain_list(marker=ret['marker'], limit=limit): + yield result + def put_httpsconf(self, name, certid, forceHttps): """ 修改证书，文档 https://developer.qiniu.com/fusion/api/4246/the-domain-name#11 From ba7c884067cd56e376f7a8d0adf885a09a2f92fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=94=A1=E5=8A=A0=E5=BC=BA?= Date: Tue, 17 Jul 2018 14:37:36 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=BC=82=E6=AD=A5?= =?UTF-8?q?=E6=8A=93=E5=8F=96async=5Ffetch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/auth.py | 5 ++- qiniu/config.py | 1 + qiniu/http.py | 14 ++++++--- qiniu/services/storage/bucket.py | 53 ++++++++++++++++++++++++++++++++ qiniu/zone.py | 9 ++++++ 5 files changed, 76 insertions(+), 6 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index c25ad6d0..1651af21 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -6,7 +6,7 @@ from requests.auth import AuthBase -from .compat import urlparse, json, b +from .compat import urlparse, json, b, s from .utils import urlsafe_base64_encode @@ -56,6 +56,9 @@ def __init__(self, access_key, secret_key): self.__access_key = access_key self.__secret_key = b(secret_key) + def get_secret_key(self): + return s(self.__secret_key) + def get_access_key(self): return self.__access_key diff --git a/qiniu/config.py b/qiniu/config.py index 9b827962..d116d223 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -5,6 +5,7 @@ RS_HOST = 'http://rs.qbox.me' # 管理操作Host RSF_HOST = 'http://rsf.qbox.me' # 列举操作Host API_HOST = 'http://api.qiniu.com' # 数据处理操作Host +BUCKET_RS_HOST = 'http://rs.qiniu.com' # 获取bucket信息 _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 diff --git a/qiniu/http.py b/qiniu/http.py index 80596700..99462ab5 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -111,14 +111,18 @@ def _post_with_auth_and_headers(url, data, auth, headers): def _put_with_auth(url, data, auth): return _put(url, data, None, qiniu.auth.RequestsAuth(auth)) - def _put_with_auth_and_headers(url, data, auth, headers): return _put(url, data, None, qiniu.auth.RequestsAuth(auth), headers) - -def _post_with_qiniu_mac(url, data, auth): +def _post_with_qiniu_mac(url, data, auth, headers=None): + post_headers = _headers.copy() + if headers is not None: + for k, v in headers.items(): + post_headers.update({k: v}) + access_key = auth.get_access_key() + secret_key = auth.get_secret_key() qn_auth = qiniu.auth.QiniuMacRequestsAuth( - auth) if auth is not None else None + qiniu.auth.QiniuMacAuth(access_key, secret_key)) if auth is not None else None timeout = config.get_default('connection_timeout') try: @@ -127,7 +131,7 @@ def _post_with_qiniu_mac(url, data, auth): json=data, auth=qn_auth, timeout=timeout, - headers=_headers) + headers=post_headers) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 30e0d100..fa26ffb9 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -161,6 +161,49 @@ def copy(self, bucket, key, bucket_to, key_to, force='false'): to = entry(bucket_to, key_to) return self.__rs_do('copy', resource, to, 'force/{0}'.format(force)) + def async_fetch(self, urls, bucket, host=None, key=None, md5=None, etag=None, callback_url=None, + callback_body=None, callback_body_type=None, callback_host=None, file_type=0): + """异步抓取文件: + 从指定URL抓取资源，并将该资源存储到指定空间中，具体规格参考： + https://developer.qiniu.com/kodo/api/4097/asynch-fetch + + Args: + url: 指定的URL + bucket: 目标资源空间 + key: 目标资源文件名 + + Returns: + 一个dict变量，成功返回NULL，失败返回{"error": ""} + 一个ResponseInfo对象 + """ + data = {"url": ";".join(urls), "bucket": bucket} + if host: + data["host"] = host + if md5: + data["md5"] = md5 + if key: + data["key"] = key + if etag: + data["etag"] = etag + if callback_url: + data["callbackurl"] = callback_url + if callback_body: + data["callbackbody"] = callback_body + if callback_body_type: + data["callbackbodytype"] = callback_body_type + if callback_host: + data["callbackhost"] = callback_host + if file_type: + data["file_type"] = file_type + + api_host = self.zone.get_api_host(bucket, self.auth) + url = "http://%s/%s" % (api_host, "sisyphus/fetch") + return http._post_with_qiniu_mac(url, data, self.auth, headers={ + "Host": api_host, + "Content-Type": "application/json", + }) + + def fetch(self, url, bucket, key=None): """抓取文件: 从指定URL抓取资源，并将该资源存储到指定空间中，具体规格参考： @@ -348,3 +391,13 @@ def _two_key_batch(operation, source_bucket, key_pairs, target_bucket, force='fa target_bucket = source_bucket return [_build_op(operation, entry(source_bucket, k), entry(target_bucket, v), 'force/{0}'.format(force)) for k, v in key_pairs.items()] + + +def get_bucket_info(bucket, auth): + """ + Args: + bucket - 存储空间名字 + 获取存储空间所在的存储区域 + """ + url = '%s/bucket/%s' % (config.BUCKET_RS_HOST, bucket) + return http._post_with_auth_and_headers(url, None, auth, headers={"Content-Type": "application/x-www-form-urlencoded"}) diff --git a/qiniu/zone.py b/qiniu/zone.py index b817f212..ea3bec9a 100644 --- a/qiniu/zone.py +++ b/qiniu/zone.py @@ -5,6 +5,7 @@ import requests from qiniu import compat from qiniu import utils +from qiniu.services.storage.bucket import get_bucket_info UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host @@ -24,6 +25,7 @@ def __init__( up_host=None, up_host_backup=None, io_host=None, + api_host=None, host_cache={}, scheme="http", home_dir=os.getcwd()): @@ -58,6 +60,13 @@ def get_up_host(self, ak, bucket): up_hosts = bucket_hosts['upHosts'] return up_hosts + def get_api_host(self, bucket, auth): + ret, resp_info = get_bucket_info(bucket, auth) + if ret: + return "api-%s.qiniu.com" % ret['region'] + else: + raise ValueError(ret) + def unmarshal_up_token(self, up_token): token = up_token.split(':') if (len(token) != 3):