Skip to content

Commit fca7849

Browse files
authored
Bump to v7.15.0 (#452)
1 parent d8ed878 commit fca7849

File tree

17 files changed

+437
-194
lines changed

17 files changed

+437
-194
lines changed

.github/workflows/ci-test.yml

+97-4
Original file line numberDiff line numberDiff line change
@@ -60,14 +60,107 @@ jobs:
6060
QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}}
6161
QINIU_TEST_ENV: "travis"
6262
MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000"
63-
PYTHONPATH: "$PYTHONPATH:."
6463
run: |
6564
flake8 --show-source --max-line-length=160 ./qiniu
66-
coverage run -m pytest ./test_qiniu.py ./tests/cases
67-
ocular --data-file .coverage
68-
codecov
65+
python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml
66+
- name: Post Setup mock server
67+
if: ${{ always() }}
68+
shell: bash
69+
run: |
70+
set +e
6971
cat mock-server.pid | xargs kill
72+
rm mock-server.pid
7073
- name: Print mock server log
7174
if: ${{ failure() }}
7275
run: |
7376
cat py-mock-server.log
77+
- name: Upload results to Codecov
78+
uses: codecov/codecov-action@v4
79+
with:
80+
token: ${{ secrets.CODECOV_TOKEN }}
81+
test-win:
82+
strategy:
83+
fail-fast: false
84+
max-parallel: 1
85+
matrix:
86+
python_version: ['2.7', '3.5', '3.9']
87+
runs-on: windows-2019
88+
# make sure only one test running,
89+
# remove this when cases could run in parallel.
90+
needs: test
91+
steps:
92+
- name: Checkout repo
93+
uses: actions/checkout@v2
94+
with:
95+
ref: ${{ github.ref }}
96+
- name: Setup miniconda
97+
uses: conda-incubator/setup-miniconda@v2
98+
with:
99+
auto-update-conda: true
100+
channels: conda-forge
101+
python-version: ${{ matrix.python_version }}
102+
activate-environment: qiniu-sdk
103+
auto-activate-base: false
104+
- name: Setup pip
105+
env:
106+
PYTHON_VERSION: ${{ matrix.python_version }}
107+
PIP_BOOTSTRAP_SCRIPT_PREFIX: https://bootstrap.pypa.io/pip
108+
run: |
109+
# reinstall pip by some python(<3.7) not compatible
110+
$pyversion = [Version]"$ENV:PYTHON_VERSION"
111+
if ($pyversion -lt [Version]"3.7") {
112+
Invoke-WebRequest "$ENV:PIP_BOOTSTRAP_SCRIPT_PREFIX/$($pyversion.Major).$($pyversion.Minor)/get-pip.py" -OutFile "$ENV:TEMP\get-pip.py"
113+
python $ENV:TEMP\get-pip.py --user
114+
Remove-Item -Path "$ENV:TEMP\get-pip.py"
115+
}
116+
- name: Install dependencies
117+
run: |
118+
python -m pip install --upgrade pip
119+
python -m pip install -I -e ".[dev]"
120+
- name: Run cases
121+
env:
122+
QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }}
123+
QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }}
124+
QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }}
125+
QINIU_TEST_NO_ACC_BUCKET: ${{ secrets.QINIU_TEST_NO_ACC_BUCKET }}
126+
QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }}
127+
QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}}
128+
QINIU_TEST_ENV: "github"
129+
MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000"
130+
PYTHONPATH: "$PYTHONPATH:."
131+
run: |
132+
Write-Host "======== Setup Mock Server ========="
133+
conda create -y -n mock-server python=3.10
134+
conda activate mock-server
135+
python --version
136+
$processOptions = @{
137+
FilePath="python"
138+
ArgumentList="tests\mock_server\main.py", "--port", "9000"
139+
PassThru=$true
140+
RedirectStandardOutput="py-mock-server.log"
141+
}
142+
$mocksrvp = Start-Process @processOptions
143+
$mocksrvp.Id | Out-File -FilePath "mock-server.pid"
144+
conda deactivate
145+
Sleep 3
146+
Write-Host "======== Running Test ========="
147+
python --version
148+
python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml
149+
- name: Post Setup mock server
150+
if: ${{ always() }}
151+
run: |
152+
Try {
153+
$mocksrvpid = Get-Content -Path "mock-server.pid"
154+
Stop-Process -Id $mocksrvpid
155+
Remove-Item -Path "mock-server.pid"
156+
} Catch {
157+
Write-Host -Object $_
158+
}
159+
- name: Print mock server log
160+
if: ${{ failure() }}
161+
run: |
162+
Get-Content -Path "py-mock-server.log" | Write-Host
163+
- name: Upload results to Codecov
164+
uses: codecov/codecov-action@v4
165+
with:
166+
token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ pip-log.txt
3636
.coverage
3737
.tox
3838
nosetests.xml
39+
coverage.xml
3940

4041
# Translations
4142
*.mo
@@ -45,4 +46,4 @@ nosetests.xml
4546
.project
4647
.pydevproject
4748
/.idea
48-
/.venv
49+
/.venv*

CHANGELOG.md

+5-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
# Changelog
2+
## 7.15.0
3+
* 对象存储,持久化处理支持工作流模版
4+
* 对象存储,修复 Windows 平台兼容性问题
5+
26
## 7.14.0
37
* 对象存储,空间管理、上传文件新增备用域名重试逻辑
48
* 对象存储,调整查询区域主备域名
@@ -44,7 +48,7 @@
4448
## 7.9.0(2022-07-20)
4549
* 对象存储,支持使用时不配置区域信息,SDK 自动获取;
4650
* 对象存储,新增 list_domains API 用于查询空间绑定的域名
47-
* 对象存储,上传 API 新增支持设置自定义元数据,详情见 put_data, put_file, put_stream API
51+
* 对象存储,上传 API 新增支持设置自定义元数据,详情见 put_data, put_file, put_stream API
4852
* 解决部分已知问题
4953

5054
## 7.8.0(2022-06-08)
@@ -237,5 +241,3 @@
237241
* 代码覆盖度报告
238242
* policy改为dict, 便于灵活增加,并加入过期字段检查
239243
* 文件列表支持目录形式
240-
241-

codecov.yml

+8-8
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,28 @@
11
codecov:
22
ci:
3-
- prow.qiniu.io # prow 里面运行需添加,其他 CI 不要
4-
require_ci_to_pass: no # 改为 no,否则 codecov 会等待其他 GitHub 上所有 CI 通过才会留言。
3+
- prow.qiniu.io # prow need this. seems useless
4+
require_ci_to_pass: no # `no` means the bot will comment on the PR even before all ci passed
55

6-
github_checks: #关闭github checks
6+
github_checks: # close github checks
77
annotations: false
88

99
comment:
1010
layout: "reach, diff, flags, files"
11-
behavior: new # 默认是更新旧留言,改为 new,删除旧的,增加新的。
11+
behavior: new # `new` means the bot will comment a new message instead of edit the old one
1212
require_changes: false # if true: only post the comment if coverage changes
1313
require_base: no # [yes :: must have a base report to post]
1414
require_head: yes # [yes :: must have a head report to post]
1515
branches: # branch names that can post comment
1616
- "master"
1717

1818
coverage:
19-
status: # 评判 pr 通过的标准
19+
status: # check coverage status to pass or fail
2020
patch: off
21-
project: # project 统计所有代码x
21+
project: # project analyze all code in the project
2222
default:
2323
# basic
24-
target: 73.5% # 总体通过标准
25-
threshold: 3% # 允许单次下降的幅度
24+
target: 73.5% # the minimum coverage ratio that the commit must meet
25+
threshold: 3% # allow the coverage to drop
2626
base: auto
2727
if_not_found: success
2828
if_ci_failed: error

qiniu/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
# flake8: noqa
1111

12-
__version__ = '7.14.0'
12+
__version__ = '7.15.0'
1313

1414
from .auth import Auth, QiniuMacAuth
1515

qiniu/auth.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,11 @@
3434
str('fsizeMin'), # 上传文件最少字节数
3535
str('keylimit'), # 设置允许上传的key列表,字符串数组类型,数组长度不可超过20个,如果设置了这个字段,上传时必须提供key
3636

37-
str('persistentOps'), # 持久化处理操作
37+
str('persistentOps'), # 持久化处理操作,与 persistentWorkflowTemplateID 二选一
3838
str('persistentNotifyUrl'), # 持久化处理结果通知URL
3939
str('persistentPipeline'), # 持久化处理独享队列
4040
str('persistentType'), # 为 `1` 时,开启闲时任务,必须是 int 类型
41+
str('persistentWorkflowTemplateID'), # 工作流模板 ID,与 persistentOps 二选一
4142

4243
str('deleteAfterDays'), # 文件多少天后自动删除
4344
str('fileType'), # 文件的存储类型,0为标准存储,1为低频存储,2为归档存储,3为深度归档存储,4为归档直读存储

qiniu/http/regions_provider.py

+7-4
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,10 @@
55
import logging
66
import tempfile
77
import os
8+
import shutil
89

910
from qiniu.compat import json, b as to_bytes
10-
from qiniu.utils import io_md5
11+
from qiniu.utils import io_md5, dt2ts
1112

1213
from .endpoint import Endpoint
1314
from .region import Region, ServiceName
@@ -264,7 +265,7 @@ def _persist_region(region):
264265
},
265266
ttl=region.ttl,
266267
# use datetime.datetime.timestamp() when min version of python >= 3
267-
createTime=int(float(region.create_time.strftime('%s.%f')) * 1000)
268+
createTime=dt2ts(region.create_time)
268269
)._asdict()
269270

270271

@@ -338,8 +339,10 @@ def _walk_persist_cache_file(persist_path, ignore_parse_error=False):
338339

339340
with open(persist_path, 'r') as f:
340341
for line in f:
342+
if not line.strip():
343+
continue
341344
try:
342-
cache_key, regions = _parse_persisted_regions(line)
345+
cache_key, regions = _parse_persisted_regions(line.strip())
343346
yield cache_key, regions
344347
except Exception as err:
345348
if not ignore_parse_error:
@@ -655,7 +658,7 @@ def __shrink_cache(self):
655658
)
656659

657660
# rename file
658-
os.rename(shrink_file_path, self._cache_scope.persist_path)
661+
shutil.move(shrink_file_path, self._cache_scope.persist_path)
659662
except FileAlreadyLocked:
660663
pass
661664
finally:

qiniu/region.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77

88
from .compat import json, s as str_from_bytes
9-
from .utils import urlsafe_base64_decode
9+
from .utils import urlsafe_base64_decode, dt2ts
1010
from .config import UC_HOST, is_customized_default, get_default
1111
from .http.endpoint import Endpoint as _HTTPEndpoint
1212
from .http.regions_provider import Region as _HTTPRegion, ServiceName, get_default_regions_provider
@@ -190,7 +190,7 @@ def get_bucket_hosts(self, ak, bucket, home_dir=None, force=False):
190190

191191
ttl = region.ttl if region.ttl > 0 else 24 * 3600 # 1 day
192192
# use datetime.datetime.timestamp() when min version of python >= 3
193-
create_time = int(float(region.create_time.strftime('%s.%f')) * 1000)
193+
create_time = dt2ts(region.create_time)
194194
bucket_hosts['deadline'] = create_time + ttl
195195

196196
return bucket_hosts

qiniu/services/processing/pfop.py

+15-4
Original file line numberDiff line numberDiff line change
@@ -24,36 +24,47 @@ def __init__(self, auth, bucket, pipeline=None, notify_url=None):
2424
self.pipeline = pipeline
2525
self.notify_url = notify_url
2626

27-
def execute(self, key, fops, force=None, persistent_type=None):
27+
def execute(self, key, fops=None, force=None, persistent_type=None, workflow_template_id=None):
2828
"""
2929
执行持久化处理
3030
3131
Parameters
3232
----------
3333
key: str
3434
待处理的源文件
35-
fops: list[str]
35+
fops: list[str], optional
3636
处理详细操作,规格详见 https://developer.qiniu.com/dora/manual/1291/persistent-data-processing-pfop
37+
与 template_id 二选一
3738
force: int or str, optional
3839
强制执行持久化处理开关
3940
persistent_type: int or str, optional
4041
持久化处理类型,为 '1' 时开启闲时任务
42+
template_id: str, optional
43+
与 fops 二选一
4144
Returns
4245
-------
4346
ret: dict
4447
持久化处理的 persistentId,类似 {"persistentId": 5476bedf7823de4068253bae};
4548
resp: ResponseInfo
4649
"""
47-
ops = ';'.join(fops)
48-
data = {'bucket': self.bucket, 'key': key, 'fops': ops}
50+
if not fops and not workflow_template_id:
51+
raise ValueError('Must provide one of fops or template_id')
52+
data = {
53+
'bucket': self.bucket,
54+
'key': key,
55+
}
4956
if self.pipeline:
5057
data['pipeline'] = self.pipeline
5158
if self.notify_url:
5259
data['notifyURL'] = self.notify_url
60+
if fops:
61+
data['fops'] = ';'.join(fops)
5362
if force == 1 or force == '1':
5463
data['force'] = str(force)
5564
if persistent_type and type(int(persistent_type)) is int:
5665
data['type'] = str(persistent_type)
66+
if workflow_template_id:
67+
data['workflowTemplateID'] = workflow_template_id
5768

5869
url = '{0}/pfop'.format(config.get_default('default_api_host'))
5970
return http._post_with_auth(url, data, self.auth)

qiniu/utils.py

+29-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
# -*- coding: utf-8 -*-
22
from hashlib import sha1, new as hashlib_new
33
from base64 import urlsafe_b64encode, urlsafe_b64decode
4-
from datetime import datetime
4+
from datetime import datetime, tzinfo, timedelta
5+
56
from .compat import b, s
67

78
try:
@@ -236,3 +237,30 @@ def canonical_mime_header_key(field_name):
236237
result += ch
237238
upper = ch == "-"
238239
return result
240+
241+
242+
class _UTC_TZINFO(tzinfo):
243+
def utcoffset(self, dt):
244+
return timedelta(hours=0)
245+
246+
def tzname(self, dt):
247+
return "UTC"
248+
249+
def dst(self, dt):
250+
return timedelta(0)
251+
252+
253+
def dt2ts(dt):
254+
"""
255+
converte datetime to timestamp
256+
257+
Parameters
258+
----------
259+
dt: datetime.datetime
260+
"""
261+
if not dt.tzinfo:
262+
st = (dt - datetime(1970, 1, 1)).total_seconds()
263+
else:
264+
st = (dt - datetime(1970, 1, 1, tzinfo=_UTC_TZINFO())).total_seconds()
265+
266+
return int(st)

setup.py

-4
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,8 @@ def find_version(*file_paths):
4242
'Operating System :: OS Independent',
4343
'Programming Language :: Python',
4444
'Programming Language :: Python :: 2',
45-
'Programming Language :: Python :: 2.6',
4645
'Programming Language :: Python :: 2.7',
4746
'Programming Language :: Python :: 3',
48-
'Programming Language :: Python :: 3.3',
4947
'Programming Language :: Python :: 3.4',
5048
'Programming Language :: Python :: 3.5',
5149
'Programming Language :: Python :: 3.6',
@@ -66,8 +64,6 @@ def find_version(*file_paths):
6664
'pytest',
6765
'pytest-cov',
6866
'freezegun',
69-
'scrutinizer-ocular',
70-
'codecov'
7167
]
7268
},
7369

0 commit comments

Comments
 (0)