Skip to content

Commit a337cd5

Browse files
wadefelixwadefelix
authored
Check func args in document with its FullArgSpec (PaddlePaddle#4427)
* directive py:function and toctree, role :ref: * there are `py:class`, `py:method` and `py:function` directives all. * check the function type * check parameters description paragraphs. * check with FullArgSpec * check with the args in fullargspec * failed the pipeline if no json or parameters not found in rst file. * empty list * paramsstr may be emptpy * bug in find parameters description
1 parent 12d77f7 commit a337cd5

File tree

3 files changed

+205
-14
lines changed

3 files changed

+205
-14
lines changed

‎ci_scripts/check_api_parameters.py

+122-14
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,20 @@
1717
import argparse
1818
import os.path as osp
1919
import re
20+
import sys
21+
import inspect
22+
import paddle
23+
24+
25+
def add_path(path):
26+
if path not in sys.path:
27+
sys.path.insert(0, path)
28+
29+
30+
this_dir = osp.dirname(__file__)
31+
# Add docs/api to PYTHONPATH
32+
add_path(osp.abspath(osp.join(this_dir, '..', 'docs', 'api')))
33+
from extract_api_from_docs import extract_params_desc_from_rst_file
2034

2135
arguments = [
2236
# flags, dest, type, default, help
@@ -43,49 +57,143 @@ def parse_args():
4357
return args
4458

4559

60+
def _check_params_in_description(rstfilename, paramstr):
61+
flag = True
62+
params_intitle = []
63+
if paramstr:
64+
params_intitle = paramstr.split(
65+
', '
66+
) # is there any parameter with default value of type list/tuple? may break this.
67+
funcdescnode = extract_params_desc_from_rst_file(rstfilename)
68+
if funcdescnode:
69+
items = funcdescnode.children[1].children[0].children
70+
if len(items) != len(params_intitle):
71+
flag = False
72+
print(f'check failed (parammeters description): {rstfilename}')
73+
else:
74+
for i in range(len(items)):
75+
pname_intitle = params_intitle[i].split('=')[0].strip()
76+
mo = re.match(r'(\w+)\b.*', items[i].children[0].astext())
77+
if mo:
78+
pname_indesc = mo.group(1)
79+
if pname_indesc != pname_intitle:
80+
flag = False
81+
print(
82+
f'check failed (parammeters description): {rstfilename}, {pname_indesc} != {pname_intitle}'
83+
)
84+
else:
85+
flag = False
86+
print(
87+
f'check failed (parammeters description): {rstfilename}, param name not found in {i} paragraph.'
88+
)
89+
else:
90+
if params_intitle:
91+
print(
92+
f'check failed (parameters description not found): {rstfilename}, {params_intitle}.'
93+
)
94+
flag = False
95+
return flag
96+
97+
98+
def _check_params_in_description_with_fullargspec(rstfilename, funcname):
99+
flag = True
100+
funcspec = inspect.getfullargspec(eval(funcname))
101+
funcdescnode = extract_params_desc_from_rst_file(rstfilename)
102+
if funcdescnode:
103+
items = funcdescnode.children[1].children[0].children
104+
params_inspec = funcspec.args
105+
if len(items) != len(params_inspec):
106+
flag = False
107+
print(f'check failed (parammeters description): {rstfilename}')
108+
else:
109+
for i in range(len(items)):
110+
pname_intitle = params_inspec[i]
111+
mo = re.match(r'(\w+)\b.*', items[i].children[0].astext())
112+
if mo:
113+
pname_indesc = mo.group(1)
114+
if pname_indesc != pname_intitle:
115+
flag = False
116+
print(
117+
f'check failed (parammeters description): {rstfilename}, {pname_indesc} != {pname_intitle}'
118+
)
119+
else:
120+
flag = False
121+
print(
122+
f'check failed (parammeters description): {rstfilename}, param name not found in {i} paragraph.'
123+
)
124+
else:
125+
if funcspec.args:
126+
print(
127+
f'check failed (parameters description not found): {rstfilename}, {funcspec.args}.'
128+
)
129+
flag = False
130+
return flag
131+
132+
46133
def check_api_parameters(rstfiles, apiinfo):
47134
"""check function's parameters same as its origin definition.
48135
49136
such as `.. py:function:: paddle.version.cuda()`
137+
138+
class类别的文档，其成员函数的说明有好多。且class标题还有好多不写参数，暂时都跳过吧
50139
"""
51-
pat = re.compile(r'^\.\.\s+py:function::\s+(\S+)\s*\(\s*(.*)\s*\)\s*$')
140+
pat = re.compile(
141+
r'^\.\.\s+py:(method|function|class)::\s+(\S+)\s*\(\s*(.*)\s*\)\s*$')
52142
check_passed = []
53143
check_failed = []
54144
api_notfound = []
55145
for rstfile in rstfiles:
56-
with open(osp.join('../docs', rstfile), 'r') as rst_fobj:
146+
rstfilename = osp.join('../docs', rstfile)
147+
print(f'checking : {rstfile}')
148+
with open(rstfilename, 'r') as rst_fobj:
57149
func_found = False
58150
for line in rst_fobj:
59151
mo = pat.match(line)
60152
if mo:
61153
func_found = True
62-
funcname = mo.group(1)
63-
paramstr = mo.group(2)
154+
functype = mo.group(1)
155+
if functype not in ('function', 'method'):
156+
check_passed.append(rstfile)
157+
funcname = mo.group(2)
158+
paramstr = mo.group(3)
64159
flag = False
65160
for apiobj in apiinfo.values():
66161
if 'all_names' in apiobj and funcname in apiobj[
67162
'all_names']:
68-
if 'args' in apiobj and paramstr == apiobj['args']:
69-
flag = True
163+
if 'args' in apiobj:
164+
if paramstr == apiobj['args']:
165+
print(
166+
f'check func:{funcname} in {rstfilename} with {paramstr}'
167+
)
168+
flag = _check_params_in_description(
169+
rstfilename, paramstr)
170+
else:
171+
print(
172+
f'check func:{funcname} in {rstfilename} with {paramstr}, but different with json\'s {apiobj["args"]}'
173+
)
174+
flag = _check_params_in_description(
175+
rstfilename, paramstr)
176+
else: # paddle.abs class_method does not have `args` in its json item.
177+
print(
178+
f'check func:{funcname} in {rstfilename} with its FullArgSpec'
179+
)
180+
flag = _check_params_in_description_with_fullargspec(
181+
rstfilename, funcname)
70182
break
71183
if flag:
72184
check_passed.append(rstfile)
185+
print(f'check success: {rstfile}')
73186
else:
74187
check_failed.append(rstfile)
188+
print(f'check failed: {rstfile}')
75189
break
76190
if not func_found:
77191
api_notfound.append(rstfile)
192+
print(f'check failed (object not found): {rstfile}')
193+
print(f'checking done: {rstfile}')
78194
return check_passed, check_failed, api_notfound
79195

80196

81-
def check_api_params_desc():
82-
"""chech the Args Segment.
83-
84-
是不是用docutils来解析rst文件的好？不要暴力正则表达式了？
85-
"""
86-
...
87-
88-
89197
if __name__ == '__main__':
90198
args = parse_args()
91199
rstfiles = [fn for fn in args.rst_files.split(' ') if fn]

‎ci_scripts/ci_start.sh

+4
Original file line numberDiff line numberDiff line change
@@ -105,8 +105,12 @@ else
105105
if [ -f $jsonfn ] ; then
106106
echo "$jsonfn exists."
107107
/bin/bash ${DIR_PATH}/check_api_parameters.sh "${need_check_cn_doc_files}" ${jsonfn}
108+
if [ $? -ne 0 ];then
109+
exit 1
110+
fi
108111
else
109112
echo "$jsonfn not exists."
113+
exit 1
110114
fi
111115
fi
112116
# 5 Approval check

‎docs/api/extract_api_from_docs.py

+79
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import docutils
2525
import docutils.core
2626
import docutils.nodes
27+
import docutils.parsers.rst
2728
import markdown
2829

2930
logger = logging.getLogger()
@@ -224,6 +225,34 @@ def extract_rst_title(filename):
224225
return None
225226

226227

228+
def extract_params_desc_from_rst_file(filename, section_title='参数'):
229+
overrides = {
230+
# Disable the promotion of a lone top-level section title to document
231+
# title (and subsequent section title to document subtitle promotion).
232+
'docinfo_xform': 0,
233+
'initial_header_level': 2,
234+
}
235+
with open(filename, 'r') as fileobj:
236+
doctree = docutils.core.publish_doctree(
237+
fileobj.read(), settings_overrides=overrides)
238+
found = False
239+
for child in doctree.children:
240+
if isinstance(child, docutils.nodes.section) and isinstance(
241+
child.children[0], docutils.nodes.title):
242+
sectitle = child.children[0].astext()
243+
if isinstance(section_title, (list, tuple)):
244+
for st in section_title:
245+
if sectitle.startswith(st):
246+
found = True
247+
break
248+
else:
249+
if sectitle.startswith(section_title):
250+
found = True
251+
if found:
252+
return child
253+
return None
254+
255+
227256
def extract_md_title(filename):
228257
with open(filename, 'r') as fileobj:
229258
html = markdown.markdown(fileobj.read())
@@ -263,6 +292,56 @@ def extract_all_infos(docdirs):
263292
return apis_dict, file_titles
264293

265294

295+
def ref_role(role, rawtext, text, lineno, inliner, options=None, content=None):
296+
'''dummy ref role'''
297+
ref_target = text
298+
node = docutils.nodes.reference(rawtext, text)
299+
return [node], []
300+
301+
302+
docutils.parsers.rst.roles.register_canonical_role('ref', ref_role)
303+
docutils.parsers.rst.roles.register_local_role('ref', ref_role)
304+
305+
306+
class PyFunctionDirective(docutils.parsers.rst.Directive):
307+
'''dummy py:function directive
308+
309+
see https://docutils-zh-cn.readthedocs.io/zh_CN/latest/howto/rst-roles.html
310+
'''
311+
required_arguments = 0
312+
optional_arguments = 0
313+
final_argument_whitespace = True
314+
option_spec = {}
315+
has_content = True
316+
317+
def run(self):
318+
text = '\n'.join(self.content)
319+
thenode = docutils.nodes.title(text, text)
320+
return [thenode]
321+
322+
323+
docutils.parsers.rst.directives.register_directive(
324+
'py:function', PyFunctionDirective) # as abs_cn.rst
325+
docutils.parsers.rst.directives.register_directive(
326+
'py:class', PyFunctionDirective) # as Tensor_cn.rst
327+
docutils.parsers.rst.directives.register_directive(
328+
'py:method', PyFunctionDirective) # as grad_cn.rst
329+
330+
331+
class ToctreeDirective(docutils.parsers.rst.Directive):
332+
'''dummy toctree directive'''
333+
required_arguments = 1
334+
optional_arguments = 5
335+
has_content = True
336+
337+
def run(self):
338+
text = self.arguments[0]
339+
thenode = None
340+
return []
341+
342+
343+
docutils.parsers.rst.directives.register_directive('toctree', ToctreeDirective)
344+
266345
arguments = [
267346
# flags, dest, type, default, help
268347
[

0 commit comments

Comments
 (0)