From ec6c907c59d99f7766901da8238f8de248351950 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 2 Feb 2025 14:26:47 -0500 Subject: [PATCH 01/21] Make the linker always output link tags only. No tags. The tags are now added by the html translator when the document is a docstring. Otherwise it does not add the enclosing tags because we're already in the middle of a code tag or similar . Adjust the themes so the tags and are really equivalent. --- pydoctor/epydoc/docutils.py | 11 +++++++---- pydoctor/epydoc/markup/__init__.py | 7 ++++--- pydoctor/epydoc/markup/_pyval_repr.py | 2 +- pydoctor/epydoc/markup/epytext.py | 2 +- pydoctor/epydoc/markup/plaintext.py | 2 +- pydoctor/epydoc/markup/restructuredtext.py | 2 +- pydoctor/epydoc2stan.py | 2 +- pydoctor/linker.py | 14 +++++++++----- pydoctor/node2stan.py | 14 +++++++++++++- pydoctor/test/test_astbuilder.py | 14 +++++++------- pydoctor/test/test_epydoc2stan.py | 2 +- pydoctor/test/test_zopeinterface.py | 12 ++++++------ pydoctor/themes/base/apidocs.css | 7 ++++--- pydoctor/themes/readthedocs/readthedocstheme.css | 4 ++-- 14 files changed, 58 insertions(+), 37 deletions(-) diff --git a/pydoctor/epydoc/docutils.py b/pydoctor/epydoc/docutils.py index 66442b2de..8d974f539 100644 --- a/pydoctor/epydoc/docutils.py +++ b/pydoctor/epydoc/docutils.py @@ -3,7 +3,10 @@ """ from __future__ import annotations -from typing import Iterable, Iterator, Optional, TypeVar, cast +from typing import Iterable, Iterator, Optional, TypeVar, cast, TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Literal import optparse @@ -14,11 +17,11 @@ _DEFAULT_DOCUTILS_SETTINGS: Optional[optparse.Values] = None -def new_document(source_path: str, settings: Optional[optparse.Values] = None) -> nodes.document: +def new_document(source: Literal['docstring', 'code'], settings: Optional[optparse.Values] = None) -> nodes.document: """ Create a new L{nodes.document} using the provided settings or cached default settings. - @returns: L{nodes.document} + @returns: L{nodes.document} which a C{source} attribute that matches the provided source. """ global _DEFAULT_DOCUTILS_SETTINGS # If we have docutils >= 0.19 we use get_default_settings to calculate and cache @@ -29,7 +32,7 @@ def new_document(source_path: str, settings: Optional[optparse.Values] = None) - settings = _DEFAULT_DOCUTILS_SETTINGS - return utils.new_document(source_path, settings) + return utils.new_document(source, settings) def _set_nodes_parent(nodes: Iterable[nodes.Node], parent: nodes.Element) -> Iterator[nodes.Node]: """ diff --git a/pydoctor/epydoc/markup/__init__.py b/pydoctor/epydoc/markup/__init__.py index 613443aed..318851cc9 100644 --- a/pydoctor/epydoc/markup/__init__.py +++ b/pydoctor/epydoc/markup/__init__.py @@ -150,7 +150,8 @@ def __init__(self, fields: Sequence['Field']): self._stan: Optional[Tag] = None self._summary: Optional['ParsedDocstring'] = None - @abc.abstractproperty + @property + @abc.abstractmethod def has_body(self) -> bool: """ Does this docstring have a non-empty body? @@ -168,7 +169,7 @@ def get_toc(self, depth: int) -> Optional['ParsedDocstring']: except NotImplementedError: return None contents = build_table_of_content(document, depth=depth) - docstring_toc = new_document('toc') + docstring_toc = new_document('docstring') if contents: docstring_toc.extend(contents) from pydoctor.epydoc.markup.restructuredtext import ParsedRstDocstring @@ -439,7 +440,7 @@ def visit_paragraph(self, node: nodes.paragraph) -> None: self.other_docs = True raise nodes.StopTraversal() - summary_doc = new_document('summary') + summary_doc = new_document('docstring') summary_pieces: list[nodes.Node] = [] # Extract the first sentences from the first paragraph until maximum number diff --git a/pydoctor/epydoc/markup/_pyval_repr.py b/pydoctor/epydoc/markup/_pyval_repr.py index 1275385cb..daa802872 100644 --- a/pydoctor/epydoc/markup/_pyval_repr.py +++ b/pydoctor/epydoc/markup/_pyval_repr.py @@ -333,7 +333,7 @@ def colorize(self, pyval: Any) -> ColorizedPyvalRepr: is_complete = True # Put it all together. - document = new_document('pyval_repr') + document = new_document('code') # This ensure the .parent and .document attributes of the child nodes are set correcly. set_node_attributes(document, children=[set_node_attributes(node, document=document) for node in state.result]) return ColorizedPyvalRepr(document, is_complete, state.warnings) diff --git a/pydoctor/epydoc/markup/epytext.py b/pydoctor/epydoc/markup/epytext.py index 414d8c75d..fdc35679c 100644 --- a/pydoctor/epydoc/markup/epytext.py +++ b/pydoctor/epydoc/markup/epytext.py @@ -1379,7 +1379,7 @@ def to_node(self) -> nodes.document: if self._document is not None: return self._document - self._document = new_document('epytext') + self._document = new_document('docstring') if self._tree is not None: node, = self._to_node(self._tree) diff --git a/pydoctor/epydoc/markup/plaintext.py b/pydoctor/epydoc/markup/plaintext.py index aefb7fe3f..1c8d3a35d 100644 --- a/pydoctor/epydoc/markup/plaintext.py +++ b/pydoctor/epydoc/markup/plaintext.py @@ -62,7 +62,7 @@ def to_node(self) -> nodes.document: return self._document else: # create document - _document = new_document('plaintext') + _document = new_document('docstring') # split text into paragraphs paragraphs = [set_node_attributes(nodes.paragraph('',''), children=[ diff --git a/pydoctor/epydoc/markup/restructuredtext.py b/pydoctor/epydoc/markup/restructuredtext.py index fd3f47784..4902428c1 100644 --- a/pydoctor/epydoc/markup/restructuredtext.py +++ b/pydoctor/epydoc/markup/restructuredtext.py @@ -176,7 +176,7 @@ def get_transforms(self) -> List[Transform]: if t != frontmatter.DocInfo] def new_document(self) -> nodes.document: - document = new_document(self.source.source_path, self.settings) + document = new_document('docstring', self.settings) # Capture all warning messages. document.reporter.attach_observer(self.report) # Return the new document. diff --git a/pydoctor/epydoc2stan.py b/pydoctor/epydoc2stan.py index 6e60a4eaa..02807bccd 100644 --- a/pydoctor/epydoc2stan.py +++ b/pydoctor/epydoc2stan.py @@ -1155,7 +1155,7 @@ def get_constructors_extra(cls:model.Class) -> ParsedDocstring | None: if not constructors: return None - document = new_document('constructors') + document = new_document('docstring') elements: list[nodes.Node] = [] plural = 's' if len(constructors)>1 else '' diff --git a/pydoctor/linker.py b/pydoctor/linker.py index a569107b2..dc3bdadb3 100644 --- a/pydoctor/linker.py +++ b/pydoctor/linker.py @@ -153,14 +153,14 @@ def link_xref(self, target: str, label: "Flattenable", lineno: int) -> Tag: try: resolved = self._resolve_identifier_xref(target, lineno) except LookupError: - xref = label + xref = tags.transparent(label) else: if isinstance(resolved, str): xref = intersphinx_link(label, url=resolved) else: xref = taglink(resolved, self.page_url, label) - return tags.code(xref) + return xref def _resolve_identifier_xref(self, identifier: str, @@ -286,13 +286,17 @@ def switch_context(self, ob:Optional['model.Documentable']) -> Iterator[None]: yield class NotFoundLinker(DocstringLinker): - """A DocstringLinker implementation that cannot find any links.""" + """ + A DocstringLinker implementation that cannot find any links. + + It will always output link tag with no C{href} attribute. + """ def link_to(self, target: str, label: "Flattenable") -> Tag: - return tags.transparent(label) + return tags.a(label) def link_xref(self, target: str, label: "Flattenable", lineno: int) -> Tag: - return tags.code(label) + return tags.a(label) @contextlib.contextmanager def switch_context(self, ob: Optional[model.Documentable]) -> Iterator[None]: diff --git a/pydoctor/node2stan.py b/pydoctor/node2stan.py index 8b2f00665..884e70c73 100644 --- a/pydoctor/node2stan.py +++ b/pydoctor/node2stan.py @@ -102,10 +102,20 @@ def __init__(self, # h1 is reserved for the page nodes.title. self.section_level += 1 + # All documents should be created with pydoctor.epydoc.docutils.new_document() helper + # such that the source attribute will always be one of the supported values. + self._document_is_code = is_code = document.attributes.get('source') == 'code' + if is_code: + # Do not wrap links in tags if we're renderring a code-like parsed element. + self._link_xref = self._linker.link_xref + else: + self._link_xref = lambda target, label, lineno: Tag('code')(self._linker.link_xref(target, label, lineno)) + + # Handle interpreted text (crossreferences) def visit_title_reference(self, node: nodes.title_reference) -> None: lineno = get_lineno(node) - self._handle_reference(node, link_func=lambda target, label: self._linker.link_xref(target, label, lineno)) + self._handle_reference(node, link_func=lambda target, label: self._link_xref(target, label, lineno)) # Handle internal references def visit_obj_reference(self, node: obj_reference) -> None: @@ -134,6 +144,8 @@ def _handle_reference(self, node: nodes.title_reference, link_func: Callable[[st def should_be_compact_paragraph(self, node: nodes.Element) -> bool: if self.document.children == [node]: return True + elif self._document_is_code: + return True else: return super().should_be_compact_paragraph(node) # type: ignore[no-any-return] diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index 60b3320ee..38ed9babe 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -1339,25 +1339,25 @@ def __init__(self): assert type2html(b) == 'string' c = C.contents['c'] assert c.docstring == """third""" - assert type2html(c) == 'str' + assert type2html(c) == 'str' d = C.contents['d'] assert d.docstring == """fourth""" - assert type2html(d) == 'str' + assert type2html(d) == 'str' e = C.contents['e'] assert e.docstring == """fifth""" - assert type2html(e) == 'List[C]' + assert type2html(e) == 'List[C]' f = C.contents['f'] assert f.docstring == """sixth""" - assert type2html(f) == 'List[C]' + assert type2html(f) == 'List[C]' g = C.contents['g'] assert g.docstring == """seventh""" - assert type2html(g) == 'List[C]' + assert type2html(g) == 'List[C]' s = C.contents['s'] assert s.docstring == """instance""" - assert type2html(s) == 'List[str]' + assert type2html(s) == 'List[str]' m = mod.contents['m'] assert m.docstring == """module-level""" - assert type2html(m) == 'bytes' + assert type2html(m) == 'bytes' @systemcls_param def test_type_comment(systemcls: Type[model.System], capsys: CapSys) -> None: diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index 3610c9a48..0096a2579 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -1101,7 +1101,7 @@ def test_EpydocLinker_adds_intersphinx_link_css_class() -> None: sut = target.docstring_linker assert isinstance(sut, linker._EpydocLinker) - result1 = sut.link_xref('base.module.other', 'base.module.other', 0).children[0] # wrapped in a code tag + result1 = sut.link_xref('base.module.other', 'base.module.other', 0) result2 = sut.link_to('base.module.other', 'base.module.other') res = flatten(result2) diff --git a/pydoctor/test/test_zopeinterface.py b/pydoctor/test/test_zopeinterface.py index f4de8929f..092c6239c 100644 --- a/pydoctor/test/test_zopeinterface.py +++ b/pydoctor/test/test_zopeinterface.py @@ -198,15 +198,15 @@ class IMyInterface(interface.Interface): mod = fromText(src, modname='mod', systemcls=systemcls) text = mod.contents['IMyInterface'].contents['text'] assert text.docstring == 'fun in a bun' - assert type2html(text)== "schema.TextLine" + assert type2html(text)== "schema.TextLine" assert text.kind is model.DocumentableKind.SCHEMA_FIELD undoc = mod.contents['IMyInterface'].contents['undoc'] assert undoc.docstring is None - assert type2html(undoc) == "schema.Bool" + assert type2html(undoc) == "schema.Bool" assert undoc.kind is model.DocumentableKind.SCHEMA_FIELD bad = mod.contents['IMyInterface'].contents['bad'] assert bad.docstring is None - assert type2html(bad) == "schema.ASCII" + assert type2html(bad) == "schema.ASCII" assert bad.kind is model.DocumentableKind.SCHEMA_FIELD captured = capsys.readouterr().out assert captured == 'mod:6: description of field "bad" is not a string literal\n' @@ -241,14 +241,14 @@ class IMyInterface(interface.Interface): mod = fromText(src, modname='mod', systemcls=systemcls) mytext = mod.contents['IMyInterface'].contents['mytext'] assert mytext.docstring == 'fun in a bun' - assert flatten(cast(ParsedDocstring, mytext.parsed_type).to_stan(NotFoundLinker())) == "MyTextLine" + assert flatten(cast(ParsedDocstring, mytext.parsed_type).to_stan(NotFoundLinker())) == 'MyTextLine' assert mytext.kind is model.DocumentableKind.SCHEMA_FIELD myothertext = mod.contents['IMyInterface'].contents['myothertext'] assert myothertext.docstring == 'fun in another bun' - assert flatten(cast(ParsedDocstring, myothertext.parsed_type).to_stan(NotFoundLinker())) == "MyOtherTextLine" + assert flatten(cast(ParsedDocstring, myothertext.parsed_type).to_stan(NotFoundLinker())) == "MyOtherTextLine" assert myothertext.kind is model.DocumentableKind.SCHEMA_FIELD myint = mod.contents['IMyInterface'].contents['myint'] - assert flatten(cast(ParsedDocstring, myint.parsed_type).to_stan(NotFoundLinker())) == "INTEGERSCHMEMAFIELD" + assert flatten(cast(ParsedDocstring, myint.parsed_type).to_stan(NotFoundLinker())) == "INTEGERSCHMEMAFIELD" assert myint.kind is model.DocumentableKind.SCHEMA_FIELD @zope_interface_systemcls_param diff --git a/pydoctor/themes/base/apidocs.css b/pydoctor/themes/base/apidocs.css index b073bd9aa..9174ae58a 100644 --- a/pydoctor/themes/base/apidocs.css +++ b/pydoctor/themes/base/apidocs.css @@ -264,7 +264,7 @@ ul ul ul ul ul ul ul { } /* parameters types (in parameters table) */ -.fieldTable tr td.fieldArgContainer > code { +.fieldTable tr td.fieldArgContainer > code, .fieldTable tr td.fieldArgContainer > .rst-literal { /* we don't want word break for the types because we already add tags inside the type HTML, and that should suffice. */ word-break: normal; display: inline; @@ -427,7 +427,7 @@ code, .rst-literal, .pre, #childList > div .functionHeader, #splitTables > table tr td:nth-child(2), .fieldArg { font-family: Menlo, Monaco, Consolas, "Courier New", monospace; } -code, #childList > div .functionHeader, .fieldArg { +code, .rst-literal, #childList > div .functionHeader, .fieldArg { color: #222222; } @@ -469,7 +469,8 @@ It also overwrite the default values inherited from bootstrap min code, .rst-literal { padding:2px 4px; background-color: #f4f4f4; - border-radius:4px + border-radius:4px; + font-size: 90%; /* and class="rst-literal" are closely equivalent */ } diff --git a/pydoctor/themes/readthedocs/readthedocstheme.css b/pydoctor/themes/readthedocs/readthedocstheme.css index af0e097a9..d0cde9d03 100644 --- a/pydoctor/themes/readthedocs/readthedocstheme.css +++ b/pydoctor/themes/readthedocs/readthedocstheme.css @@ -92,12 +92,12 @@ code, .pre, #childList > div .functionHeader, font-family: Menlo, Monaco, Consolas, "Courier New", monospace; } -code, .literal { +code, .literal, .rst-literal { border-radius:2px; font-size: 14px; } -#main code, .literal { +#main code, .literal, .rst-literal { border: 1px solid rgb(225, 228, 229); padding:1px 2px; } From f06cd122e79e69bbbb3280496737d14d9ab1f88e Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 2 Feb 2025 14:29:58 -0500 Subject: [PATCH 02/21] Fix #723 and #581 --- pydoctor/epydoc/markup/_types.py | 126 ++++---------- pydoctor/napoleon/docstring.py | 6 +- pydoctor/test/epydoc/test_epytext2node.py | 6 +- pydoctor/test/epydoc/test_google_numpy.py | 4 +- pydoctor/test/epydoc/test_pyval_repr.py | 163 +++++++++--------- pydoctor/test/epydoc/test_restructuredtext.py | 4 +- pydoctor/test/test_napoleon_docstring.py | 69 +++++++- pydoctor/test/test_templatewriter.py | 31 ++-- pydoctor/test/test_type_fields.py | 108 ++++++------ 9 files changed, 257 insertions(+), 260 deletions(-) diff --git a/pydoctor/epydoc/markup/_types.py b/pydoctor/epydoc/markup/_types.py index 8e94243d6..d2e10e98f 100644 --- a/pydoctor/epydoc/markup/_types.py +++ b/pydoctor/epydoc/markup/_types.py @@ -5,15 +5,16 @@ """ from __future__ import annotations -from typing import Any, Callable, Dict, List, Tuple, Union, cast +from typing import Callable, Dict, List, Union, cast -from pydoctor.epydoc.markup import DocstringLinker, ParseError, ParsedDocstring, get_parser_by_name -from pydoctor.node2stan import node2stan +from pydoctor.epydoc.markup import ParseError, ParsedDocstring, get_parser_by_name +from pydoctor.epydoc.markup._pyval_repr import PyvalColorizer from pydoctor.napoleon.docstring import TokenType, TypeDocstring +from pydoctor.epydoc.docutils import new_document, set_node_attributes from docutils import nodes -from twisted.web.template import Tag, tags +# TODO: This class should use composition instead of multiple inheritence... class ParsedTypeDocstring(TypeDocstring, ParsedDocstring): """ Add L{ParsedDocstring} interface on top of L{TypeDocstring} and @@ -38,25 +39,15 @@ def __init__(self, annotation: Union[nodes.document, str], else: TypeDocstring.__init__(self, annotation, warns_on_unknown_tokens) - - # We need to store the line number because we need to pass it to DocstringLinker.link_xref self._lineno = lineno + self._document = self._parse_tokens() @property def has_body(self) -> bool: return len(self._tokens)>0 def to_node(self) -> nodes.document: - """ - Not implemented. - """ - raise NotImplementedError() - - def to_stan(self, docstring_linker: DocstringLinker) -> Tag: - """ - Present the type as a stan tree. - """ - return self._convert_type_spec_to_stan(docstring_linker) + return self._document def _tokenize_node_type_spec(self, spec: nodes.document) -> List[Union[str, nodes.Node]]: def _warn_not_supported(n:nodes.Node) -> None: @@ -84,97 +75,42 @@ def _warn_not_supported(n:nodes.Node) -> None: return tokens - def _convert_obj_tokens_to_stan(self, tokens: List[Tuple[Any, TokenType]], - docstring_linker: DocstringLinker) -> list[tuple[Any, TokenType]]: - """ - Convert L{TokenType.OBJ} and PEP 484 like L{TokenType.DELIMITER} type to stan, merge them together. Leave the rest untouched. - - Exemple: - - >>> tokens = [("list", TokenType.OBJ), ("(", TokenType.DELIMITER), ("int", TokenType.OBJ), (")", TokenType.DELIMITER)] - >>> ann._convert_obj_tokens_to_stan(tokens, NotFoundLinker()) - ... [(Tag('code', children=['list', '(', 'int', ')']), TokenType.OBJ)] - - @param tokens: List of tuples: C{(token, type)} + def _parse_tokens(self) -> nodes.document: """ - - combined_tokens: list[tuple[Any, TokenType]] = [] - - open_parenthesis = 0 - open_square_braces = 0 - - for _token, _type in tokens: - # The actual type of_token is str | Tag | Node. - - if (_type is TokenType.DELIMITER and _token in ('[', '(', ')', ']')) \ - or _type is TokenType.OBJ: - if _token == "[": open_square_braces += 1 - elif _token == "(": open_parenthesis += 1 - - if _type is TokenType.OBJ: - _token = docstring_linker.link_xref( - _token, _token, self._lineno) - - if open_square_braces + open_parenthesis > 0: - try: last_processed_token = combined_tokens[-1] - except IndexError: - combined_tokens.append((_token, _type)) - else: - if last_processed_token[1] is TokenType.OBJ \ - and isinstance(last_processed_token[0], Tag): - # Merge with last Tag - if _type is TokenType.OBJ: - assert isinstance(_token, Tag) - last_processed_token[0](*_token.children) - else: - last_processed_token[0](_token) - else: - combined_tokens.append((_token, _type)) - else: - combined_tokens.append((_token, _type)) - - if _token == "]": open_square_braces -= 1 - elif _token == ")": open_parenthesis -= 1 - - else: - # the token will be processed in _convert_type_spec_to_stan() method. - combined_tokens.append((_token, _type)) - - return combined_tokens - - def _convert_type_spec_to_stan(self, docstring_linker: DocstringLinker) -> Tag: - """ - Convert type to L{Tag} object. + Convert type to docutils document object. """ - tokens = self._convert_obj_tokens_to_stan(self._tokens, docstring_linker) - + document = new_document('code') warnings: List[ParseError] = [] - converters: Dict[TokenType, Callable[[Union[str, Tag]], Union[str, Tag]]] = { - TokenType.LITERAL: lambda _token: tags.span(_token, class_="literal"), - TokenType.CONTROL: lambda _token: tags.em(_token), - # We don't use safe_to_stan() here, if these converter functions raise an exception, - # the whole type docstring will be rendered as plaintext. - # it does not crash on invalid xml entities - TokenType.REFERENCE: lambda _token: get_parser_by_name('restructuredtext')(_token, warnings).to_stan(docstring_linker) if isinstance(_token, str) else _token, - TokenType.UNKNOWN: lambda _token: get_parser_by_name('restructuredtext')(_token, warnings).to_stan(docstring_linker) if isinstance(_token, str) else _token, - TokenType.OBJ: lambda _token: _token, # These convertions (OBJ and DELIMITER) are done in _convert_obj_tokens_to_stan(). - TokenType.DELIMITER: lambda _token: _token, + converters: Dict[TokenType, Callable[[str | nodes.Node], str | nodes.Node | list[nodes.Node]]] = { + # we're re-using the variable string css class for the whole literal token, it's the + # best approximation we have for now. + TokenType.LITERAL: lambda _token: nodes.inline(_token, _token, classes=[PyvalColorizer.STRING_TAG]), + TokenType.CONTROL: lambda _token: nodes.emphasis(_token, _token), + TokenType.REFERENCE: lambda _token: get_parser_by_name('restructuredtext')(_token, warnings).to_node().children if isinstance(_token, str) else _token, + TokenType.UNKNOWN: lambda _token: get_parser_by_name('restructuredtext')(_token, warnings).to_node().children if isinstance(_token, str) else _token, + TokenType.OBJ: lambda _token: nodes.title_reference(_token, _token, line=self._lineno), + TokenType.DELIMITER: lambda _token: nodes.Text(_token), TokenType.ANY: lambda _token: _token, } for w in warnings: self.warnings.append(w.descr()) - converted = Tag('') + elements = [] - for token, type_ in tokens: + for token, type_ in self._tokens: assert token is not None - if isinstance(token, nodes.Node): - token = node2stan(token, docstring_linker) - assert isinstance(token, (str, Tag)) converted_token = converters[type_](token) - converted(converted_token) + if isinstance(converted_token, list): + elements.extend(converted_token) + elif isinstance(converted_token, str) and not isinstance(converted_token, nodes.Text): + elements.append(nodes.Text(converted_token)) + else: + elements.append(converted_token) - return converted + return set_node_attributes(document, children=[ + set_node_attributes(nodes.inline('', '', + classes=['literal']), + children=elements)]) \ No newline at end of file diff --git a/pydoctor/napoleon/docstring.py b/pydoctor/napoleon/docstring.py index 0ab64dd59..990b1a067 100644 --- a/pydoctor/napoleon/docstring.py +++ b/pydoctor/napoleon/docstring.py @@ -184,13 +184,13 @@ class TypeDocstring: """ _natural_language_delimiters_regex_str = ( - r",\sor\s|\sor\s|\sof\s|:\s|\sto\s|,\sand\s|\sand\s" + r",\sor\s|\sor\s|\sof\s|:\s|\sto\s|,\sand\s|\sand\s|,\s|," ) _natural_language_delimiters_regex = re.compile( f"({_natural_language_delimiters_regex_str})" ) - _ast_like_delimiters_regex_str = r",\s|,|[\[]|[\]]|[\(|\)]" + _ast_like_delimiters_regex_str = r"[\[]|[\]]|[\(|\)]" _ast_like_delimiters_regex = re.compile(f"({_ast_like_delimiters_regex_str})") _token_regex = re.compile( @@ -407,7 +407,7 @@ def is_numeric(token: str) -> bool: return type_ - # add espaced space when necessary + # add escaped space when necessary def _convert_type_spec_to_rst(self) -> str: def _convert( _token: Tuple[str, TokenType], diff --git a/pydoctor/test/epydoc/test_epytext2node.py b/pydoctor/test/epydoc/test_epytext2node.py index fbd0c9d56..e7df5bce0 100644 --- a/pydoctor/test/epydoc/test_epytext2node.py +++ b/pydoctor/test/epydoc/test_epytext2node.py @@ -8,7 +8,7 @@ def test_nested_markup() -> None: I{B{Inline markup} may be nested; and it may span} multiple lines. ''' - expected = ''' + expected = ''' @@ -22,7 +22,7 @@ def test_nested_markup() -> None: doc = ''' It becomes a little bit complicated with U{B{custom} links } ''' - expected = ''' + expected = ''' It becomes a little bit complicated with @@ -36,7 +36,7 @@ def test_nested_markup() -> None: doc = ''' It becomes a little bit complicated with L{B{custom} links } ''' - expected = ''' + expected = ''' It becomes a little bit complicated with diff --git a/pydoctor/test/epydoc/test_google_numpy.py b/pydoctor/test/epydoc/test_google_numpy.py index 3918ff615..e3948f483 100644 --- a/pydoctor/test/epydoc/test_google_numpy.py +++ b/pydoctor/test/epydoc/test_google_numpy.py @@ -28,7 +28,7 @@ def test_get_google_parser_attribute(self) -> None: actual = flatten(parsed_doc.fields[-1].body().to_stan(NotFoundLinker())) - expected = """numpy.ndarray""" + expected = """numpy.ndarray""" self.assertEqual(expected, actual) self.assertEqual(errors, []) @@ -65,7 +65,7 @@ def test_get_numpy_parser_attribute(self) -> None: actual = flatten(parsed_doc.fields[-1].body().to_stan(NotFoundLinker())) - expected = """numpy.ndarray""" + expected = """numpy.ndarray""" self.assertEqual(expected, actual) self.assertEqual(errors, []) diff --git a/pydoctor/test/epydoc/test_pyval_repr.py b/pydoctor/test/epydoc/test_pyval_repr.py index f34d7e144..d5e285544 100644 --- a/pydoctor/test/epydoc/test_pyval_repr.py +++ b/pydoctor/test/epydoc/test_pyval_repr.py @@ -27,15 +27,15 @@ def test_simple_types() -> None: Integers, floats, None, and complex numbers get printed using str, with no syntax highlighting. """ - assert color(1) == """ + assert color(1) == """ 1\n""" - assert color(0) == """ + assert color(0) == """ 0\n""" - assert color(100) == """ + assert color(100) == """ 100\n""" - assert color(1./4) == """ + assert color(1./4) == """ 0.25\n""" - assert color(None) == """ + assert color(None) == """ None\n""" @@ -43,9 +43,9 @@ def test_long_numbers() -> None: """ Long ints will get wrapped if they're big enough. """ - assert color(10000000) == """ + assert color(10000000) == """ 10000000\n""" - assert color(10**90) == """ + assert color(10**90) == """ 1000000000000000000000000000000000000000 ↵ @@ -61,7 +61,7 @@ def test_strings() -> None: Strings have their quotation marks tagged as 'quote'. Characters are escaped using the 'string-escape' encoding. """ - assert color(bytes(range(255)), maxlines=9999) == r""" + assert color(bytes(range(255)), maxlines=9999) == r""" b ''' @@ -179,7 +179,7 @@ def test_strings_quote() -> None: Currently, the "'" quote is always used, because that's what the 'string-escape' encoding expects. """ - assert color('Hello') == """ + assert color('Hello') == """ ' @@ -188,7 +188,7 @@ def test_strings_quote() -> None: ' """ - assert color('"Hello"') == """ + assert color('"Hello"') == """ ' @@ -197,7 +197,7 @@ def test_strings_quote() -> None: ' """ - assert color("'Hello'") == r""" + assert color("'Hello'") == r""" ' @@ -207,7 +207,7 @@ def test_strings_quote() -> None: """ def test_strings_special_chars() -> None: - assert color("'abc \t\r\n\f\v \xff 😀'\x0c\x0b\t\r \\") == r""" + assert color("'abc \t\r\n\f\v \xff 😀'\x0c\x0b\t\r \\") == r""" ''' @@ -224,7 +224,7 @@ def test_strings_multiline() -> None: """Strings containing newlines are automatically rendered as multiline strings.""" - assert color("This\n is a multiline\n string!") == """ + assert color("This\n is a multiline\n string!") == """ ''' @@ -240,7 +240,7 @@ def test_strings_multiline() -> None: # Unless we ask for them not to be: - assert color("This\n is a multiline\n string!", linebreakok=False) == r""" + assert color("This\n is a multiline\n string!", linebreakok=False) == r""" ' @@ -253,7 +253,7 @@ def test_bytes_multiline() -> None: # The same should work also for binary strings (bytes): - assert color(b"This\n is a multiline\n string!") == """ + assert color(b"This\n is a multiline\n string!") == """ b ''' @@ -268,7 +268,7 @@ def test_bytes_multiline() -> None: '''\n""" - assert color(b"This\n is a multiline\n string!", linebreakok=False) == r""" + assert color(b"This\n is a multiline\n string!", linebreakok=False) == r""" b ' @@ -281,7 +281,7 @@ def test_bytes_multiline() -> None: def test_unicode_str() -> None: """Unicode strings are handled properly. """ - assert color("\uaaaa And \ubbbb") == """ + assert color("\uaaaa And \ubbbb") == """ ' @@ -289,7 +289,7 @@ def test_unicode_str() -> None: '\n""" - assert color("ÉéèÈÜÏïü") == """ + assert color("ÉéèÈÜÏïü") == """ ' @@ -301,7 +301,7 @@ def test_bytes_str() -> None: """ Binary strings (bytes) are handled properly:""" - assert color(b"Hello world") == """ + assert color(b"Hello world") == """ b ' @@ -310,7 +310,7 @@ def test_bytes_str() -> None: '\n""" - assert color(b"\x00 And \xff") == r""" + assert color(b"\x00 And \xff") == r""" b ' @@ -326,7 +326,7 @@ def test_inline_list() -> None: current line, it is displayed on one line. Otherwise, each value is listed on a separate line, indented by the size of the open-bracket.""" - assert color(list(range(10))) == """ + assert color(list(range(10))) == """ [ 0 @@ -361,7 +361,7 @@ def test_inline_list() -> None: def test_multiline_list() -> None: - assert color(list(range(100))) == """ + assert color(list(range(100))) == """ [ 0 @@ -392,7 +392,7 @@ def test_multiline_list() -> None: def test_multiline_list2() -> None: - assert color([1,2,[5,6,[(11,22,33),9],10],11]+[99,98,97,96,95]) == """ + assert color([1,2,[5,6,[(11,22,33),9],10],11]+[99,98,97,96,95]) == """ [ 1 @@ -450,7 +450,7 @@ def test_multiline_list2() -> None: def test_multiline_set() -> None: - assert color(set(range(20))) == """ + assert color(set(range(20))) == """ set([ 0 @@ -481,7 +481,7 @@ def test_multiline_set() -> None: def test_frozenset() -> None: - assert color(frozenset([1, 2, 3])) == """ + assert color(frozenset([1, 2, 3])) == """ frozenset([ 1 @@ -498,7 +498,7 @@ class Custom: def __repr__(self) -> str: return '123' - assert color(Custom()) == """ + assert color(Custom()) == """ 123\n""" def test_buggy_live_object() -> None: @@ -506,13 +506,13 @@ class Buggy: def __repr__(self) -> str: raise NotImplementedError() - assert color(Buggy()) == """ + assert color(Buggy()) == """ ??\n""" def test_tuples_one_value() -> None: """Tuples that contains only one value need an ending comma.""" - assert color((1,)) == """ + assert color((1,)) == """ ( 1 @@ -527,7 +527,7 @@ def extract_expr(_ast: ast.Module) -> ast.AST: def test_ast_constants() -> None: assert color(extract_expr(ast.parse(dedent(""" 'Hello' - """)))) == """ + """)))) == """ ' @@ -538,40 +538,40 @@ def test_ast_constants() -> None: def test_ast_unary_op() -> None: assert color(extract_expr(ast.parse(dedent(""" not True - """)))) == """ + """)))) == """ not True\n""" assert color(extract_expr(ast.parse(dedent(""" +3.0 - """)))) == """ + """)))) == """ + 3.0\n""" assert color(extract_expr(ast.parse(dedent(""" -3.0 - """)))) == """ + """)))) == """ - 3.0\n""" assert color(extract_expr(ast.parse(dedent(""" ~3.0 - """)))) == """ + """)))) == """ ~ 3.0\n""" def test_ast_bin_op() -> None: assert color(extract_expr(ast.parse(dedent(""" 2.3*6 - """)))) == """ + """)))) == """ 2.3 * 6\n""" assert color(extract_expr(ast.parse(dedent(""" (3-6)*2 - """)))) == """ + """)))) == """ ( 3 - @@ -582,7 +582,7 @@ def test_ast_bin_op() -> None: assert color(extract_expr(ast.parse(dedent(""" 101//4+101%4 - """)))) == """ + """)))) == """ 101 // 4 @@ -593,42 +593,42 @@ def test_ast_bin_op() -> None: assert color(extract_expr(ast.parse(dedent(""" 1 & 0 - """)))) == """ + """)))) == """ 1 & 0\n""" assert color(extract_expr(ast.parse(dedent(""" 1 | 0 - """)))) == """ + """)))) == """ 1 | 0\n""" assert color(extract_expr(ast.parse(dedent(""" 1 ^ 0 - """)))) == """ + """)))) == """ 1 ^ 0\n""" assert color(extract_expr(ast.parse(dedent(""" 1 << 0 - """)))) == """ + """)))) == """ 1 << 0\n""" assert color(extract_expr(ast.parse(dedent(""" 1 >> 0 - """)))) == """ + """)))) == """ 1 >> 0\n""" assert color(extract_expr(ast.parse(dedent(""" H @ beta - """)))) == """ + """)))) == """ H @ @@ -639,7 +639,7 @@ def test_operator_precedences() -> None: assert color(extract_expr(ast.parse(dedent(""" (2 ** 3) ** 2 - """)))) == """ + """)))) == """ ( 2 ** @@ -650,7 +650,7 @@ def test_operator_precedences() -> None: assert color(extract_expr(ast.parse(dedent(""" 2 ** 3 ** 2 - """)))) == """ + """)))) == """ 2 ** ( @@ -661,7 +661,7 @@ def test_operator_precedences() -> None: assert color(extract_expr(ast.parse(dedent(""" (1 + 2) * 3 / 4 - """)))) == """ + """)))) == """ ( 1 + @@ -674,7 +674,7 @@ def test_operator_precedences() -> None: assert color(extract_expr(ast.parse(dedent(""" ((1 + 2) * 3) / 4 - """)))) == """ + """)))) == """ ( 1 + @@ -687,7 +687,7 @@ def test_operator_precedences() -> None: assert color(extract_expr(ast.parse(dedent(""" (1 + 2) * 3 / 4 - """)))) == """ + """)))) == """ ( 1 + @@ -700,7 +700,7 @@ def test_operator_precedences() -> None: assert color(extract_expr(ast.parse(dedent(""" 1 + 2 * 3 / 4 - 1 - """)))) == """ + """)))) == """ 1 + 2 @@ -714,7 +714,7 @@ def test_operator_precedences() -> None: def test_ast_bool_op() -> None: assert color(extract_expr(ast.parse(dedent(""" True and 9 - """)))) == """ + """)))) == """ True and @@ -722,7 +722,7 @@ def test_ast_bool_op() -> None: assert color(extract_expr(ast.parse(dedent(""" 1 or 0 and 2 or 3 or 1 - """)))) == """ + """)))) == """ 1 or 0 @@ -736,7 +736,7 @@ def test_ast_bool_op() -> None: def test_ast_list_tuple() -> None: assert color(extract_expr(ast.parse(dedent(""" [1,2,[5,6,[(11,22,33),9],10],11]+[99,98,97,96,95] - """)))) == """ + """)))) == """ [ 1 @@ -804,7 +804,7 @@ def test_ast_list_tuple() -> None: assert color(extract_expr(ast.parse(dedent(""" (('1', 2, 3.14), (4, '5', 6.66)) - """)))) == """ + """)))) == """ ( ( @@ -847,7 +847,7 @@ def test_ast_dict() -> None: """ assert color(extract_expr(ast.parse(dedent(""" {'1':33, '2':[1,2,3,{7:'oo'*20}]} - """))), linelen=45) == """ + """))), linelen=45) == """ { @@ -897,7 +897,7 @@ def test_ast_dict() -> None: def test_ast_annotation() -> None: assert color(extract_expr(ast.parse(dedent(""" bar[typing.Sequence[dict[str, bytes]]] - """))), linelen=999) == """ + """))), linelen=999) == """ bar [ @@ -923,7 +923,7 @@ def test_ast_annotation() -> None: def test_ast_call() -> None: assert color(extract_expr(ast.parse(dedent(""" list(range(100)) - """)))) == """ + """)))) == """ list ( @@ -939,7 +939,7 @@ def test_ast_call() -> None: def test_ast_call_args() -> None: assert color(extract_expr(ast.parse(dedent(""" list(func(1, *two, three=2, **args)) - """)))) == """ + """)))) == """ list ( @@ -970,14 +970,14 @@ def test_ast_call_args() -> None: def test_ast_ellipsis() -> None: assert color(extract_expr(ast.parse(dedent(""" ... - """)))) == """ + """)))) == """ ...\n""" def test_ast_set() -> None: assert color(extract_expr(ast.parse(dedent(""" {1, 2} - """)))) == """ + """)))) == """ set([ 1 @@ -988,7 +988,7 @@ def test_ast_set() -> None: assert color(extract_expr(ast.parse(dedent(""" set([1, 2]) - """)))) == """ + """)))) == """ set ( @@ -1005,7 +1005,7 @@ def test_ast_set() -> None: def test_ast_slice() -> None: assert color(extract_expr(ast.parse(dedent(""" o[x:y] - """)))) == """ + """)))) == """ o [ @@ -1015,7 +1015,7 @@ def test_ast_slice() -> None: assert color(extract_expr(ast.parse(dedent(""" o[x:y,z] - """)))) == """ + """)))) == """ o [ @@ -1030,21 +1030,21 @@ def test_ast_slice() -> None: def test_ast_attribute() -> None: assert color(extract_expr(ast.parse(dedent(""" mod.attr - """)))) == (""" + """)))) == (""" mod.attr\n""") # ast.Attribute nodes that contains something else as ast.Name nodes are not handled explicitely. assert color(extract_expr(ast.parse(dedent(""" func().attr - """)))) == (""" + """)))) == (""" func().attr\n""") def test_ast_regex() -> None: # invalid arguments assert color(extract_expr(ast.parse(dedent(r""" re.compile(invalidarg='[A-Za-z0-9]+') - """)))) == """ + """)))) == """ re.compile ( @@ -1062,7 +1062,7 @@ def test_ast_regex() -> None: # invalid arguments 2 assert color(extract_expr(ast.parse(dedent(""" re.compile() - """)))) == """ + """)))) == """ re.compile ( @@ -1071,7 +1071,7 @@ def test_ast_regex() -> None: # invalid arguments 3 assert color(extract_expr(ast.parse(dedent(""" re.compile(None) - """)))) == """ + """)))) == """ re.compile ( @@ -1083,7 +1083,7 @@ def test_ast_regex() -> None: # cannot colorize regex, be can't infer value assert color(extract_expr(ast.parse(dedent(""" re.compile(get_re()) - """)))) == """ + """)))) == """ re.compile ( @@ -1097,7 +1097,7 @@ def test_ast_regex() -> None: # cannot colorize regex, not a valid regex assert color(extract_expr(ast.parse(dedent(""" re.compile(r"[.*") - """)))) == """ + """)))) == """ re.compile ( @@ -1113,7 +1113,7 @@ def test_ast_regex() -> None: # actually colorize regex, with flags assert color(extract_expr(ast.parse(dedent(""" re.compile(r"[A-Za-z0-9]+", re.X) - """)))) == """ + """)))) == """ re.compile ( @@ -1147,6 +1147,12 @@ def test_ast_regex() -> None: re.X )\n""" +# hard-coded repr constants +_RE_BEGIN = 13 +_RE_R_PREFIX_EXPECTED = 11 +_RE_COMPILE_LEN = len(_re_compile:='re.compile(') +_RE_COMPILE_SUFFIX_LEN = len(_re_compile_suffix:='') + def color_re(s: Union[bytes, str], check_roundtrip:bool=True) -> str: @@ -1155,10 +1161,11 @@ def color_re(s: Union[bytes, str], if check_roundtrip: raw_text = ''.join(gettext(val.to_node())) - re_begin = 13 + re_begin = _RE_BEGIN + re_end = -2 raw_string = True - if raw_text[11] != 'r': + if raw_text[_RE_R_PREFIX_EXPECTED] != 'r': # the regex has failed to be colorized since we can't find the r prefix # meaning the string has been rendered as plaintext instead. raw_string = False @@ -1166,7 +1173,6 @@ def color_re(s: Union[bytes, str], if isinstance(s, bytes): re_begin += 1 - re_end = -2 round_trip: Union[bytes, str] = raw_text[re_begin:re_end] if isinstance(s, bytes): @@ -1181,7 +1187,10 @@ def color_re(s: Union[bytes, str], assert round_trip == expected, "%s != %s" % (repr(round_trip), repr(s)) - return flatten(val.to_stan(NotFoundLinker()))[17:-8] + html = flatten(val.to_stan(NotFoundLinker())) + assert html.startswith(_re_compile) + assert html.endswith(_re_compile_suffix) + return html[_RE_COMPILE_LEN:-(_RE_COMPILE_SUFFIX_LEN+1)] def test_re_literals() -> None: @@ -1326,7 +1335,7 @@ def test_re_multiline() -> None: assert color(extract_expr(ast.parse(dedent(r'''re.compile(r"""\d + # the integral part \. # the decimal point - \d * # some fractional digits""")''')))) == r""" + \d * # some fractional digits""")''')))) == r""" re.compile ( @@ -1350,7 +1359,7 @@ def test_re_multiline() -> None: assert color(extract_expr(ast.parse(dedent(r'''re.compile(rb"""\d + # the integral part \. # the decimal point - \d * # some fractional digits""")'''))), linelen=70) == r""" + \d * # some fractional digits""")'''))), linelen=70) == r""" re.compile ( @@ -1375,7 +1384,7 @@ def test_line_wrapping() -> None: # If a line goes beyond linelen, it is wrapped using the ``↵`` element. # Check that the last line gets a ``↵`` when maxlines is exceeded: - assert color('x'*1000) == """ + assert color('x'*1000) == """ ' @@ -1408,7 +1417,7 @@ def test_line_wrapping() -> None: # If linebreakok is False, then line wrapping gives an ellipsis instead: - assert color('x'*100, linebreakok=False) == """ + assert color('x'*100, linebreakok=False) == """ ' diff --git a/pydoctor/test/epydoc/test_restructuredtext.py b/pydoctor/test/epydoc/test_restructuredtext.py index 2234c8880..af34bab06 100644 --- a/pydoctor/test/epydoc/test_restructuredtext.py +++ b/pydoctor/test/epydoc/test_restructuredtext.py @@ -105,12 +105,12 @@ def test_rst_anon_link_email() -> None: def test_rst_xref_with_target() -> None: src = "`mapping `" html = rst2html(src) - assert html.startswith('mapping') + assert html == 'mapping' def test_rst_xref_implicit_target() -> None: src = "`func()`" html = rst2html(src) - assert html.startswith('func()') + assert html == 'func()' def test_rst_directive_adnomitions() -> None: expected_html_multiline=""" diff --git a/pydoctor/test/test_napoleon_docstring.py b/pydoctor/test/test_napoleon_docstring.py index a2188dcdf..485a0c25e 100644 --- a/pydoctor/test/test_napoleon_docstring.py +++ b/pydoctor/test/test_napoleon_docstring.py @@ -148,6 +148,21 @@ def test_tokenize_type_spec(self): specs = ( "str", "defaultdict", + + "defaultdict, defaultlist,defaultset or object", + "defaultdict, and x", + "defaultdict, or x", + "defaultdict of x", + "defaultdict of x to y", + "defaultdict, and ", + "defaultdict, or ", + "defaultdict of ", + "defaultdict of x to ", + "defaultdict, and", + "defaultdict, or", + "defaultdict of", + "defaultdict of x to", + "int, float, or complex", "int or float or None, optional", '{"F", "C", "N"}', @@ -164,6 +179,21 @@ def test_tokenize_type_spec(self): tokens = ( ["str"], ["defaultdict"], + + ['defaultdict', ', ', 'defaultlist', ', ', 'defaultset', ' or ', 'object'], + ['defaultdict', ', and ', 'x'], + ['defaultdict', ', or ', 'x'], + ['defaultdict', ' of ', 'x'], + ['defaultdict', ' of ', 'x', ' to ', 'y'], + ['defaultdict', ', and '], + ['defaultdict', ', or '], + ['defaultdict', ' of '], + ['defaultdict', ' of ', 'x', ' to '], + ['defaultdict', ', ', 'and'], + ['defaultdict', ', ', 'or'], + ['defaultdict of'], + ['defaultdict', ' of ', 'x to'], + ["int", ", ", "float", ", or ", "complex"], ["int", " or ", "float", " or ", "None", ", ", "optional"], ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"], @@ -178,8 +208,9 @@ def test_tokenize_type_spec(self): ) for spec, expected in zip(specs, tokens): - actual = TypeDocstring._tokenize_type_spec(spec) - self.assertEqual(expected, actual) + with self.subTest(f'tokenize type {spec!r}'): + actual = TypeDocstring._tokenize_type_spec(spec) + self.assertEqual(expected, actual) def test_recombine_set_tokens(self): tokens = ( @@ -277,6 +308,40 @@ def test_convert_numpy_type_spec(self): for spec, expected in zip(specs, converted): actual = str(TypeDocstring(spec)) self.assertEqual(expected, actual) + + def test_natural_language_delimiters_parsed_tokens(self): + specs = [ + "defaultdict, and x", + "defaultdict, or x", + "defaultdict of x", + "defaultdict of x to y", + "defaultdict, and ", + "defaultdict, or ", + "defaultdict of ", + "defaultdict of x to ", + "defaultdict, and", + "defaultdict, or", + "defaultdict of", + "defaultdict of x to", + ] + expected = [ + [('defaultdict', TokenType.OBJ), (', and ', TokenType.DELIMITER), ('x', TokenType.OBJ)], + [('defaultdict', TokenType.OBJ), (', or ', TokenType.DELIMITER), ('x', TokenType.OBJ)], + [('defaultdict', TokenType.OBJ), (' of ', TokenType.DELIMITER), ('x', TokenType.OBJ)], + [('defaultdict', TokenType.OBJ), (' of ', TokenType.DELIMITER), ('x', TokenType.OBJ), (' to ', TokenType.DELIMITER), ('y', TokenType.OBJ)], + [('defaultdict', TokenType.OBJ), (', and ', TokenType.DELIMITER)], + [('defaultdict', TokenType.OBJ), (', or ', TokenType.DELIMITER)], + [('defaultdict', TokenType.OBJ), (' of ', TokenType.DELIMITER)], + [('defaultdict', TokenType.OBJ), (' of ', TokenType.DELIMITER), ('x', TokenType.OBJ), (' to ', TokenType.DELIMITER)], + [('defaultdict', TokenType.OBJ), (', ', TokenType.DELIMITER), ('and', TokenType.OBJ)], + [('defaultdict', TokenType.OBJ), (', ', TokenType.DELIMITER), ('or', TokenType.OBJ)], + [('defaultdict of', TokenType.UNKNOWN)], + [('defaultdict', TokenType.OBJ), (' of ', TokenType.DELIMITER), ('x to', TokenType.UNKNOWN)], + ] + for spec, expected in zip(specs, expected): + with self.subTest(f'parsed tokens: {spec!r}'): + actual = TypeDocstring(spec)._tokens + self.assertEqual(expected, actual) def test_token_type_invalid(self): tokens = ( diff --git a/pydoctor/test/test_templatewriter.py b/pydoctor/test/test_templatewriter.py index fb2c6d27d..caf6a9655 100644 --- a/pydoctor/test/test_templatewriter.py +++ b/pydoctor/test/test_templatewriter.py @@ -804,7 +804,7 @@ def test_crash_xmlstring_entities(capsys:CapSys, processtypes:bool) -> None: epydoc2stan.ensure_parsed_docstring(o) getHTMLOf(mod) getHTMLOf(mod.contents['C']) - out = capsys.readouterr().out + warnings = '''\ test:2: bad docstring: SAXParseException: .+ undefined entity test:25: bad signature: SAXParseException: .+ undefined entity @@ -814,15 +814,12 @@ def test_crash_xmlstring_entities(capsys:CapSys, processtypes:bool) -> None: test:8: bad annotation: SAXParseException: :.+ undefined entity test:10: bad rendering of constant: SAXParseException: .+ undefined entity test:14: bad docstring: SAXParseException: .+ undefined entity -test:36: bad rendering of class signature: SAXParseException: .+ undefined entity -'''.splitlines() - - # Some how the type processing get rid of the non breaking spaces, but it's more an implementation - # detail rather than a fix for the bug. - if processtypes is True: - warnings.remove('test:30: bad docstring: SAXParseException: .+ undefined entity') +test:36: bad rendering of class signature: SAXParseException: .+ undefined entity'''.splitlines() - assert re.match('\n'.join(warnings), out) + actual = [a for a in capsys.readouterr().out.splitlines() if a] + assert len(warnings) == len(actual) + for a,e in zip(actual, warnings): + assert re.match(e, a), (f'{a!r} doesn not match {e}') @pytest.mark.parametrize('processtypes', [True, False]) def test_crash_xmlstring_entities_rst(capsys:CapSys, processtypes:bool) -> None: @@ -836,8 +833,8 @@ def test_crash_xmlstring_entities_rst(capsys:CapSys, processtypes:bool) -> None: epydoc2stan.ensure_parsed_docstring(o) getHTMLOf(mod) getHTMLOf(mod.contents['C']) - out = capsys.readouterr().out - warn_str = '''\ + + warnings = '''\ test:2: bad docstring: SAXParseException: .+ undefined entity test:25: bad signature: SAXParseException: .+ undefined entity test:17: bad rendering of decorators: SAXParseException: .+ undefined entity @@ -846,14 +843,12 @@ def test_crash_xmlstring_entities_rst(capsys:CapSys, processtypes:bool) -> None: test:8: bad annotation: SAXParseException: .+ undefined entity test:10: bad rendering of constant: SAXParseException: .+ undefined entity test:14: bad docstring: SAXParseException: .+ undefined entity -test:36: bad rendering of class signature: SAXParseException: .+ undefined entity -''' - warnings = warn_str.splitlines() +test:36: bad rendering of class signature: SAXParseException: .+ undefined entity'''.splitlines() - if processtypes is True: - warnings.remove('test:30: bad docstring: SAXParseException: .+ undefined entity') - - assert re.match('\n'.join(warnings), out) + actual = [a for a in capsys.readouterr().out.splitlines() if a] + assert len(warnings) == len(actual) + for a,e in zip(actual, warnings): + assert re.match(e, a), (f'{a!r} doesn not match {e}') def test_constructor_renders(capsys:CapSys) -> None: src = '''\ diff --git a/pydoctor/test/test_type_fields.py b/pydoctor/test/test_type_fields.py index 103d402c6..4f492d542 100644 --- a/pydoctor/test/test_type_fields.py +++ b/pydoctor/test/test_type_fields.py @@ -1,4 +1,4 @@ -from typing import List +from typing import Any, List from textwrap import dedent from pydoctor.epydoc.markup import ParseError, get_parser_by_name from pydoctor.test.epydoc.test_restructuredtext import prettify @@ -12,7 +12,6 @@ from pydoctor.epydoc.markup._types import ParsedTypeDocstring import pydoctor.epydoc.markup from pydoctor import model -from twisted.web.template import Tag def doc2html(doc: str, markup: str, processtypes: bool = False) -> str: @@ -40,23 +39,6 @@ def test_to_node_markup() -> None: for epystr, rststr in cases: assert doc2html(rststr, 'restructuredtext') == doc2html(epystr, 'epytext') -def test_parsed_type_convert_obj_tokens_to_stan() -> None: - - convert_obj_tokens_cases = [ - ([("list", TokenType.OBJ), ("(", TokenType.DELIMITER), ("int", TokenType.OBJ), (")", TokenType.DELIMITER)], - [(Tag('code', children=['list', '(', 'int', ')']), TokenType.OBJ)]), - - ([("list", TokenType.OBJ), ("(", TokenType.DELIMITER), ("int", TokenType.OBJ), (")", TokenType.DELIMITER), (", ", TokenType.DELIMITER), ("optional", TokenType.CONTROL)], - [(Tag('code', children=['list', '(', 'int', ')']), TokenType.OBJ), (", ", TokenType.DELIMITER), ("optional", TokenType.CONTROL)]), - ] - - ann = ParsedTypeDocstring("") - - for tokens_types, expected_token_types in convert_obj_tokens_cases: - - assert str(ann._convert_obj_tokens_to_stan(tokens_types, NotFoundLinker()))==str(expected_token_types) - - def typespec2htmlvianode(s: str, markup: str) -> str: err: List[ParseError] = [] parsed_doc = get_parser_by_name(markup)(s, err) @@ -72,23 +54,23 @@ def typespec2htmlviastr(s: str) -> str: assert not ann.warnings return html -def test_parsed_type() -> None: +def test_parsed_type(subtests: Any) -> None: parsed_type_cases = [ ('list of int or float or None', - 'list of int or float or None'), + 'list of int or float or None'), ("{'F', 'C', 'N'}, default 'N'", - """{'F', 'C', 'N'}, default 'N'"""), + """{'F', 'C', 'N'}, default 'N'"""), ("DataFrame, optional", - "DataFrame, optional"), + """DataFrame, optional"""), ("List[str] or list(bytes), optional", - "List[str] or list(bytes), optional"), + """List[str] or list(bytes), optional"""), (('`complicated string` or `strIO `', 'L{complicated string} or L{strIO }'), - 'complicated string or strIO'), + 'complicated string or strIO'), ] for string, excepted_html in parsed_type_cases: @@ -99,10 +81,12 @@ def test_parsed_type() -> None: rst_string, epy_string = string elif isinstance(string, str): rst_string = epy_string = string + + with subtests.test('parse type', rst=rst_string, epy=epy_string): - assert typespec2htmlviastr(rst_string) == excepted_html - assert typespec2htmlvianode(rst_string, 'restructuredtext') == excepted_html - assert typespec2htmlvianode(epy_string, 'epytext') == excepted_html + assert typespec2htmlviastr(rst_string) == excepted_html + assert typespec2htmlvianode(rst_string, 'restructuredtext') == excepted_html + assert typespec2htmlvianode(epy_string, 'epytext') == excepted_html def test_processtypes(capsys: CapSys) -> None: """ @@ -137,7 +121,7 @@ def test_processtypes(capsys: CapSys) -> None: ), ("list of int or float or None", - "list of int or float or None") + 'list of int or float or None') ), @@ -166,8 +150,8 @@ def test_processtypes(capsys: CapSys) -> None: """, ), - ("complicated string or strIO, optional", - "complicated string or strIO, optional") + ("complicated string or strIO, optional", + 'complicated string or strIO, optional') ), @@ -199,8 +183,8 @@ def test_processtypes_more() -> None: Whether it's not working. """, """
    -
  • working: bool - Whether it's working.
    • -
  • not_working: bool - Whether it's not working.
    • +
  • working: bool - Whether it's working.
    • +
  • not_working: bool - Whether it's not working.
"""), (""" @@ -212,8 +196,8 @@ def test_processtypes_more() -> None: the content description. """, """
    -
  • name: str - the name description.
    • -
  • content: str - the content description.
    • +
  • name: str - the name description.
    • +
  • content: str - the content description.
"""), ] @@ -240,10 +224,10 @@ def test_processtypes_with_system(capsys: CapSys) -> None: captured = capsys.readouterr().out assert not captured - assert "list of int or float or None" == fmt + assert 'list of int or float or None' == fmt -def test_processtypes_corner_cases(capsys: CapSys) -> None: +def test_processtypes_corner_cases(capsys: CapSys, subtests: Any) -> None: """ The corner cases does not trigger any warnings because they are still valid types. @@ -251,7 +235,7 @@ def test_processtypes_corner_cases(capsys: CapSys) -> None: we should be careful with triggering warnings because whether the type spec triggers warnings is used to check is a string is a valid type or not. """ - def process(typestr: str) -> str: + def _process(typestr: str) -> str: system = model.System() system.options.processtypes = True mod = fromText(f''' @@ -265,32 +249,40 @@ def process(typestr: str) -> str: assert isinstance(a.parsed_type, ParsedTypeDocstring) fmt = flatten(a.parsed_type.to_stan(NotFoundLinker())) + assert fmt.startswith('') + assert fmt.endswith('') + fmt = fmt[26:-7] captured = capsys.readouterr().out assert not captured return fmt - assert process('default[str]') == "default[str]" - assert process('[str]') == "[str]" - assert process('[,]') == "[, ]" - assert process('[[]]') == "[[]]" - assert process(', [str]') == ", [str]" - assert process(' of [str]') == "of[str]" - assert process(' or [str]') == "or[str]" - assert process(': [str]') == ": [str]" - assert process("'hello'[str]") == "'hello'[str]" - assert process('"hello"[str]') == "\"hello\"[str]" - assert process('`hello`[str]') == "hello[str]" - assert process('`hello `_[str]') == """hello[str]""" - assert process('**hello**[str]') == "hello[str]" - assert process('["hello" or str, default: 2]') == """["hello" or str, default: 2]""" + def process(input:str, expected:str) -> None: + with subtests.test(msg="processtypes", input=input): + actual = _process(input) + assert actual == expected + + process('default[str]', "default[str]") + process('[str]', "[str]") + process('[,]', "[, ]") + process('[[]]', "[[]]") + process(', [str]', ", [str]") + process(' of [str]', "of[str]") + process(' or [str]', "or[str]") + process(': [str]', ': [str]') + process("'hello'[str]", "'hello'[str]") + process('"hello"[str]', "\"hello\"[str]") + process('`hello`[str]', "hello[str]") + process('`hello `_[str]', """hello[str]""") + process('**hello**[str]', "hello[str]") + process('["hello" or str, default: 2]', """["hello" or str, default: 2]""") # HTML ids for problematic elements changed in docutils 0.18.0, and again in 0.19.0, so we're not testing for the exact content anymore. - - problematic = process('Union[`hello <>`_[str]]') - assert "`hello <>`_" in problematic - assert "str" in problematic + with subtests.test(msg="processtypes", input='Union[`hello <>`_[str]]'): + problematic = _process('Union[`hello <>`_[str]]') + assert "`hello <>`_" in problematic + assert "str" in problematic def test_processtypes_warning_unexpected_element(capsys: CapSys) -> None: @@ -311,7 +303,7 @@ def test_processtypes_warning_unexpected_element(capsys: CapSys) -> None: >>> print('example') """ - expected = """complicated string or strIO, optional""" + expected = """complicated string or strIO, optional""" # Test epytext epy_errors: List[ParseError] = [] @@ -422,5 +414,5 @@ class V: # Filter docstring linker warnings lines = [line for line in capsys.readouterr().out.splitlines() if 'Cannot find link target' not in line] assert not lines - assert 'int' in html + assert 'int' in html \ No newline at end of file From 788957d6b30d7fdd2623f849edb9ffcadd6b376d Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 2 Feb 2025 15:24:25 -0500 Subject: [PATCH 03/21] Fix the linenumber issue in the new references --- pydoctor/epydoc/markup/_types.py | 40 ++++++++++++++++++------------- pydoctor/test/test_type_fields.py | 21 ++++++++++++++-- 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/pydoctor/epydoc/markup/_types.py b/pydoctor/epydoc/markup/_types.py index d2e10e98f..97fde8b25 100644 --- a/pydoctor/epydoc/markup/_types.py +++ b/pydoctor/epydoc/markup/_types.py @@ -7,7 +7,8 @@ from typing import Callable, Dict, List, Union, cast -from pydoctor.epydoc.markup import ParseError, ParsedDocstring, get_parser_by_name +from pydoctor.epydoc.markup import ParseError, ParsedDocstring +from pydoctor.epydoc.markup.restructuredtext import parse_docstring from pydoctor.epydoc.markup._pyval_repr import PyvalColorizer from pydoctor.napoleon.docstring import TokenType, TypeDocstring from pydoctor.epydoc.docutils import new_document, set_node_attributes @@ -83,16 +84,17 @@ def _parse_tokens(self) -> nodes.document: document = new_document('code') warnings: List[ParseError] = [] - converters: Dict[TokenType, Callable[[str | nodes.Node], str | nodes.Node | list[nodes.Node]]] = { - # we're re-using the variable string css class for the whole literal token, it's the - # best approximation we have for now. + converters: Dict[TokenType, Callable[[str], str | nodes.Node | list[nodes.Node]]] = { + # we're re-using the variable string css + # class for the whole literal token, it's the + # best approximation we have for now. TokenType.LITERAL: lambda _token: nodes.inline(_token, _token, classes=[PyvalColorizer.STRING_TAG]), TokenType.CONTROL: lambda _token: nodes.emphasis(_token, _token), - TokenType.REFERENCE: lambda _token: get_parser_by_name('restructuredtext')(_token, warnings).to_node().children if isinstance(_token, str) else _token, - TokenType.UNKNOWN: lambda _token: get_parser_by_name('restructuredtext')(_token, warnings).to_node().children if isinstance(_token, str) else _token, - TokenType.OBJ: lambda _token: nodes.title_reference(_token, _token, line=self._lineno), - TokenType.DELIMITER: lambda _token: nodes.Text(_token), - TokenType.ANY: lambda _token: _token, + TokenType.REFERENCE: lambda _token: parse_docstring(_token, warnings).to_node().children, + TokenType.UNKNOWN: lambda _token: parse_docstring(_token, warnings).to_node().children, + TokenType.OBJ: lambda _token: set_node_attributes(nodes.title_reference(_token, _token), lineno=self._lineno), + TokenType.DELIMITER: lambda _token: _token, + } for w in warnings: @@ -102,15 +104,21 @@ def _parse_tokens(self) -> nodes.document: for token, type_ in self._tokens: assert token is not None - converted_token = converters[type_](token) + if type_ is TokenType.ANY: + assert isinstance(token, nodes.Inline) + converted_token = token + else: + assert isinstance(token, str) + converted_token = converters[type_](token) + if isinstance(converted_token, list): - elements.extend(converted_token) + elements.extend((set_node_attributes(t, document=document) for t in converted_token)) elif isinstance(converted_token, str) and not isinstance(converted_token, nodes.Text): - elements.append(nodes.Text(converted_token)) + elements.append(set_node_attributes(nodes.Text(converted_token), document=document)) else: - elements.append(converted_token) + elements.append(set_node_attributes(converted_token, document=document)) return set_node_attributes(document, children=[ - set_node_attributes(nodes.inline('', '', - classes=['literal']), - children=elements)]) \ No newline at end of file + set_node_attributes(nodes.inline('', '', classes=['literal']), + children=elements, + lineno=self._lineno)]) \ No newline at end of file diff --git a/pydoctor/test/test_type_fields.py b/pydoctor/test/test_type_fields.py index 4f492d542..c1489705a 100644 --- a/pydoctor/test/test_type_fields.py +++ b/pydoctor/test/test_type_fields.py @@ -8,7 +8,6 @@ from pydoctor.test.test_epydoc2stan import docstring2html from pydoctor.test.test_astbuilder import fromText from pydoctor.stanutils import flatten -from pydoctor.napoleon.docstring import TokenType from pydoctor.epydoc.markup._types import ParsedTypeDocstring import pydoctor.epydoc.markup from pydoctor import model @@ -415,4 +414,22 @@ class V: lines = [line for line in capsys.readouterr().out.splitlines() if 'Cannot find link target' not in line] assert not lines assert 'int' in html - \ No newline at end of file + +def test_process_types_doesnt_mess_with_warning_linenumber(capsys: CapSys) -> None: + src = ''' + __docformat__ = 'google' + class ConfigFileParser(object): + """doc""" + + def parse(self, stream): + """Parses the keys and values from a config file. + + Note: blablabla + + Args: + stream (notfound): A config file input stream (such as an open file object). + """ + ''' + mod = fromText(src) + docstring2html(mod.contents['ConfigFileParser'].contents['parse']) + assert capsys.readouterr().out == ':12: Cannot find link target for "notfound"\n' \ No newline at end of file From aefe3c7a727f1b1bf62fe73caf2dd36a1ea5dd89 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 2 Feb 2025 15:29:51 -0500 Subject: [PATCH 04/21] Help mypy --- pydoctor/epydoc/markup/_types.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pydoctor/epydoc/markup/_types.py b/pydoctor/epydoc/markup/_types.py index 97fde8b25..ddeba857f 100644 --- a/pydoctor/epydoc/markup/_types.py +++ b/pydoctor/epydoc/markup/_types.py @@ -84,7 +84,7 @@ def _parse_tokens(self) -> nodes.document: document = new_document('code') warnings: List[ParseError] = [] - converters: Dict[TokenType, Callable[[str], str | nodes.Node | list[nodes.Node]]] = { + converters: Dict[TokenType, Callable[[str], nodes.Node | list[nodes.Node]]] = { # we're re-using the variable string css # class for the whole literal token, it's the # best approximation we have for now. @@ -93,28 +93,27 @@ def _parse_tokens(self) -> nodes.document: TokenType.REFERENCE: lambda _token: parse_docstring(_token, warnings).to_node().children, TokenType.UNKNOWN: lambda _token: parse_docstring(_token, warnings).to_node().children, TokenType.OBJ: lambda _token: set_node_attributes(nodes.title_reference(_token, _token), lineno=self._lineno), - TokenType.DELIMITER: lambda _token: _token, - + TokenType.DELIMITER: lambda _token: nodes.Text(_token), } for w in warnings: self.warnings.append(w.descr()) - elements = [] + elements: list[nodes.Node] = [] for token, type_ in self._tokens: assert token is not None + converted_token: nodes.Node | list[nodes.Node] + if type_ is TokenType.ANY: assert isinstance(token, nodes.Inline) converted_token = token else: assert isinstance(token, str) converted_token = converters[type_](token) - + if isinstance(converted_token, list): elements.extend((set_node_attributes(t, document=document) for t in converted_token)) - elif isinstance(converted_token, str) and not isinstance(converted_token, nodes.Text): - elements.append(set_node_attributes(nodes.Text(converted_token), document=document)) else: elements.append(set_node_attributes(converted_token, document=document)) From c6ffcef99c298d9b6f688104fa7a4321b6af6794 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 2 Feb 2025 16:16:32 -0500 Subject: [PATCH 05/21] trying this... --- pydoctor/epydoc/markup/_types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydoctor/epydoc/markup/_types.py b/pydoctor/epydoc/markup/_types.py index ddeba857f..6ef5d8113 100644 --- a/pydoctor/epydoc/markup/_types.py +++ b/pydoctor/epydoc/markup/_types.py @@ -40,7 +40,7 @@ def __init__(self, annotation: Union[nodes.document, str], else: TypeDocstring.__init__(self, annotation, warns_on_unknown_tokens) - self._lineno = lineno + self._lineno = lineno + 1 self._document = self._parse_tokens() @property @@ -102,7 +102,7 @@ def _parse_tokens(self) -> nodes.document: elements: list[nodes.Node] = [] for token, type_ in self._tokens: - assert token is not None + converted_token: nodes.Node | list[nodes.Node] if type_ is TokenType.ANY: From a8d9af8a1d00a6922696d563ce0000150d6580d0 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 2 Feb 2025 16:25:46 -0500 Subject: [PATCH 06/21] Revert "trying this..." This reverts commit c6ffcef99c298d9b6f688104fa7a4321b6af6794. --- pydoctor/epydoc/markup/_types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydoctor/epydoc/markup/_types.py b/pydoctor/epydoc/markup/_types.py index 6ef5d8113..ddeba857f 100644 --- a/pydoctor/epydoc/markup/_types.py +++ b/pydoctor/epydoc/markup/_types.py @@ -40,7 +40,7 @@ def __init__(self, annotation: Union[nodes.document, str], else: TypeDocstring.__init__(self, annotation, warns_on_unknown_tokens) - self._lineno = lineno + 1 + self._lineno = lineno self._document = self._parse_tokens() @property @@ -102,7 +102,7 @@ def _parse_tokens(self) -> nodes.document: elements: list[nodes.Node] = [] for token, type_ in self._tokens: - + assert token is not None converted_token: nodes.Node | list[nodes.Node] if type_ is TokenType.ANY: From 30bdf2bc9fc1546b602b005b9c2fcc7c4afb0b46 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 2 Feb 2025 16:51:00 -0500 Subject: [PATCH 07/21] Turns out this refactors fixes an obscure bug that would trigger unexpected warnings --- pydoctor/test/test_epydoc2stan.py | 16 +- pydoctor/test/testpackages/numpy/_machar.py | 350 ++++++++++++++++++++ 2 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 pydoctor/test/testpackages/numpy/_machar.py diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index 0096a2579..062e27572 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -11,6 +11,7 @@ from pydoctor.epydoc.markup.epytext import ParsedEpytextDocstring from pydoctor.sphinx import SphinxInventory from pydoctor.test.test_astbuilder import fromText, unwrap +from pydoctor.test.test_packages import processPackage from pydoctor.test import CapSys, NotFoundLinker from pydoctor.templatewriter.search import stem_identifier from pydoctor.templatewriter.pages import format_signature, format_class_signature @@ -2194,4 +2195,17 @@ def __init__(self): # the link not found warnings. getHTMLOf(mod.contents['C']) assert capsys.readouterr().out == (':16: Existing docstring at line 10 is overriden\n' - ':10: Cannot find link target for "bool"\n') \ No newline at end of file + ':10: Cannot find link target for "bool"\n') + +def test_numpydoc_warns_about_unknown_types_in_attribute_section_file(capsys: CapSys): + system = processPackage('numpy/_machar.py', + lambda: model.System(model.Options.from_args('-q'))) + + for o in system.allobjects.values(): + docstring2html(o) + # there are exactly 30 references that needs a --intersphinx ootion to be resolvable. + # all the other things are good. + warnings = capsys.readouterr().out.strip().splitlines() + assert len(warnings) == 30 + assert len(set(warnings)) == 30 + diff --git a/pydoctor/test/testpackages/numpy/_machar.py b/pydoctor/test/testpackages/numpy/_machar.py new file mode 100644 index 000000000..50a00f4cb --- /dev/null +++ b/pydoctor/test/testpackages/numpy/_machar.py @@ -0,0 +1,350 @@ +""" +Machine arithmetic - determine the parameters of the +floating-point arithmetic system + +Author: Pearu Peterson, September 2003 + +""" +__all__ = ['MachAr'] +__docformat__ = 'numpy' + +class MachAr: + """ + Diagnosing machine parameters. + + Attributes + ---------- + ibeta : int + Radix in which numbers are represented. + it : int + Number of base-`ibeta` digits in the floating point mantissa M. + machep : int + Exponent of the smallest (most negative) power of `ibeta` that, + added to 1.0, gives something different from 1.0 + eps : float + Floating-point number ``beta**machep`` (floating point precision) + negep : int + Exponent of the smallest power of `ibeta` that, subtracted + from 1.0, gives something different from 1.0. + epsneg : float + Floating-point number ``beta**negep``. + iexp : int + Number of bits in the exponent (including its sign and bias). + minexp : int + Smallest (most negative) power of `ibeta` consistent with there + being no leading zeros in the mantissa. + xmin : float + Floating-point number ``beta**minexp`` (the smallest [in + magnitude] positive floating point number with full precision). + maxexp : int + Smallest (positive) power of `ibeta` that causes overflow. + xmax : float + ``(1-epsneg) * beta**maxexp`` (the largest [in magnitude] + usable floating value). + irnd : int + In ``range(6)``, information on what kind of rounding is done + in addition, and on how underflow is handled. + ngrd : int + Number of 'guard digits' used when truncating the product + of two mantissas to fit the representation. + epsilon : float + Same as `eps`. + tiny : float + An alias for `smallest_normal`, kept for backwards compatibility. + huge : float + Same as `xmax`. + precision : float + ``- int(-log10(eps))`` + resolution : float + ``- 10**(-precision)`` + smallest_normal : float + The smallest positive floating point number with 1 as leading bit in + the mantissa following IEEE-754. Same as `xmin`. + smallest_subnormal : float + The smallest positive floating point number with 0 as leading bit in + the mantissa following IEEE-754. + + Parameters + ---------- + float_conv : function, optional + Function that converts an integer or integer array to a float + or float array. Default is `float`. + int_conv : function, optional + Function that converts a float or float array to an integer or + integer array. Default is `int`. + float_to_float : function, optional + Function that converts a float array to float. Default is `float`. + Note that this does not seem to do anything useful in the current + implementation. + float_to_str : function, optional + Function that converts a single float to a string. Default is + ``lambda v:'%24.16e' %v``. + title : str, optional + Title that is printed in the string representation of `MachAr`. + + See Also + -------- + finfo : Machine limits for floating point types. + iinfo : Machine limits for integer types. + + References + ---------- + .. [1] Press, Teukolsky, Vetterling and Flannery, + "Numerical Recipes in C++," 2nd ed, + Cambridge University Press, 2002, p. 31. + + """ + + def __init__(self, float_conv=float, int_conv=int, + float_to_float=float, + float_to_str=lambda v: '%24.16e' % v, + title='Python floating point number'): + """ + + float_conv - convert integer to float (array) + int_conv - convert float (array) to integer + float_to_float - convert float array to float + float_to_str - convert array float to str + title - description of used floating point numbers + + """ + # We ignore all errors here because we are purposely triggering + # underflow to detect the properties of the running arch. + with errstate(under='ignore'): + self._do_init(float_conv, int_conv, float_to_float, float_to_str, title) + + def _do_init(self, float_conv, int_conv, float_to_float, float_to_str, title): + max_iterN = 10000 + msg = "Did not converge after %d tries with %s" + one = float_conv(1) + two = one + one + zero = one - one + + # Do we really need to do this? Aren't they 2 and 2.0? + # Determine ibeta and beta + a = one + for _ in range(max_iterN): + a = a + a + temp = a + one + temp1 = temp - a + if any(temp1 - one != zero): + break + else: + raise RuntimeError(msg % (_, one.dtype)) + b = one + for _ in range(max_iterN): + b = b + b + temp = a + b + itemp = int_conv(temp - a) + if any(itemp != 0): + break + else: + raise RuntimeError(msg % (_, one.dtype)) + ibeta = itemp + beta = float_conv(ibeta) + + # Determine it and irnd + it = -1 + b = one + for _ in range(max_iterN): + it = it + 1 + b = b * beta + temp = b + one + temp1 = temp - b + if any(temp1 - one != zero): + break + else: + raise RuntimeError(msg % (_, one.dtype)) + + betah = beta / two + a = one + for _ in range(max_iterN): + a = a + a + temp = a + one + temp1 = temp - a + if any(temp1 - one != zero): + break + else: + raise RuntimeError(msg % (_, one.dtype)) + temp = a + betah + irnd = 0 + if any(temp - a != zero): + irnd = 1 + tempa = a + beta + temp = tempa + betah + if irnd == 0 and any(temp - tempa != zero): + irnd = 2 + + # Determine negep and epsneg + negep = it + 3 + betain = one / beta + a = one + for i in range(negep): + a = a * betain + b = a + for _ in range(max_iterN): + temp = one - a + if any(temp - one != zero): + break + a = a * beta + negep = negep - 1 + # Prevent infinite loop on PPC with gcc 4.0: + if negep < 0: + raise RuntimeError("could not determine machine tolerance " + "for 'negep', locals() -> %s" % (locals())) + else: + raise RuntimeError(msg % (_, one.dtype)) + negep = -negep + epsneg = a + + # Determine machep and eps + machep = - it - 3 + a = b + + for _ in range(max_iterN): + temp = one + a + if any(temp - one != zero): + break + a = a * beta + machep = machep + 1 + else: + raise RuntimeError(msg % (_, one.dtype)) + eps = a + + # Determine ngrd + ngrd = 0 + temp = one + eps + if irnd == 0 and any(temp * one - one != zero): + ngrd = 1 + + # Determine iexp + i = 0 + k = 1 + z = betain + t = one + eps + nxres = 0 + for _ in range(max_iterN): + y = z + z = y * y + a = z * one # Check here for underflow + temp = z * t + if any(a + a == zero) or any(abs(z) >= y): + break + temp1 = temp * betain + if any(temp1 * beta == z): + break + i = i + 1 + k = k + k + else: + raise RuntimeError(msg % (_, one.dtype)) + if ibeta != 10: + iexp = i + 1 + mx = k + k + else: + iexp = 2 + iz = ibeta + while k >= iz: + iz = iz * ibeta + iexp = iexp + 1 + mx = iz + iz - 1 + + # Determine minexp and xmin + for _ in range(max_iterN): + xmin = y + y = y * betain + a = y * one + temp = y * t + if any((a + a) != zero) and any(abs(y) < xmin): + k = k + 1 + temp1 = temp * betain + if any(temp1 * beta == y) and any(temp != y): + nxres = 3 + xmin = y + break + else: + break + else: + raise RuntimeError(msg % (_, one.dtype)) + minexp = -k + + # Determine maxexp, xmax + if mx <= k + k - 3 and ibeta != 10: + mx = mx + mx + iexp = iexp + 1 + maxexp = mx + minexp + irnd = irnd + nxres + if irnd >= 2: + maxexp = maxexp - 2 + i = maxexp + minexp + if ibeta == 2 and not i: + maxexp = maxexp - 1 + if i > 20: + maxexp = maxexp - 1 + if any(a != y): + maxexp = maxexp - 2 + xmax = one - epsneg + if any(xmax * one != xmax): + xmax = one - beta * epsneg + xmax = xmax / (xmin * beta * beta * beta) + i = maxexp + minexp + 3 + for j in range(i): + if ibeta == 2: + xmax = xmax + xmax + else: + xmax = xmax * beta + + smallest_subnormal = abs(xmin / beta ** (it)) + + self.ibeta = ibeta + self.it = it + self.negep = negep + self.epsneg = float_to_float(epsneg) + self._str_epsneg = float_to_str(epsneg) + self.machep = machep + self.eps = float_to_float(eps) + self._str_eps = float_to_str(eps) + self.ngrd = ngrd + self.iexp = iexp + self.minexp = minexp + self.xmin = float_to_float(xmin) + self._str_xmin = float_to_str(xmin) + self.maxexp = maxexp + self.xmax = float_to_float(xmax) + self._str_xmax = float_to_str(xmax) + self.irnd = irnd + + self.title = title + # Commonly used parameters + self.epsilon = self.eps + self.tiny = self.xmin + self.huge = self.xmax + self.smallest_normal = self.xmin + self._str_smallest_normal = float_to_str(self.xmin) + self.smallest_subnormal = float_to_float(smallest_subnormal) + self._str_smallest_subnormal = float_to_str(smallest_subnormal) + + import math + self.precision = int(-math.log10(float_to_float(self.eps))) + ten = two + two + two + two + two + resolution = ten ** (-self.precision) + self.resolution = float_to_float(resolution) + self._str_resolution = float_to_str(resolution) + + def __str__(self): + fmt = ( + 'Machine parameters for %(title)s\n' + '---------------------------------------------------------------------\n' + 'ibeta=%(ibeta)s it=%(it)s iexp=%(iexp)s ngrd=%(ngrd)s irnd=%(irnd)s\n' + 'machep=%(machep)s eps=%(_str_eps)s (beta**machep == epsilon)\n' + 'negep =%(negep)s epsneg=%(_str_epsneg)s (beta**epsneg)\n' + 'minexp=%(minexp)s xmin=%(_str_xmin)s (beta**minexp == tiny)\n' + 'maxexp=%(maxexp)s xmax=%(_str_xmax)s ((1-epsneg)*beta**maxexp == huge)\n' + 'smallest_normal=%(smallest_normal)s ' + 'smallest_subnormal=%(smallest_subnormal)s\n' + '---------------------------------------------------------------------\n' + ) + return fmt % self.__dict__ + + +if __name__ == '__main__': + print(MachAr()) \ No newline at end of file From ec37bff79572a11295688e17c4c3c2b0eb8f495b Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 2 Feb 2025 16:56:05 -0500 Subject: [PATCH 08/21] Try to fix mypy --- pydoctor/epydoc/markup/_types.py | 2 +- pydoctor/test/test_epydoc2stan.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pydoctor/epydoc/markup/_types.py b/pydoctor/epydoc/markup/_types.py index ddeba857f..a6c0a40b5 100644 --- a/pydoctor/epydoc/markup/_types.py +++ b/pydoctor/epydoc/markup/_types.py @@ -106,7 +106,7 @@ def _parse_tokens(self) -> nodes.document: converted_token: nodes.Node | list[nodes.Node] if type_ is TokenType.ANY: - assert isinstance(token, nodes.Inline) + assert isinstance(token, nodes.Node) converted_token = token else: assert isinstance(token, str) diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index 062e27572..af2b3d4fe 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -2197,7 +2197,7 @@ def __init__(self): assert capsys.readouterr().out == (':16: Existing docstring at line 10 is overriden\n' ':10: Cannot find link target for "bool"\n') -def test_numpydoc_warns_about_unknown_types_in_attribute_section_file(capsys: CapSys): +def test_numpydoc_warns_about_unknown_types_in_attribute_section_file(capsys: CapSys) -> None: system = processPackage('numpy/_machar.py', lambda: model.System(model.Options.from_args('-q'))) From 36f6ebad144fe144ce832b16dd3537c8acd91d80 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sun, 2 Feb 2025 17:18:54 -0500 Subject: [PATCH 09/21] Fix tests of docs --- docs/tests/test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/tests/test.py b/docs/tests/test.py index 0b957c1e0..9ca3b1a1b 100644 --- a/docs/tests/test.py +++ b/docs/tests/test.py @@ -195,7 +195,6 @@ def test_search(query:str, expected:List[str], order_is_important:bool=True) -> to_stan_results = [ 'pydoctor.epydoc.markup.ParsedDocstring.to_stan', 'pydoctor.epydoc.markup.plaintext.ParsedPlaintextDocstring.to_stan', - 'pydoctor.epydoc.markup._types.ParsedTypeDocstring.to_stan', 'pydoctor.epydoc.markup._pyval_repr.ColorizedPyvalRepr.to_stan', 'pydoctor.epydoc2stan.ParsedStanOnly.to_stan', ] From bd9d457db973b15036eaf3ee71a2f5db39de8d5a Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 6 Feb 2025 09:55:33 -0500 Subject: [PATCH 10/21] Showcase the literak choices of google/numpy in the demo --- docs/google_demo/__init__.py | 2 +- docs/numpy_demo/__init__.py | 2 +- docs/source/conf.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/google_demo/__init__.py b/docs/google_demo/__init__.py index c0c0908d9..4f9b572cf 100644 --- a/docs/google_demo/__init__.py +++ b/docs/google_demo/__init__.py @@ -55,7 +55,7 @@ def function_with_types_in_docstring(param1, param2): Args: param1 (int): The first parameter. - param2 (str): The second parameter. + param2 (str, one of {"html", "json", "xml"}, optional): The second parameter. Returns: bool: The return value. True for success, False otherwise. diff --git a/docs/numpy_demo/__init__.py b/docs/numpy_demo/__init__.py index 943b66a4f..bba34b3e3 100644 --- a/docs/numpy_demo/__init__.py +++ b/docs/numpy_demo/__init__.py @@ -73,7 +73,7 @@ def function_with_types_in_docstring(param1, param2): ---------- param1 : int The first parameter. - param2 : str + param2 : str, one of {"html", "json", "xml"}, optional) The second parameter. Returns diff --git a/docs/source/conf.py b/docs/source/conf.py index eeba964ff..814ac1199 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -36,7 +36,7 @@ "sphinx_rtd_theme", "sphinx.ext.intersphinx", "pydoctor.sphinx_ext.build_apidocs", - "sphinxcontrib.spelling", + # "sphinxcontrib.spelling", "sphinxarg.ext", ] From ed1e6d9199ad3a4a26b10b4784dd96e3ee9fe569 Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Thu, 13 Feb 2025 18:19:00 -0500 Subject: [PATCH 11/21] Re-enable spelling extension --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 814ac1199..eeba964ff 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -36,7 +36,7 @@ "sphinx_rtd_theme", "sphinx.ext.intersphinx", "pydoctor.sphinx_ext.build_apidocs", - # "sphinxcontrib.spelling", + "sphinxcontrib.spelling", "sphinxarg.ext", ] From 5f5542a513866576d7ec3ebdc4efd504c711d81b Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 14 Feb 2025 10:37:49 -0500 Subject: [PATCH 12/21] Fix the numpy-style type in the demo --- docs/google_demo/__init__.py | 2 +- docs/numpy_demo/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/google_demo/__init__.py b/docs/google_demo/__init__.py index 4f9b572cf..6c474e818 100644 --- a/docs/google_demo/__init__.py +++ b/docs/google_demo/__init__.py @@ -55,7 +55,7 @@ def function_with_types_in_docstring(param1, param2): Args: param1 (int): The first parameter. - param2 (str, one of {"html", "json", "xml"}, optional): The second parameter. + param2 (str : {"html", "json", "xml"}, optional): The second parameter. Returns: bool: The return value. True for success, False otherwise. diff --git a/docs/numpy_demo/__init__.py b/docs/numpy_demo/__init__.py index bba34b3e3..db07eec8a 100644 --- a/docs/numpy_demo/__init__.py +++ b/docs/numpy_demo/__init__.py @@ -73,7 +73,7 @@ def function_with_types_in_docstring(param1, param2): ---------- param1 : int The first parameter. - param2 : str, one of {"html", "json", "xml"}, optional) + param2 : str : {"html", "json", "xml"}, optional The second parameter. Returns From 042f9dc656d9dfd9257ae5bd54563fd6196e6345 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 14 Feb 2025 10:40:27 -0500 Subject: [PATCH 13/21] Actually fix a couple of issues: - fix the linenumber of reported type docstring issues by 1. - the nested warnings for unknown token are now properly propagated and reported. --- pydoctor/epydoc/markup/_types.py | 59 ++++++++++++++++++++----------- pydoctor/test/test_type_fields.py | 22 ++++++------ 2 files changed, 50 insertions(+), 31 deletions(-) diff --git a/pydoctor/epydoc/markup/_types.py b/pydoctor/epydoc/markup/_types.py index a6c0a40b5..a26060963 100644 --- a/pydoctor/epydoc/markup/_types.py +++ b/pydoctor/epydoc/markup/_types.py @@ -76,6 +76,33 @@ def _warn_not_supported(n:nodes.Node) -> None: return tokens + _converters: Dict[TokenType, Callable[[str, list[ParseError], int], nodes.Node]] = { + # we're re-using the variable string css + # class for the whole literal token, it's the + # best approximation we have for now. + TokenType.LITERAL: lambda _token, _, __: \ + nodes.inline(_token, _token, classes=[PyvalColorizer.STRING_TAG]), + + TokenType.CONTROL: lambda _token, _, __: \ + nodes.emphasis(_token, _token), + + TokenType.REFERENCE: lambda _token, warnings, _: \ + parse_docstring(_token, warnings).to_node(), + + TokenType.UNKNOWN: lambda _token, warnings, _: \ + parse_docstring(_token, warnings).to_node(), + + TokenType.OBJ: lambda _token, _, lineno: \ + set_node_attributes(nodes.title_reference(_token, _token), + # the +1 here is coping with the fact that + # ParseErrors are 1-based but the doutils + # line we're getting form get_lineno() is zero-based. + lineno=lineno+1), + + TokenType.DELIMITER: lambda _token, _, __: \ + nodes.Text(_token), + } + def _parse_tokens(self) -> nodes.document: """ Convert type to docutils document object. @@ -83,39 +110,31 @@ def _parse_tokens(self) -> nodes.document: document = new_document('code') warnings: List[ParseError] = [] - - converters: Dict[TokenType, Callable[[str], nodes.Node | list[nodes.Node]]] = { - # we're re-using the variable string css - # class for the whole literal token, it's the - # best approximation we have for now. - TokenType.LITERAL: lambda _token: nodes.inline(_token, _token, classes=[PyvalColorizer.STRING_TAG]), - TokenType.CONTROL: lambda _token: nodes.emphasis(_token, _token), - TokenType.REFERENCE: lambda _token: parse_docstring(_token, warnings).to_node().children, - TokenType.UNKNOWN: lambda _token: parse_docstring(_token, warnings).to_node().children, - TokenType.OBJ: lambda _token: set_node_attributes(nodes.title_reference(_token, _token), lineno=self._lineno), - TokenType.DELIMITER: lambda _token: nodes.Text(_token), - } - - for w in warnings: - self.warnings.append(w.descr()) + converters = self._converters + lineno = self._lineno elements: list[nodes.Node] = [] for token, type_ in self._tokens: assert token is not None - converted_token: nodes.Node | list[nodes.Node] + converted_token: nodes.Node if type_ is TokenType.ANY: assert isinstance(token, nodes.Node) converted_token = token else: assert isinstance(token, str) - converted_token = converters[type_](token) + converted_token = converters[type_](token, warnings, lineno) - if isinstance(converted_token, list): - elements.extend((set_node_attributes(t, document=document) for t in converted_token)) + if isinstance(converted_token, nodes.document): + elements.extend((set_node_attributes(t, document=document) + for t in converted_token.children)) else: - elements.append(set_node_attributes(converted_token, document=document)) + elements.append(set_node_attributes(converted_token, + document=document)) + # warnings should be appended once we have called all converters. + for w in warnings: + self.warnings.append(w.descr()) return set_node_attributes(document, children=[ set_node_attributes(nodes.inline('', '', classes=['literal']), diff --git a/pydoctor/test/test_type_fields.py b/pydoctor/test/test_type_fields.py index c1489705a..f3507d4f4 100644 --- a/pydoctor/test/test_type_fields.py +++ b/pydoctor/test/test_type_fields.py @@ -234,7 +234,7 @@ def test_processtypes_corner_cases(capsys: CapSys, subtests: Any) -> None: we should be careful with triggering warnings because whether the type spec triggers warnings is used to check is a string is a valid type or not. """ - def _process(typestr: str) -> str: + def _process(typestr: str, fails:bool=False) -> str: system = model.System() system.options.processtypes = True mod = fromText(f''' @@ -252,8 +252,9 @@ def _process(typestr: str) -> str: assert fmt.endswith('') fmt = fmt[26:-7] - captured = capsys.readouterr().out - assert not captured + if not fails: + captured = capsys.readouterr().out + assert not captured return fmt @@ -279,7 +280,7 @@ def process(input:str, expected:str) -> None: # HTML ids for problematic elements changed in docutils 0.18.0, and again in 0.19.0, so we're not testing for the exact content anymore. with subtests.test(msg="processtypes", input='Union[`hello <>`_[str]]'): - problematic = _process('Union[`hello <>`_[str]]') + problematic = _process('Union[`hello <>`_[str]]', fails=True) assert "`hello <>`_" in problematic assert "str" in problematic @@ -417,19 +418,18 @@ class V: def test_process_types_doesnt_mess_with_warning_linenumber(capsys: CapSys) -> None: src = ''' - __docformat__ = 'google' + __docformat__ = 'epytext' class ConfigFileParser(object): """doc""" def parse(self, stream): - """Parses the keys and values from a config file. - - Note: blablabla + """ + Parses the keys and values from a config file. - Args: - stream (notfound): A config file input stream (such as an open file object). + @param stream: A config file input stream (such as an open file object). + @type stream: (notfound, thing[) """ ''' mod = fromText(src) docstring2html(mod.contents['ConfigFileParser'].contents['parse']) - assert capsys.readouterr().out == ':12: Cannot find link target for "notfound"\n' \ No newline at end of file + assert all(l.startswith(':11:') for l in capsys.readouterr().out.splitlines()) \ No newline at end of file From 6703a1f25ce586b6b323c1718a73aab3bab93c3e Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 14 Feb 2025 15:56:40 -0500 Subject: [PATCH 14/21] This changes simplify the parsed type docstring and makes the logic linenumber correct for epytext and restructuredtext. --- pydoctor/epydoc/markup/__init__.py | 4 +- pydoctor/epydoc/markup/_types.py | 36 +++----- pydoctor/test/epydoc/test_google_numpy.py | 7 +- pydoctor/test/test_epydoc2stan.py | 34 +++++++ pydoctor/test/test_type_fields.py | 104 ++++++++++++++++------ 5 files changed, 128 insertions(+), 57 deletions(-) diff --git a/pydoctor/epydoc/markup/__init__.py b/pydoctor/epydoc/markup/__init__.py index 318851cc9..b354ecf9a 100644 --- a/pydoctor/epydoc/markup/__init__.py +++ b/pydoctor/epydoc/markup/__init__.py @@ -87,7 +87,7 @@ def get_supported_docformats() -> Iterator[str]: def get_parser_by_name(docformat: str, objclass: ObjClass | None = None) -> ParserFunction: """ - Get the C{parse_docstring(str, List[ParseError], bool) -> ParsedDocstring} function based on a parser name. + Get the C{parse_docstring(str, List[ParseError]) -> ParsedDocstring} function based on a parser name. @raises ImportError: If the parser could not be imported, probably meaning that your are missing a dependency or it could be that the docformat name do not match any know L{pydoctor.epydoc.markup} submodules. @@ -112,7 +112,7 @@ def _processtypes(doc: 'ParsedDocstring', errs: List['ParseError']) -> None: for field in doc.fields: if field.tag() in ParsedTypeDocstring.FIELDS: body = ParsedTypeDocstring(field.body().to_node(), lineno=field.lineno) - append_warnings(body.warnings, errs, lineno=field.lineno+1) + append_warnings(body.warnings, errs, lineno=field.lineno) field.replace_body(body) def parse_and_processtypes(doc:str, errs:List['ParseError']) -> 'ParsedDocstring': diff --git a/pydoctor/epydoc/markup/_types.py b/pydoctor/epydoc/markup/_types.py index a26060963..eb0091e30 100644 --- a/pydoctor/epydoc/markup/_types.py +++ b/pydoctor/epydoc/markup/_types.py @@ -27,18 +27,15 @@ class ParsedTypeDocstring(TypeDocstring, ParsedDocstring): # yes this overrides the superclass type! _tokens: list[tuple[str | nodes.Node, TokenType]] # type: ignore - def __init__(self, annotation: Union[nodes.document, str], + def __init__(self, annotation: nodes.document, warns_on_unknown_tokens: bool = False, lineno: int = 0) -> None: ParsedDocstring.__init__(self, ()) - if isinstance(annotation, nodes.document): - TypeDocstring.__init__(self, '', warns_on_unknown_tokens) + TypeDocstring.__init__(self, '', warns_on_unknown_tokens) - _tokens = self._tokenize_node_type_spec(annotation) - self._tokens = cast('list[tuple[str | nodes.Node, TokenType]]', - self._build_tokens(_tokens)) - self._trigger_warnings() - else: - TypeDocstring.__init__(self, annotation, warns_on_unknown_tokens) + _tokens = self._tokenize_node_type_spec(annotation) + self._tokens = cast('list[tuple[str | nodes.Node, TokenType]]', + self._build_tokens(_tokens)) + self._trigger_warnings() self._lineno = lineno self._document = self._parse_tokens() @@ -86,21 +83,10 @@ def _warn_not_supported(n:nodes.Node) -> None: TokenType.CONTROL: lambda _token, _, __: \ nodes.emphasis(_token, _token), - TokenType.REFERENCE: lambda _token, warnings, _: \ - parse_docstring(_token, warnings).to_node(), - - TokenType.UNKNOWN: lambda _token, warnings, _: \ - parse_docstring(_token, warnings).to_node(), TokenType.OBJ: lambda _token, _, lineno: \ set_node_attributes(nodes.title_reference(_token, _token), - # the +1 here is coping with the fact that - # ParseErrors are 1-based but the doutils - # line we're getting form get_lineno() is zero-based. - lineno=lineno+1), - - TokenType.DELIMITER: lambda _token, _, __: \ - nodes.Text(_token), + lineno=lineno), } def _parse_tokens(self) -> nodes.document: @@ -114,6 +100,7 @@ def _parse_tokens(self) -> nodes.document: lineno = self._lineno elements: list[nodes.Node] = [] + default = lambda _token, _, __: nodes.Text(_token) for token, type_ in self._tokens: assert token is not None @@ -124,7 +111,7 @@ def _parse_tokens(self) -> nodes.document: converted_token = token else: assert isinstance(token, str) - converted_token = converters[type_](token, warnings, lineno) + converted_token = converters.get(type_, default)(token, warnings, lineno) if isinstance(converted_token, nodes.document): elements.extend((set_node_attributes(t, document=document) @@ -139,4 +126,7 @@ def _parse_tokens(self) -> nodes.document: return set_node_attributes(document, children=[ set_node_attributes(nodes.inline('', '', classes=['literal']), children=elements, - lineno=self._lineno)]) \ No newline at end of file + # the +1 here is coping with the fact that + # ParseErrors are 1-based but the doutils + # line we're getting form get_lineno() is zero-based. + lineno=lineno+1)]) diff --git a/pydoctor/test/epydoc/test_google_numpy.py b/pydoctor/test/epydoc/test_google_numpy.py index e3948f483..75f194469 100644 --- a/pydoctor/test/epydoc/test_google_numpy.py +++ b/pydoctor/test/epydoc/test_google_numpy.py @@ -170,8 +170,9 @@ def test_warnings(self) -> None: self.assertIn("invalid value set (missing closing brace)", errors[1].descr()) self.assertIn("malformed string literal (missing opening quote)", errors[0].descr()) - self.assertEqual(errors[2].linenum(), 21) # #FIXME: It should be 23 actually... - self.assertEqual(errors[1].linenum(), 18) - self.assertEqual(errors[0].linenum(), 14) + #FIXME: It should be 23 actually: https://github.com/twisted/pydoctor/issues/807 + self.assertEqual(errors[2].linenum(), 20) + self.assertEqual(errors[1].linenum(), 17) + self.assertEqual(errors[0].linenum(), 13) diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index af2b3d4fe..b7f23bf56 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -2209,3 +2209,37 @@ def test_numpydoc_warns_about_unknown_types_in_attribute_section_file(capsys: Ca assert len(warnings) == 30 assert len(set(warnings)) == 30 +def test_numpydoc_warns_about_unknown_types_in_explicit_references_at_line(capsys: CapSys) -> None: + # we don't have a good knowledge of linenumber in numpy or google docstring + # because of https://github.com/twisted/pydoctor/issues/807 + # But this regression test tries to ensure we're not making it worse. + # it might need to be adjusted when we fix #807. + + src = ''' + import numpy as np + __docformat__ = 'numpy' + def find(a, sub, start=0, end=None): + """ + For each element, return the lowest index in the string where + substring ``sub`` is found, such that ``sub`` is contained in the + range [``start``, ``end``). + + Parameters + ---------- + a : array_like, with ``StringDType``, ``bytes_`` or ``str_`` dtype + sub : array_like, with `np.bytes_` or `np.str_` dtype + The substring to search for. + """ + ''' + system = model.System(model.Options.from_args(['-q'])) + builder = system.systemBuilder(system) + builder.addModuleString('', modname='numpy', is_package=True) + builder.addModuleString('', modname='_core', is_package=True, parent_name='numpy') + builder.addModuleString(src, modname='strings.py', parent_name='numpy._core') + builder.buildModules() + for o in system.allobjects.values(): + docstring2html(o) + assert capsys.readouterr().out == ('numpy._core.strings.py:11: Cannot find link target for "array_like"\n' + 'numpy._core.strings.py:13: Cannot find link target for "array_like"\n' + 'numpy._core.strings.py:13: Cannot find link target for "numpy.bytes_", resolved from "np.bytes_"\n' + 'numpy._core.strings.py:13: Cannot find link target for "numpy.str_", resolved from "np.str_"\n') \ No newline at end of file diff --git a/pydoctor/test/test_type_fields.py b/pydoctor/test/test_type_fields.py index f3507d4f4..72006f264 100644 --- a/pydoctor/test/test_type_fields.py +++ b/pydoctor/test/test_type_fields.py @@ -9,6 +9,7 @@ from pydoctor.test.test_astbuilder import fromText from pydoctor.stanutils import flatten from pydoctor.epydoc.markup._types import ParsedTypeDocstring +from pydoctor.napoleon.docstring import TypeDocstring import pydoctor.epydoc.markup from pydoctor import model @@ -47,12 +48,6 @@ def typespec2htmlvianode(s: str, markup: str) -> str: assert not ann.warnings return html -def typespec2htmlviastr(s: str) -> str: - ann = ParsedTypeDocstring(s, warns_on_unknown_tokens=True) - html = flatten(ann.to_stan(NotFoundLinker())) - assert not ann.warnings - return html - def test_parsed_type(subtests: Any) -> None: parsed_type_cases = [ @@ -83,7 +78,6 @@ def test_parsed_type(subtests: Any) -> None: with subtests.test('parse type', rst=rst_string, epy=epy_string): - assert typespec2htmlviastr(rst_string) == excepted_html assert typespec2htmlvianode(rst_string, 'restructuredtext') == excepted_html assert typespec2htmlvianode(epy_string, 'epytext') == excepted_html @@ -234,13 +228,20 @@ def test_processtypes_corner_cases(capsys: CapSys, subtests: Any) -> None: we should be careful with triggering warnings because whether the type spec triggers warnings is used to check is a string is a valid type or not. """ - def _process(typestr: str, fails:bool=False) -> str: + def _process(typestr: str, fails:bool=False, docformat:str='both') -> str: + if docformat == 'both': + str1 = _process(typestr, fails, 'epytext') + str2 = _process(typestr, fails, 'restructuredtext') + assert str1 == str2 + return str1 + system = model.System() system.options.processtypes = True mod = fromText(f''' + __docformat__ = '{docformat}' a = None """ - @type: {typestr} + {'@' if docformat == 'epytext' else ':'}type: {typestr} """ ''', modname='test', system=system) a = mod.contents['a'] @@ -258,9 +259,10 @@ def _process(typestr: str, fails:bool=False) -> str: return fmt - def process(input:str, expected:str) -> None: + def process(input:str, expected:str, fails:bool=False, docformat:str='both') -> None: + # both is for epytext and restructuredtext with subtests.test(msg="processtypes", input=input): - actual = _process(input) + actual = _process(input, fails=fails, docformat=docformat) assert actual == expected process('default[str]', "default[str]") @@ -268,19 +270,20 @@ def process(input:str, expected:str) -> None: process('[,]', "[, ]") process('[[]]', "[[]]") process(', [str]', ", [str]") - process(' of [str]', "of[str]") - process(' or [str]', "or[str]") + process(' of [str]', "of [str]") + process(' or [str]', "or [str]") process(': [str]', ': [str]') process("'hello'[str]", "'hello'[str]") process('"hello"[str]', "\"hello\"[str]") - process('`hello`[str]', "hello[str]") - process('`hello `_[str]', """hello[str]""") - process('**hello**[str]', "hello[str]") process('["hello" or str, default: 2]', """["hello" or str, default: 2]""") - + + process('`hello`[str]', "`hello`[str]", fails=True, docformat='restructuredtext') + process('`hello `_[str]', """`hello <https://github.com>`_[str]""", fails=True, docformat='restructuredtext') + process('**hello**[str]', "**hello**[str]", fails=True, docformat='restructuredtext') + # HTML ids for problematic elements changed in docutils 0.18.0, and again in 0.19.0, so we're not testing for the exact content anymore. with subtests.test(msg="processtypes", input='Union[`hello <>`_[str]]'): - problematic = _process('Union[`hello <>`_[str]]', fails=True) + problematic = _process('Union[`hello <>`_[str]]', fails=True, docformat='restructuredtext') assert "`hello <>`_" in problematic assert "str" in problematic @@ -379,14 +382,14 @@ def foo(**args): # which includes much more lines because of the :type arg: fields. assert '\n'.join(lines) == '''\ warns:13: bad docstring: invalid type: 'docformatCan be one of'. Probably missing colon. -warns:7: bad docstring: unbalanced parenthesis in type expression -warns:9: bad docstring: unbalanced square braces in type expression -warns:11: bad docstring: invalid value set (missing closing brace): {1 -warns:13: bad docstring: invalid value set (missing opening brace): 3} -warns:15: bad docstring: malformed string literal (missing closing quote): '2 -warns:17: bad docstring: malformed string literal (missing opening quote): 2" -warns:24: bad docstring: Unexpected element in type specification field: element 'doctest_block'. This value should only contain text or inline markup. -warns:28: bad docstring: Unexpected element in type specification field: element 'paragraph'. This value should only contain text or inline markup.''' +warns:6: bad docstring: unbalanced parenthesis in type expression +warns:8: bad docstring: unbalanced square braces in type expression +warns:10: bad docstring: invalid value set (missing closing brace): {1 +warns:12: bad docstring: invalid value set (missing opening brace): 3} +warns:14: bad docstring: malformed string literal (missing closing quote): '2 +warns:16: bad docstring: malformed string literal (missing opening quote): 2" +warns:23: bad docstring: Unexpected element in type specification field: element 'doctest_block'. This value should only contain text or inline markup. +warns:27: bad docstring: Unexpected element in type specification field: element 'paragraph'. This value should only contain text or inline markup.''' def test_process_types_with_consolidated_fields(capsys: CapSys) -> None: """ @@ -422,14 +425,57 @@ def test_process_types_doesnt_mess_with_warning_linenumber(capsys: CapSys) -> No class ConfigFileParser(object): """doc""" - def parse(self, stream): + def parse(self, stream, stuff): """ Parses the keys and values from a config file. @param stream: A config file input stream (such as an open file object). @type stream: (notfound, thing[) + @param stuff: Stuff + @type stuff: array_like, with L{np.bytes_} or L{np.str_} dtype """ ''' - mod = fromText(src) + system = model.System() + system.options.processtypes = True + mod = fromText(src, system=system) docstring2html(mod.contents['ConfigFileParser'].contents['parse']) - assert all(l.startswith(':11:') for l in capsys.readouterr().out.splitlines()) \ No newline at end of file + # These linenumbers, are correct. + assert capsys.readouterr().out.splitlines() == [ + ':11: bad docstring: unbalanced square braces in type expression', + ':11: Cannot find link target for "notfound"', + ':11: Cannot find link target for "thing"', + ':13: Cannot find link target for "array_like"', + ':13: Cannot find link target for "np.bytes_" (you can link to external docs with --intersphinx)', + ':13: Cannot find link target for "np.str_" (you can link to external docs with --intersphinx)' + ] + +def test_process_types_doesnt_mess_with_warning_linenumber_rst(capsys: CapSys) -> None: + src = ''' + __docformat__ = 'restructuredtext' + class ConfigFileParser(object): + """doc""" + + def parse(self, stream, stuff): + """ + Parses the keys and values from a config file. + + :param stream: A config file input stream (such as an open file object). + :type stream: (notfound, thing[) + :param stuff: Stuff + :type stuff: array_like, with `np.bytes_` or `np.str_` dtype + """ + ''' + system = model.System() + system.options.processtypes = True + mod = fromText(src, system=system) + html = docstring2html(mod.contents['ConfigFileParser'].contents['parse']) + assert 'np.bytes_' in html + # These linenumbers, are correct. + assert capsys.readouterr().out.splitlines() == [ + ':11: bad docstring: unbalanced square braces in type expression', + ':11: Cannot find link target for "notfound"', + ':11: Cannot find link target for "thing"', + ':13: Cannot find link target for "array_like"', + ':13: Cannot find link target for "np.bytes_" (you can link to external docs with --intersphinx)', + ':13: Cannot find link target for "np.str_" (you can link to external docs with --intersphinx)' + ] \ No newline at end of file From a554adc4b526d78260cc9775df85c08e66e40fb6 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 14 Feb 2025 16:25:46 -0500 Subject: [PATCH 15/21] Properly add regression test for the duplicated type attribute bug --- pydoctor/test/test_epydoc2stan.py | 13 - pydoctor/test/test_type_fields.py | 20 +- pydoctor/test/testpackages/numpy/_machar.py | 350 -------------------- 3 files changed, 19 insertions(+), 364 deletions(-) delete mode 100644 pydoctor/test/testpackages/numpy/_machar.py diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index b7f23bf56..7f928194d 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -11,7 +11,6 @@ from pydoctor.epydoc.markup.epytext import ParsedEpytextDocstring from pydoctor.sphinx import SphinxInventory from pydoctor.test.test_astbuilder import fromText, unwrap -from pydoctor.test.test_packages import processPackage from pydoctor.test import CapSys, NotFoundLinker from pydoctor.templatewriter.search import stem_identifier from pydoctor.templatewriter.pages import format_signature, format_class_signature @@ -2197,18 +2196,6 @@ def __init__(self): assert capsys.readouterr().out == (':16: Existing docstring at line 10 is overriden\n' ':10: Cannot find link target for "bool"\n') -def test_numpydoc_warns_about_unknown_types_in_attribute_section_file(capsys: CapSys) -> None: - system = processPackage('numpy/_machar.py', - lambda: model.System(model.Options.from_args('-q'))) - - for o in system.allobjects.values(): - docstring2html(o) - # there are exactly 30 references that needs a --intersphinx ootion to be resolvable. - # all the other things are good. - warnings = capsys.readouterr().out.strip().splitlines() - assert len(warnings) == 30 - assert len(set(warnings)) == 30 - def test_numpydoc_warns_about_unknown_types_in_explicit_references_at_line(capsys: CapSys) -> None: # we don't have a good knowledge of linenumber in numpy or google docstring # because of https://github.com/twisted/pydoctor/issues/807 diff --git a/pydoctor/test/test_type_fields.py b/pydoctor/test/test_type_fields.py index 72006f264..dc409e415 100644 --- a/pydoctor/test/test_type_fields.py +++ b/pydoctor/test/test_type_fields.py @@ -478,4 +478,22 @@ def parse(self, stream, stuff): ':13: Cannot find link target for "array_like"', ':13: Cannot find link target for "np.bytes_" (you can link to external docs with --intersphinx)', ':13: Cannot find link target for "np.str_" (you can link to external docs with --intersphinx)' - ] \ No newline at end of file + ] + +def test_bug_attribute_type_not_found_reports_only_once(capsys:CapSys) -> None: + src = ''' + __docformat__ = 'numpy' + class MachAr: + """ + Diagnosing machine parameters. + + Attributes + ---------- + ibeta : int + Radix in which numbers are represented. + """ + ''' + + mod = fromText(src) + [docstring2html(o) for o in mod.system.allobjects.values()] + assert capsys.readouterr().out.splitlines() == [':8: Cannot find link target for "int"'] \ No newline at end of file diff --git a/pydoctor/test/testpackages/numpy/_machar.py b/pydoctor/test/testpackages/numpy/_machar.py deleted file mode 100644 index 50a00f4cb..000000000 --- a/pydoctor/test/testpackages/numpy/_machar.py +++ /dev/null @@ -1,350 +0,0 @@ -""" -Machine arithmetic - determine the parameters of the -floating-point arithmetic system - -Author: Pearu Peterson, September 2003 - -""" -__all__ = ['MachAr'] -__docformat__ = 'numpy' - -class MachAr: - """ - Diagnosing machine parameters. - - Attributes - ---------- - ibeta : int - Radix in which numbers are represented. - it : int - Number of base-`ibeta` digits in the floating point mantissa M. - machep : int - Exponent of the smallest (most negative) power of `ibeta` that, - added to 1.0, gives something different from 1.0 - eps : float - Floating-point number ``beta**machep`` (floating point precision) - negep : int - Exponent of the smallest power of `ibeta` that, subtracted - from 1.0, gives something different from 1.0. - epsneg : float - Floating-point number ``beta**negep``. - iexp : int - Number of bits in the exponent (including its sign and bias). - minexp : int - Smallest (most negative) power of `ibeta` consistent with there - being no leading zeros in the mantissa. - xmin : float - Floating-point number ``beta**minexp`` (the smallest [in - magnitude] positive floating point number with full precision). - maxexp : int - Smallest (positive) power of `ibeta` that causes overflow. - xmax : float - ``(1-epsneg) * beta**maxexp`` (the largest [in magnitude] - usable floating value). - irnd : int - In ``range(6)``, information on what kind of rounding is done - in addition, and on how underflow is handled. - ngrd : int - Number of 'guard digits' used when truncating the product - of two mantissas to fit the representation. - epsilon : float - Same as `eps`. - tiny : float - An alias for `smallest_normal`, kept for backwards compatibility. - huge : float - Same as `xmax`. - precision : float - ``- int(-log10(eps))`` - resolution : float - ``- 10**(-precision)`` - smallest_normal : float - The smallest positive floating point number with 1 as leading bit in - the mantissa following IEEE-754. Same as `xmin`. - smallest_subnormal : float - The smallest positive floating point number with 0 as leading bit in - the mantissa following IEEE-754. - - Parameters - ---------- - float_conv : function, optional - Function that converts an integer or integer array to a float - or float array. Default is `float`. - int_conv : function, optional - Function that converts a float or float array to an integer or - integer array. Default is `int`. - float_to_float : function, optional - Function that converts a float array to float. Default is `float`. - Note that this does not seem to do anything useful in the current - implementation. - float_to_str : function, optional - Function that converts a single float to a string. Default is - ``lambda v:'%24.16e' %v``. - title : str, optional - Title that is printed in the string representation of `MachAr`. - - See Also - -------- - finfo : Machine limits for floating point types. - iinfo : Machine limits for integer types. - - References - ---------- - .. [1] Press, Teukolsky, Vetterling and Flannery, - "Numerical Recipes in C++," 2nd ed, - Cambridge University Press, 2002, p. 31. - - """ - - def __init__(self, float_conv=float, int_conv=int, - float_to_float=float, - float_to_str=lambda v: '%24.16e' % v, - title='Python floating point number'): - """ - - float_conv - convert integer to float (array) - int_conv - convert float (array) to integer - float_to_float - convert float array to float - float_to_str - convert array float to str - title - description of used floating point numbers - - """ - # We ignore all errors here because we are purposely triggering - # underflow to detect the properties of the running arch. - with errstate(under='ignore'): - self._do_init(float_conv, int_conv, float_to_float, float_to_str, title) - - def _do_init(self, float_conv, int_conv, float_to_float, float_to_str, title): - max_iterN = 10000 - msg = "Did not converge after %d tries with %s" - one = float_conv(1) - two = one + one - zero = one - one - - # Do we really need to do this? Aren't they 2 and 2.0? - # Determine ibeta and beta - a = one - for _ in range(max_iterN): - a = a + a - temp = a + one - temp1 = temp - a - if any(temp1 - one != zero): - break - else: - raise RuntimeError(msg % (_, one.dtype)) - b = one - for _ in range(max_iterN): - b = b + b - temp = a + b - itemp = int_conv(temp - a) - if any(itemp != 0): - break - else: - raise RuntimeError(msg % (_, one.dtype)) - ibeta = itemp - beta = float_conv(ibeta) - - # Determine it and irnd - it = -1 - b = one - for _ in range(max_iterN): - it = it + 1 - b = b * beta - temp = b + one - temp1 = temp - b - if any(temp1 - one != zero): - break - else: - raise RuntimeError(msg % (_, one.dtype)) - - betah = beta / two - a = one - for _ in range(max_iterN): - a = a + a - temp = a + one - temp1 = temp - a - if any(temp1 - one != zero): - break - else: - raise RuntimeError(msg % (_, one.dtype)) - temp = a + betah - irnd = 0 - if any(temp - a != zero): - irnd = 1 - tempa = a + beta - temp = tempa + betah - if irnd == 0 and any(temp - tempa != zero): - irnd = 2 - - # Determine negep and epsneg - negep = it + 3 - betain = one / beta - a = one - for i in range(negep): - a = a * betain - b = a - for _ in range(max_iterN): - temp = one - a - if any(temp - one != zero): - break - a = a * beta - negep = negep - 1 - # Prevent infinite loop on PPC with gcc 4.0: - if negep < 0: - raise RuntimeError("could not determine machine tolerance " - "for 'negep', locals() -> %s" % (locals())) - else: - raise RuntimeError(msg % (_, one.dtype)) - negep = -negep - epsneg = a - - # Determine machep and eps - machep = - it - 3 - a = b - - for _ in range(max_iterN): - temp = one + a - if any(temp - one != zero): - break - a = a * beta - machep = machep + 1 - else: - raise RuntimeError(msg % (_, one.dtype)) - eps = a - - # Determine ngrd - ngrd = 0 - temp = one + eps - if irnd == 0 and any(temp * one - one != zero): - ngrd = 1 - - # Determine iexp - i = 0 - k = 1 - z = betain - t = one + eps - nxres = 0 - for _ in range(max_iterN): - y = z - z = y * y - a = z * one # Check here for underflow - temp = z * t - if any(a + a == zero) or any(abs(z) >= y): - break - temp1 = temp * betain - if any(temp1 * beta == z): - break - i = i + 1 - k = k + k - else: - raise RuntimeError(msg % (_, one.dtype)) - if ibeta != 10: - iexp = i + 1 - mx = k + k - else: - iexp = 2 - iz = ibeta - while k >= iz: - iz = iz * ibeta - iexp = iexp + 1 - mx = iz + iz - 1 - - # Determine minexp and xmin - for _ in range(max_iterN): - xmin = y - y = y * betain - a = y * one - temp = y * t - if any((a + a) != zero) and any(abs(y) < xmin): - k = k + 1 - temp1 = temp * betain - if any(temp1 * beta == y) and any(temp != y): - nxres = 3 - xmin = y - break - else: - break - else: - raise RuntimeError(msg % (_, one.dtype)) - minexp = -k - - # Determine maxexp, xmax - if mx <= k + k - 3 and ibeta != 10: - mx = mx + mx - iexp = iexp + 1 - maxexp = mx + minexp - irnd = irnd + nxres - if irnd >= 2: - maxexp = maxexp - 2 - i = maxexp + minexp - if ibeta == 2 and not i: - maxexp = maxexp - 1 - if i > 20: - maxexp = maxexp - 1 - if any(a != y): - maxexp = maxexp - 2 - xmax = one - epsneg - if any(xmax * one != xmax): - xmax = one - beta * epsneg - xmax = xmax / (xmin * beta * beta * beta) - i = maxexp + minexp + 3 - for j in range(i): - if ibeta == 2: - xmax = xmax + xmax - else: - xmax = xmax * beta - - smallest_subnormal = abs(xmin / beta ** (it)) - - self.ibeta = ibeta - self.it = it - self.negep = negep - self.epsneg = float_to_float(epsneg) - self._str_epsneg = float_to_str(epsneg) - self.machep = machep - self.eps = float_to_float(eps) - self._str_eps = float_to_str(eps) - self.ngrd = ngrd - self.iexp = iexp - self.minexp = minexp - self.xmin = float_to_float(xmin) - self._str_xmin = float_to_str(xmin) - self.maxexp = maxexp - self.xmax = float_to_float(xmax) - self._str_xmax = float_to_str(xmax) - self.irnd = irnd - - self.title = title - # Commonly used parameters - self.epsilon = self.eps - self.tiny = self.xmin - self.huge = self.xmax - self.smallest_normal = self.xmin - self._str_smallest_normal = float_to_str(self.xmin) - self.smallest_subnormal = float_to_float(smallest_subnormal) - self._str_smallest_subnormal = float_to_str(smallest_subnormal) - - import math - self.precision = int(-math.log10(float_to_float(self.eps))) - ten = two + two + two + two + two - resolution = ten ** (-self.precision) - self.resolution = float_to_float(resolution) - self._str_resolution = float_to_str(resolution) - - def __str__(self): - fmt = ( - 'Machine parameters for %(title)s\n' - '---------------------------------------------------------------------\n' - 'ibeta=%(ibeta)s it=%(it)s iexp=%(iexp)s ngrd=%(ngrd)s irnd=%(irnd)s\n' - 'machep=%(machep)s eps=%(_str_eps)s (beta**machep == epsilon)\n' - 'negep =%(negep)s epsneg=%(_str_epsneg)s (beta**epsneg)\n' - 'minexp=%(minexp)s xmin=%(_str_xmin)s (beta**minexp == tiny)\n' - 'maxexp=%(maxexp)s xmax=%(_str_xmax)s ((1-epsneg)*beta**maxexp == huge)\n' - 'smallest_normal=%(smallest_normal)s ' - 'smallest_subnormal=%(smallest_subnormal)s\n' - '---------------------------------------------------------------------\n' - ) - return fmt % self.__dict__ - - -if __name__ == '__main__': - print(MachAr()) \ No newline at end of file From 742850a72d6560f8c20a7d3bbacd70d6593e013b Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 14 Feb 2025 16:27:20 -0500 Subject: [PATCH 16/21] fix pyflakes --- pydoctor/epydoc/markup/_types.py | 1 - pydoctor/test/test_type_fields.py | 1 - 2 files changed, 2 deletions(-) diff --git a/pydoctor/epydoc/markup/_types.py b/pydoctor/epydoc/markup/_types.py index eb0091e30..35b8fb2ad 100644 --- a/pydoctor/epydoc/markup/_types.py +++ b/pydoctor/epydoc/markup/_types.py @@ -8,7 +8,6 @@ from typing import Callable, Dict, List, Union, cast from pydoctor.epydoc.markup import ParseError, ParsedDocstring -from pydoctor.epydoc.markup.restructuredtext import parse_docstring from pydoctor.epydoc.markup._pyval_repr import PyvalColorizer from pydoctor.napoleon.docstring import TokenType, TypeDocstring from pydoctor.epydoc.docutils import new_document, set_node_attributes diff --git a/pydoctor/test/test_type_fields.py b/pydoctor/test/test_type_fields.py index dc409e415..71916b825 100644 --- a/pydoctor/test/test_type_fields.py +++ b/pydoctor/test/test_type_fields.py @@ -9,7 +9,6 @@ from pydoctor.test.test_astbuilder import fromText from pydoctor.stanutils import flatten from pydoctor.epydoc.markup._types import ParsedTypeDocstring -from pydoctor.napoleon.docstring import TypeDocstring import pydoctor.epydoc.markup from pydoctor import model From 2681285f6e8d272c98e32bb5f55a6f79bca07552 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Mon, 17 Feb 2025 17:47:15 -0500 Subject: [PATCH 17/21] get_lineno refactor --- pydoctor/epydoc/docutils.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/pydoctor/epydoc/docutils.py b/pydoctor/epydoc/docutils.py index 8d974f539..06791878e 100644 --- a/pydoctor/epydoc/docutils.py +++ b/pydoctor/epydoc/docutils.py @@ -129,28 +129,32 @@ def get_first_parent_lineno(_node: nodes.Element | None) -> int: if _node.line: # This line points to the start of the containing node - # Here we are removing 1 to the result because ParseError class is zero-based - # while docutils line attribute is 1-based. - line:int = _node.line-1 + line: int = _node.line # Let's figure out how many newlines we need to add to this number # to get the right line number. parent_rawsource: Optional[str] = _node.rawsource or None node_rawsource: Optional[str] = node.rawsource or None - if parent_rawsource is not None and \ - node_rawsource is not None: - if node_rawsource in parent_rawsource: - node_index = parent_rawsource.index(node_rawsource) - # Add the required number of newlines to the result - line += parent_rawsource[:node_index].count('\n') + if parent_rawsource and node_rawsource and ( + node_rawsource in parent_rawsource): + # Add the required number of newlines to the result + node_index = parent_rawsource.index(node_rawsource) + line += parent_rawsource[:node_index].count('\n') else: line = get_first_parent_lineno(_node.parent) return line if node.line: + # If the line is explicitely set, assume it's zero-based line = node.line + + # If docutils suddenly starts populating the line attribute for + # title_reference node, all RST xref warnings will off by 1. else: - line = get_first_parent_lineno(node.parent) + # We need to traverse the docutils tree, so the retreived + # line unumber will be one-based, so adjust it + # because ParseError class is zero-based + line = get_first_parent_lineno(node.parent) - 1 return line From dace95fcf95b3186b119a2d7ac5f71f75979c78b Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Mon, 17 Feb 2025 17:47:51 -0500 Subject: [PATCH 18/21] Revert "get_lineno refactor" This reverts commit 2681285f6e8d272c98e32bb5f55a6f79bca07552. --- pydoctor/epydoc/docutils.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/pydoctor/epydoc/docutils.py b/pydoctor/epydoc/docutils.py index 06791878e..8d974f539 100644 --- a/pydoctor/epydoc/docutils.py +++ b/pydoctor/epydoc/docutils.py @@ -129,32 +129,28 @@ def get_first_parent_lineno(_node: nodes.Element | None) -> int: if _node.line: # This line points to the start of the containing node - line: int = _node.line + # Here we are removing 1 to the result because ParseError class is zero-based + # while docutils line attribute is 1-based. + line:int = _node.line-1 # Let's figure out how many newlines we need to add to this number # to get the right line number. parent_rawsource: Optional[str] = _node.rawsource or None node_rawsource: Optional[str] = node.rawsource or None - if parent_rawsource and node_rawsource and ( - node_rawsource in parent_rawsource): - # Add the required number of newlines to the result - node_index = parent_rawsource.index(node_rawsource) - line += parent_rawsource[:node_index].count('\n') + if parent_rawsource is not None and \ + node_rawsource is not None: + if node_rawsource in parent_rawsource: + node_index = parent_rawsource.index(node_rawsource) + # Add the required number of newlines to the result + line += parent_rawsource[:node_index].count('\n') else: line = get_first_parent_lineno(_node.parent) return line if node.line: - # If the line is explicitely set, assume it's zero-based line = node.line - - # If docutils suddenly starts populating the line attribute for - # title_reference node, all RST xref warnings will off by 1. else: - # We need to traverse the docutils tree, so the retreived - # line unumber will be one-based, so adjust it - # because ParseError class is zero-based - line = get_first_parent_lineno(node.parent) - 1 + line = get_first_parent_lineno(node.parent) return line From 40a8623e59bb88769ddfe8bfbb025491ae5ea3b9 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Mon, 17 Feb 2025 17:48:42 -0500 Subject: [PATCH 19/21] Simplification now that the nested warnings are not useful --- pydoctor/epydoc/markup/_types.py | 58 ++++++++++++++------------------ 1 file changed, 26 insertions(+), 32 deletions(-) diff --git a/pydoctor/epydoc/markup/_types.py b/pydoctor/epydoc/markup/_types.py index 35b8fb2ad..1e54a0b5c 100644 --- a/pydoctor/epydoc/markup/_types.py +++ b/pydoctor/epydoc/markup/_types.py @@ -5,7 +5,7 @@ """ from __future__ import annotations -from typing import Callable, Dict, List, Union, cast +from typing import Callable, Dict, List, Union from pydoctor.epydoc.markup import ParseError, ParsedDocstring from pydoctor.epydoc.markup._pyval_repr import PyvalColorizer @@ -18,22 +18,22 @@ class ParsedTypeDocstring(TypeDocstring, ParsedDocstring): """ Add L{ParsedDocstring} interface on top of L{TypeDocstring} and - allow to parse types from L{nodes.Node} objects, providing the C{--process-types} option. + allow to parse types from L{nodes.Node} objects, + providing the C{--process-types} option. """ FIELDS = ('type', 'rtype', 'ytype', 'returntype', 'yieldtype') - # yes this overrides the superclass type! - _tokens: list[tuple[str | nodes.Node, TokenType]] # type: ignore - def __init__(self, annotation: nodes.document, - warns_on_unknown_tokens: bool = False, lineno: int = 0) -> None: + warns_on_unknown_tokens: bool = False, + lineno: int = 0) -> None: ParsedDocstring.__init__(self, ()) TypeDocstring.__init__(self, '', warns_on_unknown_tokens) _tokens = self._tokenize_node_type_spec(annotation) - self._tokens = cast('list[tuple[str | nodes.Node, TokenType]]', - self._build_tokens(_tokens)) + # yes this overrides the superclass type! + self._tokens: list[tuple[str | nodes.Node, TokenType]] \ + = self._build_tokens(_tokens) # type: ignore self._trigger_warnings() self._lineno = lineno @@ -46,10 +46,12 @@ def has_body(self) -> bool: def to_node(self) -> nodes.document: return self._document - def _tokenize_node_type_spec(self, spec: nodes.document) -> List[Union[str, nodes.Node]]: + def _tokenize_node_type_spec(self, spec: nodes.document + ) -> List[Union[str, nodes.Node]]: def _warn_not_supported(n:nodes.Node) -> None: - self.warnings.append(f"Unexpected element in type specification field: element '{n.__class__.__name__}'. " - "This value should only contain text or inline markup.") + self.warnings.append("Unexpected element in type specification field: " + f"element '{n.__class__.__name__}'. This value should " + "only contain text or inline markup.") tokens: List[Union[str, nodes.Node]] = [] # Determine if the content is nested inside a paragraph @@ -72,18 +74,15 @@ def _warn_not_supported(n:nodes.Node) -> None: return tokens - _converters: Dict[TokenType, Callable[[str, list[ParseError], int], nodes.Node]] = { + _converters: Dict[TokenType, Callable[[str, int], nodes.Node]] = { # we're re-using the variable string css # class for the whole literal token, it's the # best approximation we have for now. - TokenType.LITERAL: lambda _token, _, __: \ + TokenType.LITERAL: lambda _token, _: \ nodes.inline(_token, _token, classes=[PyvalColorizer.STRING_TAG]), - - TokenType.CONTROL: lambda _token, _, __: \ + TokenType.CONTROL: lambda _token, _: \ nodes.emphasis(_token, _token), - - - TokenType.OBJ: lambda _token, _, lineno: \ + TokenType.OBJ: lambda _token, lineno: \ set_node_attributes(nodes.title_reference(_token, _token), lineno=lineno), } @@ -94,12 +93,12 @@ def _parse_tokens(self) -> nodes.document: """ document = new_document('code') - warnings: List[ParseError] = [] + converters = self._converters lineno = self._lineno elements: list[nodes.Node] = [] - default = lambda _token, _, __: nodes.Text(_token) + default = lambda _token, _: nodes.Text(_token) for token, type_ in self._tokens: assert token is not None @@ -110,22 +109,17 @@ def _parse_tokens(self) -> nodes.document: converted_token = token else: assert isinstance(token, str) - converted_token = converters.get(type_, default)(token, warnings, lineno) + converted_token = converters.get(type_, default)(token, lineno) - if isinstance(converted_token, nodes.document): - elements.extend((set_node_attributes(t, document=document) - for t in converted_token.children)) - else: - elements.append(set_node_attributes(converted_token, + elements.append(set_node_attributes(converted_token, document=document)) - # warnings should be appended once we have called all converters. - for w in warnings: - self.warnings.append(w.descr()) return set_node_attributes(document, children=[ set_node_attributes(nodes.inline('', '', classes=['literal']), children=elements, - # the +1 here is coping with the fact that - # ParseErrors are 1-based but the doutils - # line we're getting form get_lineno() is zero-based. + document=document, lineno=lineno+1)]) + # the +1 here is coping with the fact that + # Field.lineno are 0-based but the docutils tree + # is supposed to be 1-based + From 32cbcfa58b20b4c257acbdce7437a23df3146db6 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Wed, 19 Feb 2025 14:53:41 -0500 Subject: [PATCH 20/21] add a comment to get_lineno --- pydoctor/epydoc/docutils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pydoctor/epydoc/docutils.py b/pydoctor/epydoc/docutils.py index 8d974f539..a31fec017 100644 --- a/pydoctor/epydoc/docutils.py +++ b/pydoctor/epydoc/docutils.py @@ -148,7 +148,11 @@ def get_first_parent_lineno(_node: nodes.Element | None) -> int: return line if node.line: + # If the line is explicitely set, assume it's zero-based line = node.line + # If docutils suddenly starts populating the line attribute for + # title_reference node, all RST xref warnings will off by 1 :/ + else: line = get_first_parent_lineno(node.parent) From 7eb22b7db31a10a1f4a89f40a6333f21aedfb5f5 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Wed, 19 Feb 2025 18:55:40 -0500 Subject: [PATCH 21/21] Remove unused import --- pydoctor/epydoc/markup/_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydoctor/epydoc/markup/_types.py b/pydoctor/epydoc/markup/_types.py index 1e54a0b5c..eaec8dbde 100644 --- a/pydoctor/epydoc/markup/_types.py +++ b/pydoctor/epydoc/markup/_types.py @@ -7,7 +7,7 @@ from typing import Callable, Dict, List, Union -from pydoctor.epydoc.markup import ParseError, ParsedDocstring +from pydoctor.epydoc.markup import ParsedDocstring from pydoctor.epydoc.markup._pyval_repr import PyvalColorizer from pydoctor.napoleon.docstring import TokenType, TypeDocstring from pydoctor.epydoc.docutils import new_document, set_node_attributes