Better static analysis

.github/workflows/static.yaml

@@ -16,7 +16,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.8'
+        python-version: '3.11.0'
 
     - name: Install tox
       run: |

pydoctor/astutils.py

@@ -6,7 +6,7 @@
 import platform
 import sys
 from numbers import Number
-from typing import Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Union
+from typing import Any, Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Union
 from inspect import BoundArguments, Signature
 import ast
 
@@ -59,7 +59,16 @@ def get_children(cls, node: ast.AST) -> Iterable[ast.AST]:
 class NodeVisitorExt(visitor.VisitorExt[ast.AST]):
     ...
 
-_AssingT = Union[ast.Assign, ast.AnnAssign]
+_AssingT = Union[ast.Assign, ast.AnnAssign, ast.AugAssign]
+
+def iterassignfull(node:_AssingT) -> Iterator[Tuple[Optional[List[str]], ast.expr]]:
+    """
+    Utility function to iterate assignments targets. 
+    Like L{iterassign} but returns the C{ast.expr} of the target as well.
+    """
+    for target in node.targets if isinstance(node, ast.Assign) else [node.target]:
+        yield node2dottedname(target) , target
+
 def iterassign(node:_AssingT) -> Iterator[Optional[List[str]]]:
     """
     Utility function to iterate assignments targets. 
@@ -80,10 +89,10 @@ def iterassign(node:_AssingT) -> Iterator[Optional[List[str]]]:
     >>> from ast import parse
     >>> node = parse('self.var = target = thing[0] = node.astext()').body[0]
     >>> list(iterassign(node))
+    [['self.var'], ['target'], None]
 
     """
-    for target in node.targets if isinstance(node, ast.Assign) else [node.target]:
-        dottedname = node2dottedname(target) 
+    for dottedname, _ in iterassignfull(node):
         yield dottedname
 
 def node2dottedname(node: Optional[ast.AST]) -> Optional[List[str]]:
@@ -107,6 +116,7 @@ def node2fullname(expr: Optional[ast.AST], ctx: 'model.Documentable') -> Optiona
         return None
     return ctx.expandName('.'.join(dottedname))
 
+
 def bind_args(sig: Signature, call: ast.Call) -> BoundArguments:
     """
     Binds the arguments of a function call to that function's signature.
@@ -402,4 +412,35 @@ def extract_docstring(node: ast.Str) -> Tuple[int, str]:
         - The docstring to be parsed, cleaned by L{inspect.cleandoc}.
     """
     lineno = extract_docstring_linenum(node)
-    return lineno, inspect.cleandoc(node.s)
+    return lineno, inspect.cleandoc(node.s)
+
+# The following code handles attaching extra meta information to AST statements.
+
+_EXTRA_FIELD = '_pydoctor'
+
+def setfield(node:ast.AST, key:str, value:Any) -> None:
+    """
+    Set an extra field on this node.
+    """
+    fields = getattr(node, _EXTRA_FIELD, {})
+    setattr(node, _EXTRA_FIELD, fields)
+    if hasfield(node, key):
+        raise ValueError(f'Node {node!r} already has field {key!r}')
+    fields[key] = value
+
+def hasfield(node:ast.AST, key:str) -> bool:
+    fields = getattr(node, _EXTRA_FIELD, {})
+    return key in fields
+
+def getfield(node:ast.AST, key:str, default:Any=None) -> Any:
+    """
+    Get an extra field from this node.
+    """
+    fields = getattr(node, _EXTRA_FIELD, {})
+    if key not in fields and default is None:
+        raise KeyError(f'Node {node!r} has no field {key!r}')
+    return fields.get(key, default)
+
+def delfield(node:ast.AST, key:str) -> None:
+    fields = getattr(node, _EXTRA_FIELD, {})
+    del fields[key]

pydoctor/model.py

@@ -26,13 +26,14 @@
 from urllib.parse import quote
 
 from pydoctor.options import Options
-from pydoctor import factory, qnmatch, utils, linker, astutils, mro
+from pydoctor import factory, qnmatch, utils, linker, astutils, mro, symbols
 from pydoctor.epydoc.markup import ParsedDocstring
 from pydoctor.sphinx import CacheT, SphinxInventory
 
 if TYPE_CHECKING:
     from typing_extensions import Literal
     from pydoctor.astbuilder import ASTBuilder, DocumentableT
+    from pydoctor.symbols import Scope
 else:
     Literal = {True: bool, False: bool}
     ASTBuilder = object
@@ -52,12 +53,6 @@
 #   Functions can't contain anything.
 
 
-_string_lineno_is_end = sys.version_info < (3,8) \
-                    and platform.python_implementation() != 'PyPy'
-"""True iff the 'lineno' attribute of an AST string node points to the last
-line in the string, rather than the first line.
-"""
-
 
 class DocLocation(Enum):
     OWN_PAGE = 1
@@ -157,6 +152,10 @@ def doctarget(self) -> 'Documentable':
     def setup(self) -> None:
         self.contents: Dict[str, Documentable] = {}
         self._linker: Optional['linker.DocstringLinker'] = None
+        self._stmt: Optional[symbols.Statement] = None
+        """
+        If this documentable represents a L{Scope}, then it's stored here by the builder.
+        """
 
     def setDocstring(self, node: ast.Str) -> None:
         lineno, doc = astutils.extract_docstring(node)
@@ -222,11 +221,7 @@ def url(self) -> str:
             return f'{page_url}#{quote(self.name)}'
 
     def fullName(self) -> str:
-        parent = self.parent
-        if parent is None:
-            return self.name
-        else:
-            return f'{parent.fullName()}.{self.name}'
+        return symbols.HasNameAndParent.fullName(self)
 
     def __repr__(self) -> str:
         return f"{self.__class__.__name__} {self.fullName()!r}"
@@ -271,6 +266,25 @@ def _handle_reparenting_post(self) -> None:
     def _localNameToFullName(self, name: str) -> str:
         raise NotImplementedError(self._localNameToFullName)
 
+    def _betterLocalNameToFullName(self, name:str) ->str:
+        """
+        Wraps L{_localNameToFullName} and fallback to L{symbols.localNameToFullName} if 
+        the name is not found. 
+        """
+        try:
+            full_name = self._localNameToFullName(name)
+        except LookupError:
+            # Fallback to symbols.localNameToFullName() if possible.
+            stmt = self._stmt
+            if stmt is not None:
+                try:
+                    full_name = symbols.localNameToFullName(stmt.scope, stmt, name)
+                except LookupError:
+                    full_name = name
+            else:
+                full_name = name
+        return full_name
+
     def expandName(self, name: str) -> str:
         """Return a fully qualified name for the possibly-dotted `name`.
 
@@ -295,7 +309,7 @@ class E:
         parts = name.split('.')
         obj: Documentable = self
         for i, p in enumerate(parts):
-            full_name = obj._localNameToFullName(p)
+            full_name = obj._betterLocalNameToFullName(p)
             if full_name == p and i != 0:
                 # The local name was not found.
                 # If we're looking at a class, we try our luck with the inherited members
@@ -305,8 +319,6 @@ class E:
                         full_name = inherited.fullName()
                 if full_name == p:
                     # We don't have a full name
-                    # TODO: Instead of returning the input, _localNameToFullName()
-                    #       should probably either return None or raise LookupError.
                     full_name = f'{obj.fullName()}.{p}'
                     break
             nxt = self.system.objForFullName(full_name)
@@ -423,6 +435,12 @@ def setup(self) -> None:
         """The live module if the module was built from introspection."""
         self._py_string: Optional[str] = None
         """The module string if the module was built from text."""
+
+        self.node: Optional[ast.Module] = None
+        """
+        The C{ast.Module} counterpart of this L{Module}. 
+        It's None when the module was built from introspection.
+        """
 
         self.all: Optional[Collection[str]] = None
         """Names listed in the C{__all__} variable of this module.
@@ -444,7 +462,7 @@ def _localNameToFullName(self, name: str) -> str:
         elif name in self._localNameToFullName_map:
             return self._localNameToFullName_map[name]
         else:
-            return name
+            raise LookupError()
 
     @property
     def module(self) -> 'Module':
@@ -832,8 +850,6 @@ def __init__(self, options: Optional['Options'] = None):
         self.needsnl = False
         self.once_msgs: Set[Tuple[str, str]] = set()
 
-        # We're using the id() of the modules as key, and not the fullName becaue modules can
-        # be reparented, generating KeyError.
         self.unprocessed_modules: List[_ModuleT] = []
 
         self.module_count = 0
@@ -863,6 +879,8 @@ def __init__(self, options: Optional['Options'] = None):
         for ext in self.extensions + self.custom_extensions:
             # Load extensions
             extensions.load_extension_module(self, ext)
+
+        self._ast_parser = self.defaultBuilder.ASTParser()
 
     @property
     def Class(self) -> Type['Class']:
@@ -1086,17 +1104,34 @@ def analyzeModule(self,
             parentPackage: Optional[_PackageT] = None,
             is_package: bool = False
             ) -> _ModuleT:
+        """
+        Create a new module to be analyze.
+        """
         factory = self.Package if is_package else self.Module
         mod = factory(self, modname, parentPackage, modpath)
         self._addUnprocessedModule(mod)
         self.setSourceHref(mod, modpath)
         return mod
 
+    def _parseModuleAST(self, mod: _ModuleT) -> None:
+        """
+        Set the L{Module.node} attribute.
+        """
+        # Parse the ast of the module early. 
+        # We parse AST only if the module has NOT been imported.
+        # the introspection happens at the time the module get processed.
+        if mod._py_mod is None:
+            if mod._py_string is not None:
+                mod.node = self._ast_parser.parseString(mod._py_string, mod)
+            else:
+                assert mod.source_path is not None
+                mod.node = self._ast_parser.parseFile(mod.source_path, mod)
+
     def _addUnprocessedModule(self, mod: _ModuleT) -> None:
         """
         First add the new module into the unprocessed_modules list. 
         Handle eventual duplication of module names, and finally add the 
-        module to the system.
+        module to the system and create the AST representation for the module.
         """
         assert mod.state is ProcessingState.UNPROCESSED
         first = self.allobjects.get(mod.fullName())
@@ -1105,10 +1140,14 @@ def _addUnprocessedModule(self, mod: _ModuleT) -> None:
             assert isinstance(first, Module)
             self._handleDuplicateModule(first, mod)
         else:
+            # parse ast early
+            self._parseModuleAST(mod)
+
+            # add the unprocessed mod
             self.unprocessed_modules.append(mod)
             self.addObject(mod)
             self.progress(
-                "analyzeModule", len(self.allobjects),
+                "_addUnprocessedModule", len(self.allobjects),
                 None, "modules and packages discovered")        
             self.module_count += 1
 
@@ -1185,7 +1224,7 @@ def introspectModule(self,
         module = factory(self, module_name, package, path)
 
         module.docstring = py_mod.__doc__
-        module._is_c_module = True
+        module._is_c_module = True # we assume an introspected module is a C module.
         module._py_mod = py_mod
 
         self._addUnprocessedModule(module)
@@ -1266,13 +1305,17 @@ def getProcessedModule(self, modname: str) -> Optional[_ModuleT]:
         return mod
 
     def processModule(self, mod: _ModuleT) -> None:
+        """
+        Triggers the *analysis* of the AST (or the introspection in the case of a C-extension)
+        """
         assert mod.state is ProcessingState.UNPROCESSED
         assert mod in self.unprocessed_modules
         mod.state = ProcessingState.PROCESSING
         self.unprocessed_modules.remove(mod)
         if mod.source_path is None:
             assert mod._py_string is not None
-        if mod._is_c_module:
+
+        if mod._py_mod is not None:
             self.processing_modules.append(mod.fullName())
             self.msg("processModule", "processing %s"%(self.processing_modules), 1)
             self._introspectThing(mod._py_mod, mod, mod)
@@ -1281,16 +1324,14 @@ def processModule(self, mod: _ModuleT) -> None:
             assert head == mod.fullName()
         else:
             builder = self.defaultBuilder(self)
-            if mod._py_string is not None:
-                ast = builder.parseString(mod._py_string, mod)
-            else:
-                assert mod.source_path is not None
-                ast = builder.parseFile(mod.source_path, mod)
-            if ast:
+            ast_mod = mod.node
+            if ast_mod:
                 self.processing_modules.append(mod.fullName())
                 if mod._py_string is None:
+                    # _py_string is only used in tests, so we use special loggin messages
+                    # just when _py_string is used.
                     self.msg("processModule", "processing %s"%(self.processing_modules), 1)
-                builder.processModuleAST(ast, mod)
+                builder.processModuleAST(ast_mod, mod)
                 mod.state = ProcessingState.PROCESSED
                 head = self.processing_modules.pop()
                 assert head == mod.fullName()