Resolve wildcard imports via __all__ or public-names rule (#126)

`from pkg import *` is now desugared at analysis time against the target
package's exports, so names reached via wildcard — including those
re-exported through __init__.py — appear as concrete edges instead of as
spurious *.* residue at the importer's module level.

Export determination:
- Literal __all__ = ["a", "b"] (list or tuple, string elements only) is
  authoritative, including leading-underscore names if listed.
- Absent __all__: public-names rule — every module-scope binding whose
  identifier does not start with underscore.
- Non-literal forms (augmented assignment, dynamic construction) fall back
  to the public-names rule with a debug log.

Implementation:
- New self.module_all dict populated from AST by _extract_dunder_all.
- New _module_public_exports helper with short-name → FQ-name candidate
  resolution, mirroring _record_import.
- visit_ImportFrom desugars wildcard aliases against the export set
  before the per-name binding loop.
- New prescan phase in CallGraphVisitor.process populates scopes and
  __all__ for every input file before pass 1, making wildcard desugaring
  order-independent (consumers of a wildcard import no longer need to
  appear after the exporting package in the filename list).

Version bumped to 2.5.0-dev — this is a new-feature change, not a patch.

Author: Technologicat

CHANGELOG.md

@@ -1,8 +1,14 @@
 # Changelog
 
-## 2.4.4 (in progress)
+## 2.5.0 (in progress)
 
-*No user-visible changes yet.*
+### New features
+
+- **Wildcard imports now resolve to actual targets.** `from pkg import *` is desugared at analysis time against the target package's `__all__` when declared as a literal list/tuple of strings, and against the public-names rule (every module-scope name not starting with `_`) otherwise. Names reached via wildcard — including those re-exported through `__init__.py` — now appear as concrete edges in the call graph instead of as spurious `*.*` residue at the importer's module level. Non-literal `__all__` forms (augmented assignment, dynamic construction) fall back to the public-names rule with a debug log. (#126)
+
+### Internal
+
+- **Prescan phase added before the two visitor passes.** `CallGraphVisitor.process` now does a lightweight scope + `__all__` walk over every input file up front, so cross-module metadata is fully populated before pass 1. This makes wildcard desugaring order-independent — the consumer of a wildcard import no longer has to appear after the exporting package in the filename list.
 
 
 ---

pyan/__init__.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-__version__ = "2.4.4-dev"
+__version__ = "2.5.0-dev"
 
 from .main import create_callgraph, main  # noqa: F401
 from .modvis import create_modulegraph  # noqa: F401

pyan/analyzer.py

@@ -156,6 +156,13 @@ def _init_common(self, logger):
         # Used by expand_unknowns to constrain wildcard expansion.
         self.namespace_imports = {}  # e.g. {"mymod.func": {"os", "foo.bar"}}
 
+        # module name → set of names exposed by `from module import *`.
+        # A value of None means __all__ was present but in a form we don't
+        # parse (augmented, dynamic, etc.) — callers should fall back to the
+        # public-names rule via ``_module_public_exports``.
+        # Populated by visit_Module when a literal __all__ assignment is seen.
+        self.module_all = {}  # e.g. {"pkg": {"fn1", "fn2"}}
+
         # current context for analysis
         self.module_name = None
         self.filename = None
@@ -167,6 +174,12 @@ def _init_common(self, logger):
 
     def process(self):
         """Analyze the set of files, twice so that any forward-references are picked up."""
+        # Prescan: populate self.scopes and self.module_all for every file
+        # before the main visitor passes. This lets cross-module lookups
+        # (wildcard desugaring, chiefly) succeed in pass 1 regardless of
+        # the order in which filenames were given.
+        for filename in self.filenames:
+            self._prescan_one(filename)
         for pas in range(2):
             for filename in self.filenames:
                 self.logger.info(f"========== pass {pas + 1}, file '{filename}' ==========")
@@ -175,6 +188,31 @@ def process(self):
                 self.resolve_base_classes()  # must be done only after all files seen
         self.postprocess()
 
+    def _prescan_one(self, filename):
+        """Populate self.scopes and self.module_all for a single file.
+
+        Reads content, runs symtable-based scope analysis, and extracts any
+        literal ``__all__``. Does **not** run the main AST visitor — this is
+        strictly metadata collection so that cross-module lookups during the
+        subsequent visitor passes find what they need.
+        """
+        if hasattr(self, "_source_texts"):
+            content = self._source_texts[filename]
+            display_name = filename.removesuffix(".__init__") if filename.endswith(".__init__") else filename
+        else:
+            with open(filename, encoding="utf-8") as f:
+                content = f.read()
+            display_name = get_module_name(filename, root=self.root)
+
+        saved_module_name = self.module_name
+        self.module_name = display_name
+        try:
+            self.analyze_scopes(content, filename)
+            tree = ast.parse(content, filename)
+            self._extract_dunder_all(tree.body)
+        finally:
+            self.module_name = saved_module_name
+
     def process_one(self, filename):
         """Analyze one source unit (file path, or module name in source mode).
 
@@ -593,12 +631,73 @@ def visit_Module(self, node):
         module_node = self.get_node("", self.module_name, node, flavor=Flavor.MODULE)
         self.associate_node(module_node, node, filename=self.filename)
 
+        # Extract __all__ so that `from module import *` can desugar against it
+        # (or against the public-names rule when absent). Done at module entry,
+        # before visiting children, so the data is available if this module is
+        # itself visited later during the same pass — see visit_ImportFrom.
+        self._extract_dunder_all(node.body)
+
         with self._module_scope(self.module_name):
             self.generic_visit(node)  # visit the **children** of node
 
         if self.add_defines_edge(module_node, None):
             self.logger.info(f"Def Module {node}")
 
+    def _extract_dunder_all(self, module_body):
+        """Record the current module's ``__all__`` in ``self.module_all``.
+
+        Parses only the simple literal forms::
+
+            __all__ = ["a", "b"]
+            __all__ = ("a", "b")
+            __all__: list[str] = ["a", "b"]   # PEP 526 annotated assignment
+
+        Anything else — augmented assignment (``__all__ += [...]``), calls
+        (``__all__ = _compute()``), or non-string elements — is skipped with
+        a debug log, leaving callers to fall back to the public-names rule.
+
+        Multiple top-level ``__all__`` assignments: the last one wins, matching
+        Python's own binding semantics.
+        """
+        names = None
+        saw_unparseable = False
+        for stmt in module_body:
+            target_value = None
+            if isinstance(stmt, ast.Assign):
+                for tgt in stmt.targets:
+                    if isinstance(tgt, ast.Name) and tgt.id == "__all__":
+                        target_value = stmt.value
+                        break
+            elif isinstance(stmt, ast.AnnAssign):
+                if (isinstance(stmt.target, ast.Name)
+                        and stmt.target.id == "__all__"
+                        and stmt.value is not None):
+                    target_value = stmt.value
+            elif isinstance(stmt, ast.AugAssign):
+                if isinstance(stmt.target, ast.Name) and stmt.target.id == "__all__":
+                    saw_unparseable = True
+                    continue
+
+            if target_value is None:
+                continue
+
+            if isinstance(target_value, (ast.List, ast.Tuple)) and all(
+                isinstance(e, ast.Constant) and isinstance(e.value, str)
+                for e in target_value.elts
+            ):
+                names = {e.value for e in target_value.elts}
+            else:
+                saw_unparseable = True
+                names = None  # discard any prior literal — last assignment wins
+
+        if saw_unparseable and names is None:
+            self.logger.debug(
+                f"__all__ in module '{self.module_name}' uses a form pyan does not parse "
+                f"(augmented/dynamic); falling back to the public-names rule"
+            )
+        if names is not None:
+            self.module_all[self.module_name] = names
+
     def visit_ClassDef(self, node):
         self.logger.debug(f"ClassDef {node.name}, {self.filename}:{node.lineno}")
 
@@ -943,8 +1042,25 @@ def visit_ImportFrom(self, node):
 
         self._record_import(tgt_name)
 
+        # Desugar `from tgt_name import *` against the target module's public
+        # exports (__all__ if literal, otherwise the public-names rule). If the
+        # target wasn't analyzed — or hasn't been visited yet in this pass —
+        # we leave the wildcard alone; pass 2 (or expand_unknowns) will handle
+        # what it can. See _module_public_exports for the lookup rules.
+        names = node.names
+        if len(names) == 1 and names[0].name == "*":
+            exports = self._module_public_exports(tgt_name)
+            if exports is not None:
+                self.logger.debug(
+                    f"Desugaring 'from {tgt_name} import *' to {sorted(exports)}, "
+                    f"{self.filename}:{node.lineno}"
+                )
+                # Synthesize aliases for each exported name. asname=None because
+                # star-import can't rebind (there's no `as` clause on `*`).
+                names = [ast.alias(name=n, asname=None) for n in exports]
+
         # link each import separately
-        for alias in node.names:
+        for alias in names:
             # check if import is module
             if tgt_name + "." + alias.name in self.module_to_filename:
                 to_node = self.get_node("", tgt_name + "." + alias.name, node, flavor=Flavor.MODULE)
@@ -959,6 +1075,46 @@ def visit_ImportFrom(self, node):
             if self.add_uses_edge(from_node, to_node):
                 self.logger.info(f"New edge added for Use from {from_node} to ImportFrom {to_node}")
 
+    def _module_public_exports(self, module_name):
+        """Return the set of names exposed by ``from module_name import *``.
+
+        If the module declares a literal ``__all__`` (see ``_extract_dunder_all``),
+        that set is authoritative — even a leading-underscore name counts if listed.
+        Otherwise the public-names rule applies: every name bound at module scope
+        whose identifier does not start with an underscore.
+
+        *module_name* may be a short name from an import statement
+        (``"common"``) or a fully qualified one (``"pkg.sub.common"``); both are
+        resolved against the analyzer's known module set, mirroring what
+        ``_record_import`` does for ``namespace_imports``.
+
+        Returns ``None`` when no matching analyzed module is found.
+        Callers are expected to fall back to the current wildcard-IMPORTEDITEM
+        behavior in that case.
+        """
+        for candidate in self._module_name_candidates(module_name):
+            if candidate in self.module_all:
+                return self.module_all[candidate]
+            scope = self.scopes.get(candidate)
+            if scope is not None:
+                return {n for n in scope.defs if not n.startswith("_")}
+        return None
+
+    def _module_name_candidates(self, module_name):
+        """Yield fully qualified module names that could match *module_name*.
+
+        First yields *module_name* itself (handles the fully qualified case),
+        then any analyzed module whose FQ name ends in ``"." + module_name``
+        (handles short names from absolute imports where the project root adds
+        extra prefix components, e.g. ``from common import *`` matching
+        ``tests.fixtures.common``).
+        """
+        yield module_name
+        suffix = "." + module_name
+        for fq_name in self.module_to_filename:
+            if fq_name.endswith(suffix) and fq_name != module_name:
+                yield fq_name
+
     def analyze_module_import(self, import_item, ast_node):
         """Analyze a names AST node inside an Import or ImportFrom AST node.
 

tests/test_code/dunder_all/consumer.py

@@ -0,0 +1,11 @@
+from pkg_with_all import *  # noqa: F401, F403
+from pkg_private import *  # noqa: F401, F403
+
+
+def use_with_all():
+    alpha()  # noqa: F405 — from pkg_with_all's __all__
+    _helper()  # noqa: F405 — leading underscore, but listed in __all__
+
+
+def use_private():
+    pub()  # noqa: F405 — non-underscore, brought in by public-names rule

tests/test_code/dunder_all/pkg_private/__init__.py

@@ -0,0 +1,3 @@
+# No __all__ — public-names rule applies: wildcard brings in everything
+# whose name does not start with an underscore.
+from .exports import pub, _priv  # noqa: F401

tests/test_code/dunder_all/pkg_private/exports.py

@@ -0,0 +1,6 @@
+def pub():
+    return "public"
+
+
+def _priv():
+    return "private"

tests/test_code/dunder_all/pkg_with_all/__init__.py

@@ -0,0 +1,5 @@
+# Literal __all__ — only these names are re-exported by `import *`,
+# even though the module binds more.
+from .exports import alpha, beta, gamma, _helper  # noqa: F401
+
+__all__ = ["alpha", "_helper"]

tests/test_code/dunder_all/pkg_with_all/exports.py

@@ -0,0 +1,14 @@
+def alpha():
+    return "alpha"
+
+
+def beta():
+    return "beta"
+
+
+def gamma():
+    return "gamma"
+
+
+def _helper():
+    return "helper"

tests/test_regressions.py

@@ -329,3 +329,69 @@ def test_issue126_wildcard_import_resolves_to_submodule():
     fn_parent = f"{ISSUE126_PREFIX}.test_sample.fn_parent"
     uses = get_in_dict(v.uses_edges, fn_parent)
     get_node(uses, f"{ISSUE126_PREFIX}.common.file3.fn3")
+
+
+def test_issue126_no_spurious_wildcard_edge_at_module_level():
+    """After wildcard desugaring (v2.5), ``from common import *`` should not
+    leave a ``*.*`` residue edge at the importer's module level."""
+    v = _issue126_visitor()
+    mod_uses = get_in_dict(v.uses_edges, f"{ISSUE126_PREFIX}.test_sample")
+    targets = {n.get_name() for n in mod_uses}
+    assert "*.*" not in targets, f"unexpected wildcard residue: {sorted(targets)}"
+
+
+# --- Wildcard imports: __all__ vs. public-names rule ---
+
+DUNDER_ALL_DIR = os.path.join(TESTS_DIR, "test_code/dunder_all")
+DUNDER_ALL_PREFIX = "test_code.dunder_all"
+
+
+def _dunder_all_visitor():
+    from glob import glob as globfunc
+
+    filenames = sorted(globfunc(os.path.join(DUNDER_ALL_DIR, "**/*.py"), recursive=True))
+    return CallGraphVisitor(filenames, root=TESTS_DIR, logger=logging.getLogger())
+
+
+def test_dunder_all_literal_governs_wildcard():
+    """With literal ``__all__ = ["alpha", "_helper"]`` in the package, a
+    downstream ``from pkg_with_all import *; alpha(); _helper()`` should
+    resolve both — ``__all__`` is authoritative and overrides the default
+    underscore-is-private rule."""
+    v = _dunder_all_visitor()
+    use = f"{DUNDER_ALL_PREFIX}.consumer.use_with_all"
+    uses = get_in_dict(v.uses_edges, use)
+    get_node(uses, f"{DUNDER_ALL_PREFIX}.pkg_with_all.exports.alpha")
+    get_node(uses, f"{DUNDER_ALL_PREFIX}.pkg_with_all.exports._helper")
+
+
+def test_dunder_all_literal_excludes_unlisted_names():
+    """Names bound in the module but absent from ``__all__`` should not be
+    reachable through ``import *``. ``beta`` and ``gamma`` are imported into
+    pkg_with_all's namespace but not listed, so consumer's wildcard shouldn't
+    bind them — there should be no edge from use_with_all to beta/gamma."""
+    v = _dunder_all_visitor()
+    use = f"{DUNDER_ALL_PREFIX}.consumer.use_with_all"
+    uses = get_in_dict(v.uses_edges, use)
+    targets = {n.get_name() for n in uses}
+    assert f"{DUNDER_ALL_PREFIX}.pkg_with_all.exports.beta" not in targets
+    assert f"{DUNDER_ALL_PREFIX}.pkg_with_all.exports.gamma" not in targets
+
+
+def test_public_names_rule_without_dunder_all():
+    """When ``__all__`` is absent, wildcard brings in every non-underscore
+    name bound at module scope. ``pub`` should resolve; ``_priv`` should not
+    — it's leading-underscore and not whitelisted."""
+    v = _dunder_all_visitor()
+    use = f"{DUNDER_ALL_PREFIX}.consumer.use_private"
+    uses = get_in_dict(v.uses_edges, use)
+    get_node(uses, f"{DUNDER_ALL_PREFIX}.pkg_private.exports.pub")
+
+
+def test_dunder_all_recorded_only_when_literal():
+    """The extractor should record pkg_with_all's __all__ but leave
+    pkg_private absent (no __all__ statement at all)."""
+    v = _dunder_all_visitor()
+    assert f"{DUNDER_ALL_PREFIX}.pkg_with_all" in v.module_all
+    assert v.module_all[f"{DUNDER_ALL_PREFIX}.pkg_with_all"] == {"alpha", "_helper"}
+    assert f"{DUNDER_ALL_PREFIX}.pkg_private" not in v.module_all