Improve performance by caching find_spec

Certain checkers upstream on pylint like import-error heavily use
find_spec. This method is IO intensive as it looks for files
across several search paths to return a ModuleSpec.

Since imports across files may repeat themselves it makes sense to cache
this method in order to speed up the linting process.

Local testing shows that caching reduces the total amount of calls to
find_module methods (used by find_spec) by about 50%. Linting the test
repository in the related issue goes from 40 seconds to 37 seconds. This
was on a NVME disk and after warmup, so timing gains may be bigger on
slower file systems like the one mentioned in the referenced issue.

Closes pylint-dev/pylint#9310.

Author: crazybolillo

astroid/interpreter/_import/spec.py

@@ -24,6 +24,8 @@
 
 from . import util
 
+modpath_cache = {}
+
 
 # The MetaPathFinder protocol comes from typeshed, which says:
 # Intentionally omits one deprecated and one optional method of `importlib.abc.MetaPathFinder`
@@ -423,6 +425,18 @@ def _find_spec_with_path(
     raise ImportError(f"No module named {'.'.join(module_parts)}")
 
 
+def cache_modpath(func):
+    def wrapper(*args):
+        key = ".".join(args[0])
+        if key not in modpath_cache:
+            modpath_cache[key] = func(*args)
+
+        return modpath_cache[key]
+
+    return wrapper
+
+
+@cache_modpath
 def find_spec(modpath: list[str], path: Sequence[str] | None = None) -> ModuleSpec:
     """Find a spec for the given module.
 

tests/test_manager.py

@@ -41,6 +41,7 @@ class AstroidManagerTest(
 ):
     def setUp(self) -> None:
         super().setUp()
+        astroid.interpreter._import.spec.modpath_cache.clear()
         self.manager = test_utils.brainless_manager()
 
     def test_ast_from_file(self) -> None:

tests/test_modutils.py

@@ -41,6 +41,7 @@ class ModuleFileTest(unittest.TestCase):
     package = "mypypa"
 
     def tearDown(self) -> None:
+        astroid.interpreter._import.spec.modpath_cache.clear()
         for k in list(sys.path_importer_cache):
             if "MyPyPa" in k:
                 del sys.path_importer_cache[k]