Skip to content

Commit c82c47d

Browse files
committed
PERF: Improve startup time by 8% with lazy loading of wrapped libraries
*** WORK IN PROGRESS: For now, you have to make sure SlicerApp-real and Slicer launcher are built to ensure the successful generation of the json files *** Startup time reduced from 3.8s to 3.5s with a "cold cache" and from 2.7s to 2.38s with a "warm cache". For each logic/mrml/dm/widgets python modules, a json files listing the associated attributes is generated. Then, when the application is initialized, the "slicer" module is created as a "lazy" module with the attributes associated with logic/mrml/dm/widgets set as "not loaded". Finally, as soon as an attribute not yet loaded is accessed, the specialized __getattribute__ load the associated python module and update the module dictionary. The "lazy" module has been adapted from "itkLazy.py" Results have been gathered on Ubuntu 15.10 on a workstation with the following specs: 64GB / M.2 PCIe NVMe SSD / Quad Core 3.80GHz
1 parent ab4521d commit c82c47d

7 files changed

+179
-24
lines changed

Base/Python/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ set(Slicer_PYTHON_SCRIPTS
77
slicer/testing
88
slicer/util
99
freesurfer
10+
lazy
1011
mrml
1112
saferef
1213
teem

Base/Python/lazy.py

+107
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import imp
2+
import json
3+
import os
4+
import sys
5+
import types
6+
7+
not_loaded = 'not loaded'
8+
9+
def library_loader(module_name):
10+
#print("Loading %s" % module_name)
11+
fp, pathname, description = imp.find_module(module_name)
12+
module = imp.load_module(module_name, fp, pathname, description)
13+
return module
14+
15+
class LazyModule(types.ModuleType):
16+
"""Subclass of ModuleType that implements a custom __getattribute__ method
17+
to allow lazy-loading of attributes from slicer sub-modules."""
18+
19+
def __init__(self, name):
20+
types.ModuleType.__init__(self, name)
21+
self.__lazy_attributes = {}
22+
#print("__lazy_attributes: %s" % len(self.__lazy_attributes))
23+
24+
def _update_lazy_attributes(self, lazy_attributes):
25+
self.__lazy_attributes.update(lazy_attributes)
26+
for k in lazy_attributes:
27+
setattr(self, k, not_loaded)
28+
29+
def __getattribute__(self, attr):
30+
value = types.ModuleType.__getattribute__(self, attr)
31+
#print("__getattribute__ %s" % (attr))
32+
if value is not_loaded:
33+
module_name = self.__lazy_attributes[attr]
34+
35+
module = library_loader(module_name)
36+
namespace = module.__dict__
37+
38+
# Load into 'namespace' first, then self.__dict__ (via setattr) to
39+
# prevent the warnings about overwriting the 'NotLoaded' values
40+
# already in self.__dict__ we would get if we just update
41+
# self.__dict__.
42+
for k, v in namespace.items():
43+
if not k.startswith('_'):
44+
setattr(self, k, v)
45+
value = namespace[attr]
46+
return value
47+
48+
def writeModuleAttributeFile(module_name, config_dir='.'):
49+
try:
50+
exec("import %s as module" % module_name)
51+
except ImportError as details:
52+
print("%s [skipped: failed to import: %s]" % (module_name, details))
53+
return
54+
attributes = []
55+
for attr in dir(module):
56+
if not attr.startswith('__'):
57+
attributes.append(attr)
58+
filename = os.path.join(config_dir, "%s.json" % module_name)
59+
with open(filename, 'w') as output:
60+
print("%s [done: %s]" % (module_name, filename))
61+
output.write(json.dumps({"attributes":attributes}, indent=4))
62+
63+
def updateLazyModule(module, input_module_names=[], config_dir=None):
64+
if isinstance(module, basestring):
65+
if module not in sys.modules:
66+
print("updateLazyModule failed: Couldn't find %s module" % module)
67+
return
68+
module = sys.modules[module]
69+
if not isinstance(module, LazyModule):
70+
print("updateLazyModule failed: module '%s' is not a LazyModule" % module)
71+
return
72+
if isinstance(input_module_names, basestring):
73+
input_module_names = [input_module_names]
74+
if config_dir is None:
75+
config_dir = os.path.dirname(module.__path__[0])
76+
for input_module_name in input_module_names:
77+
filename = os.path.join(config_dir, "%s.json" % input_module_name)
78+
with open(filename) as input:
79+
module_attributes = json.load(input)['attributes']
80+
#print("Updating %s with %d attributes" % (filename, len(module_attributes)))
81+
module._update_lazy_attributes({attribute: input_module_name for attribute in module_attributes})
82+
83+
#print("Updated %s module with %d attributes from %s" % (module, len(module._LazyModule__lazy_attributes), input_module_name))
84+
85+
def createLazyModule(module_name, module_path, input_module_names=[], config_dir=None):
86+
87+
thisModule = sys.modules[module_name] if module_name in sys.modules else None
88+
89+
if isinstance(thisModule, LazyModule):
90+
# Handle reload case where we've already done this once.
91+
# If we made a new module every time, multiple reload()s would fail
92+
# because the identity of sys.modules['itk'] would always be changing.
93+
#print("slicer: Calling ctor of LazyModule")
94+
thisModule.__init__(module_name)
95+
else:
96+
print("slicer: Creating new LazyModule")
97+
thisModule = LazyModule(module_name)
98+
99+
# Set the __path__ attribute, which is required for this module to be used as a
100+
# package
101+
setattr(thisModule, '__path__', module_path)
102+
103+
sys.modules[module_name] = thisModule
104+
105+
updateLazyModule(thisModule, input_module_names, config_dir)
106+
107+
return thisModule

Base/Python/slicer/__init__.py

+11-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
""" This module sets up root logging and loads the Slicer library modules into its namespace."""
22

3+
import lazy
4+
thisModule = lazy.createLazyModule(__name__, __path__)
5+
del lazy
6+
37
#-----------------------------------------------------------------------------
48
def _createModule(name, globals, docstring):
59
import imp
@@ -14,14 +18,14 @@ def _createModule(name, globals, docstring):
1418
#-----------------------------------------------------------------------------
1519
# Create slicer.modules and slicer.moduleNames
1620

17-
_createModule('slicer.modules', globals(),
21+
_createModule('slicer.modules', vars(thisModule),
1822
"""This module provides an access to all instantiated Slicer modules.
1923
2024
The module attributes are the lower-cased Slicer module names, the
2125
associated value is an instance of ``qSlicerAbstractCoreModule``.
2226
""")
2327

24-
_createModule('slicer.moduleNames', globals(),
28+
_createModule('slicer.moduleNames', vars(thisModule),
2529
"""This module provides an access to all instantiated Slicer module names.
2630
2731
The module attributes are the Slicer modules names, the associated
@@ -36,15 +40,19 @@ def _createModule(name, globals, docstring):
3640
except ImportError:
3741
available_kits = []
3842

43+
from .util import importModuleObjects
44+
3945
for kit in available_kits:
4046
try:
41-
exec "from %s import *" % (kit)
47+
importModuleObjects(kit, thisModule)
48+
#exec "from %s import *" % (kit)
4249
except ImportError as detail:
4350
print detail
4451

4552
#-----------------------------------------------------------------------------
4653
# Cleanup: Removing things the user shouldn't have to see.
4754

55+
del thisModule
4856
del _createModule
4957
del available_kits
5058
del kit

Base/Python/slicer/util.py

+21-17
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,11 @@ def sourceDir():
5353
# Custom Import
5454
#
5555

56-
def importVTKClassesFromDirectory(directory, dest_module_name, filematch = '*'):
57-
importClassesFromDirectory(directory, dest_module_name, 'vtkclass', filematch)
56+
def importVTKClassesFromDirectory(directory, dest_module_name, filematch = '*', lazy=False):
57+
importClassesFromDirectory(directory, dest_module_name, 'vtkclass', filematch, lazy)
5858

59-
def importQtClassesFromDirectory(directory, dest_module_name, filematch = '*'):
60-
importClassesFromDirectory(directory, dest_module_name, 'PythonQtClassWrapper', filematch)
59+
def importQtClassesFromDirectory(directory, dest_module_name, filematch = '*', lazy=False):
60+
importClassesFromDirectory(directory, dest_module_name, 'PythonQtClassWrapper', filematch, lazy)
6161

6262
# To avoid globbing multiple times the same directory, successful
6363
# call to ``importClassesFromDirectory()`` will be indicated by
@@ -66,35 +66,39 @@ def importQtClassesFromDirectory(directory, dest_module_name, filematch = '*'):
6666
# Each entry is a tuple of form (directory, dest_module_name, type_name, filematch)
6767
__import_classes_cache = set()
6868

69-
def importClassesFromDirectory(directory, dest_module_name, type_name, filematch = '*'):
69+
def importClassesFromDirectory(directory, dest_module_name, type_name, filematch = '*', lazy=False):
7070

7171
# Create entry for __import_classes_cache
7272
cache_key = ",".join([directory, dest_module_name, type_name, filematch])
7373
# Check if function has already been called with this set of parameters
7474
if cache_key in __import_classes_cache:
7575
return
7676

77-
import glob, os, re, fnmatch
77+
import glob, lazy, os, re, fnmatch
7878
re_filematch = re.compile(fnmatch.translate(filematch))
7979
for fname in glob.glob(os.path.join(directory, filematch)):
8080
if not re_filematch.match(os.path.basename(fname)):
8181
continue
82-
try:
83-
from_module_name = os.path.splitext(os.path.basename(fname))[0]
84-
importModuleObjects(from_module_name, dest_module_name, type_name)
85-
except ImportError as detail:
86-
import sys
87-
print(detail, file=sys.stderr)
82+
from_module_name = os.path.splitext(os.path.basename(fname))[0]
83+
if lazy:
84+
lazy.updateLazyModule(dest_module_name, from_module_name, os.path.dirname(fname))
85+
else:
86+
try:
87+
importModuleObjects(from_module_name, dest_module_name, type_name)
88+
except ImportError as detail:
89+
import sys
90+
print(detail, file=sys.stderr)
8891

8992
__import_classes_cache.add(cache_key)
9093

91-
def importModuleObjects(from_module_name, dest_module_name, type_name):
94+
def importModuleObjects(from_module_name, dest_module, type_name='*'):
9295
"""Import object of type 'type_name' from module identified
93-
by 'from_module_name' into the module identified by 'dest_module_name'."""
96+
by 'from_module_name' into the module identified by 'dest_module'."""
9497

95-
# Obtain a reference to the module identifed by 'dest_module_name'
98+
# Obtain a reference to the module identifed by 'dest_module'
9699
import sys
97-
dest_module = sys.modules[dest_module_name]
100+
if isinstance(dest_module, basestring):
101+
dest_module = sys.modules[dest_module]
98102

99103
# Skip if module has already been loaded
100104
if from_module_name in sys.modules:
@@ -112,7 +116,7 @@ def importModuleObjects(from_module_name, dest_module_name, type_name):
112116
item = getattr(module, item_name)
113117

114118
# Add the object to dest_module_globals_dict if any
115-
if type(item).__name__ == type_name:
119+
if type(item).__name__ == type_name or (type_name == '*' and not item_name.startswith('_')):
116120
setattr(dest_module, item_name, item)
117121

118122
#

Base/QTGUI/qSlicerLoadableModule.cxx

+4-4
Original file line numberDiff line numberDiff line change
@@ -74,13 +74,13 @@ bool qSlicerLoadableModule::importModulePythonExtensions(
7474
ctkScopedCurrentDir scopedCurrentDir(QFileInfo(modulePath).absolutePath());
7575
pythonManager->executeString(QString(
7676
"from slicer.util import importVTKClassesFromDirectory;"
77-
"importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleLogicPython.*');"
78-
"importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleMRMLPython.*');"
79-
"importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleMRMLDisplayableManagerPython.*');"
77+
"importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleLogicPython.*', lazy=True);"
78+
"importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleMRMLPython.*', lazy=True);"
79+
"importVTKClassesFromDirectory('%1', 'slicer', filematch='vtkSlicer*ModuleMRMLDisplayableManagerPython.*', lazy=True);"
8080
).arg(scopedCurrentDir.currentPath()));
8181
pythonManager->executeString(QString(
8282
"from slicer.util import importQtClassesFromDirectory;"
83-
"importQtClassesFromDirectory('%1', 'slicer', filematch='qSlicer*PythonQt.*');"
83+
"importQtClassesFromDirectory('%1', 'slicer', filematch='qSlicer*PythonQt.*', lazy=True);"
8484
).arg(scopedCurrentDir.currentPath()));
8585
return !pythonManager->pythonErrorOccured();
8686
#else

CMake/SlicerMacroBuildModuleQtLibrary.cmake

+16
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,22 @@ macro(SlicerMacroBuildModuleQtLibrary)
240240
if(NOT "${MODULEQTLIBRARY_FOLDER}" STREQUAL "")
241241
set_target_properties(${lib_name}PythonQt PROPERTIES FOLDER ${MODULEQTLIBRARY_FOLDER})
242242
endif()
243+
244+
# XXX Check if Slicer_LAUNCHER_EXECUTABLE available at during a clean build
245+
# XXX Install .json file. Should be taking care of by ctkMacroCompilePythonScript
246+
247+
# Add target to generate module attributes file to allow lazy loading
248+
set(module_name "${lib_name}PythonQt")
249+
set(config_dir "${CMAKE_BINARY_DIR}/${Slicer_QTLOADABLEMODULES_LIB_DIR}/")
250+
set(code "import sys; sys.path.append('${Slicer_SOURCE_DIR}/Base/Python/');")
251+
set(code "${code}import lazy;")
252+
set(code "${code}lazy.writeModuleAttributeFile('${module_name}', config_dir='${config_dir}')")
253+
add_custom_command(TARGET ${module_name} POST_BUILD
254+
COMMAND ${Slicer_LAUNCHER_EXECUTABLE} --no-splash -c "${code}"
255+
COMMENT "Generating ${module_name}.json"
256+
VERBATIM
257+
)
258+
243259
endif()
244260

245261
endmacro()

CMake/SlicerMacroPythonWrapModuleVTKLibrary.cmake

+19
Original file line numberDiff line numberDiff line change
@@ -81,4 +81,23 @@ macro(SlicerMacroPythonWrapModuleVTKLibrary)
8181
KIT_PYTHON_LIBRARIES ${PYTHONWRAPMODULEVTKLIBRARY_Wrapped_LIBRARIES}
8282
)
8383

84+
# XXX Check if Slicer_LAUNCHER_EXECUTABLE available at during a clean build
85+
# XXX Install .json file. Should be taking care of by ctkMacroCompilePythonScript
86+
87+
# Get path to real executable
88+
get_filename_component(python_bin_dir ${PYTHON_EXECUTABLE} PATH)
89+
set(real_python_executable ${python_bin_dir}/python${CMAKE_EXECUTABLE_SUFFIX})
90+
91+
# Add target to generate module attributes file to allow lazy loading
92+
set(module_name "${PYTHONWRAPMODULEVTKLIBRARY_NAME}Python")
93+
set(config_dir "${CMAKE_BINARY_DIR}/${Slicer_QTLOADABLEMODULES_LIB_DIR}/")
94+
set(code "import sys; sys.path.append('${Slicer_SOURCE_DIR}/Base/Python/');")
95+
set(code "${code}import lazy;")
96+
set(code "${code}lazy.writeModuleAttributeFile('${module_name}', config_dir='${config_dir}')")
97+
add_custom_command(TARGET ${module_name} POST_BUILD
98+
COMMAND ${Slicer_LAUNCHER_EXECUTABLE} --launch ${real_python_executable} -c "${code}"
99+
COMMENT "Generating ${module_name}.json"
100+
VERBATIM
101+
)
102+
84103
endmacro()

0 commit comments

Comments
 (0)