Skip to content

Import is slow #26

@JPHutchins

Description

@JPHutchins

This isn't a huge deal, but I've been on a mission to improve python import time performance lately since it's noticeable (and embarrassing!) in CLI applications.

You can see that the initialization of these Pydantic models is taking quite a bit.

Run each command or script at least once before measuring to give Python a chance to compile.

My terminal records ~300ms just to do this:

python -c "from smp import image_management"

Here's the import times:

python -X importtime -c "from smp import image_management"

import time: self [us] | cumulative | imported package
import time:       194 |        194 | winreg
import time:       271 |        271 |   _io
import time:        36 |         36 |   marshal
import time:       132 |        132 |   nt
import time:       739 |       1176 | _frozen_importlib_external
import time:       454 |        454 |   time
import time:       215 |        669 | zipimport
import time:        45 |         45 |     _codecs
import time:       533 |        578 |   codecs
import time:      1227 |       1227 |   encodings.aliases
import time:      2022 |       3826 | encodings
import time:      1018 |       1018 | encodings.utf_8
import time:       646 |        646 | encodings.cp1252
import time:        93 |         93 | _signal
import time:        67 |         67 |     _abc
import time:       258 |        324 |   abc
import time:       328 |        651 | io
import time:        94 |         94 |       _stat
import time:       139 |        232 |     stat
import time:      1371 |       1371 |     _collections_abc
import time:        94 |         94 |       genericpath
import time:       100 |        100 |       _winapi
import time:       242 |        436 |     ntpath
import time:       505 |       2543 |   os
import time:        66 |         66 |   _sitebuiltins
import time:       473 |        473 |   encodings.utf_8_sig
import time:       437 |        437 |     __future__
import time:       853 |       1290 |   _virtualenv
import time:       402 |        402 |   sitecustomize
import time:      2771 |       7543 | site
import time:       875 |        875 | smp
import time:       608 |        608 |     types
import time:        80 |         80 |       _operator
import time:      1016 |       1095 |     operator
import time:       143 |        143 |         itertools
import time:       538 |        538 |         keyword
import time:      1025 |       1025 |         reprlib
import time:       153 |        153 |         _collections
import time:      1429 |       3287 |       collections
import time:        55 |         55 |       _functools
import time:      1656 |       4997 |     functools
import time:      2361 |       9060 |   enum
import time:       747 |        747 |     collections.abc
import time:       758 |        758 |     copyreg
import time:      1242 |       1242 |     contextlib
import time:        62 |         62 |         _sre
import time:       457 |        457 |           re._constants
import time:       742 |       1199 |         re._parser
import time:       364 |        364 |         re._casefix
import time:       861 |       2485 |       re._compiler
import time:      1240 |       3725 |     re
import time:       666 |        666 |     warnings
import time:        39 |         39 |     _typing
import time:      4490 |      11665 |   typing
import time:       226 |        226 |       pydantic.version
import time:       559 |        785 |     pydantic._migration
import time:       702 |        702 |             _ast
import time:      2315 |       3016 |           ast
import time:        35 |         35 |               _opcode
import time:       795 |        830 |             opcode
import time:      1445 |       2274 |           dis
import time:      1374 |       1374 |             importlib
import time:       149 |       1523 |           importlib.machinery
import time:       437 |        437 |               token
import time:        90 |         90 |               _tokenize
import time:      1585 |       2111 |             tokenize
import time:       865 |       2976 |           linecache
import time:      1491 |       1491 |             _weakrefset
import time:      1825 |       3315 |           weakref
import time:      2848 |      15950 |         inspect
import time:      2827 |       2827 |         _socket
import time:      1697 |      20473 |       typing_extensions
import time:       685 |      21158 |     pydantic.errors
import time:      1219 |      23161 |   pydantic
import time:       567 |        567 |   copy
import time:       261 |        261 |         _datetime
import time:       560 |        821 |       datetime
import time:      1304 |       2124 |     pydantic_core._pydantic_core
import time:       769 |        769 |           numbers
import time:      1660 |       2428 |         _decimal
import time:       537 |       2964 |       decimal
import time:      6550 |       9514 |     pydantic_core.core_schema
import time:      1068 |      12705 |   pydantic_core
import time:       818 |        818 |   pydantic._internal
import time:      2004 |       2004 |       dataclasses
import time:       569 |        569 |       pydantic._internal._internal_dataclass
import time:      1263 |       3834 |     pydantic.aliases
import time:       818 |        818 |     pydantic.config
import time:       301 |        301 |     pydantic.warnings
import time:       793 |       5745 |   pydantic._internal._config
import time:       322 |        322 |         pydantic._internal._typing_extra
import time:       322 |        643 |       pydantic._internal._repr
import time:       949 |       1591 |     pydantic._internal._core_utils
import time:      3427 |       5018 |   pydantic._internal._decorators
import time:      1376 |       1376 |       textwrap
import time:       717 |       2093 |     pydantic._internal._docs_extraction
import time:       839 |       2931 |   pydantic._internal._fields
import time:      1167 |       1167 |   pydantic._internal._forward_ref
import time:       105 |        105 |       _contextvars
import time:       984 |       1088 |     contextvars
import time:      1614 |       1614 |     pydantic._internal._utils
import time:      1256 |       3958 |   pydantic._internal._generics
import time:       634 |        634 |       pydantic.plugin
import time:       494 |       1127 |     pydantic.plugin._schema_validator
import time:       718 |       1845 |   pydantic._internal._mock_val_ser
import time:       249 |        249 |       pydantic.annotated_handlers
import time:        56 |         56 |         math
import time:       349 |        349 |         pydantic._internal._core_metadata
import time:       321 |        321 |         pydantic._internal._schema_generation_shared
import time:      1811 |       2536 |       pydantic.json_schema
import time:       331 |        331 |       pydantic._internal._discriminated_union
import time:       334 |        334 |       pydantic._internal._known_annotated_metadata
import time:      1319 |       4768 |     pydantic._internal._generate_schema
import time:       261 |        261 |     pydantic._internal._signature
import time:       681 |        681 |     pydantic._internal._validate_call
import time:       829 |       6537 |   pydantic._internal._model_construction
import time:       936 |        936 |         traceback
import time:        50 |         50 |           _string
import time:      2254 |       2304 |         string
import time:       983 |        983 |         threading
import time:        36 |         36 |         atexit
import time:      2356 |       6611 |       logging
import time:        65 |         65 |             _struct
import time:       574 |        639 |           struct
import time:      1055 |       1055 |           cbor2._types
import time:       769 |       2461 |         cbor2._decoder
import time:       423 |        423 |         cbor2._encoder
import time:      1290 |       1290 |         _cbor2
import time:       900 |       5073 |       cbor2
import time:      2215 |       2215 |       smp.header
import time:       697 |        697 |       smp.exceptions
import time:      7986 |       7986 |         annotated_types
import time:      1551 |       1551 |           ipaddress
import time:       595 |       2145 |         pydantic._internal._validators
import time:        64 |         64 |             binascii
import time:       741 |        804 |           base64
import time:       347 |        347 |                 posix
import time:       190 |        537 |               posixpath
import time:       623 |       1159 |             fnmatch
import time:       154 |        154 |             errno
import time:      1045 |       1045 |               urllib
import time:      3476 |       4520 |             urllib.parse
import time:      2447 |       8278 |           pathlib
import time:       859 |        859 |             _uuid
import time:       919 |       1777 |           uuid
import time:      5563 |      16421 |         pydantic.types
import time:      3177 |      29727 |       pydantic.fields
import time:      1639 |       1639 |       pydantic._internal._std_types_schema
import time:       318 |        318 |         pydantic._internal._dataclasses
import time:       429 |        746 |       pydantic.dataclasses
import time:       260 |        260 |             _csv
import time:      1597 |       1857 |           csv
import time:      1371 |       1371 |           email
import time:       392 |        392 |               importlib._abc
import time:       274 |        666 |             importlib.util
import time:       225 |        225 |               zlib
import time:       439 |        439 |                 _compression
import time:       952 |        952 |                 _bz2
import time:       600 |       1990 |               bz2
import time:       826 |        826 |                 _lzma
import time:       709 |       1535 |               lzma
import time:      1830 |       5579 |             shutil
import time:       419 |        419 |               zipfile._path.glob
import time:       774 |       1192 |             zipfile._path
import time:      2039 |       9474 |           zipfile
import time:       437 |        437 |               quopri
import time:       654 |        654 |                     _bisect
import time:       558 |       1212 |                   bisect
import time:       103 |        103 |                   _random
import time:       110 |        110 |                   _sha2
import time:      1078 |       2501 |                 random
import time:       825 |        825 |                     select
import time:      1205 |       2030 |                   selectors
import time:      2896 |       4926 |                 socket
import time:        53 |         53 |                       _locale
import time:      1451 |       1504 |                     locale
import time:      1500 |       3003 |                   calendar
import time:       762 |       3765 |                 email._parseaddr
import time:       405 |        405 |                   email.base64mime
import time:       558 |        558 |                   email.quoprimime
import time:      1024 |       1024 |                   email.errors
import time:       859 |        859 |                   email.encoders
import time:       732 |       3576 |                 email.charset
import time:       994 |      15760 |               email.utils
import time:       625 |        625 |                 email.header
import time:       519 |       1143 |               email._policybase
import time:       469 |        469 |               email._encoded_words
import time:       424 |        424 |               email.iterators
import time:      1059 |      19291 |             email.message
import time:       337 |        337 |               importlib.metadata._functools
import time:       998 |       1335 |             importlib.metadata._text
import time:       694 |      21319 |           importlib.metadata._adapters
import time:       564 |        564 |           importlib.metadata._meta
import time:       446 |        446 |           importlib.metadata._collections
import time:       695 |        695 |           importlib.metadata._itertools
import time:       677 |        677 |                 tempfile
import time:       548 |        548 |                 importlib.resources.abc
import time:       480 |        480 |                 importlib.resources._adapters
import time:       962 |       2666 |               importlib.resources._common
import time:       922 |        922 |               importlib.resources._legacy
import time:       627 |       4214 |             importlib.resources
import time:       903 |       5116 |           importlib.abc
import time:      2414 |      43252 |         importlib.metadata
import time:       730 |      43982 |       pydantic.plugin._loader
import time:     36638 |     127326 |     smp.message
import time:      5955 |     133280 |   smp.error
import time:     40081 |     258531 | smp.image_management

Pydantic is looking at ways of improving this, so we should stay tuned and follow any suggestions: pydantic/pydantic#6748

As of writing this, smp is using Pydantic 2.8.2.

If I sprinkle defer_build=True on everything, it seems to do what it says it does:

import time:      3300 |      41647 |     smp.message
import time:      1917 |      43564 |   smp.error
import time:     46152 |     220726 | smp.image_management

Net, it does improve the speed quite a bit, but the reported 46ms for image_management is still pretty wild!

Metadata

Metadata

Assignees

No one assigned

    Labels

    enhancementNew feature or request

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions