Add instantiators parameters

CHANGELOG.rst

@@ -235,12 +235,27 @@ Removed
 v4.53.0 (unreleased)
 --------------------
 
+Added
+^^^^^
+- New keyword-only ``instantiators`` parameter of
+  ``ArgumentParser.instantiate``, ``auto_cli`` and
+  ``FromConfigMixin.from_config``, to use custom instantiators only in that
+  call. Recommended over the global ``add_instantiator`` (`#994
+  <https://github.com/mauvilsa/jsonargparse/pull/994>`__).
+
 Fixed
 ^^^^^
 - Subcommand parsers ignoring the ``allow_abbrev``, ``suggest_on_error`` and
   ``color`` given to the parent parser or to ``auto_cli``, e.g. abbreviations
   still accepted after a subcommand with ``allow_abbrev=False`` (`#992
   <https://github.com/mauvilsa/jsonargparse/pull/992>`__).
+- Custom instantiators with an ``applied_instantiation_links`` parameter
+  missing links for classes nested in the ``init_args`` of another class, and
+  for the partials that ``instantiate`` gives for callables that return a class
+  (`#994 <https://github.com/mauvilsa/jsonargparse/pull/994>`__).
+- Custom instantiators not applied to a class given as component to
+  ``auto_cli``, which was instantiated directly (`#996
+  <https://github.com/mauvilsa/jsonargparse/pull/996>`__).
 
 Deprecated
 ^^^^^^^^^^

DOCUMENTATION.rst

@@ -2332,6 +2332,35 @@ would also accept subclasses of ``MyClass``, and the config would be:
     they are not among the known subclasses shown in the help.
 
 
+.. _custom-instantiators:
+
+Custom instantiators
+--------------------
+
+By default a class is instantiated as ``class_type(*args, **kwargs)``. A
+different way to instantiate, e.g. within some context, can be given with the
+``instantiators`` parameter of :meth:`instantiate <.ArgumentParser.instantiate>`,
+:func:`.auto_cli` and ``from_config``. It is a list of ``(instantiator,
+class_type, subclasses)`` tuples that apply only to that call, including nested
+classes and a callable that returns a class, even if called afterwards. The
+first entry whose ``class_type`` matches is used:
+
+.. code-block:: python
+
+    def instantiator(class_type, *args, **kwargs):
+        with some_context():
+            return class_type(*args, **kwargs)
+
+
+    init = parser.instantiate(cfg, instantiators=[(instantiator, MyModel, True)])
+
+With ``subclasses=True`` the instantiator also applies to subclasses of
+``class_type``. The details of instantiator functions, e.g. getting values
+applied by links, are in :func:`.add_instantiator`, which registers an
+instantiator globally, for all calls. Prefer the ``instantiators`` parameter,
+since a global one affects code of other libraries that use jsonargparse.
+
+
 .. _untrusted-configs:
 
 Untrusted configs

jsonargparse/_cli.py

@@ -5,8 +5,15 @@
 from typing import Any
 
 from ._actions import ActionConfigFile, _ActionPrintConfig, remove_actions
+from ._common import parser_context
 from ._core import ArgumentParser
-from ._instantiation import bind_call, get_call_arguments
+from ._instantiation import (
+    InstantiatorsType,
+    bind_call,
+    get_call_arguments,
+    get_class_instantiator,
+    validate_instantiators,
+)
 from ._namespace import Namespace, dict_to_namespace
 from ._optionals import get_doc_short_description
 from ._signatures import FailUntyped
@@ -34,6 +41,8 @@ def auto_cli(
     return_instance: bool = False,
     fail_untyped: FailUntyped = True,
     parser_class: type[ArgumentParser] = ArgumentParser,
+    *,
+    instantiators: InstantiatorsType | None = None,
     **kwargs,
 ):
     """Simple creation of command line interfaces.
@@ -60,6 +69,7 @@ def auto_cli(
         fail_untyped: Whether to raise an exception for parameters that don't have a type:
             True for the required ones, "all" for all of them, False for none.
         parser_class: The :class:`ArgumentParser` subclass to use.
+        instantiators: Custom instantiators, see :meth:`.ArgumentParser.instantiate`.
         **kwargs: Used to instantiate :class:`.ArgumentParser`.
 
     Returns:
@@ -81,6 +91,8 @@ def auto_cli(
     if unexpected:
         raise ValueError(f"Unexpected components, not class or function: {unexpected}")
 
+    # given by context, since a parser_class could override instantiate without the instantiators parameter
+    scoped = None if instantiators is None else validate_instantiators(instantiators)
     parser = parser_class(**kwargs)
     if isinstance(components, (list, dict)) or not has_parameter(components, "config"):
         parser.add_argument("--config", action=ActionConfigFile, help=config_help)
@@ -90,8 +102,9 @@ def auto_cli(
         if set_defaults is not None:
             parser.set_defaults(set_defaults)
         cfg = parser.parse_args(args)
-        init = parser.instantiate(cfg)
-        return _run_component(components, init, parser)
+        with parser_context(scoped_class_instantiators=scoped):
+            init = parser.instantiate(cfg)
+        return _run_component(components, init, parser, scoped)
 
     elif isinstance(components, list):
         components = {c.__name__: c for c in components}
@@ -101,7 +114,8 @@ def auto_cli(
     if set_defaults is not None:
         parser.set_defaults(set_defaults)
     cfg = parser.parse_args(args)
-    init = parser.instantiate(cfg)
+    with parser_context(scoped_class_instantiators=scoped):
+        init = parser.instantiate(cfg)
     components_ns = dict_to_namespace(components)
     subcommand = init.get("subcommand")
     while isinstance(init.get(subcommand), Namespace) and isinstance(init[subcommand].get("subcommand"), str):
@@ -113,7 +127,7 @@ def auto_cli(
     component = components_ns[subcommand]
     for name in subcommand.split("."):
         parser = parser._subcommands_action._name_parser_map[name]  # type: ignore[union-attr]
-    return _run_component(component, init.get(subcommand), parser)
+    return _run_component(component, init.get(subcommand), parser, scoped)
 
 
 def auto_parser(*args, **kwargs) -> ArgumentParser:
@@ -207,13 +221,19 @@ def _get_call_values(cfg: Namespace) -> dict:
     return dict(cfg.items(branches=True, nested=False))
 
 
-def _run_component(component, cfg, parser):
+def _class_instantiator(component, scoped: tuple | None):
+    with parser_context(scoped_class_instantiators=scoped):
+        return get_class_instantiator(component)
+
+
+def _run_component(component, cfg, parser, scoped: tuple | None):
     cfg.pop("config", None)
     subcommand = cfg.pop("subcommand")
     if inspect.isclass(component) and subcommand:
         subcommand_cfg = cfg.pop(subcommand, {})
         subcommand_cfg.pop("config", None)
-        component_obj = bind_call(component, parser._call_layouts[None], _get_call_values(cfg))()
+        instantiator = _class_instantiator(component, scoped)
+        component_obj = bind_call(instantiator, parser._call_layouts[None], _get_call_values(cfg))()
         if isinstance(getattr(component, subcommand), property):
             return getattr(component_obj, subcommand)
         component = getattr(component_obj, subcommand)
@@ -222,4 +242,6 @@ def _run_component(component, cfg, parser):
     args, kwargs = get_call_arguments(parser._call_layouts[None], _get_call_values(cfg), component)
     if inspect.iscoroutinefunction(component):
         return __import__("asyncio").run(component(*args, **kwargs))
+    if inspect.isclass(component):
+        return _class_instantiator(component, scoped)(*args, **kwargs)
     return component(*args, **kwargs)

jsonargparse/_common.py

@@ -78,6 +78,7 @@ def __bool__(self):
 load_value_mode: ContextVar[str | None] = ContextVar("load_value_mode", default=None)
 nested_links: ContextVar[list[dict]] = ContextVar("nested_links", default=[])
 applied_instantiation_links: ContextVar[set | None] = ContextVar("applied_instantiation_links", default=None)
+scoped_class_instantiators: ContextVar[tuple | None] = ContextVar("scoped_class_instantiators", default=None)
 path_dump_preserve_relative: ContextVar[bool] = ContextVar("path_dump_preserve_relative", default=False)
 command_line_option: ContextVar[str | None] = ContextVar("command_line_option", default=None)
 
@@ -93,6 +94,7 @@ def __bool__(self):
     "load_value_mode": load_value_mode,
     "nested_links": nested_links,
     "applied_instantiation_links": applied_instantiation_links,
+    "scoped_class_instantiators": scoped_class_instantiators,
     "path_dump_preserve_relative": path_dump_preserve_relative,
     "command_line_option": command_line_option,
 }

jsonargparse/_from_config.py

@@ -5,6 +5,7 @@
 from typing import TypeVar
 
 from ._core import ArgumentParser
+from ._instantiation import InstantiatorsType
 from ._optionals import _get_config_read_mode
 from ._paths import Path as ConfigPath
 from ._required import clear_required, iter_required_keys
@@ -47,23 +48,33 @@ def __init_subclass__(cls, **kwargs) -> None:
         _override_init_defaults(cls, cls.__from_config_parser_kwargs__)
 
     @classmethod
-    def from_config(cls: type[T], config: str | PathLike | dict) -> T:
+    def from_config(
+        cls: type[T],
+        config: str | PathLike | dict,
+        *,
+        instantiators: InstantiatorsType | None = None,
+    ) -> T:
         """Instantiate current class based on a config file or dict.
 
         Args:
             config: Path to a config file or a dict with config values.
+            instantiators: Custom instantiators, see :meth:`.ArgumentParser.instantiate`.
         """
         parser = ArgumentParser(exit_on_error=False, **cls.__from_config_parser_kwargs__)  # type: ignore[attr-defined]
         parser.add_argument("--from_config", type=cls, required=True, sub_configs=True)
-        return _parse_and_instantiate(parser, config)
+        return _parse_and_instantiate(parser, config, instantiators)
 
 
-def _parse_and_instantiate(parser: ArgumentParser, config: str | PathLike | dict):
+def _parse_and_instantiate(
+    parser: ArgumentParser,
+    config: str | PathLike | dict,
+    instantiators: InstantiatorsType | None = None,
+):
     """Parses a config as the value of the ``from_config`` key of ``parser``, and instantiates it."""
     if not isinstance(config, dict):
         config = str(ConfigPath(config, mode=_get_config_read_mode()))  # else a subclass type takes it as a class path
     cfg = parser.parse_object({"from_config": config}, defaults=False)
-    return parser.instantiate(cfg).from_config
+    return parser.instantiate(cfg, instantiators=instantiators).from_config
 
 
 def _override_init_defaults(cls: type[T], parser_kwargs: dict) -> None:

jsonargparse/_instantiation.py

@@ -2,11 +2,19 @@
 import functools
 import inspect
 import sys
-from collections.abc import Callable, Mapping
+from collections.abc import Callable, Mapping, Sequence
 from functools import partial
+from itertools import chain
 from typing import Any, Protocol
 
-from ._common import ClassType, applied_instantiation_links, get_parsing_setting, is_subclass, parser_context
+from ._common import (
+    ClassType,
+    applied_instantiation_links,
+    get_parsing_setting,
+    is_subclass,
+    parser_context,
+    scoped_class_instantiators,
+)
 from ._namespace import Namespace, get_value_and_parent, split_key
 
 __all__ = ["add_instantiator"]
@@ -88,6 +96,7 @@ def __call__(self, class_type: type[ClassType], *args, **kwargs) -> ClassType:
 
 
 InstantiatorsDictType = dict[tuple[type, bool], InstantiatorCallable]
+InstantiatorsType = Sequence[tuple[InstantiatorCallable, type, bool]]
 
 _class_instantiators: InstantiatorsDictType = {}
 
@@ -97,6 +106,8 @@ def instantiate(
         self,
         namespace: Namespace,
         instantiate_groups: bool = True,
+        *,
+        instantiators: InstantiatorsType | None = None,
     ) -> Namespace:
         """Instantiates all signature components in a configuration namespace.
 
@@ -134,6 +145,9 @@ def instantiate(
                 by one of the ``parse_*`` methods and not modified in a way that
                 breaks the structure expected by the parser.
             instantiate_groups: Whether class, function and method groups should be instantiated.
+            instantiators: Custom instantiators for this call, a list of ``(instantiator, class_type,
+                subclasses)`` tuples, see :func:`.add_instantiator`. The first match is used, before any
+                global ones.
 
         Returns:
             A new configuration object where every registered signature
@@ -145,6 +159,11 @@ def instantiate(
         from ._subcommands import get_subcommand
         from ._typehints import ActionTypeHint
 
+        if instantiators is None:  # a nested instantiate keeps the ones of the enclosing call
+            scoped = scoped_class_instantiators.get()
+        else:
+            scoped = validate_instantiators(instantiators)
+
         components: list[ActionTypeHint | _ActionConfigLoad | ArgumentGroup] = []
         for action in filter_non_parsing_actions(self._actions):  # type: ignore[attr-defined]
             if isinstance(action, ActionTypeHint):
@@ -179,21 +198,25 @@ def instantiate(
                         with parser_context(
                             parent_parser=self,
                             nested_links=ActionLink.get_nested_links(self, component),
-                            applied_instantiation_links=cfg.get("__applied_instantiation_links__"),
+                            scoped_class_instantiators=scoped,
+                            applied_instantiation_links=get_applied_instantiation_links(cfg),
                         ):
                             parent[key] = component.instantiate_classes(value)
             elif hasattr(component, "instantiate_class"):
                 with parser_context(
                     load_value_mode=self.parser_mode,  # type: ignore[attr-defined]
-                    applied_instantiation_links=cfg.get("__applied_instantiation_links__"),
+                    scoped_class_instantiators=scoped,
+                    applied_instantiation_links=get_applied_instantiation_links(cfg),
                 ):
                     component.instantiate_class(component, cfg)
 
         ActionLink.apply_instantiation_links(self, cfg, order=order)
 
         subcommand, subparser = get_subcommand(self, cfg, fail_no_subcommand=False)  # type: ignore[arg-type]
         if subcommand is not None and subparser is not None:
-            cfg[subcommand] = subparser.instantiate(cfg[subcommand], instantiate_groups=instantiate_groups)
+            # given by context, since a subparser could override instantiate without the instantiators parameter
+            with parser_context(scoped_class_instantiators=scoped):
+                cfg[subcommand] = subparser.instantiate(cfg[subcommand], instantiate_groups=instantiate_groups)
 
         return cfg
 
@@ -204,7 +227,9 @@ def add_instantiator(
     subclasses: bool = True,
     prepend: bool = False,
 ) -> None:
-    """Adds a custom instantiator for a class type. Used by ``ArgumentParser.instantiate``.
+    """Adds a custom instantiator for a class type, globally for all ``ArgumentParser.instantiate`` calls.
+
+    Prefer the ``instantiators`` parameter of ``instantiate``, which is limited to that call.
 
     Instantiator functions are expected to have as signature ``(class_type:
     Type[ClassType], *args, **kwargs) -> ClassType``.
@@ -236,14 +261,53 @@ def add_instantiator(
         _class_instantiators[key] = instantiator
 
 
-def dynamic_class_instantiator(class_type: type[ClassType], *args, **kwargs) -> ClassType:
-    for (cls, subclasses), instantiator in _class_instantiators.items():
+def get_applied_instantiation_links(cfg: Namespace) -> set:
+    """Gets the links applied so far, for a nested ``instantiate`` including the ones of the enclosing call."""
+    return (cfg.get("__applied_instantiation_links__") or set()) | (applied_instantiation_links.get() or set())
+
+
+def validate_instantiators(instantiators: InstantiatorsType) -> tuple:
+    if (
+        not isinstance(instantiators, Sequence)
+        or isinstance(instantiators, str)
+        or not all(
+            isinstance(item, tuple)
+            and len(item) == 3
+            and callable(item[0])
+            and inspect.isclass(item[1])
+            and isinstance(item[2], bool)
+            for item in instantiators
+        )
+    ):
+        raise ValueError(
+            "Expected instantiators to be a list of tuples (instantiator, class_type, subclasses), "
+            f"got {instantiators!r}"
+        )
+    return tuple(instantiators)
+
+
+def get_class_instantiator(class_type: type[ClassType]) -> Callable[..., ClassType]:
+    """Gets the callable that instantiates class_type.
+
+    It is resolved at this point, so that a deferred call, e.g. a partial given by ``instantiate``, uses the same
+    instantiators and applied instantiation links, even though it happens outside of the ``instantiate`` call.
+    """
+    instantiator: Callable[..., ClassType] = class_type
+    scoped = ((cls, subclasses, fn) for fn, cls, subclasses in scoped_class_instantiators.get() or ())
+    registered = ((cls, subclasses, fn) for (cls, subclasses), fn in _class_instantiators.items())
+    for cls, subclasses, fn in chain(scoped, registered):
         if class_type is cls or (subclasses and is_subclass(class_type, cls)):
-            param_names = set(inspect.signature(instantiator).parameters)
-            if "applied_instantiation_links" in param_names:
+            if "applied_instantiation_links" in inspect.signature(fn).parameters:
                 applied_links = applied_instantiation_links.get() or set()
-                kwargs["applied_instantiation_links"] = {
-                    action.target[0]: action.applied_value for action in applied_links
-                }
-            return instantiator(class_type, *args, **kwargs)
-    return class_type(*args, **kwargs)
+                links = {action.target[0]: action.applied_value for action in applied_links}
+                instantiator = partial(fn, class_type, applied_instantiation_links=links)
+            else:
+                instantiator = partial(fn, class_type)
+            break
+    return partial(call_without_instantiate_context, instantiator)
+
+
+def call_without_instantiate_context(func: Callable[..., ClassType], /, *args, **kwargs) -> ClassType:
+    """Calls func without the context of the enclosing ``instantiate``, so that it doesn't affect other parsers."""
+    with parser_context(scoped_class_instantiators=None, applied_instantiation_links=None):
+        return func(*args, **kwargs)