From 5c1d992b1605a1115683cae3788c6fcfdb0c1d9f Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Fri, 22 Aug 2025 17:24:42 +0100 Subject: [PATCH 01/12] add forward_type attribute to Field --- src/ducktools/classbuilder/__init__.py | 8 +++++++- tests/test_core.py | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/ducktools/classbuilder/__init__.py b/src/ducktools/classbuilder/__init__.py index 049e2aa..edf3326 100644 --- a/src/ducktools/classbuilder/__init__.py +++ b/src/ducktools/classbuilder/__init__.py @@ -721,6 +721,7 @@ class Field(metaclass=SlotMakerMeta): :param repr: Include in the class __repr__. :param compare: Include in the class __eq__. :param kw_only: Make this a keyword only parameter in __init__. + :param forward_type: type as a ForwardRef for Python 3.14+ """ # Plain slots are required as part of bootstrapping @@ -734,6 +735,7 @@ class Field(metaclass=SlotMakerMeta): "repr", "compare", "kw_only", + "forward_type" ) # noinspection PyShadowingBuiltins @@ -748,6 +750,7 @@ def __init__( repr=True, compare=True, kw_only=False, + forward_type=None, # forward_type can be None as None is not a ForwardRef ): # The init function for 'Field' cannot be generated # as 'Field' needs to exist first. @@ -763,6 +766,7 @@ def __init__( self.repr = repr self.compare = compare self.kw_only = kw_only + self.forward_type = forward_type self.validate_field() @@ -818,6 +822,7 @@ def _build_field(): "repr": "Include this attribute in the class __repr__", "compare": "Include this attribute in the class __eq__ method", "kw_only": "Make this a keyword only parameter in __init__", + "forward_type": "A ForwardRef version of the type for Python 3.14" } fields = { @@ -828,7 +833,8 @@ def _build_field(): "init": Field(default=True, doc=field_docs["init"]), "repr": Field(default=True, doc=field_docs["repr"]), "compare": Field(default=True, doc=field_docs["compare"]), - "kw_only": Field(default=False, doc=field_docs["kw_only"]) + "kw_only": Field(default=False, doc=field_docs["kw_only"]), + "forward_type": Field(default=None, doc=field_docs["forward_type"]), } modifications = {"__slots__": field_docs} diff --git a/tests/test_core.py b/tests/test_core.py index 9e425b8..e440b79 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -127,7 +127,7 @@ def test_repr_field(): f4 = Field(default=True, type=bool) f5 = Field(default=True, doc="True or False") - repr_ending = "init=True, repr=True, compare=True, kw_only=False" + repr_ending = "init=True, repr=True, compare=True, kw_only=False, forward_type=None" nothing_repr = repr(NOTHING) @@ -505,7 +505,7 @@ class Ex: "GatheredFields(" "fields={'x': Field(" "default=1, default_factory=, type=, doc=None, " - "init=True, repr=True, compare=True, kw_only=False" + "init=True, repr=True, compare=True, kw_only=False, forward_type=None" ")}, " "modifications={'x': }" ")" From c5f57872f3685a612d549476279f89773c310b0c Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Fri, 22 Aug 2025 17:26:14 +0100 Subject: [PATCH 02/12] remove forward_type from the repr --- src/ducktools/classbuilder/__init__.py | 2 +- tests/test_core.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ducktools/classbuilder/__init__.py b/src/ducktools/classbuilder/__init__.py index edf3326..57b92e0 100644 --- a/src/ducktools/classbuilder/__init__.py +++ b/src/ducktools/classbuilder/__init__.py @@ -834,7 +834,7 @@ def _build_field(): "repr": Field(default=True, doc=field_docs["repr"]), "compare": Field(default=True, doc=field_docs["compare"]), "kw_only": Field(default=False, doc=field_docs["kw_only"]), - "forward_type": Field(default=None, doc=field_docs["forward_type"]), + "forward_type": Field(default=None, doc=field_docs["forward_type"], repr=False), } modifications = {"__slots__": field_docs} diff --git a/tests/test_core.py b/tests/test_core.py index e440b79..9e425b8 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -127,7 +127,7 @@ def test_repr_field(): f4 = Field(default=True, type=bool) f5 = Field(default=True, doc="True or False") - repr_ending = "init=True, repr=True, compare=True, kw_only=False, forward_type=None" + repr_ending = "init=True, repr=True, compare=True, kw_only=False" nothing_repr = repr(NOTHING) @@ -505,7 +505,7 @@ class Ex: "GatheredFields(" "fields={'x': Field(" "default=1, default_factory=, type=, doc=None, " - "init=True, repr=True, compare=True, kw_only=False, forward_type=None" + "init=True, repr=True, compare=True, kw_only=False" ")}, " "modifications={'x': }" ")" From 939e927e3d05b1ddcec361529778360d4da25d2e Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Fri, 22 Aug 2025 21:39:50 +0100 Subject: [PATCH 03/12] Logic changes for annotations Under 3.14 annotations all .type values for Fields are ForwardRefs --- src/ducktools/classbuilder/__init__.py | 107 +++++------ src/ducktools/classbuilder/__init__.pyi | 13 +- src/ducktools/classbuilder/annotations.py | 147 -------------- src/ducktools/classbuilder/annotations.pyi | 32 +++- .../classbuilder/annotations/__init__.py | 74 ++++++++ .../annotations/annotations_314.py | 179 ++++++++++++++++++ .../annotations/annotations_pre_314.py | 58 ++++++ src/ducktools/classbuilder/prefab.py | 16 +- src/ducktools/classbuilder/prefab.pyi | 1 - tests/annotations/test_annotated.py | 1 + tests/annotations/test_annotations_module.py | 6 +- .../test_forwardref_annotations.py | 36 +++- 12 files changed, 450 insertions(+), 220 deletions(-) delete mode 100644 src/ducktools/classbuilder/annotations.py create mode 100644 src/ducktools/classbuilder/annotations/__init__.py create mode 100644 src/ducktools/classbuilder/annotations/annotations_314.py create mode 100644 src/ducktools/classbuilder/annotations/annotations_pre_314.py diff --git a/src/ducktools/classbuilder/__init__.py b/src/ducktools/classbuilder/__init__.py index 57b92e0..6c019f1 100644 --- a/src/ducktools/classbuilder/__init__.py +++ b/src/ducktools/classbuilder/__init__.py @@ -33,7 +33,19 @@ import os import sys -from .annotations import get_ns_annotations, is_classvar, make_annotate_func +try: + # Use the internal C module if it is available + from _types import ( # type: ignore + MemberDescriptorType as _MemberDescriptorType, + MappingProxyType as _MappingProxyType + ) +except ImportError: + from types import ( + MemberDescriptorType as _MemberDescriptorType, + MappingProxyType as _MappingProxyType, + ) + +from .annotations import get_ns_annotations, is_classvar, make_annotate_func, evaluate_forwardref from ._version import __version__, __version_tuple__ # noqa: F401 # Change this name if you make heavy modifications @@ -45,13 +57,6 @@ # overwritten. When running this is a performance penalty so it is not required. _UNDER_TESTING = os.environ.get("PYTEST_VERSION") is not None -# Obtain types the same way types.py does in pypy -# See: https://github.com/pypy/pypy/blob/19d9fa6be11165116dd0839b9144d969ab426ae7/lib-python/3/types.py#L61-L73 -class _C: __slots__ = 's' # noqa -_MemberDescriptorType = type(_C.s) # type: ignore -_MappingProxyType = type(type.__dict__) -del _C - def get_fields(cls, *, local=False): """ @@ -132,22 +137,18 @@ class GeneratedCode: This class provides a return value for the generated output from source code generators. """ - __slots__ = ("source_code", "globs", "annotations", "extra_annotation_func") + __slots__ = ("source_code", "globs", "annotations") - def __init__(self, source_code, globs, annotations=None, extra_annotation_func=None): + def __init__(self, source_code, globs, annotations=None): self.source_code = source_code self.globs = globs self.annotations = annotations - # extra annotation function to evaluate if needed, required for post_init - self.extra_annotation_func = extra_annotation_func - def __repr__(self): first_source_line = self.source_code.split("\n")[0] return ( f"GeneratorOutput(source_code='{first_source_line} ...', " - f"globs={self.globs!r}, annotations={self.annotations!r}, " - f"extra_annotation_func={self.extra_annotation_func!r})" + f"globs={self.globs!r}, annotations={self.annotations!r})" ) def __eq__(self, other): @@ -156,12 +157,10 @@ def __eq__(self, other): self.source_code, self.globs, self.annotations, - self.extra_annotation_func ) == ( other.source_code, other.globs, other.annotations, - other.extra_annotation_func ) return NotImplemented @@ -227,11 +226,7 @@ def __get__(self, inst, cls): if "__annotations__" in gen_cls.__dict__: method.__annotations__ = gen.annotations else: - anno_func = make_annotate_func( - gen_cls, - gen.annotations, - gen.extra_annotation_func, - ) + anno_func = make_annotate_func(gen.annotations) anno_func.__qualname__ = f"{gen_cls.__qualname__}.{self.funcname}.__annotate__" method.__annotate__ = anno_func else: @@ -532,6 +527,10 @@ def builder(cls=None, /, *, gatherer, methods, flags=None, fix_signature=True): """ The main builder for class generation + If the GATHERED_DATA attribute exists on the class it will be used instead of + the provided gatherer and 3.14 annotations will be updated with links to + the class. + :param cls: Class to be analysed and have methods generated :param gatherer: Function to gather field information :type gatherer: Callable[[type], tuple[dict[str, Field], dict[str, Any]]] @@ -557,7 +556,18 @@ def builder(cls=None, /, *, gatherer, methods, flags=None, fix_signature=True): internals = {} setattr(cls, INTERNALS_DICT, internals) - cls_fields, modifications = gatherer(cls) + cls_gathered = cls.__dict__.get(GATHERED_DATA) + + if cls_gathered: + cls_fields, modifications = cls_gathered + # Reconnect the forwardrefs in types to the class so they can evaluate. + if sys.version_info >= (3, 14): + annos = annotations.get_ns_annotations(cls.__dict__, cls=cls) + for k, v in cls_fields.items(): + if annotations.is_forwardref(v.type): + cls_fields[k] = type(v).from_field(v, type=annos[k]) + else: + cls_fields, modifications = gatherer(cls) for name, value in modifications.items(): if value is NOTHING: @@ -721,7 +731,6 @@ class Field(metaclass=SlotMakerMeta): :param repr: Include in the class __repr__. :param compare: Include in the class __eq__. :param kw_only: Make this a keyword only parameter in __init__. - :param forward_type: type as a ForwardRef for Python 3.14+ """ # Plain slots are required as part of bootstrapping @@ -735,7 +744,6 @@ class Field(metaclass=SlotMakerMeta): "repr", "compare", "kw_only", - "forward_type" ) # noinspection PyShadowingBuiltins @@ -750,7 +758,6 @@ def __init__( repr=True, compare=True, kw_only=False, - forward_type=None, # forward_type can be None as None is not a ForwardRef ): # The init function for 'Field' cannot be generated # as 'Field' needs to exist first. @@ -766,7 +773,6 @@ def __init__( self.repr = repr self.compare = compare self.kw_only = kw_only - self.forward_type = forward_type self.validate_field() @@ -806,6 +812,12 @@ def from_field(cls, fld, /, **kwargs): return cls(**argument_dict) + @property + def type_eval(self): + if sys.version_info >= (3, 14): + return annotations.evaluate_forwardref(self.type) + return self.type + def _build_field(): # Complete the construction of the Field class @@ -822,7 +834,6 @@ def _build_field(): "repr": "Include this attribute in the class __repr__", "compare": "Include this attribute in the class __eq__ method", "kw_only": "Make this a keyword only parameter in __init__", - "forward_type": "A ForwardRef version of the type for Python 3.14" } fields = { @@ -834,7 +845,6 @@ def _build_field(): "repr": Field(default=True, doc=field_docs["repr"]), "compare": Field(default=True, doc=field_docs["compare"]), "kw_only": Field(default=False, doc=field_docs["kw_only"]), - "forward_type": Field(default=None, doc=field_docs["forward_type"], repr=False), } modifications = {"__slots__": field_docs} @@ -854,21 +864,6 @@ def _build_field(): del _build_field -def pre_gathered_gatherer(cls_or_ns): - """ - Retrieve fields previously gathered by SlotMakerMeta - - :param cls_or_ns: Class to gather field information from (or class namespace) - :return: dict of field_name: Field(...) and modifications to be performed by the builder - """ - if isinstance(cls_or_ns, (_MappingProxyType, dict)): - cls_dict = cls_or_ns - else: - cls_dict = cls_or_ns.__dict__ - - return cls_dict[GATHERED_DATA] - - def make_slot_gatherer(field_type=Field): """ Create a new annotation gatherer that will work with `Field` instances @@ -951,8 +946,10 @@ def make_annotation_gatherer( """ def field_annotation_gatherer(cls_or_ns): if isinstance(cls_or_ns, (_MappingProxyType, dict)): + cls = None cls_dict = cls_or_ns else: + cls = cls_or_ns cls_dict = cls_or_ns.__dict__ # This should really be dict[str, field_type] but static analysis @@ -960,7 +957,7 @@ def field_annotation_gatherer(cls_or_ns): cls_fields: dict[str, Field] = {} modifications = {} - cls_annotations = get_ns_annotations(cls_dict) + cls_annotations = get_ns_annotations(cls_dict, cls=cls) kw_flag = False @@ -969,7 +966,9 @@ def field_annotation_gatherer(cls_or_ns): if is_classvar(v): continue - if v is KW_ONLY or (isinstance(v, str) and v == "KW_ONLY"): + v_eval = evaluate_forwardref(v) + + if v_eval is KW_ONLY or (isinstance(v, str) and v == "KW_ONLY"): if kw_flag: raise SyntaxError("KW_ONLY sentinel may only appear once.") kw_flag = True @@ -1012,8 +1011,10 @@ def make_field_gatherer( def field_attribute_gatherer(cls_or_ns): if isinstance(cls_or_ns, (_MappingProxyType, dict)): cls_dict = cls_or_ns + cls = None else: cls_dict = cls_or_ns.__dict__ + cls = cls_or_ns cls_attributes = { k: v @@ -1022,7 +1023,7 @@ def field_attribute_gatherer(cls_or_ns): } if assign_types: - cls_annotations = get_ns_annotations(cls_dict) + cls_annotations = get_ns_annotations(cls_dict, cls=cls) else: cls_annotations = {} @@ -1064,12 +1065,10 @@ def make_unified_gatherer( def field_unified_gatherer(cls_or_ns): if isinstance(cls_or_ns, (_MappingProxyType, dict)): cls_dict = cls_or_ns + cls = None else: cls_dict = cls_or_ns.__dict__ - - cls_gathered = cls_dict.get(GATHERED_DATA) - if cls_gathered: - return pre_gathered_gatherer(cls_dict) + cls = cls_or_ns cls_slots = cls_dict.get("__slots__") @@ -1082,7 +1081,7 @@ def field_unified_gatherer(cls_or_ns): # To choose between annotation and attribute gatherers # compare sets of names. # Don't bother evaluating string annotations, as we only need names - cls_annotations = get_ns_annotations(cls_dict) + cls_annotations = get_ns_annotations(cls_dict, cls=cls) cls_attributes = { k: v for k, v in cls_dict.items() if isinstance(v, field_type) } @@ -1092,7 +1091,9 @@ def field_unified_gatherer(cls_or_ns): if set(cls_annotation_names).issuperset(set(cls_attribute_names)): # All `Field` values have annotations, so use annotation gatherer - return anno_g(cls_dict) + # Pass the original cls_or_ns object + + return anno_g(cls_or_ns) return attrib_g(cls_dict) diff --git a/src/ducktools/classbuilder/__init__.pyi b/src/ducktools/classbuilder/__init__.pyi index d633360..7802fae 100644 --- a/src/ducktools/classbuilder/__init__.pyi +++ b/src/ducktools/classbuilder/__init__.pyi @@ -1,13 +1,19 @@ +import sys import types import typing import typing_extensions -import inspect from collections.abc import Callable from types import MappingProxyType -_py_type = type | str # Alias for type hint values +if sys.version_info >= (3, 14): + import annotationlib + + _py_type = annotationlib.ForwardRef | type | str +else: + _py_type = type | str + _CopiableMappings = dict[str, typing.Any] | MappingProxyType[str, typing.Any] __version__: str @@ -265,9 +271,6 @@ class GatheredFields: fields: dict[str, Field] modifications: dict[str, typing.Any] - __classbuilder_internals__: dict - __signature__: inspect.Signature - def __init__( self, fields: dict[str, Field], diff --git a/src/ducktools/classbuilder/annotations.py b/src/ducktools/classbuilder/annotations.py deleted file mode 100644 index cbdd388..0000000 --- a/src/ducktools/classbuilder/annotations.py +++ /dev/null @@ -1,147 +0,0 @@ -# MIT License -# -# Copyright (c) 2024 David C Ellis -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -import sys - - -class _LazyAnnotationLib: - def __getattr__(self, item): - global _lazy_annotationlib - import annotationlib # type: ignore - _lazy_annotationlib = annotationlib - return getattr(annotationlib, item) - - -_lazy_annotationlib = _LazyAnnotationLib() - - -def get_func_annotations(func): - """ - Given a function, return the annotations dictionary - - :param func: function object - :return: dictionary of annotations - """ - # This method exists for use by prefab in getting annotations from - # the __prefab_post_init__ function - try: - annotations = func.__annotations__ - except Exception: - if sys.version_info >= (3, 14): - annotations = _lazy_annotationlib.get_annotations( - func, - format=_lazy_annotationlib.Format.FORWARDREF, - ) - else: - raise - - return annotations - - -def get_ns_annotations(ns): - """ - Given a class namespace, attempt to retrieve the - annotations dictionary. - - :param ns: Class namespace (eg cls.__dict__) - :return: dictionary of annotations - """ - - annotations = ns.get("__annotations__") - if annotations is not None: - annotations = annotations.copy() - elif sys.version_info >= (3, 14): - # See if we're using PEP-649 annotations - annotate = _lazy_annotationlib.get_annotate_from_class_namespace(ns) - if annotate: - annotations = _lazy_annotationlib.call_annotate_function( - annotate, - format=_lazy_annotationlib.Format.FORWARDREF - ) - - if annotations is None: - annotations = {} - - return annotations - - -def make_annotate_func(cls, annos, extra_annotation_func=None): - # Only used in 3.14 or later so no sys.version_info gate - - get_annotations = _lazy_annotationlib.get_annotations - type_repr = _lazy_annotationlib.type_repr - Format = _lazy_annotationlib.Format - ForwardRef = _lazy_annotationlib.ForwardRef - # Construct an annotation function from __annotations__ - def __annotate__(format, /): - match format: - case Format.VALUE | Format.FORWARDREF | Format.STRING: - cls_annotations = {} - - for base in reversed(cls.__mro__): - cls_annotations.update( - get_annotations(base, format=format) - ) - - if extra_annotation_func: - cls_annotations.update( - get_annotations(extra_annotation_func, format=format) - ) - - new_annos = {} - for k, v in annos.items(): - try: - new_annos[k] = cls_annotations[k] - except KeyError: - # Likely a return value - if format == Format.STRING: - new_annos[k] = type_repr(v) - else: - new_annos[k] = v - - return new_annos - case _: - raise NotImplementedError(format) - return __annotate__ - - -def is_classvar(hint): - if isinstance(hint, str): - # String annotations, just check if the string 'ClassVar' is in there - # This is overly broad and could be smarter. - return "ClassVar" in hint - elif (annotationlib := sys.modules.get("annotationlib")) and isinstance(hint, annotationlib.ForwardRef): - return "ClassVar" in hint.__arg__ - else: - _typing = sys.modules.get("typing") - if _typing: - _Annotated = _typing.Annotated - _get_origin = _typing.get_origin - - if _Annotated and _get_origin(hint) is _Annotated: - hint = getattr(hint, "__origin__", None) - - if ( - hint is _typing.ClassVar - or getattr(hint, "__origin__", None) is _typing.ClassVar - ): - return True - return False diff --git a/src/ducktools/classbuilder/annotations.pyi b/src/ducktools/classbuilder/annotations.pyi index 31ce86f..e350053 100644 --- a/src/ducktools/classbuilder/annotations.pyi +++ b/src/ducktools/classbuilder/annotations.pyi @@ -1,9 +1,35 @@ from collections.abc import Callable import typing import types +import sys _CopiableMappings = dict[str, typing.Any] | types.MappingProxyType[str, typing.Any] +if sys.version_info >= (3, 14): + from annotationlib import ForwardRef, Format + + # def get_ns_forwardrefs(ns: _CopiableMappings) -> dict[str, ForwardRef]: ... + + @typing.overload + def evaluate_forwardref(ref: ForwardRef, format: Format | None = ...) -> ForwardRef | typing.Any: ... + @typing.overload + def evaluate_forwardref(ref: str, format: Format | None = ...) -> str: ... + + def is_forwardref(obj: object) -> bool: ... + + def make_annotate_func( + cls: type, + annos: dict[str, typing.Any], + extra_annotation_func: types.FunctionType | None = ..., + ) -> Callable[[int], dict[str, typing.Any]]: ... + +else: + # def get_ns_forwardrefs(ns: _CopiableMappings) -> dict: ... # Actually always empty + def evaluate_forwardref[T](ref: T, format: None = None) -> T: ... + def is_forwardref(obj: object) -> typing.Literal[False]: ... + def make_annotate_func(cls: type, annos: dict[str, typing.Any]) -> typing.Never: ... + + def get_func_annotations( func: types.FunctionType, ) -> dict[str, typing.Any]: ... @@ -12,12 +38,6 @@ def get_ns_annotations( ns: _CopiableMappings, ) -> dict[str, typing.Any]: ... -def make_annotate_func( - cls: type, - annos: dict[str, typing.Any], - extra_annotation_func: types.FunctionType | None = ..., -) -> Callable[[int], dict[str, typing.Any]]: ... - def is_classvar( hint: object, ) -> bool: ... diff --git a/src/ducktools/classbuilder/annotations/__init__.py b/src/ducktools/classbuilder/annotations/__init__.py new file mode 100644 index 0000000..d59deb3 --- /dev/null +++ b/src/ducktools/classbuilder/annotations/__init__.py @@ -0,0 +1,74 @@ +# MIT License +# +# Copyright (c) 2024 David C Ellis +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import sys + +if sys.version_info >= (3, 14): + from .annotations_314 import ( + evaluate_forwardref, + is_forwardref, + make_annotate_func, + get_func_annotations, + get_ns_annotations, + ) +else: + from .annotations_pre_314 import ( + evaluate_forwardref, + is_forwardref, + make_annotate_func, + get_func_annotations, + get_ns_annotations, + ) + + +__all__ = [ + "evaluate_forwardref", + "get_ns_forwardrefs", + "is_forwardref", + "make_annotate_func", + "get_func_annotations", + "get_ns_annotations", +] + + +def is_classvar(hint): + if isinstance(hint, str): + # String annotations, just check if the string 'ClassVar' is in there + # This is overly broad and could be smarter. + return "ClassVar" in hint + else: + _typing = sys.modules.get("typing") + if _typing: + _Annotated = _typing.Annotated + _get_origin = _typing.get_origin + + hint = evaluate_forwardref(hint) + + if _Annotated and _get_origin(hint) is _Annotated: + hint = getattr(hint, "__origin__", None) + + if ( + hint is _typing.ClassVar + or getattr(hint, "__origin__", None) is _typing.ClassVar + ): + return True + return False diff --git a/src/ducktools/classbuilder/annotations/annotations_314.py b/src/ducktools/classbuilder/annotations/annotations_314.py new file mode 100644 index 0000000..98a0d2c --- /dev/null +++ b/src/ducktools/classbuilder/annotations/annotations_314.py @@ -0,0 +1,179 @@ +# MIT License +# +# Copyright (c) 2024 David C Ellis +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +try: + from _types import FunctionType as _FunctionType # type: ignore +except ImportError: + from types import FunctionType as _FunctionType + +class _LazyAnnotationLib: + def __getattr__(self, item): + global _lazy_annotationlib + import annotationlib # type: ignore + _lazy_annotationlib = annotationlib + return getattr(annotationlib, item) + + +_lazy_annotationlib = _LazyAnnotationLib() + + +def _call_annotate_forwardrefs(annotate, *, owner=None): + # Get all annotations as unevaluated forward references + # Logic taken from the call_annotate_function logic + + is_class = isinstance(owner, type) + + # Check that VALUE_WITH_FAKE_GLOBALS doesn't raise a NotImplementedError + try: + _ = annotate(_lazy_annotationlib.Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + return _lazy_annotationlib.call_annotate_function( + annotate, + owner=owner, + format=_lazy_annotationlib.Format.FORWARDREF, + ) + except Exception: + pass # any other error is fine + + globals = _lazy_annotationlib._StringifierDict( + {}, + globals=annotate.__globals__, + owner=owner, + is_class=is_class, + format=format, + ) + closure = _lazy_annotationlib._build_closure( + annotate, owner, is_class, globals, allow_evaluation=False + ) + func = _FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + result = func(_lazy_annotationlib.Format.VALUE_WITH_FAKE_GLOBALS) + + globals.transmogrify() + + return result + + +def make_annotate_func(annos): + type_repr = _lazy_annotationlib.type_repr + Format = _lazy_annotationlib.Format + # Construct an annotation function from __annotations__ + def __annotate__(format, /): + match format: + case Format.VALUE | Format.FORWARDREF | Format.STRING: + new_annos = {} + for k, v in annos.items(): + v = evaluate_forwardref(v, format=format) + if not isinstance(v, str) and format == Format.STRING: + v = type_repr(v) + new_annos[k] = v + return new_annos + case _: + raise NotImplementedError(format) + return __annotate__ + + +def is_forwardref(obj): + return isinstance(obj, _lazy_annotationlib.ForwardRef) + + +def evaluate_forwardref(ref, format=None): + # A special forwardref evaluation that tries to include closure variables if they exist + # It also places globals in the locals dict to assist in partial evaluation + if isinstance(ref, str): + return ref + elif is_forwardref(ref): + format = _lazy_annotationlib.Format.FORWARDREF if format is None else format + + if (owner := ref.__owner__): + annotate = owner.__annotate__ + + # Add globals first + closure_and_locals = {**annotate.__globals__} + if annotate.__closure__: + for name, value in zip(annotate.__code__.co_freevars, annotate.__closure__): + try: + closure_and_locals[name] = value.cell_contents + except ValueError: + pass + + closure_and_locals.update(vars(owner)) + + return ref.evaluate( + globals=annotate.__globals__, + locals=closure_and_locals, + format=format + ) + else: + return ref.evaluate(format=format) + + return ref + + +def get_func_annotations(func): + """ + Given a function, return the annotations dictionary + + :param func: function object + :return: dictionary of annotations + """ + # This method exists for use by prefab in getting annotations from + # the __prefab_post_init__ function + try: + annotations = func.__annotations__ + except Exception: + annotations = _call_annotate_forwardrefs(func.__annotate__, owner=func) + + return annotations + + +def get_ns_annotations(ns, cls=None): + """ + Given a class namespace, attempt to retrieve the + annotations dictionary. + + :param ns: Class namespace (eg cls.__dict__) + :param cls: Class if available + :return: dictionary of annotations + """ + + annotations = ns.get("__annotations__") + if annotations is not None: + annotations = annotations.copy() + else: + # See if we're using PEP-649 annotations + annotate = _lazy_annotationlib.get_annotate_from_class_namespace(ns) + if annotate: + if cls is None: + annotations = _call_annotate_forwardrefs(annotate) + else: + annotations = _call_annotate_forwardrefs(annotate, owner=cls) + + if annotations is None: + annotations = {} + + return annotations diff --git a/src/ducktools/classbuilder/annotations/annotations_pre_314.py b/src/ducktools/classbuilder/annotations/annotations_pre_314.py new file mode 100644 index 0000000..e5e7e29 --- /dev/null +++ b/src/ducktools/classbuilder/annotations/annotations_pre_314.py @@ -0,0 +1,58 @@ +# MIT License +# +# Copyright (c) 2024 David C Ellis +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +import sys + + +# These exist so 3.13 and earlier work, but have no real function +def is_forwardref(obj): + return False + + +def evaluate_forwardref(ref, format=None): + return ref + + +def make_annotate_func(cls, annos): + verno = ".".join(str(v) for v in sys.version_info[:3]) + raise RuntimeError(f"make_annotate_function should never be used in Python {verno}") + + +def get_func_annotations(func): + """ + Given a function, return the annotations dictionary + + :param func: function object + :return: dictionary of annotations + """ + annotations = func.__annotations__ + return annotations + + +# This is simplified under 3.13 or earlier +def get_ns_annotations(ns): + annotations = ns.get("__annotations__") + if annotations is not None: + annotations = annotations.copy() + else: + annotations = {} + return annotations + diff --git a/src/ducktools/classbuilder/prefab.py b/src/ducktools/classbuilder/prefab.py index d01131c..e459f24 100644 --- a/src/ducktools/classbuilder/prefab.py +++ b/src/ducktools/classbuilder/prefab.py @@ -25,6 +25,8 @@ Includes pre and post init functions along with other methods. """ +import sys + from . import ( INTERNALS_DICT, NOTHING, FIELD_NOTHING, Field, MethodMaker, GatheredFields, GeneratedCode, SlotMakerMeta, @@ -34,7 +36,7 @@ get_repr_generator, ) -from .annotations import get_func_annotations +from .annotations import get_func_annotations, make_annotate_func # These aren't used but are re-exported for ease of use # noinspection PyUnresolvedReferences @@ -203,13 +205,11 @@ def init_generator(cls, funcname="__init__"): if annotations: annotations["return"] = None - extra_annotation_func = getattr(cls, POST_INIT_FUNC, None) else: # If there are no annotations, return an unannotated init function annotations = None - extra_annotation_func = None - return GeneratedCode(code, globs, annotations, extra_annotation_func) + return GeneratedCode(code, globs, annotations) def iter_generator(cls, funcname="__iter__"): @@ -329,7 +329,7 @@ def attribute( :param private: Short for init, repr, compare, iter, serialize = False, must have default or factory :param doc: Parameter documentation for slotted classes :param metadata: Dictionary for additional non-construction metadata - :param type: Type of this attribute (for slotted classes) + :param type: Type of this attribute :return: Attribute generated with these parameters. """ @@ -704,7 +704,11 @@ def build_prefab( if slots: class_dict["__slots__"] = class_slots - class_dict["__annotations__"] = class_annotations + if sys.version_info >= (3, 14): + class_dict["__annotate__"] = make_annotate_func(class_annotations) + else: + class_dict["__annotations__"] = class_annotations + cls = type(class_name, bases, class_dict) gathered_fields = GatheredFields(fields, {}) diff --git a/src/ducktools/classbuilder/prefab.pyi b/src/ducktools/classbuilder/prefab.pyi index b66fca0..156f3e2 100644 --- a/src/ducktools/classbuilder/prefab.pyi +++ b/src/ducktools/classbuilder/prefab.pyi @@ -2,7 +2,6 @@ import typing from types import MappingProxyType from typing_extensions import dataclass_transform -import inspect # Suppress weird pylance error from collections.abc import Callable # type: ignore diff --git a/tests/annotations/test_annotated.py b/tests/annotations/test_annotated.py index 411e6f6..59835a1 100644 --- a/tests/annotations/test_annotated.py +++ b/tests/annotations/test_annotated.py @@ -89,6 +89,7 @@ class ExampleAnnotated: h: Annotated[CV[str], ''] = "h" annos, modifications = gatherer(ExampleAnnotated) + annotations = get_ns_annotations(vars(ExampleAnnotated)) assert annos["blank_field"] == NewField(type=str) diff --git a/tests/annotations/test_annotations_module.py b/tests/annotations/test_annotations_module.py index 5174a81..fc91343 100644 --- a/tests/annotations/test_annotations_module.py +++ b/tests/annotations/test_annotations_module.py @@ -1,13 +1,17 @@ # This module commits intentional typing related crimes, ignore any errors # type: ignore +import sys +from typing import Annotated, ClassVar from ducktools.classbuilder.annotations import ( get_ns_annotations, is_classvar, ) -from typing import Annotated, ClassVar + +import pytest +@pytest.mark.skipif(sys.version_info >= (3, 14), reason="Result has changed in 3.14") def test_ns_annotations(): CV = ClassVar diff --git a/tests/py314_tests/test_forwardref_annotations.py b/tests/py314_tests/test_forwardref_annotations.py index 5a84cd6..9bbcd62 100644 --- a/tests/py314_tests/test_forwardref_annotations.py +++ b/tests/py314_tests/test_forwardref_annotations.py @@ -3,8 +3,9 @@ from ducktools.classbuilder.annotations import get_ns_annotations, get_func_annotations from pathlib import Path +from typing import Annotated, ClassVar -from _test_support import EqualToForwardRef +from _test_support import EqualToForwardRef, SimpleEqualToForwardRef global_type = int @@ -57,3 +58,36 @@ def forwardref_func(x: unknown) -> str: 'x': EqualToForwardRef("unknown", owner=forwardref_func), 'return': str } + + +def test_ns_annotations(): + # The 3.14 annotations version of test_ns_annotations + CV = ClassVar + + class AnnotatedClass: + a: str + b: "str" + c: list[str] + d: "list[str]" + e: ClassVar[str] + f: "ClassVar[str]" + g: "ClassVar[forwardref]" + h: "Annotated[ClassVar[str], '']" + i: "Annotated[ClassVar[forwardref], '']" + j: "CV[str]" + + annos = get_ns_annotations(vars(AnnotatedClass)) + + assert annos == { + 'a': SimpleEqualToForwardRef("str"), + 'b': "str", + 'c': SimpleEqualToForwardRef("list[str]"), + 'd': "list[str]", + 'e': SimpleEqualToForwardRef("ClassVar[str]"), + 'f': "ClassVar[str]", + 'g': "ClassVar[forwardref]", + 'h': "Annotated[ClassVar[str], '']", + 'i': "Annotated[ClassVar[forwardref], '']", + 'j': "CV[str]", + } + From 5fc4dd73b28c246987314edac43fd2714f7602a8 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Fri, 22 Aug 2025 22:11:56 +0100 Subject: [PATCH 04/12] Fix types in tests --- tests/_type_support.py | 27 ++++++++++++++++ tests/annotations/test_annotated.py | 7 +++-- tests/prefab/test_creation.py | 12 ++++--- tests/prefab/test_internals_dict.py | 7 +++-- tests/py312_tests/test_generic_annotations.py | 4 ++- .../test_forwardref_annotations.py | 31 ++++++++++--------- tests/test_core.py | 4 +-- tests/test_slotmakermeta.py | 11 ++++--- 8 files changed, 73 insertions(+), 30 deletions(-) create mode 100644 tests/_type_support.py diff --git a/tests/_type_support.py b/tests/_type_support.py new file mode 100644 index 0000000..07a7d7a --- /dev/null +++ b/tests/_type_support.py @@ -0,0 +1,27 @@ +import sys + +if sys.version_info >= (3, 14): + from annotationlib import ForwardRef, type_repr + + class SimpleEqualToForwardRef: + def __init__(self, arg): + self.__forward_arg__ = arg + + def __eq__(self, other): + if not isinstance(other, (SimpleEqualToForwardRef, ForwardRef)): + return NotImplemented + else: + return self.__forward_arg__ == other.__forward_arg__ + + def __repr__(self): + return f"SimpleEqualToForwardRef({self.__forward_arg__!r})" + + + def matches_type(arg): + if isinstance(arg, str): + return SimpleEqualToForwardRef(arg) + + return SimpleEqualToForwardRef(type_repr(arg)) +else: + def matches_type(arg): + return arg diff --git a/tests/annotations/test_annotated.py b/tests/annotations/test_annotated.py index 59835a1..a8ecb6d 100644 --- a/tests/annotations/test_annotated.py +++ b/tests/annotations/test_annotated.py @@ -20,6 +20,9 @@ get_ns_annotations, ) +from _type_support import matches_type + + CV = ClassVar @@ -90,9 +93,9 @@ class ExampleAnnotated: annos, modifications = gatherer(ExampleAnnotated) - annotations = get_ns_annotations(vars(ExampleAnnotated)) + annotations = get_ns_annotations(vars(ExampleAnnotated), ExampleAnnotated) - assert annos["blank_field"] == NewField(type=str) + assert annos["blank_field"] == NewField(type=matches_type(str)) # ABC should be present in annos and in the class for key in "abc": diff --git a/tests/prefab/test_creation.py b/tests/prefab/test_creation.py index 9b9e63b..666252e 100644 --- a/tests/prefab/test_creation.py +++ b/tests/prefab/test_creation.py @@ -4,12 +4,14 @@ from typing import Annotated, ClassVar from ducktools.classbuilder.prefab import prefab, attribute -from ducktools.classbuilder.annotations import get_ns_annotations +from ducktools.classbuilder.annotations import get_ns_annotations, evaluate_forwardref import pytest -# These classes are defined at module level for easier +from _type_support import matches_type + +# These classes are defined at module level for easier # REPR testing @prefab class Empty: @@ -205,7 +207,7 @@ class SplitVarRedef: for cls in [SplitVarDef, SplitVarDefReverseOrder, SplitVarRedef]: - assert get_ns_annotations(cls.__dict__)["x"] == str + assert evaluate_forwardref(get_ns_annotations(cls.__dict__)["x"]) == str inst = cls() assert inst.x == "test" @@ -233,7 +235,9 @@ def test_horriblemess(self): assert inst.x == "true_test" assert repr(inst) == "HorribleMess(x='true_test', y='test_2')" - assert get_ns_annotations(HorribleMess.__dict__) == {"x": str, "y": str} + assert get_ns_annotations(HorribleMess.__dict__, HorribleMess) == { + "x": matches_type(str), "y": matches_type(str) + } def test_call_mistaken(): diff --git a/tests/prefab/test_internals_dict.py b/tests/prefab/test_internals_dict.py index 0d024f0..23b284e 100644 --- a/tests/prefab/test_internals_dict.py +++ b/tests/prefab/test_internals_dict.py @@ -1,6 +1,7 @@ from ducktools.classbuilder.prefab import prefab, attribute from ducktools.classbuilder import INTERNALS_DICT +from _type_support import matches_type def test_internals_dict(): @prefab @@ -12,9 +13,9 @@ class X: class Z(X): z: int = 3 - x_attrib = attribute(type=int) - y_attrib = attribute(default=2, type=int) - z_attrib = attribute(default=3, type=int) + x_attrib = attribute(type=matches_type(int)) + y_attrib = attribute(default=2, type=matches_type(int)) + z_attrib = attribute(default=3, type=matches_type(int)) assert hasattr(X, INTERNALS_DICT) diff --git a/tests/py312_tests/test_generic_annotations.py b/tests/py312_tests/test_generic_annotations.py index 26ed1a8..938297b 100644 --- a/tests/py312_tests/test_generic_annotations.py +++ b/tests/py312_tests/test_generic_annotations.py @@ -1,6 +1,8 @@ # This syntax only exists in Python 3.12 or later. from ducktools.classbuilder.annotations import get_ns_annotations +from _type_support import matches_type + def test_312_generic(): class X[T]: @@ -10,6 +12,6 @@ class X[T]: y: "list[T]" assert get_ns_annotations(vars(X)) == { - "x": list[X.test_var], + "x": matches_type(list[X.test_var]), "y": "list[T]", } diff --git a/tests/py314_tests/test_forwardref_annotations.py b/tests/py314_tests/test_forwardref_annotations.py index 9bbcd62..ef386ff 100644 --- a/tests/py314_tests/test_forwardref_annotations.py +++ b/tests/py314_tests/test_forwardref_annotations.py @@ -1,11 +1,13 @@ # Bare forwardrefs only work in 3.14 or later -from ducktools.classbuilder.annotations import get_ns_annotations, get_func_annotations +from ducktools.classbuilder.annotations import get_ns_annotations, get_func_annotations, evaluate_forwardref + +import pathlib -from pathlib import Path from typing import Annotated, ClassVar from _test_support import EqualToForwardRef, SimpleEqualToForwardRef +from _type_support import matches_type global_type = int @@ -13,12 +15,16 @@ def test_bare_forwardref(): class Ex: a: str - b: Path + b: pathlib.Path c: plain_forwardref annos = get_ns_annotations(Ex.__dict__) - assert annos == {'a': str, 'b': Path, 'c': EqualToForwardRef("plain_forwardref")} + assert annos == { + 'a': matches_type(str), + 'b': matches_type(pathlib.Path), + 'c': matches_type("plain_forwardref") + } def test_inner_outer_ref(): @@ -36,17 +42,14 @@ class Inner: hyper_type = float - return Inner, annos + return annos - cls, annos = make_func() + annos = make_func() - # Forwardref given as string if used before it can be evaluated - assert annos == {"a_val": str, "b_val": int, "c_val": EqualToForwardRef("hyper_type")} - - # Correctly evaluated if it exists - assert get_ns_annotations(cls.__dict__) == { - "a_val": str, "b_val": int, "c_val": float - } + # Confirm the annotations all evaluate + assert evaluate_forwardref(annos['a_val']) == str + assert evaluate_forwardref(annos["b_val"]) == int + assert evaluate_forwardref(annos["c_val"]) == float def test_func_annotations(): @@ -56,7 +59,7 @@ def forwardref_func(x: unknown) -> str: annos = get_func_annotations(forwardref_func) assert annos == { 'x': EqualToForwardRef("unknown", owner=forwardref_func), - 'return': str + 'return': matches_type(str), } diff --git a/tests/test_core.py b/tests/test_core.py index 9e425b8..d1df0c3 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -28,7 +28,7 @@ from ducktools.classbuilder.annotations import get_ns_annotations from utils import graalpy_fails # type: ignore - +from _type_support import matches_type def test_get_fields_flags_methods(): local_fields = {"Example": Field()} @@ -217,7 +217,7 @@ class SlotsExample: assert slots == fields assert modifications["__slots__"] == {"a": None, "b": None, "c": "a list", "d": None} - assert get_ns_annotations(SlotsExample.__dict__) == {"a": int} # Original annotations dict unmodified + assert get_ns_annotations(SlotsExample.__dict__) == {"a": matches_type(int)} # Original annotations dict unmodified def test_slot_gatherer_failure(): diff --git a/tests/test_slotmakermeta.py b/tests/test_slotmakermeta.py index 73eb45a..49d308c 100644 --- a/tests/test_slotmakermeta.py +++ b/tests/test_slotmakermeta.py @@ -1,3 +1,4 @@ +import typing from typing import Annotated, ClassVar, List from ducktools.classbuilder import Field, SlotFields, NOTHING, SlotMakerMeta, GATHERED_DATA @@ -6,13 +7,15 @@ from utils import graalpy_fails # type: ignore +from _type_support import matches_type + @graalpy_fails def test_slots_created(): class ExampleAnnotated(metaclass=SlotMakerMeta): a: str = "a" b: "List[str]" = "b" # Yes this is the wrong type, I know. - c: Annotated[str, ""] = "c" + c: typing.Annotated[str, ""] = "c" d: ClassVar[str] = "d" e: Annotated[ClassVar[str], ""] = "e" @@ -27,9 +30,9 @@ class ExampleAnnotated(metaclass=SlotMakerMeta): assert slots == expected_slots expected_fields = { - "a": Field(default="a", type=str), - "b": Field(default="b", type="List[str]"), - "c": Field(default="c", type=Annotated[str, ""]), + "a": Field(default="a", type=matches_type(str)), + "b": Field(default="b", type="List[str]"), + "c": Field(default="c", type=matches_type(typing.Annotated[str, ""])), } fields, modifications = getattr(ExampleAnnotated, GATHERED_DATA) From d15e12aba1b1750c715832ff2de24150e9ff6878 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Fri, 22 Aug 2025 22:58:05 +0100 Subject: [PATCH 05/12] Fix missing argument --- src/ducktools/classbuilder/annotations/annotations_pre_314.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ducktools/classbuilder/annotations/annotations_pre_314.py b/src/ducktools/classbuilder/annotations/annotations_pre_314.py index e5e7e29..b240879 100644 --- a/src/ducktools/classbuilder/annotations/annotations_pre_314.py +++ b/src/ducktools/classbuilder/annotations/annotations_pre_314.py @@ -48,7 +48,7 @@ def get_func_annotations(func): # This is simplified under 3.13 or earlier -def get_ns_annotations(ns): +def get_ns_annotations(ns, cls=None): annotations = ns.get("__annotations__") if annotations is not None: annotations = annotations.copy() From 04bc442575b2ff4367595daebfa47f4a608ec934 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Fri, 22 Aug 2025 23:02:06 +0100 Subject: [PATCH 06/12] Fix stubs and __all__ --- src/ducktools/classbuilder/__init__.pyi | 9 ++------- src/ducktools/classbuilder/annotations.pyi | 1 + src/ducktools/classbuilder/annotations/__init__.py | 2 +- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/ducktools/classbuilder/__init__.pyi b/src/ducktools/classbuilder/__init__.pyi index 7802fae..0f91afa 100644 --- a/src/ducktools/classbuilder/__init__.pyi +++ b/src/ducktools/classbuilder/__init__.pyi @@ -51,14 +51,12 @@ class GeneratedCode: source_code: str globs: dict[str, typing.Any] annotations: dict[str, typing.Any] - extra_annotation_func: None | types.FunctionType def __init__( self, source_code: str, globs: dict[str, typing.Any], annotations: dict[str, typing.Any] | None = ..., - extra_annotation_func: None | types.FunctionType = ..., ) -> None: ... def __repr__(self) -> str: ... @@ -175,17 +173,14 @@ class Field(metaclass=SlotMakerMeta): def validate_field(self) -> None: ... @classmethod def from_field(cls, fld: Field, /, **kwargs: typing.Any) -> Field: ... - + @property + def type_eval(self) -> typing.Any: ... # type[Field] doesn't work due to metaclass # This is not really precise enough because isinstance is used _ReturnsField = Callable[..., Field] _FieldType = typing.TypeVar("_FieldType", bound=Field) -def pre_gathered_gatherer( - cls_or_ns: type | _CopiableMappings -) -> tuple[dict[str, Field | _FieldType], dict[str, typing.Any]]: ... - @typing.overload def make_slot_gatherer( field_type: type[_FieldType] diff --git a/src/ducktools/classbuilder/annotations.pyi b/src/ducktools/classbuilder/annotations.pyi index e350053..45aefc4 100644 --- a/src/ducktools/classbuilder/annotations.pyi +++ b/src/ducktools/classbuilder/annotations.pyi @@ -36,6 +36,7 @@ def get_func_annotations( def get_ns_annotations( ns: _CopiableMappings, + cls: type | None = ... ) -> dict[str, typing.Any]: ... def is_classvar( diff --git a/src/ducktools/classbuilder/annotations/__init__.py b/src/ducktools/classbuilder/annotations/__init__.py index d59deb3..8fc0749 100644 --- a/src/ducktools/classbuilder/annotations/__init__.py +++ b/src/ducktools/classbuilder/annotations/__init__.py @@ -42,11 +42,11 @@ __all__ = [ "evaluate_forwardref", - "get_ns_forwardrefs", "is_forwardref", "make_annotate_func", "get_func_annotations", "get_ns_annotations", + "is_classvar", ] From d07f77f1184ccf3ab1709503335d2c4fca40e1ed Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Sat, 23 Aug 2025 09:52:01 +0100 Subject: [PATCH 07/12] Modify logic for calling non-cpython generated annotate functions --- src/ducktools/classbuilder/__init__.py | 4 +- .../annotations/annotations_314.py | 50 +++++++++---------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/ducktools/classbuilder/__init__.py b/src/ducktools/classbuilder/__init__.py index 6c019f1..bafd391 100644 --- a/src/ducktools/classbuilder/__init__.py +++ b/src/ducktools/classbuilder/__init__.py @@ -561,7 +561,9 @@ def builder(cls=None, /, *, gatherer, methods, flags=None, fix_signature=True): if cls_gathered: cls_fields, modifications = cls_gathered # Reconnect the forwardrefs in types to the class so they can evaluate. - if sys.version_info >= (3, 14): + # If there are forwardrefs then annotationlib should be in modules + # No need to do this if __future__ annotations are used + if sys.version_info >= (3, 14) and sys.modules.get("annotationlib"): annos = annotations.get_ns_annotations(cls.__dict__, cls=cls) for k, v in cls_fields.items(): if annotations.is_forwardref(v.type): diff --git a/src/ducktools/classbuilder/annotations/annotations_314.py b/src/ducktools/classbuilder/annotations/annotations_314.py index 98a0d2c..5adcd69 100644 --- a/src/ducktools/classbuilder/annotations/annotations_314.py +++ b/src/ducktools/classbuilder/annotations/annotations_314.py @@ -19,6 +19,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import sys try: from _types import FunctionType as _FunctionType # type: ignore @@ -42,17 +43,14 @@ def _call_annotate_forwardrefs(annotate, *, owner=None): is_class = isinstance(owner, type) - # Check that VALUE_WITH_FAKE_GLOBALS doesn't raise a NotImplementedError + # Attempt to call the annotate function directly + # Usually this should fail, success would indicate this + # is not a method generated by CPython and can't be run + # in the fake globals context. try: - _ = annotate(_lazy_annotationlib.Format.VALUE_WITH_FAKE_GLOBALS) + return annotate(_lazy_annotationlib.Format.FORWARDREF) except NotImplementedError: - return _lazy_annotationlib.call_annotate_function( - annotate, - owner=owner, - format=_lazy_annotationlib.Format.FORWARDREF, - ) - except Exception: - pass # any other error is fine + pass globals = _lazy_annotationlib._StringifierDict( {}, @@ -81,19 +79,20 @@ def _call_annotate_forwardrefs(annotate, *, owner=None): def make_annotate_func(annos): type_repr = _lazy_annotationlib.type_repr Format = _lazy_annotationlib.Format + # Construct an annotation function from __annotations__ def __annotate__(format, /): - match format: - case Format.VALUE | Format.FORWARDREF | Format.STRING: - new_annos = {} - for k, v in annos.items(): - v = evaluate_forwardref(v, format=format) - if not isinstance(v, str) and format == Format.STRING: - v = type_repr(v) - new_annos[k] = v - return new_annos - case _: - raise NotImplementedError(format) + if format in {Format.VALUE, Format.FORWARDREF, Format.STRING}: + new_annos = {} + for k, v in annos.items(): + v = evaluate_forwardref(v, format=format) + if not isinstance(v, str) and format == Format.STRING: + v = type_repr(v) + new_annos[k] = v + return new_annos + else: + raise NotImplementedError(format) + return __annotate__ @@ -141,12 +140,11 @@ def get_func_annotations(func): :param func: function object :return: dictionary of annotations """ - # This method exists for use by prefab in getting annotations from - # the __prefab_post_init__ function - try: - annotations = func.__annotations__ - except Exception: - annotations = _call_annotate_forwardrefs(func.__annotate__, owner=func) + # functions have '__annotations__' defined so check for '__annotate__' instead + if annotate := getattr(func, "__annotate__", None): + annotations = _call_annotate_forwardrefs(annotate, owner=func) + else: + annotations = getattr(func, "__annotations__", {}).copy() return annotations From a2ce0a9370c8b85df7eb9d861fff39d34e90f51d Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Sat, 23 Aug 2025 22:16:10 +0100 Subject: [PATCH 08/12] Make the test pass on 3.14 so it no longer needs to be skipped. --- tests/annotations/test_annotations_module.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/annotations/test_annotations_module.py b/tests/annotations/test_annotations_module.py index fc91343..62687ff 100644 --- a/tests/annotations/test_annotations_module.py +++ b/tests/annotations/test_annotations_module.py @@ -1,6 +1,7 @@ # This module commits intentional typing related crimes, ignore any errors # type: ignore import sys +import typing from typing import Annotated, ClassVar from ducktools.classbuilder.annotations import ( @@ -10,8 +11,9 @@ import pytest +from _type_support import matches_type + -@pytest.mark.skipif(sys.version_info >= (3, 14), reason="Result has changed in 3.14") def test_ns_annotations(): CV = ClassVar @@ -20,7 +22,7 @@ class AnnotatedClass: b: "str" c: list[str] d: "list[str]" - e: ClassVar[str] + e: typing.ClassVar[str] f: "ClassVar[str]" g: "ClassVar[forwardref]" h: "Annotated[ClassVar[str], '']" @@ -30,11 +32,11 @@ class AnnotatedClass: annos = get_ns_annotations(vars(AnnotatedClass)) assert annos == { - 'a': str, + 'a': matches_type(str), 'b': "str", - 'c': list[str], + 'c': matches_type(list[str]), 'd': "list[str]", - 'e': ClassVar[str], + 'e': matches_type(typing.ClassVar[str]), 'f': "ClassVar[str]", 'g': "ClassVar[forwardref]", 'h': "Annotated[ClassVar[str], '']", From 915032fdb7da2cc25daaae7647f5cda2f131408a Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Tue, 26 Aug 2025 16:56:51 +0100 Subject: [PATCH 09/12] wrap fix_signature --- src/ducktools/classbuilder/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ducktools/classbuilder/__init__.py b/src/ducktools/classbuilder/__init__.py index bafd391..6336e88 100644 --- a/src/ducktools/classbuilder/__init__.py +++ b/src/ducktools/classbuilder/__init__.py @@ -551,6 +551,7 @@ def builder(cls=None, /, *, gatherer, methods, flags=None, fix_signature=True): gatherer=gatherer, methods=methods, flags=flags, + fix_signature=fix_signature, ) internals = {} From ad80e81449c6126a3bf9c5be8992c96ad9835393 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Tue, 26 Aug 2025 17:49:43 +0100 Subject: [PATCH 10/12] Alternate check for fake_globals capability --- .../annotations/annotations_314.py | 17 +++++++++----- tests/py314_tests/test_314_annotations.py | 22 +++++++++++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 tests/py314_tests/test_314_annotations.py diff --git a/src/ducktools/classbuilder/annotations/annotations_314.py b/src/ducktools/classbuilder/annotations/annotations_314.py index 5adcd69..b308a6e 100644 --- a/src/ducktools/classbuilder/annotations/annotations_314.py +++ b/src/ducktools/classbuilder/annotations/annotations_314.py @@ -19,7 +19,6 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import sys try: from _types import FunctionType as _FunctionType # type: ignore @@ -43,13 +42,19 @@ def _call_annotate_forwardrefs(annotate, *, owner=None): is_class = isinstance(owner, type) - # Attempt to call the annotate function directly - # Usually this should fail, success would indicate this - # is not a method generated by CPython and can't be run - # in the fake globals context. + # Attempt to call with VALUE_WITH_FAKE_GLOBALS + # If this fails with a NotImplementedError then return + # the FORWARDREF format as the best we can do as this means + # the fake globals method can't be applied. try: - return annotate(_lazy_annotationlib.Format.FORWARDREF) + annotate(_lazy_annotationlib.Format.VALUE_WITH_FAKE_GLOBALS) except NotImplementedError: + return _lazy_annotationlib.call_annotate_function( + annotate, + format=_lazy_annotationlib.Format.FORWARDREF, + owner=owner, + ) + except Exception: pass globals = _lazy_annotationlib._StringifierDict( diff --git a/tests/py314_tests/test_314_annotations.py b/tests/py314_tests/test_314_annotations.py new file mode 100644 index 0000000..6d410e1 --- /dev/null +++ b/tests/py314_tests/test_314_annotations.py @@ -0,0 +1,22 @@ +from annotationlib import Format + +from ducktools.classbuilder.annotations.annotations_314 import _call_annotate_forwardrefs + + +from unittest.mock import MagicMock, call + + +def test_user_annotate_called_with_forwardref(): + def annotate(format, /): + if format == Format.VALUE_WITH_FAKE_GLOBALS: + raise NotImplementedError(format) + return {} + + annotate_mock = MagicMock(wraps=annotate) + + _call_annotate_forwardrefs(annotate_mock) + + first_call = call(Format.VALUE_WITH_FAKE_GLOBALS) + second_call = call(Format.FORWARDREF) + + annotate_mock.assert_has_calls([first_call, second_call]) From e35a622c0576a94da9e78dfa3c2e4d389a5bc080 Mon Sep 17 00:00:00 2001 From: David Ellis Date: Fri, 29 Aug 2025 12:14:23 +0100 Subject: [PATCH 11/12] Move _build_closure into the annotations_314 module in case it is changed in CPython --- .../annotations/annotations_314.py | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/src/ducktools/classbuilder/annotations/annotations_314.py b/src/ducktools/classbuilder/annotations/annotations_314.py index b308a6e..f1478aa 100644 --- a/src/ducktools/classbuilder/annotations/annotations_314.py +++ b/src/ducktools/classbuilder/annotations/annotations_314.py @@ -21,9 +21,9 @@ # SOFTWARE. try: - from _types import FunctionType as _FunctionType # type: ignore + from _types import FunctionType as _FunctionType, CellType as _CellType # type: ignore except ImportError: - from types import FunctionType as _FunctionType + from types import FunctionType as _FunctionType, CellType as _CellType class _LazyAnnotationLib: def __getattr__(self, item): @@ -36,6 +36,40 @@ def __getattr__(self, item): _lazy_annotationlib = _LazyAnnotationLib() +def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluation): + # Looks like Jelle might be changing the return type, copy this in here for now. + if not annotate.__closure__: + return None + freevars = annotate.__code__.co_freevars + new_closure = [] + for i, cell in enumerate(annotate.__closure__): + if i < len(freevars): + name = freevars[i] + else: + name = "__cell__" + new_cell = None + if allow_evaluation: + try: + cell.cell_contents + except ValueError: + pass + else: + new_cell = cell + if new_cell is None: + fwdref = _lazy_annotationlib._Stringifier( + name, + cell=cell, + owner=owner, + globals=annotate.__globals__, + is_class=is_class, + stringifier_dict=stringifier_dict, + ) + stringifier_dict.stringifiers.append(fwdref) + new_cell = _CellType(fwdref) + new_closure.append(new_cell) + return tuple(new_closure) + + def _call_annotate_forwardrefs(annotate, *, owner=None): # Get all annotations as unevaluated forward references # Logic taken from the call_annotate_function logic @@ -64,7 +98,7 @@ def _call_annotate_forwardrefs(annotate, *, owner=None): is_class=is_class, format=format, ) - closure = _lazy_annotationlib._build_closure( + closure = _build_closure( annotate, owner, is_class, globals, allow_evaluation=False ) func = _FunctionType( From 9ab32c2d06f7aa4f6da4976e5545b0290d73c69d Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Tue, 16 Sep 2025 12:59:57 +0100 Subject: [PATCH 12/12] use the match case syntax but provide the arguments correctly. --- .../annotations/annotations_314.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/ducktools/classbuilder/annotations/annotations_314.py b/src/ducktools/classbuilder/annotations/annotations_314.py index f1478aa..4d4e973 100644 --- a/src/ducktools/classbuilder/annotations/annotations_314.py +++ b/src/ducktools/classbuilder/annotations/annotations_314.py @@ -120,17 +120,18 @@ def make_annotate_func(annos): Format = _lazy_annotationlib.Format # Construct an annotation function from __annotations__ - def __annotate__(format, /): - if format in {Format.VALUE, Format.FORWARDREF, Format.STRING}: - new_annos = {} - for k, v in annos.items(): - v = evaluate_forwardref(v, format=format) - if not isinstance(v, str) and format == Format.STRING: - v = type_repr(v) - new_annos[k] = v - return new_annos - else: - raise NotImplementedError(format) + def __annotate__(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + match format: + case __Format.VALUE | __Format.FORWARDREF | __Format.STRING: + new_annos = {} + for k, v in annos.items(): + v = evaluate_forwardref(v, format=format) + if not isinstance(v, str) and format == __Format.STRING: + v = type_repr(v) + new_annos[k] = v + return new_annos + case _: + raise __NotImplementedError(format) return __annotate__