Skip to content

fix(types): type hints from future python versions #5693

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 23 commits into
base: master
Choose a base branch
from
Open

fix(types): type hints from future python versions #5693

Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
f430803
fix future type hints
InvincibleRMC May 25, 2025
14f9c7d
Merge branch 'master' into fix-remaining-future-type-hints
InvincibleRMC May 25, 2025
e374a2f
style: pre-commit fixes
pre-commit-ci[bot] May 25, 2025
325f080
remove unused var
InvincibleRMC May 25, 2025
6dcc1d9
remove union_helper
InvincibleRMC May 25, 2025
8570b25
fix speelling error
InvincibleRMC May 25, 2025
5ea93d4
base case for union_concat
InvincibleRMC May 25, 2025
f514886
style: pre-commit fixes
pre-commit-ci[bot] May 25, 2025
0d2de7a
add case for one descr
InvincibleRMC May 25, 2025
c489b95
weakref and final test
InvincibleRMC May 25, 2025
55c2999
Merge branch 'master' into fix-remaining-future-type-hints
InvincibleRMC May 26, 2025
41e2bbe
Add acrpss_version_type_hint_checker
InvincibleRMC May 26, 2025
bd41651
cleanup
InvincibleRMC May 26, 2025
3d4185a
style: pre-commit fixes
pre-commit-ci[bot] May 26, 2025
df38f99
remove test.pyi
InvincibleRMC May 26, 2025
eecf91d
Merge branch 'master' into fix-remaining-future-type-hints
InvincibleRMC May 27, 2025
8746f74
use new unions and add fixture
InvincibleRMC May 27, 2025
d2c2e60
Merge branch 'master' into fix-remaining-future-type-hints
InvincibleRMC May 27, 2025
f733d21
timohl suggested cleanup
InvincibleRMC May 27, 2025
62a2ee6
style: pre-commit fixes
pre-commit-ci[bot] May 27, 2025
06a4188
add missing auto
InvincibleRMC May 27, 2025
c287018
style: pre-commit fixes
pre-commit-ci[bot] May 27, 2025
08ff0d3
move operator| def
InvincibleRMC May 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions include/pybind11/cast.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ class type_caster<void> : public type_caster<void_type> {
template <typename T>
using cast_op_type = void *&;
explicit operator void *&() { return value; }
static constexpr auto name = const_name("types.CapsuleType");
static constexpr auto name = const_name(PYBIND11_CAPSULE_TYPE_TYPE_HINT);

private:
void *value = nullptr;
Expand Down Expand Up @@ -1361,7 +1361,11 @@ struct handle_type_name<dict> {
};
template <>
struct handle_type_name<anyset> {
#if PYBIND11_USE_NEW_UNIONS
static constexpr auto name = const_name("set | frozenset");
#else
static constexpr auto name = const_name("typing.Union[set, frozenset]");
#endif
};
template <>
struct handle_type_name<set> {
Expand Down Expand Up @@ -1441,15 +1445,15 @@ struct handle_type_name<type> {
};
template <>
struct handle_type_name<capsule> {
static constexpr auto name = const_name("types.CapsuleType");
static constexpr auto name = const_name(PYBIND11_CAPSULE_TYPE_TYPE_HINT);
};
template <>
struct handle_type_name<ellipsis> {
static constexpr auto name = const_name("ellipsis");
};
template <>
struct handle_type_name<weakref> {
static constexpr auto name = const_name("weakref");
static constexpr auto name = const_name("weakref.ReferenceType");
};
template <>
struct handle_type_name<args> {
Expand Down
25 changes: 25 additions & 0 deletions include/pybind11/detail/common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,13 +261,38 @@
# define PYBIND11_HAS_SUBINTERPRETER_SUPPORT
#endif

// 3.13 Compatibility
#if 0x030D0000 <= PY_VERSION_HEX
# define PYBIND11_TYPE_IS_TYPE_HINT "typing.TypeIs"
# define PYBIND11_CAPSULE_TYPE_TYPE_HINT "types.CapsuleType"
#else
# define PYBIND11_TYPE_IS_TYPE_HINT "typing_extensions.TypeIs"
# define PYBIND11_CAPSULE_TYPE_TYPE_HINT "typing_extensions.CapsuleType"
#endif

// 3.12 Compatibility
#if 0x030C0000 <= PY_VERSION_HEX
# define PYBIND11_BUFFER_TYPE_HINT "collections.abc.Buffer"
#else
# define PYBIND11_BUFFER_TYPE_HINT "typing_extensions.Buffer"
#endif

// 3.11 Compatibility
#if 0x030B0000 <= PY_VERSION_HEX
# define PYBIND11_NEVER_TYPE_HINT "typing.Never"
#else
# define PYBIND11_NEVER_TYPE_HINT "typing_extensions.Never"
#endif

// 3.10 Compatibility
#if 0x030A0000 <= PY_VERSION_HEX
# define PYBIND11_USE_NEW_UNIONS true
# define PYBIND11_TYPE_GUARD_TYPE_HINT "typing.TypeGuard"
#else
# define PYBIND11_USE_NEW_UNIONS false
# define PYBIND11_TYPE_GUARD_TYPE_HINT "typing_extensions.TypeGuard"
#endif

// #define PYBIND11_STR_LEGACY_PERMISSIVE
// If DEFINED, pybind11::str can hold PyUnicodeObject or PyBytesObject
// (probably surprising and never documented, but this was the
Expand Down
24 changes: 24 additions & 0 deletions include/pybind11/detail/descr.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,12 +157,18 @@ constexpr descr<1, Type> _() {
#endif // #ifndef _

constexpr descr<0> concat() { return {}; }
constexpr descr<0> union_concat() { return {}; }

template <size_t N, typename... Ts>
constexpr descr<N, Ts...> concat(const descr<N, Ts...> &descr) {
return descr;
}

template <size_t N, typename... Ts>
constexpr descr<N, Ts...> union_concat(const descr<N, Ts...> &descr) {
return descr;
}

#ifdef __cpp_fold_expressions
template <size_t N1, size_t N2, typename... Ts1, typename... Ts2>
constexpr descr<N1 + N2 + 2, Ts1..., Ts2...> operator,(const descr<N1, Ts1...> &a,
Expand All @@ -174,12 +180,30 @@ template <size_t N, typename... Ts, typename... Args>
constexpr auto concat(const descr<N, Ts...> &d, const Args &...args) {
return (d, ..., args);
}

template <size_t N1, size_t N2, typename... Ts1, typename... Ts2>
constexpr descr<N1 + N2 + 3, Ts1..., Ts2...> operator|(const descr<N1, Ts1...> &a,
const descr<N2, Ts2...> &b) {
return a + const_name(" | ") + b;
}

template <size_t N, typename... Ts, typename... Args>
constexpr auto union_concat(const descr<N, Ts...> &d, const Args &...args) {
return (d | ... | args);
}

#else
template <size_t N, typename... Ts, typename... Args>
constexpr auto concat(const descr<N, Ts...> &d, const Args &...args)
-> decltype(std::declval<descr<N + 2, Ts...>>() + concat(args...)) {
return d + const_name(", ") + concat(args...);
}

template <size_t N, typename... Ts, typename... Args>
constexpr auto union_concat(const descr<N, Ts...> &d, const Args &...args)
-> decltype(std::declval<descr<N + 3, Ts...>>() + union_concat(args...)) {
return d + const_name(" | ") + union_concat(args...);
}
#endif

template <size_t N, typename... Ts>
Expand Down
4 changes: 4 additions & 0 deletions include/pybind11/stl/filesystem.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,11 @@ struct path_caster {
return true;
}

# if PYBIND11_USE_NEW_UNIONS
PYBIND11_TYPE_CASTER(T, io_name("os.PathLike | str | bytes", "pathlib.Path"));
# else
PYBIND11_TYPE_CASTER(T, io_name("typing.Union[os.PathLike, str, bytes]", "pathlib.Path"));
# endif
};

#endif // PYBIND11_HAS_FILESYSTEM || defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM)
Expand Down
19 changes: 14 additions & 5 deletions include/pybind11/typing.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,15 +218,24 @@ struct handle_type_name<typing::Type<T>> {

template <typename... Types>
struct handle_type_name<typing::Union<Types...>> {
#if PYBIND11_USE_NEW_UNIONS
static constexpr auto name = ::pybind11::detail::union_concat(make_caster<Types>::name...);
#else
static constexpr auto name = const_name("typing.Union[")
+ ::pybind11::detail::concat(make_caster<Types>::name...)
+ const_name("]");
#endif
};

template <typename T>
struct handle_type_name<typing::Optional<T>> {
#if PYBIND11_USE_NEW_UNIONS
static constexpr auto name
= ::pybind11::detail::union_concat(make_caster<T>::name, make_caster<none>::name);
#else
static constexpr auto name
= const_name("typing.Optional[") + make_caster<T>::name + const_name("]");
#endif
};

template <typename T>
Expand All @@ -244,14 +253,14 @@ struct handle_type_name<typing::ClassVar<T>> {

template <typename T>
struct handle_type_name<typing::TypeGuard<T>> {
static constexpr auto name
= const_name("typing.TypeGuard[") + make_caster<T>::name + const_name("]");
static constexpr auto name = const_name(PYBIND11_TYPE_GUARD_TYPE_HINT) + const_name("[")
+ make_caster<T>::name + const_name("]");
};

template <typename T>
struct handle_type_name<typing::TypeIs<T>> {
static constexpr auto name
= const_name("typing.TypeIs[") + make_caster<T>::name + const_name("]");
static constexpr auto name = const_name(PYBIND11_TYPE_IS_TYPE_HINT) + const_name("[")
+ make_caster<T>::name + const_name("]");
};

template <>
Expand All @@ -261,7 +270,7 @@ struct handle_type_name<typing::NoReturn> {

template <>
struct handle_type_name<typing::Never> {
static constexpr auto name = const_name("typing.Never");
static constexpr auto name = const_name(PYBIND11_NEVER_TYPE_HINT);
};

#if defined(PYBIND11_TYPING_H_HAS_STRING_LITERAL)
Expand Down
18 changes: 18 additions & 0 deletions tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,3 +242,21 @@ def pytest_report_header():
lines.append("free-threaded Python build")

return lines


def across_version_type_hint_checker(doc: str, expected: str) -> None:
if sys.version_info < (3, 13):
expected = expected.replace("typing.TypeIs", "typing_extensions.TypeIs")
expected = expected.replace(
"types.CapsuleType", "typing_extensions.CapsuleType"
)
if sys.version_info < (3, 12):
expected = expected.replace(
"collections.abc.Buffer", "typing_extensions.Buffer"
)
if sys.version_info < (3, 11):
expected = expected.replace("typing.Never", "typing_extensions.Never")
if sys.version_info < (3, 10):
expected = expected.replace("typing.TypeGuard", "typing_extensions.TypeGuard")

assert doc == expected
18 changes: 10 additions & 8 deletions tests/test_opaque_types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import annotations

import pytest
from conftest import across_version_type_hint_checker
Copy link
Collaborator

@henryiii henryiii May 27, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please, never import from conftest.py. You can put fixtures, etc. there, but don't import from it. It causes all sorts of issues (such as the issue holding up cibuildwheel 3.0 in numpy right now!); it's not meant to be an importable module for users. We sadly have one case (tests/test_docs_advanced_cast_custom.py), but I'd like to get rid of that, not add more. Edit: it's only at type check time!

Instead of adding a test/utils directory just for this, and requiring PyTest 7+, this can be wrapped in a fixture for now.

Since we do have one case of importing from conftest, I'm also okay to leave this in for now and add the test utils in a followup later, as anything broken by it will already have workarounds.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this also reduce the quality of assertion rewriting?

Copy link
Collaborator

@henryiii henryiii May 27, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FYI, wrapping it in a fixture could look like:

@pytest.fixture
def backport_typehints() -> Callable[[str], str]:
    d = {}
    if sys.version_info < (3, 13):
        d["typing.TypeIs"] = "typing_extensions.TypeIs"
        d["types.CapsuleType"] = "typing_extensions.CapsuleType"
    if sys.version_info < (3, 12):
        d["collections.abc.Buffer"] = "typing_extensions.Buffer"
    if sys.version_info < (3, 11):
        d["typing.Never"] = "typing_extensions.Never"
    if sys.version_info < (3, 10):
        d["typing.TypeGuard"] = "typing_extensions.TypeGuard"
       
    def backport(text: str) -> str:
        for old, new in d.items():
            text = text.replace(old, new)
        return text
    return backport

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please, never import from conftest.py. You can put fixtures, etc. there, but don't import from it. It causes all sorts of issues (such as the issue holding up cibuildwheel 3.0 in numpy right now!); it's not meant to be an importable module for users. We sadly have one case (tests/test_docs_advanced_cast_custom.py), but I'd like to get rid of that, not add more. Edit: it's only at type check time!

Just to clarify and to learn from it (since I added that conftest.py import in tests/test_docs_advanced_cast_custom.py):
Would you also discourage from importing conftest.py while using if TYPE_CHECKING?
Generally, I like having type hints for function arguments for better code readability and navigation.
Vscode/Pylance is capable of matching test arguments to fixtures and show gray type hints without manual type annotations, so I could see not having those in tests.


import env
from pybind11_tests import ConstructorStats, UserType
Expand Down Expand Up @@ -37,14 +38,15 @@ def test_pointers(msg):

with pytest.raises(TypeError) as excinfo:
m.get_void_ptr_value([1, 2, 3]) # This should not work
assert (
msg(excinfo.value)
== """
get_void_ptr_value(): incompatible function arguments. The following argument types are supported:
1. (arg0: types.CapsuleType) -> int

Invoked with: [1, 2, 3]
"""

across_version_type_hint_checker(
msg(excinfo.value),
"""
get_void_ptr_value(): incompatible function arguments. The following argument types are supported:
1. (arg0: types.CapsuleType) -> int

Invoked with: [1, 2, 3]
""",
)

assert m.return_null_str() is None
Expand Down
4 changes: 4 additions & 0 deletions tests/test_pytypes.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,11 @@ namespace detail {

template <>
struct type_caster<RealNumber> {
#if PYBIND11_USE_NEW_UNIONS
PYBIND11_TYPE_CASTER(RealNumber, io_name("float | int", "float"));
#else
PYBIND11_TYPE_CASTER(RealNumber, io_name("typing.Union[float, int]", "float"));
#endif

static handle cast(const RealNumber &number, return_value_policy, handle) {
return py::float_(number.value).release();
Expand Down
Loading
Loading