From 36c1048ab9f5be615cef4dfeb5eed614873f9f54 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 17 Jul 2025 10:55:08 +0100 Subject: [PATCH 1/5] Switch to heap types using PyType_FromSpec - Convert PyJuliaBase_Type creation from manual PyTypeObject to PyType_FromSpec - Add PyType_Spec and PyType_Slot structs to C API bindings - Add all Python type slot constants from typeslots.h - Use __weaklistoffset__ member definition for older Python compatibility - Implement proper slot-based type definition with buffer protocol support Closes #641 --- src/C/consts.jl | 101 ++++++++++++++++++++++++++++++++++++++++++++++ src/C/pointers.jl | 1 + src/JlWrap/C.jl | 53 ++++++++++++++++++------ 3 files changed, 142 insertions(+), 13 deletions(-) diff --git a/src/C/consts.jl b/src/C/consts.jl index 5df9be37..4560e928 100644 --- a/src/C/consts.jl +++ b/src/C/consts.jl @@ -345,3 +345,104 @@ const NPY_ARRAY_ALIGNED = 0x0100 const NPY_ARRAY_NOTSWAPPED = 0x0200 const NPY_ARRAY_WRITEABLE = 0x0400 const NPY_ARR_HAS_DESCR = 0x0800 + +# Python type slot constants +# From https://github.com/python/cpython/blob/main/Include/typeslots.h +const Py_bf_getbuffer = Cint(1) +const Py_bf_releasebuffer = Cint(2) +const Py_mp_ass_subscript = Cint(3) +const Py_mp_length = Cint(4) +const Py_mp_subscript = Cint(5) +const Py_nb_absolute = Cint(6) +const Py_nb_add = Cint(7) +const Py_nb_and = Cint(8) +const Py_nb_bool = Cint(9) +const Py_nb_divmod = Cint(10) +const Py_nb_float = Cint(11) +const Py_nb_floor_divide = Cint(12) +const Py_nb_index = Cint(13) +const Py_nb_inplace_add = Cint(14) +const Py_nb_inplace_and = Cint(15) +const Py_nb_inplace_floor_divide = Cint(16) +const Py_nb_inplace_lshift = Cint(17) +const Py_nb_inplace_multiply = Cint(18) +const Py_nb_inplace_or = Cint(19) +const Py_nb_inplace_power = Cint(20) +const Py_nb_inplace_remainder = Cint(21) +const Py_nb_inplace_rshift = Cint(22) +const Py_nb_inplace_subtract = Cint(23) +const Py_nb_inplace_true_divide = Cint(24) +const Py_nb_inplace_xor = Cint(25) +const Py_nb_int = Cint(26) +const Py_nb_invert = Cint(27) +const Py_nb_lshift = Cint(28) +const Py_nb_multiply = Cint(29) +const Py_nb_negative = Cint(30) +const Py_nb_or = Cint(31) +const Py_nb_positive = Cint(32) +const Py_nb_power = Cint(33) +const Py_nb_remainder = Cint(34) +const Py_nb_rshift = Cint(35) +const Py_nb_subtract = Cint(36) +const Py_nb_true_divide = Cint(37) +const Py_nb_xor = Cint(38) +const Py_sq_ass_item = Cint(39) +const Py_sq_concat = Cint(40) +const Py_sq_contains = Cint(41) +const Py_sq_inplace_concat = Cint(42) +const Py_sq_inplace_repeat = Cint(43) +const Py_sq_item = Cint(44) +const Py_sq_length = Cint(45) +const Py_sq_repeat = Cint(46) +const Py_tp_alloc = Cint(47) +const Py_tp_base = Cint(48) +const Py_tp_bases = Cint(49) +const Py_tp_call = Cint(50) +const Py_tp_clear = Cint(51) +const Py_tp_dealloc = Cint(52) +const Py_tp_del = Cint(53) +const Py_tp_descr_get = Cint(54) +const Py_tp_descr_set = Cint(55) +const Py_tp_doc = Cint(56) +const Py_tp_getattr = Cint(57) +const Py_tp_getattro = Cint(58) +const Py_tp_hash = Cint(59) +const Py_tp_init = Cint(60) +const Py_tp_is_gc = Cint(61) +const Py_tp_iter = Cint(62) +const Py_tp_iternext = Cint(63) +const Py_tp_methods = Cint(64) +const Py_tp_new = Cint(65) +const Py_tp_repr = Cint(66) +const Py_tp_richcompare = Cint(67) +const Py_tp_setattr = Cint(68) +const Py_tp_setattro = Cint(69) +const Py_tp_str = Cint(70) +const Py_tp_traverse = Cint(71) +const Py_tp_members = Cint(72) +const Py_tp_getset = Cint(73) +const Py_tp_free = Cint(74) +const Py_nb_matrix_multiply = Cint(75) +const Py_nb_inplace_matrix_multiply = Cint(76) +const Py_am_await = Cint(77) +const Py_am_aiter = Cint(78) +const Py_am_anext = Cint(79) +const Py_tp_finalize = Cint(80) +const Py_am_send = Cint(81) +const Py_tp_vectorcall = Cint(82) +const Py_tp_token = Cint(83) + +# PyType_Spec and PyType_Slot structs +# From https://docs.python.org/3/c-api/type.html#c.PyType_Spec +@kwdef struct PyType_Slot + slot::Cint = 0 + pfunc::Ptr{Cvoid} = C_NULL +end + +@kwdef struct PyType_Spec + name::Cstring = C_NULL + basicsize::Cint = 0 + itemsize::Cint = 0 + flags::Cuint = 0 + slots::Ptr{PyType_Slot} = C_NULL +end diff --git a/src/C/pointers.jl b/src/C/pointers.jl index 6faabb60..22343104 100644 --- a/src/C/pointers.jl +++ b/src/C/pointers.jl @@ -77,6 +77,7 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}( :PyType_IsSubtype => (PyPtr, PyPtr) => Cint, :PyType_Ready => (PyPtr,) => Cint, :PyType_GenericNew => (PyPtr, PyPtr, PyPtr) => PyPtr, + :PyType_FromSpec => (Ptr{Cvoid},) => PyPtr, # MAPPING :PyMapping_HasKeyString => (PyPtr, Ptr{Cchar}) => Cint, :PyMapping_SetItemString => (PyPtr, Ptr{Cchar}, PyPtr) => Cint, diff --git a/src/JlWrap/C.jl b/src/JlWrap/C.jl index fa96dd36..88901f8d 100644 --- a/src/JlWrap/C.jl +++ b/src/JlWrap/C.jl @@ -269,14 +269,17 @@ function _pyjl_deserialize(t::C.PyPtr, v::C.PyPtr) end const _pyjlbase_name = "juliacall.ValueBase" -const _pyjlbase_type = fill(C.PyTypeObject()) const _pyjlbase_isnull_name = "_jl_isnull" const _pyjlbase_callmethod_name = "_jl_callmethod" const _pyjlbase_reduce_name = "__reduce__" const _pyjlbase_serialize_name = "_jl_serialize" const _pyjlbase_deserialize_name = "_jl_deserialize" +const _pyjlbase_weaklistoffset_name = "__weaklistoffset__" const _pyjlbase_methods = Vector{C.PyMethodDef}() +const _pyjlbase_members = Vector{C.PyMemberDef}() const _pyjlbase_as_buffer = fill(C.PyBufferProcs()) +const _pyjlbase_slots = Vector{C.PyType_Slot}() +const _pyjlbase_spec = fill(C.PyType_Spec()) function init_c() empty!(_pyjlbase_methods) @@ -313,21 +316,45 @@ function init_c() get = @cfunction(_pyjl_get_buffer, Cint, (C.PyPtr, Ptr{C.Py_buffer}, Cint)), release = @cfunction(_pyjl_release_buffer, Cvoid, (C.PyPtr, Ptr{C.Py_buffer})), ) - _pyjlbase_type[] = C.PyTypeObject( + + # Create members for weakref support + empty!(_pyjlbase_members) + push!( + _pyjlbase_members, + C.PyMemberDef( + name = pointer(_pyjlbase_weaklistoffset_name), + typ = C.Py_T_PYSSIZET, + offset = fieldoffset(PyJuliaValueObject, 3), + flags = C.Py_READONLY, + ), + C.PyMemberDef(), # NULL terminator + ) + + # Create slots for PyType_Spec + empty!(_pyjlbase_slots) + push!( + _pyjlbase_slots, + C.PyType_Slot(slot = C.Py_tp_new, pfunc = @cfunction(_pyjl_new, C.PyPtr, (C.PyPtr, C.PyPtr, C.PyPtr))), + C.PyType_Slot(slot = C.Py_tp_dealloc, pfunc = @cfunction(_pyjl_dealloc, Cvoid, (C.PyPtr,))), + C.PyType_Slot(slot = C.Py_tp_methods, pfunc = pointer(_pyjlbase_methods)), + C.PyType_Slot(slot = C.Py_tp_members, pfunc = pointer(_pyjlbase_members)), + C.PyType_Slot(slot = C.Py_bf_getbuffer, pfunc = @cfunction(_pyjl_get_buffer, Cint, (C.PyPtr, Ptr{C.Py_buffer}, Cint))), + C.PyType_Slot(slot = C.Py_bf_releasebuffer, pfunc = @cfunction(_pyjl_release_buffer, Cvoid, (C.PyPtr, Ptr{C.Py_buffer}))), + C.PyType_Slot(), # NULL terminator + ) + + # Create PyType_Spec + _pyjlbase_spec[] = C.PyType_Spec( name = pointer(_pyjlbase_name), - basicsize = sizeof(PyJuliaValueObject), - # new = C.POINTERS.PyType_GenericNew, - new = @cfunction(_pyjl_new, C.PyPtr, (C.PyPtr, C.PyPtr, C.PyPtr)), - dealloc = @cfunction(_pyjl_dealloc, Cvoid, (C.PyPtr,)), + basicsize = Cint(sizeof(PyJuliaValueObject)), + itemsize = Cint(0), flags = C.Py_TPFLAGS_BASETYPE | C.Py_TPFLAGS_HAVE_VERSION_TAG, - weaklistoffset = fieldoffset(PyJuliaValueObject, 3), - # getattro = C.POINTERS.PyObject_GenericGetAttr, - # setattro = C.POINTERS.PyObject_GenericSetAttr, - methods = pointer(_pyjlbase_methods), - as_buffer = pointer(_pyjlbase_as_buffer), + slots = pointer(_pyjlbase_slots), ) - o = PyJuliaBase_Type[] = C.PyPtr(pointer(_pyjlbase_type)) - if C.PyType_Ready(o) == -1 + + # Create type using PyType_FromSpec + o = PyJuliaBase_Type[] = C.PyType_FromSpec(pointer(_pyjlbase_spec)) + if o == C.PyNULL C.PyErr_Print() error("Error initializing 'juliacall.ValueBase'") end From 0ebe0de86d6af32e39a7e49ddab38d84e726eba3 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 17 Jul 2025 11:07:15 +0100 Subject: [PATCH 2/5] Clean up PyType_Spec creation - Remove unnecessary itemsize field (defaults to 0) - Remove unnecessary Cint conversion for basicsize (happens automatically) --- src/JlWrap/C.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/JlWrap/C.jl b/src/JlWrap/C.jl index 88901f8d..299be3bc 100644 --- a/src/JlWrap/C.jl +++ b/src/JlWrap/C.jl @@ -346,8 +346,7 @@ function init_c() # Create PyType_Spec _pyjlbase_spec[] = C.PyType_Spec( name = pointer(_pyjlbase_name), - basicsize = Cint(sizeof(PyJuliaValueObject)), - itemsize = Cint(0), + basicsize = sizeof(PyJuliaValueObject), flags = C.Py_TPFLAGS_BASETYPE | C.Py_TPFLAGS_HAVE_VERSION_TAG, slots = pointer(_pyjlbase_slots), ) From 5323048227cd21b99f599fcbf41102ec8eeb77dc Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 17 Jul 2025 11:11:08 +0100 Subject: [PATCH 3/5] update release notes --- docs/src/releasenotes.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/src/releasenotes.md b/docs/src/releasenotes.md index 8db2325b..841ae268 100644 --- a/docs/src/releasenotes.md +++ b/docs/src/releasenotes.md @@ -1,5 +1,8 @@ # Release Notes +## Unreleased +* Internal: Use heap-allocated types (PyType_FromSpec) to improve ABI compatibility. + ## 0.9.26 (2025-07-15) * Added PySide6 support to the GUI compatibility layer. * Added FAQ on interactive threads. From 2ab39a4f92b225b1fc83b55e437e4300a7066f94 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 17 Jul 2025 12:08:10 +0100 Subject: [PATCH 4/5] Remove unused _pyjlbase_as_buffer constant Buffer protocol methods are now handled through PyType_Slot mechanism, making the old _pyjlbase_as_buffer constant unnecessary. --- pysrc/juliacall/juliapkg.json | 4 +++- src/JlWrap/C.jl | 5 ----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pysrc/juliacall/juliapkg.json b/pysrc/juliacall/juliapkg.json index 2315c1b6..bb4167bc 100644 --- a/pysrc/juliacall/juliapkg.json +++ b/pysrc/juliacall/juliapkg.json @@ -3,7 +3,9 @@ "packages": { "PythonCall": { "uuid": "6099a3de-0909-46bc-b1f4-468b9a2dfc0d", - "version": "=0.9.26" + "version": "=0.9.26", + "path": "../..", + "dev": true }, "OpenSSL_jll": { "uuid": "458c3c95-2e84-50aa-8efc-19380b2a3a95", diff --git a/src/JlWrap/C.jl b/src/JlWrap/C.jl index 299be3bc..50c17987 100644 --- a/src/JlWrap/C.jl +++ b/src/JlWrap/C.jl @@ -277,7 +277,6 @@ const _pyjlbase_deserialize_name = "_jl_deserialize" const _pyjlbase_weaklistoffset_name = "__weaklistoffset__" const _pyjlbase_methods = Vector{C.PyMethodDef}() const _pyjlbase_members = Vector{C.PyMemberDef}() -const _pyjlbase_as_buffer = fill(C.PyBufferProcs()) const _pyjlbase_slots = Vector{C.PyType_Slot}() const _pyjlbase_spec = fill(C.PyType_Spec()) @@ -312,10 +311,6 @@ function init_c() ), C.PyMethodDef(), ) - _pyjlbase_as_buffer[] = C.PyBufferProcs( - get = @cfunction(_pyjl_get_buffer, Cint, (C.PyPtr, Ptr{C.Py_buffer}, Cint)), - release = @cfunction(_pyjl_release_buffer, Cvoid, (C.PyPtr, Ptr{C.Py_buffer})), - ) # Create members for weakref support empty!(_pyjlbase_members) From 12a0d2c78a681c87b7567ea68abae127237208d7 Mon Sep 17 00:00:00 2001 From: Christopher Doris Date: Thu, 17 Jul 2025 14:02:46 +0100 Subject: [PATCH 5/5] revert --- pysrc/juliacall/juliapkg.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pysrc/juliacall/juliapkg.json b/pysrc/juliacall/juliapkg.json index bb4167bc..2315c1b6 100644 --- a/pysrc/juliacall/juliapkg.json +++ b/pysrc/juliacall/juliapkg.json @@ -3,9 +3,7 @@ "packages": { "PythonCall": { "uuid": "6099a3de-0909-46bc-b1f4-468b9a2dfc0d", - "version": "=0.9.26", - "path": "../..", - "dev": true + "version": "=0.9.26" }, "OpenSSL_jll": { "uuid": "458c3c95-2e84-50aa-8efc-19380b2a3a95",