Switch to heap types

.github/workflows/tests.yml

@@ -19,7 +19,7 @@ jobs:
       matrix:
         arch: [x64] # x86 unsupported by MicroMamba
         os: [ubuntu-latest, windows-latest, macos-latest]
-        jlversion: ['1','1.9']
+        jlversion: ['1','1.10']
 
     steps:
       - uses: actions/checkout@v4
@@ -56,7 +56,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        pyversion: ["3", "3.8"]
+        pyversion: ["3", "3.9"]
 
     steps:
       - uses: actions/checkout@v4

pyproject.toml

@@ -12,7 +12,7 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
 ]
-requires-python = ">=3.8, <4"
+requires-python = ">=3.9, <4"
 dependencies = ["juliapkg >=0.1.17, <0.2"]
 
 [dependency-groups]

src/C/consts.jl

@@ -249,79 +249,32 @@ end
     weakreflist::PyPtr = PyNULL
 end
 
-@kwdef struct PyTypeObject
-    ob_base::PyVarObject = PyVarObject()
-    name::Cstring = C_NULL
-
-    basicsize::Py_ssize_t = 0
-    itemsize::Py_ssize_t = 0
-
-    dealloc::Ptr{Cvoid} = C_NULL
-    vectorcall_offset::Py_ssize_t = 0
-    getattr::Ptr{Cvoid} = C_NULL
-    setattr::Ptr{Cvoid} = C_NULL
-    as_async::Ptr{Cvoid} = C_NULL
-    repr::Ptr{Cvoid} = C_NULL
-
-    as_number::Ptr{PyNumberMethods} = C_NULL
-    as_sequence::Ptr{PySequenceMethods} = C_NULL
-    as_mapping::Ptr{PyMappingMethods} = C_NULL
-
-    hash::Ptr{Cvoid} = C_NULL
-    call::Ptr{Cvoid} = C_NULL
-    str::Ptr{Cvoid} = C_NULL
-    getattro::Ptr{Cvoid} = C_NULL
-    setattro::Ptr{Cvoid} = C_NULL
-
-    as_buffer::Ptr{PyBufferProcs} = C_NULL
-
-    flags::Culong = 0
-
-    doc::Cstring = C_NULL
+@kwdef struct PyType_Slot
+    slot::Cint = 0
+    pfunc::Ptr{Cvoid} = C_NULL
+end
 
-    traverse::Ptr{Cvoid} = C_NULL
-
-    clear::Ptr{Cvoid} = C_NULL
-
-    richcompare::Ptr{Cvoid} = C_NULL
-
-    weaklistoffset::Py_ssize_t = 0
-
-    iter::Ptr{Cvoid} = C_NULL
-    iternext::Ptr{Cvoid} = C_NULL
-
-    methods::Ptr{PyMethodDef} = C_NULL
-    members::Ptr{PyMemberDef} = C_NULL
-    getset::Ptr{PyGetSetDef} = C_NULL
-    base::PyPtr = C_NULL
-    dict::PyPtr = C_NULL
-    descr_get::Ptr{Cvoid} = C_NULL
-    descr_set::Ptr{Cvoid} = C_NULL
-    dictoffset::Py_ssize_t = 0
-    init::Ptr{Cvoid} = C_NULL
-    alloc::Ptr{Cvoid} = C_NULL
-    new::Ptr{Cvoid} = C_NULL
-    free::Ptr{Cvoid} = C_NULL
-    is_gc::Ptr{Cvoid} = C_NULL
-    bases::PyPtr = C_NULL
-    mro::PyPtr = C_NULL
-    cache::PyPtr = C_NULL
-    subclasses::PyPtr = C_NULL
-    weaklist::PyPtr = C_NULL
-    del::Ptr{Cvoid} = C_NULL
-
-    version_tag::Cuint = 0
-
-    finalize::Ptr{Cvoid} = C_NULL
-    vectorcall::Ptr{Cvoid} = C_NULL
-
-    # Python 3.12+ fields
-    watched::Cuchar = 0
-
-    # Python 3.13+ fields  
-    versions_used::UInt16 = 0
+@kwdef struct PyType_Spec
+    name::Cstring = C_NULL
+    basicsize::Cint = 0
+    itemsize::Cint = 0
+    flags::Cuint = 0
+    slots::Ptr{PyType_Slot} = C_NULL
 end
 
+# These numbers are part of the CPython stable ABI and
+# are guaranteed to be the same.
+# https://raw.githubusercontent.com/python/cpython/main/Include/typeslots.h
+const Py_bf_getbuffer = 1
+const Py_bf_releasebuffer = 2
+const Py_tp_alloc = 47
+const Py_tp_dealloc = 52
+const Py_tp_methods = 64
+const Py_tp_new = 65
+const Py_tp_members = 72
+const Py_tp_getset = 73
+const Py_tp_free = 74
+
 @kwdef struct PySimpleObject{T}
     ob_base::PyObject = PyObject()
     value::T

src/C/extras.jl

@@ -8,37 +8,51 @@ Py_TypeCheck(o, t) = Base.GC.@preserve o t PyType_IsSubtype(Py_Type(asptr(o)), a
 Py_TypeCheckFast(o, f::Integer) = Base.GC.@preserve o PyType_IsSubtypeFast(Py_Type(asptr(o)), f)
 
 PyType_IsSubtypeFast(t, f::Integer) =
-    Base.GC.@preserve t Cint(!iszero(UnsafePtr{PyTypeObject}(asptr(t)).flags[] & f))
+    Base.GC.@preserve t Cint(!iszero(PyType_GetFlags(asptr(t)) & f))
+
+function pytypename(t::PyPtr)
+    name_obj = PyObject_GetAttrString(t, "__name__")
+    name_obj == C_NULL && return "(unknown type)"
+    cstr = PyUnicode_AsUTF8(name_obj)
+    if cstr == C_NULL
+        Py_DecRef(name_obj)
+        return "(unknown type)"
+    else
+        str = unsafe_string(cstr)
+        Py_DecRef(name_obj)
+        return str
+    end
+end
 
 PyMemoryView_GET_BUFFER(m) = Base.GC.@preserve m Ptr{Py_buffer}(UnsafePtr{PyMemoryViewObject}(asptr(m)).view)
 
 PyType_CheckBuffer(t) = Base.GC.@preserve t begin
-    p = UnsafePtr{PyTypeObject}(asptr(t)).as_buffer[]
-    return p != C_NULL && p.get[!] != C_NULL
+    p = PyType_GetSlot(asptr(t), Py_bf_getbuffer)
+    return p != C_NULL
 end
 
 PyObject_CheckBuffer(o) = Base.GC.@preserve o PyType_CheckBuffer(Py_Type(asptr(o)))
 
 PyObject_GetBuffer(_o, b, flags) = Base.GC.@preserve _o begin
     o = asptr(_o)
-    p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[]
-    if p == C_NULL || p.get[!] == C_NULL
+    getbuf = PyType_GetSlot(Py_Type(o), Py_bf_getbuffer)
+    if getbuf == C_NULL
         PyErr_SetString(
             POINTERS.PyExc_TypeError,
-            "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'",
+            "a bytes-like object is required, not '$(pytypename(Py_Type(o)))'",
         )
         return Cint(-1)
     end
-    return ccall(p.get[!], Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags)
+    return ccall(getbuf, Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags)
 end
 
 PyBuffer_Release(_b) = begin
     b = UnsafePtr(Base.unsafe_convert(Ptr{Py_buffer}, _b))
     o = b.obj[]
     o == C_NULL && return
-    p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[]
-    if (p != C_NULL && p.release[!] != C_NULL)
-        ccall(p.release[!], Cvoid, (PyPtr, Ptr{Py_buffer}), o, b)
+    releasebuf = PyType_GetSlot(Py_Type(o), Py_bf_releasebuffer)
+    if releasebuf != C_NULL
+        ccall(releasebuf, Cvoid, (PyPtr, Ptr{Py_buffer}), o, b)
     end
     b.obj[] = C_NULL
     Py_DecRef(o)

src/C/pointers.jl

@@ -77,6 +77,11 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}(
     :PyType_IsSubtype => (PyPtr, PyPtr) => Cint,
     :PyType_Ready => (PyPtr,) => Cint,
     :PyType_GenericNew => (PyPtr, PyPtr, PyPtr) => PyPtr,
+    :PyType_FromSpec => (Ptr{PyType_Spec},) => PyPtr,
+    :PyType_FromSpecWithBases => (Ptr{PyType_Spec}, PyPtr) => PyPtr,
+    :PyType_GetSlot => (PyPtr, Cint) => Ptr{Cvoid},
+    :PyType_GetFlags => (PyPtr,) => Culong,
+    # TODO: PyType_GetName, once we are Python 3.11+ exclusively
     # MAPPING
     :PyMapping_HasKeyString => (PyPtr, Ptr{Cchar}) => Cint,
     :PyMapping_SetItemString => (PyPtr, Ptr{Cchar}, PyPtr) => Cint,
@@ -139,6 +144,7 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}(
     :PyComplex_AsCComplex => (PyPtr,) => Py_complex,
     # STR
     :PyUnicode_DecodeUTF8 => (Ptr{Cchar}, Py_ssize_t, Ptr{Cchar}) => PyPtr,
+    :PyUnicode_AsUTF8 => (PyPtr,) => Ptr{Cchar},
     :PyUnicode_AsUTF8String => (PyPtr,) => PyPtr,
     :PyUnicode_InternInPlace => (Ptr{PyPtr},) => Cvoid,
     # BYTES

src/JlWrap/C.jl

@@ -9,7 +9,6 @@ using Serialization: serialize, deserialize
 @kwdef struct PyJuliaValueObject
     ob_base::C.PyObject = C.PyObject()
     value::Int = 0
-    weaklist::C.PyPtr = C_NULL
 end
 
 const PyJuliaBase_Type = Ref(C.PyNULL)
@@ -21,9 +20,9 @@ const PYJLVALUES = []
 const PYJLFREEVALUES = Int[]
 
 function _pyjl_new(t::C.PyPtr, ::C.PyPtr, ::C.PyPtr)
-    o = ccall(UnsafePtr{C.PyTypeObject}(t).alloc[!], C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0)
+    allocptr = C.PyType_GetSlot(t, C.Py_tp_alloc)
+    o = ccall(allocptr, C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0)
     o == C.PyNULL && return C.PyNULL
-    UnsafePtr{PyJuliaValueObject}(o).weaklist[] = C.PyNULL
     UnsafePtr{PyJuliaValueObject}(o).value[] = 0
     return o
 end
@@ -34,8 +33,8 @@ function _pyjl_dealloc(o::C.PyPtr)
         PYJLVALUES[idx] = nothing
         push!(PYJLFREEVALUES, idx)
     end
-    UnsafePtr{PyJuliaValueObject}(o).weaklist[!] == C.PyNULL || C.PyObject_ClearWeakRefs(o)
-    ccall(UnsafePtr{C.PyTypeObject}(C.Py_Type(o)).free[!], Cvoid, (C.PyPtr,), o)
+    freeptr = C.PyType_GetSlot(C.Py_Type(o), C.Py_tp_free)
+    ccall(freeptr, Cvoid, (C.PyPtr,), o)
     nothing
 end
 
@@ -269,14 +268,12 @@ function _pyjl_deserialize(t::C.PyPtr, v::C.PyPtr)
 end
 
 const _pyjlbase_name = "juliacall.ValueBase"
-const _pyjlbase_type = fill(C.PyTypeObject())
 const _pyjlbase_isnull_name = "_jl_isnull"
 const _pyjlbase_callmethod_name = "_jl_callmethod"
 const _pyjlbase_reduce_name = "__reduce__"
 const _pyjlbase_serialize_name = "_jl_serialize"
 const _pyjlbase_deserialize_name = "_jl_deserialize"
 const _pyjlbase_methods = Vector{C.PyMethodDef}()
-const _pyjlbase_as_buffer = fill(C.PyBufferProcs())
 
 function init_c()
     empty!(_pyjlbase_methods)
@@ -309,28 +306,37 @@ function init_c()
         ),
         C.PyMethodDef(),
     )
-    _pyjlbase_as_buffer[] = C.PyBufferProcs(
-        get = @cfunction(_pyjl_get_buffer, Cint, (C.PyPtr, Ptr{C.Py_buffer}, Cint)),
-        release = @cfunction(_pyjl_release_buffer, Cvoid, (C.PyPtr, Ptr{C.Py_buffer})),
-    )
-    _pyjlbase_type[] = C.PyTypeObject(
+    slots = C.PyType_Slot[
+        C.PyType_Slot(slot = C.Py_tp_dealloc,
+                      pfunc = @cfunction(_pyjl_dealloc, Cvoid, (C.PyPtr,))),
+        C.PyType_Slot(slot = C.Py_tp_new,
+                      pfunc = @cfunction(_pyjl_new, C.PyPtr,
+                                         (C.PyPtr, C.PyPtr, C.PyPtr))),
+        C.PyType_Slot(slot = C.Py_tp_methods, pfunc = pointer(_pyjlbase_methods)),
+        C.PyType_Slot(slot = C.Py_bf_getbuffer,
+                      pfunc = @cfunction(_pyjl_get_buffer, Cint,
+                                         (C.PyPtr, Ptr{C.Py_buffer}, Cint))),
+        C.PyType_Slot(slot = C.Py_bf_releasebuffer,
+                      pfunc = @cfunction(_pyjl_release_buffer, Cvoid,
+                                         (C.PyPtr, Ptr{C.Py_buffer}))),
+        C.PyType_Slot(),
+    ]
+    spec = C.PyType_Spec(
         name = pointer(_pyjlbase_name),
         basicsize = sizeof(PyJuliaValueObject),
-        # new = C.POINTERS.PyType_GenericNew,
-        new = @cfunction(_pyjl_new, C.PyPtr, (C.PyPtr, C.PyPtr, C.PyPtr)),
-        dealloc = @cfunction(_pyjl_dealloc, Cvoid, (C.PyPtr,)),
+        itemsize = 0,
         flags = C.Py_TPFLAGS_BASETYPE | C.Py_TPFLAGS_HAVE_VERSION_TAG,
-        weaklistoffset = fieldoffset(PyJuliaValueObject, 3),
-        # getattro = C.POINTERS.PyObject_GenericGetAttr,
-        # setattro = C.POINTERS.PyObject_GenericSetAttr,
-        methods = pointer(_pyjlbase_methods),
-        as_buffer = pointer(_pyjlbase_as_buffer),
+        slots = pointer(slots),
     )
-    o = PyJuliaBase_Type[] = C.PyPtr(pointer(_pyjlbase_type))
-    if C.PyType_Ready(o) == -1
+    spec_ref = Ref(spec)
+    o = GC.@preserve spec slots _pyjlbase_methods spec_ref begin
+        C.PyType_FromSpec(Base.unsafe_convert(Ptr{C.PyType_Spec}, spec_ref))
+    end
+    if o == C.PyNULL
         C.PyErr_Print()
         error("Error initializing 'juliacall.ValueBase'")
     end
+    PyJuliaBase_Type[] = o
 end
 
 function __init__()