convert module initialization to use multi-phase-init

.github/workflows/ci.yml

@@ -269,9 +269,10 @@ jobs:
             "3.13t",
             "3.14-dev",
             "3.14t-dev",
-            "pypy3.9",
-            "pypy3.10",
-            "pypy3.11",
+            # FIXME: using nightly for 5142, should consider what to do before merging
+            #"pypy3.9",
+            "pypy3.10-nightly",
+            "pypy3.11-nightly",
             "graalpy24.2",
           ]
         platform:

pyo3-ffi/src/modsupport.rs

@@ -73,6 +73,7 @@ extern "C" {
     // skipped PyModule_AddStringMacro
     pub fn PyModule_SetDocString(arg1: *mut PyObject, arg2: *const c_char) -> c_int;
     pub fn PyModule_AddFunctions(arg1: *mut PyObject, arg2: *mut PyMethodDef) -> c_int;
+    #[cfg_attr(PyPy, link_name = "PyPyModule_ExecDef")]
     pub fn PyModule_ExecDef(module: *mut PyObject, def: *mut PyModuleDef) -> c_int;
 }
 

pyo3-macros-backend/src/module.rs

@@ -10,7 +10,7 @@ use crate::{
     get_doc,
     pyclass::PyClassPyO3Option,
     pyfunction::{impl_wrap_pyfunction, PyFunctionOptions},
-    utils::{has_attribute, has_attribute_with_namespace, Ctx, IdentOrStr, LitCStr},
+    utils::{has_attribute, has_attribute_with_namespace, Ctx, IdentOrStr, LitCStr, PythonDoc},
 };
 use proc_macro2::{Span, TokenStream};
 use quote::quote;
@@ -357,23 +357,15 @@ pub fn pymodule_module_impl(
     #[cfg(not(feature = "experimental-inspect"))]
     let introspection_id = quote! {};
 
-    let module_def = quote! {{
-        use #pyo3_path::impl_::pymodule as impl_;
-        const INITIALIZER: impl_::ModuleInitializer = impl_::ModuleInitializer(__pyo3_pymodule);
-        unsafe {
-           impl_::ModuleDef::new(
-                __PYO3_NAME,
-                #doc,
-                INITIALIZER
-            )
-        }
-    }};
+    let gil_used = options.gil_used.map_or(true, |op| op.value.value);
+
     let initialization = module_initialization(
         &name,
         ctx,
-        module_def,
+        quote! { __pyo3_pymodule },
         options.submodule.is_some(),
-        options.gil_used.map_or(true, |op| op.value.value),
+        gil_used,
+        doc,
     );
 
     let module_consts_names = module_consts.iter().map(|i| i.unraw().to_string());
@@ -423,12 +415,15 @@ pub fn pymodule_function_impl(
     let vis = &function.vis;
     let doc = get_doc(&function.attrs, None, ctx);
 
+    let gil_used = options.gil_used.map_or(true, |op| op.value.value);
+
     let initialization = module_initialization(
         &name,
         ctx,
-        quote! { MakeDef::make_def() },
+        quote! { ModuleExec::__pyo3_module_exec },
         false,
-        options.gil_used.map_or(true, |op| op.value.value),
+        gil_used,
+        doc,
     );
 
     #[cfg(feature = "experimental-inspect")]
@@ -461,20 +456,9 @@ pub fn pymodule_function_impl(
         // (and `super` doesn't always refer to the outer scope, e.g. if the `#[pymodule] is
         // inside a function body)
         #[allow(unknown_lints, non_local_definitions)]
-        impl #ident::MakeDef {
-            const fn make_def() -> #pyo3_path::impl_::pymodule::ModuleDef {
-                fn __pyo3_pymodule(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> {
-                    #ident(#(#module_args),*)
-                }
-
-                const INITIALIZER: #pyo3_path::impl_::pymodule::ModuleInitializer = #pyo3_path::impl_::pymodule::ModuleInitializer(__pyo3_pymodule);
-                unsafe {
-                    #pyo3_path::impl_::pymodule::ModuleDef::new(
-                        #ident::__PYO3_NAME,
-                        #doc,
-                        INITIALIZER
-                    )
-                }
+        impl #ident::ModuleExec {
+            fn __pyo3_module_exec(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> {
+                #ident(#(#module_args),*)
             }
         }
     })
@@ -483,9 +467,10 @@ pub fn pymodule_function_impl(
 fn module_initialization(
     name: &syn::Ident,
     ctx: &Ctx,
-    module_def: TokenStream,
+    module_exec: TokenStream,
     is_submodule: bool,
     gil_used: bool,
+    doc: PythonDoc,
 ) -> TokenStream {
     let Ctx { pyo3_path, .. } = ctx;
     let pyinit_symbol = format!("PyInit_{name}");
@@ -496,9 +481,27 @@ fn module_initialization(
         #[doc(hidden)]
         pub const __PYO3_NAME: &'static ::std::ffi::CStr = #pyo3_name;
 
-        pub(super) struct MakeDef;
+        // This structure exists for `fn` modules declared within `fn` bodies, where due to the hidden
+        // module (used for importing) the `fn` to initialize the module cannot be seen from the #module_def
+        // declaration just below.
         #[doc(hidden)]
-        pub static _PYO3_DEF: #pyo3_path::impl_::pymodule::ModuleDef = #module_def;
+        pub(super) struct ModuleExec;
+
+        #[doc(hidden)]
+        pub static _PYO3_DEF: #pyo3_path::impl_::pymodule::ModuleDef = {
+            use #pyo3_path::impl_::pymodule as impl_;
+
+            unsafe extern "C" fn __pyo3_module_exec(module: *mut #pyo3_path::ffi::PyObject) -> ::std::os::raw::c_int {
+                #pyo3_path::impl_::trampoline::module_exec(module, #module_exec)
+            }
+
+            static SLOTS: impl_::PyModuleSlots<4> = impl_::PyModuleSlotsBuilder::new()
+                .with_mod_exec(__pyo3_module_exec)
+                .with_gil_used(#gil_used)
+                .build();
+
+            impl_::ModuleDef::new(__PYO3_NAME, #doc, &SLOTS)
+        };
         #[doc(hidden)]
         // so wrapped submodules can see what gil_used is
         pub static __PYO3_GIL_USED: bool = #gil_used;
@@ -510,7 +513,10 @@ fn module_initialization(
             #[doc(hidden)]
             #[export_name = #pyinit_symbol]
             pub unsafe extern "C" fn __pyo3_init() -> *mut #pyo3_path::ffi::PyObject {
-                unsafe { #pyo3_path::impl_::trampoline::module_init(|py| _PYO3_DEF.make_module(py, #gil_used)) }
+                _PYO3_DEF.init_multi_phase(
+                    unsafe { #pyo3_path::Python::assume_gil_acquired() },
+                    #gil_used
+                )
             }
         });
     }

pytests/tests/test_misc.py

@@ -24,6 +24,7 @@ def test_multiple_imports_same_interpreter_ok():
     spec = importlib.util.find_spec("pyo3_pytests.pyo3_pytests")
 
     module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
     assert dir(module) == dir(pyo3_pytests.pyo3_pytests)