From 086ded34264c674e5f20be5a276d03bd574fdc32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 13 Jul 2024 21:13:39 +0200 Subject: [PATCH 01/45] explain how to add modules --- developer-workflow/extension-modules.rst | 314 ++++++++++++++++++++++- 1 file changed, 312 insertions(+), 2 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 0384c2b382..a749b7f322 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -5,9 +5,13 @@ Standard library extension modules ================================== -In this section, we could explain how to write a CPython extension with the C language, but the topic can take a complete book. +In this section, we could explain how to write a CPython extension with the +C language, but the topic can take a complete book. We will however explain +how to add a new extension module to the standard library, e.g., a module +responsible for accelerating some parts of the library. -For this reason, we prefer to give you some links where you can read a very good documentation. +For writing a CPython extension itself, we prefer to give you some links +where you can read a very good documentation. Read the following references: @@ -15,3 +19,309 @@ Read the following references: * https://docs.python.org/dev/extending/ * :pep:`399` * https://pythonextensionpatterns.readthedocs.io/en/latest/ + +===================================== +Adding an extension module to CPython +===================================== + +In this section, we assume that the extension module to be added +does not rely on external dependencies and is not a frozen module. + +Let us assume that the standard library has the :mod:`!foo` module +which contains some function :func:`!foo.bar`: + +.. code-block:: python + + def bar(): + return "Hello World!" + +Instead of using the Python implementation of :func:`!foo.bar`, we want to +use its C implementation that we would have written somewhere else. Ideally, +we want to modify ``foo.py`` as follows: + +.. code-block:: python + + try: + # use the C implementation if possible + from _foo import bar + except ImportError: + # fallback to the pure Python implementation + def bar(): + return "Hello World!" + +Some modules in the standard library are implemented both in C and in Python, +such as :mod:`functools` or :mod:`io`, and the C implementation should offer +improved performances when available (such modules are usually referred to as +*accelerator modules*). In our example, we need to + +- determine where the extension module is to be placed; +- determine which files to modify in order to compile the project; +- determine which Makefile rules to invoke in the end. + +In general, accelerator modules are added in the ``Modules/`` directory +of the CPython project. If more than one file is needed for the extension +module, it is convenient to create a sub-directory in ``Modules/`` and place +the files inside it. For instance, + +.. code-block:: c + + // Modules/foo/foomodule.h: file containing shared prototypes + + #ifndef FOOMODULE_H + #define FOOMODULE_H + + #include "Python.h" + + typedef struct { + /* ... */ + } foomodule_state; + + static inline foomodule_state * + get_foomodule_state(PyObject *module) + { + void *state = PyModule_GetState(module); + assert(state != NULL); + return (foomodule_state *)state; + } + + /* helper implemented somewhere else */ + extern PyObject *_Py_fast_bar(); + + #endif // FOOMODULE_H + +The actual implementation of the module is in the corresponding ``.c`` file: + +.. code-block:: c + + // Modules/foo/foomodule.c + + #include "foomodule.h" + #include "clinic/foomodule.c.h" + + /* Functions for the module's state */ + static int + foomodule_exec(PyObject *module) + { + // imports, static attributes, exported classes, etc + return 0; + } + + static int + foomodule_traverse(PyObject *m, visitproc visit, void *arg) + { + foomodule_state *st = get_foomodule_state(m); + // call Py_VISIT() on the state attributes + return 0; + } + + static int + foomodule_clear(PyObject *m) + { + foomodule_state *st = get_foomodule_state(m); + // call Py_CLEAR() on the state attributes + return 0; + } + + static void + foomodule_free(void *m) { + (void)foomodule_clear((PyObject *)m); + } + + /* Implementation of publicly exported functions */ + + /*[clinic input] + module foo + [clinic start generated code]*/ + /*[clinic end generated code: output=... input=...]*/ + + /*[clinic input] + foo.bar -> object + + [clinic start generated code]*/ + static PyObject * + foo_bar_impl(PyObject *module) + /*[clinic end generated code: output=... input=...]*/ + { + return _Py_fast_bar(); + } + + /* Exported module's data */ + + static PyMethodDef foomodule_methods[] = { + FOO_BAR_METHODDEF // this becomes available after running 'make clinic' + {NULL, NULL} + }; + + static struct PyModuleDef_Slot foomodule_slots[] = { + {Py_mod_exec, foomodule_exec}, // foomodule_exec may be NULL if the state is trivial + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, + }; + + static struct PyModuleDef foomodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_foo", + .m_doc = "some doc", // or NULL if not needed + .m_size = sizeof(foomodule_state), + .m_methods = foomodule_methods, + .m_slots = foomodule_slots, + .m_traverse = foomodule_traverse, // or NULL if the state is trivial + .m_clear = foomodule_clear, // or NULL if the state is trivial + .m_free = foomodule_free, // or NULL if the state is trivial + }; + + PyMODINIT_FUNC + PyInit_foo(void) + { + return PyModuleDef_Init(&_foomodule); + } + +In a separate file, we would have the implementation of ``Py_fast_bar``: + +.. code-block:: c + + // Modules/foo/helper.c + + #include "foomodule.h" + + PyObject *_Py_fast_bar() { + return PyUnicode_FromString("Hello World!"); + } + +Now, to summarize, we have the following files: + +- ``Modules/foo/foomodule.h`` -- the shared prototypes for our mini-project. +- ``Modules/foo/foomodule.c`` -- the actual module's implementation. +- ``Modules/foo/helper.c`` -- some helper's implementation. + +One could imagine having more ``.h`` files, or no ``helper.c`` file if it is +not needed. Here, we wanted to illustrate a simple example without making it +too trivial. + +### Make the CPython project compile + +Now that we have our files, we need to update the ``Makefile.pre.in`` file. +First, define the following the variables: + +```makefile +FOO_H = Modules/foo/foomodule.h + +FOO_OBJS = \ + Modules/foo/foomodule.o \ + Modules/foo/helper.o +``` + +and place them somewhere in the file (usually where other variables of the +same kind are). + +Then, add the following rule in the '# Special rules for object files' section: + +```makefile +$(FOO_OBJS): $(FOO_H) +``` + +and the following rule in the dependencies section: + +```makefile +MODULE_FOO_DEPS=$(srcdir)/Modules/foo/foomodule.h +``` + +.. note:: + + The ``FOO_OBJS`` and ``FOO_H`` are not necessarily needed and the rule + ``$(FOO_OBJS): $(FOO_H)`` could be hard-coded. Using Makefile variables + is generally better if more than multiple files need to be compiled. + +Finally, we need to modify the configuration for Windows platforms: + +- Open ``PC/config.c`` and add the prototype: + + .. code-block:: c + + extern PyObject* PyInit_foo(void); + + and the entry ``{"foo", PyInit_foo}`` to ``_PyImport_Inittab``. + +- Open ``PCbuild/pythoncore.vcxproj`` and add the following line to + the ``ItemGroup`` containing the ``..\Modules\*.h`` files: + + .. code-block:: xml + + + + In addition, add the following lines to the ``ItemGroup`` containing + the ``..\Modules\*.c`` files: + + .. code-block:: xml + + + + +- Open ``PCbuild/pythoncore.vcxproj.filters`` and add the following line to + the ``ItemGroup`` containing the ``..\Modules\*.h`` files: + + .. code-block:: xml + + + Modules\foo + + + In addition, add the following lines to the ``ItemGroup`` containing + the ``..\Modules\*.c`` files: + + .. code-block:: xml + + + Modules\foo + + + Modules\foo + + +Observe that ``.h`` files use ```` whereas ``.c`` files +use ```` tags. + +### Compile the CPython project + +Now that everything is in place, it remains to compile everything. To that +end, run the following commands: + +.. code-block:: shell + + make regen-configure + make regen-all + make regen-stdlib-module-names + +.. tip:: Use ``make -j12`` to speed-up the compilation. + +- The ``make regen-configure`` step regenerates the configure script. + +- The ``make regen-all`` is responsible for running Arguments Clinic, + regenerating global objects, etc. It is useful to run when you do not + know which files should be updated. + +- The ``regen-stdlib-module-names`` updates the standard module names, + making ``_foo`` discoverable and importable via ``import _foo``! + +You can now compile the entire project by running the following commands: + +.. code-block:: shell + + ./configure --with-pydebug + make + +#### Troubleshooting: ``make regen-configure`` does not work! + +Since this rule requires Docker to be running and a Docker instance, +the following can be done on Linux platforms (systemctl-based): + +.. code-block:: shell + + $ systemctl status docker # is the docker service running? + $ sudo systemctl start docker # start it if not! + $ sudo systemctl restart docker # or restart it! + +If docker complains about missing permissions, the following StackOverflow post +could be useful in solving the issue: `How to fix docker: permission denied +`_ From 96fb998737f68713299886b6d915f832495feee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 13 Jul 2024 21:28:24 +0200 Subject: [PATCH 02/45] Update extension-modules.rst --- developer-workflow/extension-modules.rst | 27 +++++++++++------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index a749b7f322..ffb6f56df6 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -167,8 +167,8 @@ The actual implementation of the module is in the corresponding ``.c`` file: .m_methods = foomodule_methods, .m_slots = foomodule_slots, .m_traverse = foomodule_traverse, // or NULL if the state is trivial - .m_clear = foomodule_clear, // or NULL if the state is trivial - .m_free = foomodule_free, // or NULL if the state is trivial + .m_clear = foomodule_clear, // or NULL if the state is trivial + .m_free = foomodule_free, // or NULL if the state is trivial }; PyMODINIT_FUNC @@ -204,28 +204,25 @@ too trivial. Now that we have our files, we need to update the ``Makefile.pre.in`` file. First, define the following the variables: -```makefile -FOO_H = Modules/foo/foomodule.h - -FOO_OBJS = \ - Modules/foo/foomodule.o \ - Modules/foo/helper.o -``` +.. code-block:: makefile + + FOO_H = Modules/foo/foomodule.h + FOO_OBJS = Modules/foo/foomodule.o Modules/foo/helper.o and place them somewhere in the file (usually where other variables of the same kind are). Then, add the following rule in the '# Special rules for object files' section: -```makefile -$(FOO_OBJS): $(FOO_H) -``` +.. code-block:: makefile + + $(FOO_OBJS): $(FOO_H) and the following rule in the dependencies section: -```makefile -MODULE_FOO_DEPS=$(srcdir)/Modules/foo/foomodule.h -``` +.. code-block:: makefile + + MODULE_FOO_DEPS=$(srcdir)/Modules/foo/foomodule.h .. note:: From 5f8797c802106b01510384e885ae785de66a3723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 13 Jul 2024 21:29:46 +0200 Subject: [PATCH 03/45] Update extension-modules.rst --- developer-workflow/extension-modules.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index ffb6f56df6..7104ba7987 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -205,7 +205,7 @@ Now that we have our files, we need to update the ``Makefile.pre.in`` file. First, define the following the variables: .. code-block:: makefile - + FOO_H = Modules/foo/foomodule.h FOO_OBJS = Modules/foo/foomodule.o Modules/foo/helper.o @@ -221,7 +221,7 @@ Then, add the following rule in the '# Special rules for object files' section: and the following rule in the dependencies section: .. code-block:: makefile - + MODULE_FOO_DEPS=$(srcdir)/Modules/foo/foomodule.h .. note:: From e5d41f8a59505ed19371e65c5924265845995f66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 13 Jul 2024 21:35:19 +0200 Subject: [PATCH 04/45] Update extension-modules.rst --- developer-workflow/extension-modules.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 7104ba7987..f1f8198490 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -199,7 +199,8 @@ One could imagine having more ``.h`` files, or no ``helper.c`` file if it is not needed. Here, we wanted to illustrate a simple example without making it too trivial. -### Make the CPython project compile +Make the CPython project compile +-------------------------------- Now that we have our files, we need to update the ``Makefile.pre.in`` file. First, define the following the variables: @@ -279,7 +280,8 @@ Finally, we need to modify the configuration for Windows platforms: Observe that ``.h`` files use ```` whereas ``.c`` files use ```` tags. -### Compile the CPython project +Compile the CPython project +--------------------------- Now that everything is in place, it remains to compile everything. To that end, run the following commands: @@ -308,7 +310,8 @@ You can now compile the entire project by running the following commands: ./configure --with-pydebug make -#### Troubleshooting: ``make regen-configure`` does not work! +Troubleshooting: ``make regen-configure`` does not work! +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Since this rule requires Docker to be running and a Docker instance, the following can be done on Linux platforms (systemctl-based): From b740114b8d87bb713a6324b5fff26722cf399cd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 13 Jul 2024 22:08:12 +0200 Subject: [PATCH 05/45] Update extension-modules.rst --- developer-workflow/extension-modules.rst | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index f1f8198490..c90ccbd622 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -20,9 +20,8 @@ Read the following references: * :pep:`399` * https://pythonextensionpatterns.readthedocs.io/en/latest/ -===================================== Adding an extension module to CPython -===================================== +------------------------------------- In this section, we assume that the extension module to be added does not rely on external dependencies and is not a frozen module. @@ -200,7 +199,7 @@ not needed. Here, we wanted to illustrate a simple example without making it too trivial. Make the CPython project compile --------------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Now that we have our files, we need to update the ``Makefile.pre.in`` file. First, define the following the variables: @@ -242,14 +241,14 @@ Finally, we need to modify the configuration for Windows platforms: and the entry ``{"foo", PyInit_foo}`` to ``_PyImport_Inittab``. - Open ``PCbuild/pythoncore.vcxproj`` and add the following line to - the ``ItemGroup`` containing the ``..\Modules\*.h`` files: + the ```` containing the ``..\Modules\*.h`` files: .. code-block:: xml - In addition, add the following lines to the ``ItemGroup`` containing - the ``..\Modules\*.c`` files: + In addition, add the following lines to the ```` + containing the ``..\Modules\*.c`` files: .. code-block:: xml @@ -281,7 +280,7 @@ Observe that ``.h`` files use ```` whereas ``.c`` files use ```` tags. Compile the CPython project ---------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^ Now that everything is in place, it remains to compile everything. To that end, run the following commands: @@ -311,7 +310,7 @@ You can now compile the entire project by running the following commands: make Troubleshooting: ``make regen-configure`` does not work! -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +........................................................ Since this rule requires Docker to be running and a Docker instance, the following can be done on Linux platforms (systemctl-based): @@ -324,4 +323,4 @@ the following can be done on Linux platforms (systemctl-based): If docker complains about missing permissions, the following StackOverflow post could be useful in solving the issue: `How to fix docker: permission denied -`_ +`_. From c74df674e872a0165bf074c9c1af95d7f01ff3b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 13 Jul 2024 22:25:22 +0200 Subject: [PATCH 06/45] Update extension-modules.rst --- developer-workflow/extension-modules.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index c90ccbd622..ab196c2ade 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -309,8 +309,11 @@ You can now compile the entire project by running the following commands: ./configure --with-pydebug make -Troubleshooting: ``make regen-configure`` does not work! -........................................................ +Troubleshooting +^^^^^^^^^^^^^^^ + +``make regen-configure`` does not work! +....................................... Since this rule requires Docker to be running and a Docker instance, the following can be done on Linux platforms (systemctl-based): From e645869d2226099e32e351fbd225faa87f86a183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 14 Jul 2024 02:45:54 +0200 Subject: [PATCH 07/45] Update extension-modules.rst --- developer-workflow/extension-modules.rst | 83 +++++++++++++----------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index ab196c2ade..e39e7177d1 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -35,8 +35,8 @@ which contains some function :func:`!foo.bar`: return "Hello World!" Instead of using the Python implementation of :func:`!foo.bar`, we want to -use its C implementation that we would have written somewhere else. Ideally, -we want to modify ``foo.py`` as follows: +use its corresponding C implementation exposed as the :mod:`!_foo` module. +Ideally, we want to modify ``foo.py`` as follows: .. code-block:: python @@ -49,25 +49,37 @@ we want to modify ``foo.py`` as follows: return "Hello World!" Some modules in the standard library are implemented both in C and in Python, -such as :mod:`functools` or :mod:`io`, and the C implementation should offer -improved performances when available (such modules are usually referred to as +such as :class:`decimal` or :class:`io`, and the C implementation is expected +to improve performance when available (such modules are usually referred to as *accelerator modules*). In our example, we need to - determine where the extension module is to be placed; - determine which files to modify in order to compile the project; - determine which Makefile rules to invoke in the end. -In general, accelerator modules are added in the ``Modules/`` directory -of the CPython project. If more than one file is needed for the extension -module, it is convenient to create a sub-directory in ``Modules/`` and place -the files inside it. For instance, +Usually, accelerator modules are added in the ``Modules/`` directory of +the CPython project. If more than one file is needed for the extension +module, it is convenient to create a sub-directory in ``Modules/``, and +place the files inside it. In our example, we will assume that we have +the following structure: + +- ``Modules/foo/foomodule.h`` -- the shared prototypes for our mini-project. +- ``Modules/foo/foomodule.c`` -- the actual module's implementation. +- ``Modules/foo/helper.c`` -- some helper's implementation. + +.. note:: + + If ``Modules/foo/foomodule.c`` contains some Argument Clinic directives, + the corresponding header file is written to ``Modules/clinic/foomodule.c.h``. + +The following code snippets illustrate the possible contents of the above files: .. code-block:: c - // Modules/foo/foomodule.h: file containing shared prototypes + // Modules/foo/foomodule.h - #ifndef FOOMODULE_H - #define FOOMODULE_H + #ifndef FOO_FOOMODULE_H + #define FOO_FOOMODULE_H #include "Python.h" @@ -86,7 +98,7 @@ the files inside it. For instance, /* helper implemented somewhere else */ extern PyObject *_Py_fast_bar(); - #endif // FOOMODULE_H + #endif // FOO_FOOMODULE_H The actual implementation of the module is in the corresponding ``.c`` file: @@ -147,12 +159,14 @@ The actual implementation of the module is in the corresponding ``.c`` file: /* Exported module's data */ static PyMethodDef foomodule_methods[] = { - FOO_BAR_METHODDEF // this becomes available after running 'make clinic' + // the following macro is available in 'Modules/foo/clinic/foomodule.c.h' + // after running 'make clinic' + FOO_BAR_METHODDEF {NULL, NULL} }; static struct PyModuleDef_Slot foomodule_slots[] = { - {Py_mod_exec, foomodule_exec}, // foomodule_exec may be NULL if the state is trivial + {Py_mod_exec, foomodule_exec}, // 'foomodule_exec' may be NULL if the state is trivial {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, @@ -160,7 +174,7 @@ The actual implementation of the module is in the corresponding ``.c`` file: static struct PyModuleDef foomodule = { PyModuleDef_HEAD_INIT, - .m_name = "_foo", + .m_name = "_foo", // name to use in 'import' statements .m_doc = "some doc", // or NULL if not needed .m_size = sizeof(foomodule_state), .m_methods = foomodule_methods, @@ -188,12 +202,6 @@ In a separate file, we would have the implementation of ``Py_fast_bar``: return PyUnicode_FromString("Hello World!"); } -Now, to summarize, we have the following files: - -- ``Modules/foo/foomodule.h`` -- the shared prototypes for our mini-project. -- ``Modules/foo/foomodule.c`` -- the actual module's implementation. -- ``Modules/foo/helper.c`` -- some helper's implementation. - One could imagine having more ``.h`` files, or no ``helper.c`` file if it is not needed. Here, we wanted to illustrate a simple example without making it too trivial. @@ -201,7 +209,8 @@ too trivial. Make the CPython project compile ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Now that we have our files, we need to update the ``Makefile.pre.in`` file. +Now that we have our files, we need to update the ``Makefile.pre.in`` file. + First, define the following the variables: .. code-block:: makefile @@ -209,16 +218,15 @@ First, define the following the variables: FOO_H = Modules/foo/foomodule.h FOO_OBJS = Modules/foo/foomodule.o Modules/foo/helper.o -and place them somewhere in the file (usually where other variables of the -same kind are). - -Then, add the following rule in the '# Special rules for object files' section: +and place them in the **Modules** section where other pre-defined objects live such +as ``MODULE_OBJS`` and ``IO_OBJS``. Then, add the following rule in the section for +**Special rules for object files**: .. code-block:: makefile $(FOO_OBJS): $(FOO_H) -and the following rule in the dependencies section: +and the following rule in the section for **Module dependencies and platform-specific files**: .. code-block:: makefile @@ -228,7 +236,7 @@ and the following rule in the dependencies section: The ``FOO_OBJS`` and ``FOO_H`` are not necessarily needed and the rule ``$(FOO_OBJS): $(FOO_H)`` could be hard-coded. Using Makefile variables - is generally better if more than multiple files need to be compiled. + is generally better if multiple files need to be compiled. Finally, we need to modify the configuration for Windows platforms: @@ -282,16 +290,13 @@ use ```` tags. Compile the CPython project ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Now that everything is in place, it remains to compile everything. To that -end, run the following commands: +Now that everything is in place, it remains to compile the project: .. code-block:: shell - make regen-configure - make regen-all - make regen-stdlib-module-names - -.. tip:: Use ``make -j12`` to speed-up the compilation. + $ make regen-configure + $ make regen-all + $ make regen-stdlib-module-names - The ``make regen-configure`` step regenerates the configure script. @@ -306,12 +311,16 @@ You can now compile the entire project by running the following commands: .. code-block:: shell - ./configure --with-pydebug - make + $ ./configure --with-pydebug + $ make + +.. tip:: Use ``make -j12`` to speed-up the compilation if you have enough CPU cores. Troubleshooting ^^^^^^^^^^^^^^^ +This section addresses common issues that developers may face when following this tutorial. + ``make regen-configure`` does not work! ....................................... From 5a9a57c97f83ff1b0c9f71c291f6bb25f96a1b2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 14 Jul 2024 02:48:51 +0200 Subject: [PATCH 08/45] Update extension-modules.rst --- developer-workflow/extension-modules.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index e39e7177d1..f541c7f1ea 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -49,9 +49,9 @@ Ideally, we want to modify ``foo.py`` as follows: return "Hello World!" Some modules in the standard library are implemented both in C and in Python, -such as :class:`decimal` or :class:`io`, and the C implementation is expected -to improve performance when available (such modules are usually referred to as -*accelerator modules*). In our example, we need to +such as :mod:`decimal` or :mod:`io`, and the C implementation is expected +to improve performance when available (such modules are usually referred to as +*accelerator modules*). In our example, we need to: - determine where the extension module is to be placed; - determine which files to modify in order to compile the project; @@ -209,8 +209,7 @@ too trivial. Make the CPython project compile ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Now that we have our files, we need to update the ``Makefile.pre.in`` file. - +Now that we have our files, we need to update the ``Makefile.pre.in`` file. First, define the following the variables: .. code-block:: makefile @@ -219,7 +218,7 @@ First, define the following the variables: FOO_OBJS = Modules/foo/foomodule.o Modules/foo/helper.o and place them in the **Modules** section where other pre-defined objects live such -as ``MODULE_OBJS`` and ``IO_OBJS``. Then, add the following rule in the section for +as ``MODULE_OBJS`` and ``IO_OBJS``. Then, add the following rule in the section for **Special rules for object files**: .. code-block:: makefile From 26a18eb948bf014642c91c84146b49e2bc92a469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 14 Jul 2024 02:55:38 +0200 Subject: [PATCH 09/45] Update extension-modules.rst --- developer-workflow/extension-modules.rst | 26 ++++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index f541c7f1ea..90526aa35b 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -57,9 +57,9 @@ to improve performance when available (such modules are usually referred to as - determine which files to modify in order to compile the project; - determine which Makefile rules to invoke in the end. -Usually, accelerator modules are added in the ``Modules/`` directory of +Usually, accelerator modules are added in the :cpy-file:`Modules` directory of the CPython project. If more than one file is needed for the extension -module, it is convenient to create a sub-directory in ``Modules/``, and +module, it is convenient to create a sub-directory in :cpy-file:`Modules`, and place the files inside it. In our example, we will assume that we have the following structure: @@ -70,7 +70,7 @@ the following structure: .. note:: If ``Modules/foo/foomodule.c`` contains some Argument Clinic directives, - the corresponding header file is written to ``Modules/clinic/foomodule.c.h``. + the corresponding header file is written to ``Modules/foo/clinic/foomodule.c.h``. The following code snippets illustrate the possible contents of the above files: @@ -209,7 +209,7 @@ too trivial. Make the CPython project compile ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Now that we have our files, we need to update the ``Makefile.pre.in`` file. +Now that we have our files, we need to update the :cpy-file:`Makefile.pre.in` file. First, define the following the variables: .. code-block:: makefile @@ -239,7 +239,7 @@ and the following rule in the section for **Module dependencies and platform-spe Finally, we need to modify the configuration for Windows platforms: -- Open ``PC/config.c`` and add the prototype: +- Open :cpy-file:`PC/config.c` and add the prototype: .. code-block:: c @@ -247,23 +247,23 @@ Finally, we need to modify the configuration for Windows platforms: and the entry ``{"foo", PyInit_foo}`` to ``_PyImport_Inittab``. -- Open ``PCbuild/pythoncore.vcxproj`` and add the following line to - the ```` containing the ``..\Modules\*.h`` files: +- Open :cpy-file:`PCbuild/pythoncore.vcxproj` and add the following line to + the ```` containing the other ``..\Modules\*.h`` files: .. code-block:: xml In addition, add the following lines to the ```` - containing the ``..\Modules\*.c`` files: + containing the the other ``..\Modules\*.c`` files: .. code-block:: xml -- Open ``PCbuild/pythoncore.vcxproj.filters`` and add the following line to - the ``ItemGroup`` containing the ``..\Modules\*.h`` files: +- Open :cpy-file:`PCbuild/pythoncore.vcxproj.filters` and add the following line to + the ``ItemGroup`` containing the the other ``..\Modules\*.h`` files: .. code-block:: xml @@ -272,7 +272,7 @@ Finally, we need to modify the configuration for Windows platforms: In addition, add the following lines to the ``ItemGroup`` containing - the ``..\Modules\*.c`` files: + the the other ``..\Modules\*.c`` files: .. code-block:: xml @@ -304,7 +304,7 @@ Now that everything is in place, it remains to compile the project: know which files should be updated. - The ``regen-stdlib-module-names`` updates the standard module names, - making ``_foo`` discoverable and importable via ``import _foo``! + making ``_foo`` discoverable and importable via ``import _foo``. You can now compile the entire project by running the following commands: @@ -332,6 +332,6 @@ the following can be done on Linux platforms (systemctl-based): $ sudo systemctl start docker # start it if not! $ sudo systemctl restart docker # or restart it! -If docker complains about missing permissions, the following StackOverflow post +If Docker complains about missing permissions, the following StackOverflow post could be useful in solving the issue: `How to fix docker: permission denied `_. From 1b4d73f4da97eb0414ddd39b4a997c41fb952ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 14 Jul 2024 03:02:17 +0200 Subject: [PATCH 10/45] Update extension-modules.rst --- developer-workflow/extension-modules.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 90526aa35b..e742af7348 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -95,7 +95,7 @@ The following code snippets illustrate the possible contents of the above files: return (foomodule_state *)state; } - /* helper implemented somewhere else */ + /* Helper implemented somewhere else. */ extern PyObject *_Py_fast_bar(); #endif // FOO_FOOMODULE_H @@ -190,7 +190,7 @@ The actual implementation of the module is in the corresponding ``.c`` file: return PyModuleDef_Init(&_foomodule); } -In a separate file, we would have the implementation of ``Py_fast_bar``: +In a separate file, we would have the implementation of ``_Py_fast_bar``: .. code-block:: c @@ -202,6 +202,11 @@ In a separate file, we would have the implementation of ``Py_fast_bar``: return PyUnicode_FromString("Hello World!"); } +.. tip:: + + Do not forget that symbols exported by libpython must start + with "Py" or "_Py", which is verified via ``make smelly``. + One could imagine having more ``.h`` files, or no ``helper.c`` file if it is not needed. Here, we wanted to illustrate a simple example without making it too trivial. From 94086ddeffaf326a3dab420a801f30dccb5835b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 14 Jul 2024 11:08:17 +0200 Subject: [PATCH 11/45] Address Hugo's feedback --- developer-workflow/extension-modules.rst | 40 +++++++++++------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index e742af7348..c10d67e3bc 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -7,13 +7,11 @@ Standard library extension modules In this section, we could explain how to write a CPython extension with the C language, but the topic can take a complete book. We will however explain -how to add a new extension module to the standard library, e.g., a module -responsible for accelerating some parts of the library. +how to add a new extension module to the standard library, for instance, a +module responsible for accelerating some parts of the library. For writing a CPython extension itself, we prefer to give you some links -where you can read a very good documentation. - -Read the following references: +where you can read good documentation: * https://docs.python.org/dev/c-api/ * https://docs.python.org/dev/extending/ @@ -53,9 +51,9 @@ such as :mod:`decimal` or :mod:`io`, and the C implementation is expected to improve performance when available (such modules are usually referred to as *accelerator modules*). In our example, we need to: -- determine where the extension module is to be placed; +- determine where to place the extension module; - determine which files to modify in order to compile the project; -- determine which Makefile rules to invoke in the end. +- determine which ``Makefile`` rules to invoke at the end. Usually, accelerator modules are added in the :cpy-file:`Modules` directory of the CPython project. If more than one file is needed for the extension @@ -204,8 +202,8 @@ In a separate file, we would have the implementation of ``_Py_fast_bar``: .. tip:: - Do not forget that symbols exported by libpython must start - with "Py" or "_Py", which is verified via ``make smelly``. + Do not forget that symbols exported by ``libpython`` must start + with ``Py`` or ``_Py``, which is verified via ``make smelly``. One could imagine having more ``.h`` files, or no ``helper.c`` file if it is not needed. Here, we wanted to illustrate a simple example without making it @@ -298,9 +296,9 @@ Now that everything is in place, it remains to compile the project: .. code-block:: shell - $ make regen-configure - $ make regen-all - $ make regen-stdlib-module-names + make regen-configure + make regen-all + make regen-stdlib-module-names - The ``make regen-configure`` step regenerates the configure script. @@ -315,28 +313,28 @@ You can now compile the entire project by running the following commands: .. code-block:: shell - $ ./configure --with-pydebug - $ make + ./configure --with-pydebug + make -.. tip:: Use ``make -j12`` to speed-up the compilation if you have enough CPU cores. +.. tip:: Use ``make -j12`` to speed-up compilation if you have enough CPU cores. Troubleshooting ^^^^^^^^^^^^^^^ -This section addresses common issues that developers may face when following this tutorial. +This section addresses common issues that you may face when following this tutorial. ``make regen-configure`` does not work! ....................................... Since this rule requires Docker to be running and a Docker instance, -the following can be done on Linux platforms (systemctl-based): +the following can be done on Linux platforms (``systemctl``-based): .. code-block:: shell - $ systemctl status docker # is the docker service running? - $ sudo systemctl start docker # start it if not! - $ sudo systemctl restart docker # or restart it! + systemctl status docker # is the docker service running? + sudo systemctl start docker # start it if not! + sudo systemctl restart docker # or restart it! -If Docker complains about missing permissions, the following StackOverflow post +If Docker complains about missing permissions, this Stack Overflow post could be useful in solving the issue: `How to fix docker: permission denied `_. From 9a78a3be5f840a2c455bb9b1e7ecfab7d0973a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 14 Jul 2024 11:44:39 +0200 Subject: [PATCH 12/45] Update extension-modules.rst --- developer-workflow/extension-modules.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index c10d67e3bc..f85f61d75d 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -21,11 +21,8 @@ where you can read good documentation: Adding an extension module to CPython ------------------------------------- -In this section, we assume that the extension module to be added -does not rely on external dependencies and is not a frozen module. - -Let us assume that the standard library has the :mod:`!foo` module -which contains some function :func:`!foo.bar`: +Assume that the standard library contains the module :mod:`!foo` +together with some :func:`!foo.bar` function: .. code-block:: python @@ -48,8 +45,8 @@ Ideally, we want to modify ``foo.py`` as follows: Some modules in the standard library are implemented both in C and in Python, such as :mod:`decimal` or :mod:`io`, and the C implementation is expected -to improve performance when available (such modules are usually referred to as -*accelerator modules*). In our example, we need to: +to improve performance when available (such modules are commonly referred +to as *accelerator modules*). In our example, we need to: - determine where to place the extension module; - determine which files to modify in order to compile the project; @@ -58,8 +55,9 @@ to improve performance when available (such modules are usually referred to as Usually, accelerator modules are added in the :cpy-file:`Modules` directory of the CPython project. If more than one file is needed for the extension module, it is convenient to create a sub-directory in :cpy-file:`Modules`, and -place the files inside it. In our example, we will assume that we have -the following structure: +place the files inside it. + +For our extension, we will create the following files: - ``Modules/foo/foomodule.h`` -- the shared prototypes for our mini-project. - ``Modules/foo/foomodule.c`` -- the actual module's implementation. @@ -70,7 +68,9 @@ the following structure: If ``Modules/foo/foomodule.c`` contains some Argument Clinic directives, the corresponding header file is written to ``Modules/foo/clinic/foomodule.c.h``. -The following code snippets illustrate the possible contents of the above files: +For simplicity, we assume that the extension module does not rely on external dependencies +and is not a frozen module. The following code snippets illustrate the possible contents of +the above files: .. code-block:: c From e22c2785ac7844d06f0082e38a3bf18d1fc90c09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 14 Jul 2024 12:06:30 +0200 Subject: [PATCH 13/45] Update extension-modules.rst --- developer-workflow/extension-modules.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index f85f61d75d..07dfde748f 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -55,7 +55,7 @@ to as *accelerator modules*). In our example, we need to: Usually, accelerator modules are added in the :cpy-file:`Modules` directory of the CPython project. If more than one file is needed for the extension module, it is convenient to create a sub-directory in :cpy-file:`Modules`, and -place the files inside it. +place the files inside it. For our extension, we will create the following files: From 1f51497d34d0a387a5d0c90a95b1b8703fb8a2f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 15 Jul 2024 13:43:41 +0200 Subject: [PATCH 14/45] improvements - use distinct names for files to highlight differences in configurations - be more precise on the terminology of an extension module - explain how to build a required module (always built-in) vs an optional module (built-in or dynamic) - address Ezio's review --- developer-workflow/extension-modules.rst | 315 +++++++++++++++-------- 1 file changed, 212 insertions(+), 103 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 07dfde748f..ab70d7b237 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -5,12 +5,11 @@ Standard library extension modules ================================== -In this section, we could explain how to write a CPython extension with the -C language, but the topic can take a complete book. We will however explain -how to add a new extension module to the standard library, for instance, a -module responsible for accelerating some parts of the library. +In this section, we are interested in extending the CPython project with +an :term:`extension module`. We will not explain how to write the module +in C but rather explain how to configure the project and make it compile. -For writing a CPython extension itself, we prefer to give you some links +For writing an extension module in C, we prefer to give you some links where you can read good documentation: * https://docs.python.org/dev/c-api/ @@ -19,65 +18,69 @@ where you can read good documentation: * https://pythonextensionpatterns.readthedocs.io/en/latest/ Adding an extension module to CPython -------------------------------------- +===================================== -Assume that the standard library contains the module :mod:`!foo` -together with some :func:`!foo.bar` function: +Assume that the standard library contains a pure Python module :mod:`foo` +together with the following :func:`!foo.greet` function: .. code-block:: python - def bar(): + def greet(): return "Hello World!" -Instead of using the Python implementation of :func:`!foo.bar`, we want to -use its corresponding C implementation exposed as the :mod:`!_foo` module. -Ideally, we want to modify ``foo.py`` as follows: +Instead of using the Python implementation of :func:`!foo.greet`, we want to +use its corresponding C implementation exposed in some :mod:`!fastfoo` module +written in C. Ideally, we want to modify ``foo.py`` as follows: .. code-block:: python try: # use the C implementation if possible - from _foo import bar + from fastfoo import greet except ImportError: # fallback to the pure Python implementation - def bar(): + def greet(): return "Hello World!" Some modules in the standard library are implemented both in C and in Python, -such as :mod:`decimal` or :mod:`io`, and the C implementation is expected +such as :mod:`decimal` or :mod:`itertools`, and the C implementation is expected to improve performance when available (such modules are commonly referred -to as *accelerator modules*). In our example, we need to: +to as *accelerator modules*). In our example, we need to determine: -- determine where to place the extension module; -- determine which files to modify in order to compile the project; -- determine which ``Makefile`` rules to invoke at the end. +- where to place the extension module source code in the CPython project tree; +- which files to modify in order to compile the CPython project; +- which :cpy-file:`!Makefile` rules to invoke at the end. + +Updating the CPython project tree +--------------------------------- Usually, accelerator modules are added in the :cpy-file:`Modules` directory of -the CPython project. If more than one file is needed for the extension -module, it is convenient to create a sub-directory in :cpy-file:`Modules`, and -place the files inside it. +the CPython project. If more than one file is needed for the extension module, +it is more convenient to create a sub-directory in :cpy-file:`Modules`. For our extension, we will create the following files: -- ``Modules/foo/foomodule.h`` -- the shared prototypes for our mini-project. -- ``Modules/foo/foomodule.c`` -- the actual module's implementation. -- ``Modules/foo/helper.c`` -- some helper's implementation. +- ``Modules/cfoo/foomodule.h`` --- the shared prototypes for our mini-project. +- ``Modules/cfoo/foomodule.c`` --- the actual module's implementation. +- ``Modules/cfoo/helper.c`` --- helpers implementation. + +We deliberately named the mini-project directory and files with names distinct +from the actual Python module to import (whether it is the pure Python module +or its C implementation) to highlight the differences in configuration files. .. note:: - If ``Modules/foo/foomodule.c`` contains some Argument Clinic directives, - the corresponding header file is written to ``Modules/foo/clinic/foomodule.c.h``. + If ``Modules/cfoo/foomodule.c`` contains Argument Clinic directives, + ``make clinic`` creates the file ``Modules/cfoo/clinic/foomodule.c.h``. -For simplicity, we assume that the extension module does not rely on external dependencies -and is not a frozen module. The following code snippets illustrate the possible contents of -the above files: +The following code snippets illustrate the possible contents of the above files: .. code-block:: c - // Modules/foo/foomodule.h + // Modules/cfoo/foomodule.h - #ifndef FOO_FOOMODULE_H - #define FOO_FOOMODULE_H + #ifndef CFOO_FOOMODULE_H + #define CFOO_FOOMODULE_H #include "Python.h" @@ -93,16 +96,19 @@ the above files: return (foomodule_state *)state; } - /* Helper implemented somewhere else. */ - extern PyObject *_Py_fast_bar(); + /* Helper used in Modules/cfoo/foomodule.c + * but implemented in Modules/cfoo/helper.c. + */ + extern PyObject *_Py_greet_fast(); + + #endif // CFOO_FOOMODULE_H - #endif // FOO_FOOMODULE_H The actual implementation of the module is in the corresponding ``.c`` file: .. code-block:: c - // Modules/foo/foomodule.c + // Modules/cfoo/foomodule.c #include "foomodule.h" #include "clinic/foomodule.c.h" @@ -136,7 +142,7 @@ The actual implementation of the module is in the corresponding ``.c`` file: (void)foomodule_clear((PyObject *)m); } - /* Implementation of publicly exported functions */ + /* Implementation of publicly exported functions. */ /*[clinic input] module foo @@ -144,27 +150,27 @@ The actual implementation of the module is in the corresponding ``.c`` file: /*[clinic end generated code: output=... input=...]*/ /*[clinic input] - foo.bar -> object + foo.greet -> object [clinic start generated code]*/ static PyObject * - foo_bar_impl(PyObject *module) + foo_greet(PyObject *module) /*[clinic end generated code: output=... input=...]*/ { - return _Py_fast_bar(); + return _Py_greet_fast(); } /* Exported module's data */ static PyMethodDef foomodule_methods[] = { - // the following macro is available in 'Modules/foo/clinic/foomodule.c.h' - // after running 'make clinic' - FOO_BAR_METHODDEF + // macro available in 'clinic/foomodule.c.h' after running 'make clinic' + FOO_GREET_METHODDEF {NULL, NULL} }; static struct PyModuleDef_Slot foomodule_slots[] = { - {Py_mod_exec, foomodule_exec}, // 'foomodule_exec' may be NULL if the state is trivial + // 'foomodule_exec' may be NULL if the state is trivial + {Py_mod_exec, foomodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, @@ -172,7 +178,7 @@ The actual implementation of the module is in the corresponding ``.c`` file: static struct PyModuleDef foomodule = { PyModuleDef_HEAD_INIT, - .m_name = "_foo", // name to use in 'import' statements + .m_name = "fastfoo", // name to use in 'import' statements .m_doc = "some doc", // or NULL if not needed .m_size = sizeof(foomodule_state), .m_methods = foomodule_methods, @@ -183,20 +189,26 @@ The actual implementation of the module is in the corresponding ``.c`` file: }; PyMODINIT_FUNC - PyInit_foo(void) + PyInit_fastfoo(void) { - return PyModuleDef_Init(&_foomodule); + return PyModuleDef_Init(&foomodule); } -In a separate file, we would have the implementation of ``_Py_fast_bar``: +.. tip:: + + Recall that the ``PyInit_`` function must be suffixed by the same + module name as defined by :c:member:`PyModuleDef.m_mod` (here, ``fastfoo``). + The other identifiers or functions do not have such naming requirements. + +In a separate file, we put the implementation of ``_Py_greet_fast``: .. code-block:: c - // Modules/foo/helper.c + // Modules/cfoo/helper.c #include "foomodule.h" - PyObject *_Py_fast_bar() { + PyObject *_Py_greet_fast() { return PyUnicode_FromString("Hello World!"); } @@ -207,134 +219,231 @@ In a separate file, we would have the implementation of ``_Py_fast_bar``: One could imagine having more ``.h`` files, or no ``helper.c`` file if it is not needed. Here, we wanted to illustrate a simple example without making it -too trivial. +too trivial. If the extension module does not require additional files, it +may directly be placed in :cpy-file:`Modules` as ``Modules/foomodule.c``. + +Extension Modules Types +----------------------- + +Extension modules can be classified into the following types: + +- A *built-in* extension module is a module built and shipped with + the Python interpreter. + + .. note:: + + A built-in module is *statically* linked into the interpreter, + and thereby lacks a :attr:`__file__` attribute. + + .. seealso:: :data:`sys.builtin_module_names` + +- A *dynamic* (or *shared*) extension module is built as a *dynamic* library, + and is *dynamically* linked into the Python interpreter. + + In particular, the corresponding ``.so`` or ``.dll`` file is described by the + module's :attr:`__file__` attribute. + +Built-in extension modules are part of the interpreter, while dynamic extension +modules might be supplied or overridden externally. In particular, the latter +provide a pure Python implementation in case of missing ``.so/.dll`` files. Make the CPython project compile -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +-------------------------------- -Now that we have our files, we need to update the :cpy-file:`Makefile.pre.in` file. -First, define the following the variables: +Now that we have our files, we first update :cpy-file:`configure.ac`: -.. code-block:: makefile +1. Add a line ``Modules/cfoo`` in - FOO_H = Modules/foo/foomodule.h - FOO_OBJS = Modules/foo/foomodule.o Modules/foo/helper.o + .. code-block:: configure -and place them in the **Modules** section where other pre-defined objects live such -as ``MODULE_OBJS`` and ``IO_OBJS``. Then, add the following rule in the section for -**Special rules for object files**: + AC_SUBST([SRCDIRS]) + SRCDIRS="\ + ... + Modules/cfoo \ + ..." -.. code-block:: makefile + .. note:: - $(FOO_OBJS): $(FOO_H) + This step is only needed when adding new source directories to + the CPython project. -and the following rule in the section for **Module dependencies and platform-specific files**: +2. Find the section containing ``PY_STDLIB_MOD_SIMPLE`` usages and + add the following line: -.. code-block:: makefile + .. code-block:: configure - MODULE_FOO_DEPS=$(srcdir)/Modules/foo/foomodule.h + PY_STDLIB_MOD_SIMPLE([fastfoo], [-I\$(srcdir)/Modules/cfoo], []) -.. note:: + The ``PY_STDLIB_MOD_SIMPLE`` macro takes as arguments: + + - the module name as specified by :c:member:`PyModuleDef.m_mod`, + - the compiler flags (CFLAGS), and + - the linker flags (LDFLAGS). + +Then, we update :cpy-file:`Makefile.pre.in` by adding to the +section **Module dependencies and platform-specific files**: - The ``FOO_OBJS`` and ``FOO_H`` are not necessarily needed and the rule - ``$(FOO_OBJS): $(FOO_H)`` could be hard-coded. Using Makefile variables - is generally better if multiple files need to be compiled. + .. code-block:: makefile -Finally, we need to modify the configuration for Windows platforms: + MODULE_FASTFOO_DEPS=$(srcdir)/Modules/cfoo/foomodule.h + +Additionally, we update the configuration files for Windows platforms: - Open :cpy-file:`PC/config.c` and add the prototype: .. code-block:: c - extern PyObject* PyInit_foo(void); + extern PyObject* PyInit_fastfoo(void); + + and update the :c:data:`!_PyImport_Inittab`: - and the entry ``{"foo", PyInit_foo}`` to ``_PyImport_Inittab``. + .. code-block:: c + + struct _inittab _PyImport_Inittab[] = { + ... + {"fastfoo", PyInit_fastfoo}, + ... + {0, 0} + }; + extern PyObject* PyInit_fastfoo(void); - Open :cpy-file:`PCbuild/pythoncore.vcxproj` and add the following line to the ```` containing the other ``..\Modules\*.h`` files: .. code-block:: xml - + In addition, add the following lines to the ```` - containing the the other ``..\Modules\*.c`` files: + containing the other ``..\Modules\*.c`` files: .. code-block:: xml - - + + -- Open :cpy-file:`PCbuild/pythoncore.vcxproj.filters` and add the following line to - the ``ItemGroup`` containing the the other ``..\Modules\*.h`` files: +- Open :cpy-file:`PCbuild/pythoncore.vcxproj.filters` and add the following + line to the ``ItemGroup`` containing the other ``..\Modules\*.h`` files: .. code-block:: xml - - Modules\foo + + Modules\cfoo In addition, add the following lines to the ``ItemGroup`` containing - the the other ``..\Modules\*.c`` files: + the other ``..\Modules\*.c`` files: .. code-block:: xml - - Modules\foo + + Modules\cfoo - - Modules\foo + + Modules\cfoo Observe that ``.h`` files use ```` whereas ``.c`` files use ```` tags. +It remains to update :cpy-file:`Modules/Setup.bootstrap.in` if the module is +required to get a functioning interpreter (such module is *always* a built-in +module) or :cpy-file:`Modules/Setup.stdlib.in` otherwise (such module can be +built-in or dynamic). + +.. note:: + + Built-in modules do not need to have a pure Python implementation + but optional extension modules should have one in case the shared + library is not present on the system. + +.. rubric:: For required extension modules (built-in) + + Open :cpy-file:`Modules/Setup.bootstrap.in` and add the following line: + + .. code-block:: text + + fastfoo cfoo/foomodule.c cfoo/helper.c + +.. rubric:: For optional extension modules + + Open :cpy-file:`Modules/Setup.stdlib.in` and add the following line: + + .. code-block:: text + + @MODULE_FASTFOO_TRUE@fastfoo cfoo/foomodule.c cfoo/helper.c + + The ``@MODULE__TRUE@`` marker requires ```` + to be the upper case form of the module name ````. + Compile the CPython project -^^^^^^^^^^^^^^^^^^^^^^^^^^^ +--------------------------- Now that everything is in place, it remains to compile the project: .. code-block:: shell - make regen-configure + ./Tools/build/regen-configure.sh + ./configure --with-pydebug make regen-all make regen-stdlib-module-names + make + +.. tip:: + + Use ``make -j12`` to speed-up compilation if you have enough CPU cores. + +- Since the shipped version of :cpy-file:`configure` may not be up-to-date for + the new extension module, ``./Tools/build/regen-configure.sh`` should always + be executed first. This is equivalent to run ``make regen-configure`` but does + not require to create a ``Makefile`` first. -- The ``make regen-configure`` step regenerates the configure script. + Alternatively, :cpy-file:`configure` can be regenerated as follows: + + .. code-block:: shell + + ./configure # for creating a Makefile + make regen-configure # for updating 'configure' + ./configure # for updating the Makefile + +- The ``./configure --with-pydebug`` step generates the new Makefile. - The ``make regen-all`` is responsible for running Arguments Clinic, regenerating global objects, etc. It is useful to run when you do not know which files should be updated. - The ``regen-stdlib-module-names`` updates the standard module names, - making ``_foo`` discoverable and importable via ``import _foo``. + making ``fastfoo`` discoverable and importable via ``import fastfoo``. -You can now compile the entire project by running the following commands: - -.. code-block:: shell - - ./configure --with-pydebug - make - -.. tip:: Use ``make -j12`` to speed-up compilation if you have enough CPU cores. +- The final ``make`` step is generally not needed since ``make regen-all`` + and ``make regen-stdlib-module-names`` may completely rebuild the project, + but it could be needed in some specific cases. Troubleshooting -^^^^^^^^^^^^^^^ +--------------- This section addresses common issues that you may face when following this tutorial. ``make regen-configure`` does not work! -....................................... +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Since this rule requires Docker to be running and a Docker instance, the following can be done on Linux platforms (``systemctl``-based): .. code-block:: shell - systemctl status docker # is the docker service running? - sudo systemctl start docker # start it if not! - sudo systemctl restart docker # or restart it! + systemctl status docker # is the docker service running? + sudo systemctl start docker # start it if not! + sudo systemctl restart docker # or restart it! If Docker complains about missing permissions, this Stack Overflow post could be useful in solving the issue: `How to fix docker: permission denied `_. + +Once the Docker service is running, check if you have an `Ubuntu 22.04 image +`_, or pull it if it is not case: + +.. code-block:: shell + + docker images ubuntu:22.04 # check for the Docker image presence + docker image pull ubuntu:22.04 # or pull the image if it does not exist! From bdf09e58517a79db5916232a8ee80883a6d0590c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:03:16 +0200 Subject: [PATCH 15/45] fixup! sphinx --- developer-workflow/extension-modules.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index ab70d7b237..d79b7ba2c6 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -359,22 +359,22 @@ built-in or dynamic). .. rubric:: For required extension modules (built-in) - Open :cpy-file:`Modules/Setup.bootstrap.in` and add the following line: +Open :cpy-file:`Modules/Setup.bootstrap.in` and add the following line: - .. code-block:: text +.. code-block:: text - fastfoo cfoo/foomodule.c cfoo/helper.c + fastfoo cfoo/foomodule.c cfoo/helper.c .. rubric:: For optional extension modules - Open :cpy-file:`Modules/Setup.stdlib.in` and add the following line: +Open :cpy-file:`Modules/Setup.stdlib.in` and add the following line: - .. code-block:: text +.. code-block:: text - @MODULE_FASTFOO_TRUE@fastfoo cfoo/foomodule.c cfoo/helper.c + @MODULE_FASTFOO_TRUE@fastfoo cfoo/foomodule.c cfoo/helper.c - The ``@MODULE__TRUE@`` marker requires ```` - to be the upper case form of the module name ````. +The ``@MODULE__TRUE@`` marker requires ```` +to be the upper case form of the module name ````. Compile the CPython project --------------------------- From ef7cf3eba5773d599d197fe447c4bfbe1f4b2e24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:03:33 +0200 Subject: [PATCH 16/45] fixup! indents --- developer-workflow/extension-modules.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index d79b7ba2c6..a61093a360 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -363,7 +363,7 @@ Open :cpy-file:`Modules/Setup.bootstrap.in` and add the following line: .. code-block:: text - fastfoo cfoo/foomodule.c cfoo/helper.c + fastfoo cfoo/foomodule.c cfoo/helper.c .. rubric:: For optional extension modules @@ -371,7 +371,7 @@ Open :cpy-file:`Modules/Setup.stdlib.in` and add the following line: .. code-block:: text - @MODULE_FASTFOO_TRUE@fastfoo cfoo/foomodule.c cfoo/helper.c + @MODULE_FASTFOO_TRUE@fastfoo cfoo/foomodule.c cfoo/helper.c The ``@MODULE__TRUE@`` marker requires ```` to be the upper case form of the module name ````. From 1a405bab0862e45186b03b1af9c4e5e483441b7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:06:15 +0200 Subject: [PATCH 17/45] fixup! warnings --- developer-workflow/extension-modules.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index a61093a360..6eb9e2e324 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -20,7 +20,7 @@ where you can read good documentation: Adding an extension module to CPython ===================================== -Assume that the standard library contains a pure Python module :mod:`foo` +Assume that the standard library contains a pure Python module :mod:`!foo` together with the following :func:`!foo.greet` function: .. code-block:: python @@ -197,7 +197,7 @@ The actual implementation of the module is in the corresponding ``.c`` file: .. tip:: Recall that the ``PyInit_`` function must be suffixed by the same - module name as defined by :c:member:`PyModuleDef.m_mod` (here, ``fastfoo``). + module name as defined by :c:member:`PyModuleDef.m_name` (here, ``fastfoo``). The other identifiers or functions do not have such naming requirements. In a separate file, we put the implementation of ``_Py_greet_fast``: @@ -254,7 +254,7 @@ Now that we have our files, we first update :cpy-file:`configure.ac`: 1. Add a line ``Modules/cfoo`` in - .. code-block:: configure + .. code-block:: text AC_SUBST([SRCDIRS]) SRCDIRS="\ @@ -270,13 +270,13 @@ Now that we have our files, we first update :cpy-file:`configure.ac`: 2. Find the section containing ``PY_STDLIB_MOD_SIMPLE`` usages and add the following line: - .. code-block:: configure + .. code-block:: text PY_STDLIB_MOD_SIMPLE([fastfoo], [-I\$(srcdir)/Modules/cfoo], []) The ``PY_STDLIB_MOD_SIMPLE`` macro takes as arguments: - - the module name as specified by :c:member:`PyModuleDef.m_mod`, + - the module name as specified by :c:member:`PyModuleDef.m_name`, - the compiler flags (CFLAGS), and - the linker flags (LDFLAGS). From e39cbc2b225655cc25a8710945d9c2720358720b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:22:56 +0200 Subject: [PATCH 18/45] improve sections --- developer-workflow/extension-modules.rst | 93 +++++++++++++----------- 1 file changed, 52 insertions(+), 41 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 6eb9e2e324..e3e9fc3d32 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -228,12 +228,8 @@ Extension Modules Types Extension modules can be classified into the following types: - A *built-in* extension module is a module built and shipped with - the Python interpreter. - - .. note:: - - A built-in module is *statically* linked into the interpreter, - and thereby lacks a :attr:`__file__` attribute. + the Python interpreter. A built-in module is *statically* linked + into the interpreter, thereby lacking a :attr:`__file__` attribute. .. seealso:: :data:`sys.builtin_module_names` @@ -244,52 +240,62 @@ Extension modules can be classified into the following types: module's :attr:`__file__` attribute. Built-in extension modules are part of the interpreter, while dynamic extension -modules might be supplied or overridden externally. In particular, the latter -provide a pure Python implementation in case of missing ``.so/.dll`` files. +modules might be supplied or overridden externally. The latter should provide +a pure Python implementation in case of missing ``.so`` or ``.dll`` files. Make the CPython project compile -------------------------------- -Now that we have our files, we first update :cpy-file:`configure.ac`: +Once we have our files, we will need to update some configuration files. -1. Add a line ``Modules/cfoo`` in +Updating :cpy-file:`configure.ac` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - .. code-block:: text +* Add a line ``Modules/cfoo`` in - AC_SUBST([SRCDIRS]) - SRCDIRS="\ - ... - Modules/cfoo \ - ..." + .. code-block:: text - .. note:: + AC_SUBST([SRCDIRS]) + SRCDIRS="\ + ... + Modules/cfoo \ + ..." + + .. note:: - This step is only needed when adding new source directories to - the CPython project. + This step is only needed when adding new source directories to + the CPython project. -2. Find the section containing ``PY_STDLIB_MOD_SIMPLE`` usages and - add the following line: +* Find the section containing ``PY_STDLIB_MOD_SIMPLE`` usages and + add the following line: - .. code-block:: text + .. code-block:: text - PY_STDLIB_MOD_SIMPLE([fastfoo], [-I\$(srcdir)/Modules/cfoo], []) + PY_STDLIB_MOD_SIMPLE([fastfoo], [-I\$(srcdir)/Modules/cfoo], []) - The ``PY_STDLIB_MOD_SIMPLE`` macro takes as arguments: + The ``PY_STDLIB_MOD_SIMPLE`` macro takes as arguments: - - the module name as specified by :c:member:`PyModuleDef.m_name`, - - the compiler flags (CFLAGS), and - - the linker flags (LDFLAGS). + - the module name as specified by :c:member:`PyModuleDef.m_name`, + - the compiler flags (CFLAGS), and + - the linker flags (LDFLAGS). -Then, we update :cpy-file:`Makefile.pre.in` by adding to the -section **Module dependencies and platform-specific files**: +Updating :cpy-file:`Makefile.pre.in` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - .. code-block:: makefile +.. code-block:: makefile - MODULE_FASTFOO_DEPS=$(srcdir)/Modules/cfoo/foomodule.h + ########################################################################## + # Module dependencies and platform-specific files + ... + MODULE_FASTFOO_DEPS=$(srcdir)/Modules/cfoo/foomodule.h + ... -Additionally, we update the configuration files for Windows platforms: +Updating Windows configuration files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Open :cpy-file:`PC/config.c` and add the prototype: +We describe the minimal steps to build our extension on Windows platforms: + +* Open :cpy-file:`PC/config.c` and add the prototype: .. code-block:: c @@ -307,7 +313,7 @@ Additionally, we update the configuration files for Windows platforms: }; extern PyObject* PyInit_fastfoo(void); -- Open :cpy-file:`PCbuild/pythoncore.vcxproj` and add the following line to +* Open :cpy-file:`PCbuild/pythoncore.vcxproj` and add the following line to the ```` containing the other ``..\Modules\*.h`` files: .. code-block:: xml @@ -322,7 +328,7 @@ Additionally, we update the configuration files for Windows platforms: -- Open :cpy-file:`PCbuild/pythoncore.vcxproj.filters` and add the following +* Open :cpy-file:`PCbuild/pythoncore.vcxproj.filters` and add the following line to the ``ItemGroup`` containing the other ``..\Modules\*.h`` files: .. code-block:: xml @@ -343,13 +349,18 @@ Additionally, we update the configuration files for Windows platforms: Modules\cfoo -Observe that ``.h`` files use ```` whereas ``.c`` files -use ```` tags. +.. tip:: + + Observe that ``.h`` files use ```` whereas ``.c`` files + use ```` tags. + +Update :cpy-file:`!Modules/Setup.*.in` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -It remains to update :cpy-file:`Modules/Setup.bootstrap.in` if the module is -required to get a functioning interpreter (such module is *always* a built-in -module) or :cpy-file:`Modules/Setup.stdlib.in` otherwise (such module can be -built-in or dynamic). +Depending on whether the module is required to required to get a functioning +interpreter, we update :cpy-file:`Modules/Setup.bootstrap.in` (in which case +the extension is built-in) or :cpy-file:`Modules/Setup.stdlib.in`, (in which +case the extension can be built-in or dynamic). .. note:: From 35f207f45d6cc0b410490419cab06545d3fe93c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:25:16 +0200 Subject: [PATCH 19/45] fix markup --- developer-workflow/extension-modules.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index e3e9fc3d32..4847c8cde5 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -286,9 +286,9 @@ Updating :cpy-file:`Makefile.pre.in` ########################################################################## # Module dependencies and platform-specific files - ... + # ... MODULE_FASTFOO_DEPS=$(srcdir)/Modules/cfoo/foomodule.h - ... + # ... Updating Windows configuration files ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 316b00db18cb96a58a72a164b50b81b00dd338e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:27:13 +0200 Subject: [PATCH 20/45] improve titles --- developer-workflow/extension-modules.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 4847c8cde5..361e3584fd 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -354,8 +354,8 @@ We describe the minimal steps to build our extension on Windows platforms: Observe that ``.h`` files use ```` whereas ``.c`` files use ```` tags. -Update :cpy-file:`!Modules/Setup.*.in` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Updating :cpy-file:`!Modules/Setup.{bootstrap,stdlib}.in` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Depending on whether the module is required to required to get a functioning interpreter, we update :cpy-file:`Modules/Setup.bootstrap.in` (in which case From 523dece7713fdd4cc0057c3369522da58f76726e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:37:47 +0200 Subject: [PATCH 21/45] improve presentation --- developer-workflow/extension-modules.rst | 76 ++++++++++++------------ 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 361e3584fd..3b5bc8746d 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -49,7 +49,7 @@ to as *accelerator modules*). In our example, we need to determine: - where to place the extension module source code in the CPython project tree; - which files to modify in order to compile the CPython project; -- which :cpy-file:`!Makefile` rules to invoke at the end. +- which ``Makefile`` rules to invoke at the end. Updating the CPython project tree --------------------------------- @@ -227,13 +227,13 @@ Extension Modules Types Extension modules can be classified into the following types: -- A *built-in* extension module is a module built and shipped with +* A *built-in* extension module is a module built and shipped with the Python interpreter. A built-in module is *statically* linked into the interpreter, thereby lacking a :attr:`__file__` attribute. .. seealso:: :data:`sys.builtin_module_names` -- A *dynamic* (or *shared*) extension module is built as a *dynamic* library, +* A *dynamic* (or *shared*) extension module is built as a *dynamic* library, and is *dynamically* linked into the Python interpreter. In particular, the corresponding ``.so`` or ``.dll`` file is described by the @@ -275,9 +275,9 @@ Updating :cpy-file:`configure.ac` The ``PY_STDLIB_MOD_SIMPLE`` macro takes as arguments: - - the module name as specified by :c:member:`PyModuleDef.m_name`, - - the compiler flags (CFLAGS), and - - the linker flags (LDFLAGS). + * the module name as specified by :c:member:`PyModuleDef.m_name`, + * the compiler flags (CFLAGS), and + * the linker flags (LDFLAGS). Updating :cpy-file:`Makefile.pre.in` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -368,24 +368,22 @@ case the extension can be built-in or dynamic). but optional extension modules should have one in case the shared library is not present on the system. -.. rubric:: For required extension modules (built-in) +* For **required** extension modules (built-in), add the following + line to :cpy-file:`Modules/Setup.bootstrap.in`: -Open :cpy-file:`Modules/Setup.bootstrap.in` and add the following line: - -.. code-block:: text - - fastfoo cfoo/foomodule.c cfoo/helper.c + .. code-block:: text -.. rubric:: For optional extension modules + fastfoo cfoo/foomodule.c cfoo/helper.c -Open :cpy-file:`Modules/Setup.stdlib.in` and add the following line: +* For optional extension modules, add the following + line to :cpy-file:`Modules/Setup.stdlib.in`: -.. code-block:: text + .. code-block:: text - @MODULE_FASTFOO_TRUE@fastfoo cfoo/foomodule.c cfoo/helper.c + @MODULE_FASTFOO_TRUE@fastfoo cfoo/foomodule.c cfoo/helper.c -The ``@MODULE__TRUE@`` marker requires ```` -to be the upper case form of the module name ````. + The ``@MODULE__TRUE@`` marker requires ```` + to be the upper case form of the module name ````. Compile the CPython project --------------------------- @@ -394,7 +392,7 @@ Now that everything is in place, it remains to compile the project: .. code-block:: shell - ./Tools/build/regen-configure.sh + make regen-configure ./configure --with-pydebug make regen-all make regen-stdlib-module-names @@ -404,29 +402,18 @@ Now that everything is in place, it remains to compile the project: Use ``make -j12`` to speed-up compilation if you have enough CPU cores. -- Since the shipped version of :cpy-file:`configure` may not be up-to-date for - the new extension module, ``./Tools/build/regen-configure.sh`` should always - be executed first. This is equivalent to run ``make regen-configure`` but does - not require to create a ``Makefile`` first. - - Alternatively, :cpy-file:`configure` can be regenerated as follows: - - .. code-block:: shell +* ``make regen-configure`` updates the :cpy-file:`configure` script. - ./configure # for creating a Makefile - make regen-configure # for updating 'configure' - ./configure # for updating the Makefile +* ``./configure --with-pydebug`` updates the ``Makefile``. -- The ``./configure --with-pydebug`` step generates the new Makefile. +* ``make regen-all`` is responsible for regenerating header files and + invoking other scripts, such as :ref:`Arguments Clinic `. + It is useful to run when you do not know which files should be updated. -- The ``make regen-all`` is responsible for running Arguments Clinic, - regenerating global objects, etc. It is useful to run when you do not - know which files should be updated. +* ``regen-stdlib-module-names`` updates the standard module names, making + :mod:`!fastfoo` discoverable and importable via ``import fastfoo``. -- The ``regen-stdlib-module-names`` updates the standard module names, - making ``fastfoo`` discoverable and importable via ``import fastfoo``. - -- The final ``make`` step is generally not needed since ``make regen-all`` +* The final ``make`` step is generally not needed since ``make regen-all`` and ``make regen-stdlib-module-names`` may completely rebuild the project, but it could be needed in some specific cases. @@ -435,6 +422,19 @@ Troubleshooting This section addresses common issues that you may face when following this tutorial. +No rule to make target `regen-configure` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This usually happens after running ``make distclean`` since this removes +the local ``Makefile``. The solution is to regenerate :cpy-file:`configure` +as follows: + +.. code-block:: shell + + ./configure # for creating a Makefile + make regen-configure # for updating 'configure' + ./configure # for updating the Makefile + ``make regen-configure`` does not work! ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From d1cdd1de56535a4fd3de1ca27af64435a77eebba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:40:26 +0200 Subject: [PATCH 22/45] fixup! markup --- developer-workflow/extension-modules.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 3b5bc8746d..f6dec09379 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -422,8 +422,8 @@ Troubleshooting This section addresses common issues that you may face when following this tutorial. -No rule to make target `regen-configure` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +No rule to make target ``regen-configure`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This usually happens after running ``make distclean`` since this removes the local ``Makefile``. The solution is to regenerate :cpy-file:`configure` From defb31e8b07c9555dbdd8c5b3fed712f3ea80583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:41:12 +0200 Subject: [PATCH 23/45] simplify snippets --- developer-workflow/extension-modules.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index f6dec09379..31795aed25 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -393,7 +393,7 @@ Now that everything is in place, it remains to compile the project: .. code-block:: shell make regen-configure - ./configure --with-pydebug + ./configure make regen-all make regen-stdlib-module-names make @@ -404,8 +404,6 @@ Now that everything is in place, it remains to compile the project: * ``make regen-configure`` updates the :cpy-file:`configure` script. -* ``./configure --with-pydebug`` updates the ``Makefile``. - * ``make regen-all`` is responsible for regenerating header files and invoking other scripts, such as :ref:`Arguments Clinic `. It is useful to run when you do not know which files should be updated. From 6213438fdca49a889a52bef0a3172c96fb2f9dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:54:27 +0200 Subject: [PATCH 24/45] improvements --- developer-workflow/extension-modules.rst | 47 ++++++++++++++++-------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 31795aed25..46bfc76c44 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -311,7 +311,6 @@ We describe the minimal steps to build our extension on Windows platforms: ... {0, 0} }; - extern PyObject* PyInit_fastfoo(void); * Open :cpy-file:`PCbuild/pythoncore.vcxproj` and add the following line to the ```` containing the other ``..\Modules\*.h`` files: @@ -334,7 +333,7 @@ We describe the minimal steps to build our extension on Windows platforms: .. code-block:: xml - Modules\cfoo + Modules\cfoo In addition, add the following lines to the ``ItemGroup`` containing @@ -357,10 +356,10 @@ We describe the minimal steps to build our extension on Windows platforms: Updating :cpy-file:`!Modules/Setup.{bootstrap,stdlib}.in` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Depending on whether the module is required to required to get a functioning -interpreter, we update :cpy-file:`Modules/Setup.bootstrap.in` (in which case -the extension is built-in) or :cpy-file:`Modules/Setup.stdlib.in`, (in which -case the extension can be built-in or dynamic). +Depending on whether the extension module is required to get a functioning +interpreter or not, we update :cpy-file:`Modules/Setup.bootstrap.in` or +:cpy-file:`Modules/Setup.stdlib.in`. In the former case, the module is +built-in and statically linked. .. note:: @@ -368,22 +367,38 @@ case the extension can be built-in or dynamic). but optional extension modules should have one in case the shared library is not present on the system. -* For **required** extension modules (built-in), add the following - line to :cpy-file:`Modules/Setup.bootstrap.in`: +For required extension modules, update :cpy-file:`Modules/Setup.bootstrap.in` +by adding the following line after the ``*static*`` marker. - .. code-block:: text +.. code-block:: text - fastfoo cfoo/foomodule.c cfoo/helper.c + *static* + ... + fastfoo cfoo/foomodule.c cfoo/helper.c + ... -* For optional extension modules, add the following - line to :cpy-file:`Modules/Setup.stdlib.in`: +For other extension modules, update :cpy-file:`Modules/Setup.stdlib.in` +by adding the following line after the ``*@MODULE_BUILDTYPE@*`` marker +but before the ``*shared*`` marker: - .. code-block:: text +.. code-block:: text + + *@MODULE_BUILDTYPE@* + ... + @MODULE_FASTFOO_TRUE@fastfoo cfoo/foomodule.c cfoo/helper.c + ... + *shared* + +The ``@MODULE__TRUE@`` marker expects ```` to be the +upper-cased module name ````. If the extension module requires to +be built as a *shared* module, the additional line must be put after the +``*shared*`` marker: - @MODULE_FASTFOO_TRUE@fastfoo cfoo/foomodule.c cfoo/helper.c +.. code-block:: text - The ``@MODULE__TRUE@`` marker requires ```` - to be the upper case form of the module name ````. + *shared* + ... + @MODULE_FASTFOO_TRUE@fastfoo cfoo/foomodule.c cfoo/helper.c Compile the CPython project --------------------------- From f6e5d556f399a566505f25cce95f5d3a9fced189 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 15 Jul 2024 17:01:52 +0200 Subject: [PATCH 25/45] improvements --- developer-workflow/extension-modules.rst | 186 +++++++++++++---------- 1 file changed, 106 insertions(+), 80 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 46bfc76c44..70d2d123b9 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -76,8 +76,7 @@ or its C implementation) to highlight the differences in configuration files. The following code snippets illustrate the possible contents of the above files: .. code-block:: c - - // Modules/cfoo/foomodule.h + :caption: Modules/cfoo/foomodule.h #ifndef CFOO_FOOMODULE_H #define CFOO_FOOMODULE_H @@ -99,7 +98,7 @@ The following code snippets illustrate the possible contents of the above files: /* Helper used in Modules/cfoo/foomodule.c * but implemented in Modules/cfoo/helper.c. */ - extern PyObject *_Py_greet_fast(); + extern PyObject *_Py_greet_fast(void); #endif // CFOO_FOOMODULE_H @@ -107,8 +106,7 @@ The following code snippets illustrate the possible contents of the above files: The actual implementation of the module is in the corresponding ``.c`` file: .. code-block:: c - - // Modules/cfoo/foomodule.c + :caption: Modules/cfoo/foomodule.c #include "foomodule.h" #include "clinic/foomodule.c.h" @@ -200,27 +198,26 @@ The actual implementation of the module is in the corresponding ``.c`` file: module name as defined by :c:member:`PyModuleDef.m_name` (here, ``fastfoo``). The other identifiers or functions do not have such naming requirements. -In a separate file, we put the implementation of ``_Py_greet_fast``: +In a separate file, we put the implementation of :c:func:`!_Py_greet_fast`: .. code-block:: c - - // Modules/cfoo/helper.c + :caption: Modules/cfoo/helper.c #include "foomodule.h" - PyObject *_Py_greet_fast() { + PyObject *_Py_greet_fast(void) { return PyUnicode_FromString("Hello World!"); } .. tip:: Do not forget that symbols exported by ``libpython`` must start - with ``Py`` or ``_Py``, which is verified via ``make smelly``. + with ``Py`` or ``_Py``, which can be verified by ``make smelly``. -One could imagine having more ``.h`` files, or no ``helper.c`` file if it is -not needed. Here, we wanted to illustrate a simple example without making it -too trivial. If the extension module does not require additional files, it -may directly be placed in :cpy-file:`Modules` as ``Modules/foomodule.c``. +One could imagine having more ``.h`` files, or no ``helper.c`` file. Here, +we wanted to illustrate a simple example without making it too trivial. If +the extension module does not require additional files, it may directly be +placed in :cpy-file:`Modules` as ``Modules/foomodule.c``. Extension Modules Types ----------------------- @@ -231,7 +228,7 @@ Extension modules can be classified into the following types: the Python interpreter. A built-in module is *statically* linked into the interpreter, thereby lacking a :attr:`__file__` attribute. - .. seealso:: :data:`sys.builtin_module_names` + .. seealso:: :data:`sys.builtin_module_names` --- names of built-in modules. * A *dynamic* (or *shared*) extension module is built as a *dynamic* library, and is *dynamically* linked into the Python interpreter. @@ -240,20 +237,21 @@ Extension modules can be classified into the following types: module's :attr:`__file__` attribute. Built-in extension modules are part of the interpreter, while dynamic extension -modules might be supplied or overridden externally. The latter should provide -a pure Python implementation in case of missing ``.so`` or ``.dll`` files. +modules might be supplied or overridden externally. Make the CPython project compile -------------------------------- -Once we have our files, we will need to update some configuration files. +Once we have our files, we need to update some configuration files. Updating :cpy-file:`configure.ac` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* Add a line ``Modules/cfoo`` in +* Locate the ``SRCDIRS`` variable and add the following line: .. code-block:: text + :caption: :cpy-file:`configure.ac` + :emphasize-lines: 4 AC_SUBST([SRCDIRS]) SRCDIRS="\ @@ -270,8 +268,13 @@ Updating :cpy-file:`configure.ac` add the following line: .. code-block:: text + :caption: :cpy-file:`configure.ac` + :emphasize-lines: 3 + dnl always enabled extension modules + ... PY_STDLIB_MOD_SIMPLE([fastfoo], [-I\$(srcdir)/Modules/cfoo], []) + ... The ``PY_STDLIB_MOD_SIMPLE`` macro takes as arguments: @@ -282,76 +285,96 @@ Updating :cpy-file:`configure.ac` Updating :cpy-file:`Makefile.pre.in` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. code-block:: makefile +If needed, add the following line to the section for module dependencies: + +.. code-block:: text + :caption: :cpy-file:`Makefile.pre.in` + :emphasize-lines: 4 ########################################################################## # Module dependencies and platform-specific files - # ... + ... MODULE_FASTFOO_DEPS=$(srcdir)/Modules/cfoo/foomodule.h - # ... + ... -Updating Windows configuration files -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Updating MSVC project files +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -We describe the minimal steps to build our extension on Windows platforms: +The minimal steps for cross-compiling on platforms using MSVC instead of +GCC/Clang are listed below. More steps may be needed depending on the +complexity of the extension module: -* Open :cpy-file:`PC/config.c` and add the prototype: +* Update :cpy-file:`PC/config.c`: .. code-block:: c + :caption: :cpy-file:`PC/config.c` + :emphasize-lines: 3, 8 + ... + // add the entry point prototype extern PyObject* PyInit_fastfoo(void); - - and update the :c:data:`!_PyImport_Inittab`: - - .. code-block:: c - + ... + // update the entry points table struct _inittab _PyImport_Inittab[] = { ... {"fastfoo", PyInit_fastfoo}, ... {0, 0} }; + ... -* Open :cpy-file:`PCbuild/pythoncore.vcxproj` and add the following line to - the ```` containing the other ``..\Modules\*.h`` files: - - .. code-block:: xml - - - - In addition, add the following lines to the ```` - containing the other ``..\Modules\*.c`` files: - - .. code-block:: xml - - - - -* Open :cpy-file:`PCbuild/pythoncore.vcxproj.filters` and add the following - line to the ``ItemGroup`` containing the other ``..\Modules\*.h`` files: +* Update :cpy-file:`PCbuild/pythoncore.vcxproj`: .. code-block:: xml - - - Modules\cfoo - - - In addition, add the following lines to the ``ItemGroup`` containing - the other ``..\Modules\*.c`` files: + :caption: :cpy-file:`PCbuild/pythoncore.vcxproj` + :emphasize-lines: 4, 11-12 + + + + ... + + ... + + + + + ... + + + ... + + +* Update :cpy-file:`PCbuild/pythoncore.vcxproj.filters`: .. code-block:: xml - - - Modules\cfoo - - - Modules\cfoo - + :caption: :cpy-file:`PCbuild/pythoncore.vcxproj.filters` + :emphasize-lines: 4-6, 13-18 + + + + ... + + Modules\cfoo + + ... + + + + + ... + + Modules\cfoo + + + Modules\cfoo + + ... + .. tip:: - Observe that ``.h`` files use ```` whereas ``.c`` files - use ```` tags. + Header files use ```` tags, whereas + source files use ```` tags. Updating :cpy-file:`!Modules/Setup.{bootstrap,stdlib}.in` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -368,9 +391,10 @@ built-in and statically linked. library is not present on the system. For required extension modules, update :cpy-file:`Modules/Setup.bootstrap.in` -by adding the following line after the ``*static*`` marker. +by adding the following line after the ``*static*`` marker: .. code-block:: text + :emphasize-lines: 3 *static* ... @@ -382,6 +406,7 @@ by adding the following line after the ``*@MODULE_BUILDTYPE@*`` marker but before the ``*shared*`` marker: .. code-block:: text + :emphasize-lines: 3 *@MODULE_BUILDTYPE@* ... @@ -390,12 +415,14 @@ but before the ``*shared*`` marker: *shared* The ``@MODULE__TRUE@`` marker expects ```` to be the -upper-cased module name ````. If the extension module requires to -be built as a *shared* module, the additional line must be put after the -``*shared*`` marker: +upper-cased module name ````. If the extension module must be built +as a *shared* module, put the ``@MODULE_FASTFOO_TRUE@fastfoo`` line after +the ``*shared*`` marker: .. code-block:: text + :emphasize-lines: 4 + ... *shared* ... @MODULE_FASTFOO_TRUE@fastfoo cfoo/foomodule.c cfoo/helper.c @@ -420,8 +447,8 @@ Now that everything is in place, it remains to compile the project: * ``make regen-configure`` updates the :cpy-file:`configure` script. * ``make regen-all`` is responsible for regenerating header files and - invoking other scripts, such as :ref:`Arguments Clinic `. - It is useful to run when you do not know which files should be updated. + invoking other scripts, such as :ref:`Argument Clinic `. + Execute this rule if you do not know which files should be updated. * ``regen-stdlib-module-names`` updates the standard module names, making :mod:`!fastfoo` discoverable and importable via ``import fastfoo``. @@ -438,9 +465,8 @@ This section addresses common issues that you may face when following this tutor No rule to make target ``regen-configure`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -This usually happens after running ``make distclean`` since this removes -the local ``Makefile``. The solution is to regenerate :cpy-file:`configure` -as follows: +This usually happens after running ``make distclean`` (which removes +the ``Makefile``). The solution is to regenerate :cpy-file:`configure`: .. code-block:: shell @@ -451,23 +477,23 @@ as follows: ``make regen-configure`` does not work! ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Since this rule requires Docker to be running and a Docker instance, -the following can be done on Linux platforms (``systemctl``-based): +Since this rule requires `Docker `_ to be +running, the following can be done on Linux platforms (``systemctl``-based): .. code-block:: shell - systemctl status docker # is the docker service running? - sudo systemctl start docker # start it if not! - sudo systemctl restart docker # or restart it! + systemctl status docker # is the Docker service running? + sudo systemctl start docker # start it if it is not + sudo systemctl restart docker # or restart it if the issue persists If Docker complains about missing permissions, this Stack Overflow post could be useful in solving the issue: `How to fix docker: permission denied `_. -Once the Docker service is running, check if you have an `Ubuntu 22.04 image +Once the Docker service is running, check that you have an `Ubuntu 22.04 image `_, or pull it if it is not case: .. code-block:: shell docker images ubuntu:22.04 # check for the Docker image presence - docker image pull ubuntu:22.04 # or pull the image if it does not exist! + docker image pull ubuntu:22.04 # or pull the image if needed From d1a1ed551859f14e58721ddbbb5ae25292a94a0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 15 Jul 2024 17:32:52 +0200 Subject: [PATCH 26/45] some rewordings and cleanups --- developer-workflow/extension-modules.rst | 30 ++++++++++-------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 70d2d123b9..62ebbe25c9 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -102,9 +102,6 @@ The following code snippets illustrate the possible contents of the above files: #endif // CFOO_FOOMODULE_H - -The actual implementation of the module is in the corresponding ``.c`` file: - .. code-block:: c :caption: Modules/cfoo/foomodule.c @@ -198,8 +195,6 @@ The actual implementation of the module is in the corresponding ``.c`` file: module name as defined by :c:member:`PyModuleDef.m_name` (here, ``fastfoo``). The other identifiers or functions do not have such naming requirements. -In a separate file, we put the implementation of :c:func:`!_Py_greet_fast`: - .. code-block:: c :caption: Modules/cfoo/helper.c @@ -230,15 +225,19 @@ Extension modules can be classified into the following types: .. seealso:: :data:`sys.builtin_module_names` --- names of built-in modules. -* A *dynamic* (or *shared*) extension module is built as a *dynamic* library, - and is *dynamically* linked into the Python interpreter. +* A *dynamic* (or *shared*) extension module is built as a *dynamic* library + (``.so`` or ``.dll`` file) and is dynamically linked into the interpreter. - In particular, the corresponding ``.so`` or ``.dll`` file is described by the - module's :attr:`__file__` attribute. + In particular, the module's :attr:`__file__` attribute contains the path + to the ``.so`` or ``.dll`` file. Built-in extension modules are part of the interpreter, while dynamic extension modules might be supplied or overridden externally. +In particular, built-in extension modules do not need to have a pure Python +implementation but shared extension modules should have one in case the shared +library is not present on the system. + Make the CPython project compile -------------------------------- @@ -382,18 +381,13 @@ Updating :cpy-file:`!Modules/Setup.{bootstrap,stdlib}.in` Depending on whether the extension module is required to get a functioning interpreter or not, we update :cpy-file:`Modules/Setup.bootstrap.in` or :cpy-file:`Modules/Setup.stdlib.in`. In the former case, the module is -built-in and statically linked. - -.. note:: - - Built-in modules do not need to have a pure Python implementation - but optional extension modules should have one in case the shared - library is not present on the system. +necessarily built as a built-in module. -For required extension modules, update :cpy-file:`Modules/Setup.bootstrap.in` +For built-in extension modules, update :cpy-file:`Modules/Setup.bootstrap.in` by adding the following line after the ``*static*`` marker: .. code-block:: text + :caption: :cpy-file:`Modules/Setup.bootstrap.in` :emphasize-lines: 3 *static* @@ -406,6 +400,7 @@ by adding the following line after the ``*@MODULE_BUILDTYPE@*`` marker but before the ``*shared*`` marker: .. code-block:: text + :caption: :cpy-file:`Modules/Setup.stdlib.in` :emphasize-lines: 3 *@MODULE_BUILDTYPE@* @@ -420,6 +415,7 @@ as a *shared* module, put the ``@MODULE_FASTFOO_TRUE@fastfoo`` line after the ``*shared*`` marker: .. code-block:: text + :caption: :cpy-file:`Modules/Setup.stdlib.in` :emphasize-lines: 4 ... From 86e3e5456674a7bdf60d64cb6635f25c8ac3a372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 16 Jul 2024 11:12:38 +0200 Subject: [PATCH 27/45] simplify wording --- developer-workflow/extension-modules.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 62ebbe25c9..29d54ea718 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -299,9 +299,7 @@ If needed, add the following line to the section for module dependencies: Updating MSVC project files ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The minimal steps for cross-compiling on platforms using MSVC instead of -GCC/Clang are listed below. More steps may be needed depending on the -complexity of the extension module: +We describe the minimal steps for compiling on Windows using MSVC. * Update :cpy-file:`PC/config.c`: From 65f62e7f332e17f63d1c40dec178295fabaa5c36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 17 Jul 2024 16:13:57 +0200 Subject: [PATCH 28/45] address Erlend's review --- developer-workflow/extension-modules.rst | 142 ++++++++++++++++------- 1 file changed, 101 insertions(+), 41 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 29d54ea718..5bf4dee4f8 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -60,9 +60,9 @@ it is more convenient to create a sub-directory in :cpy-file:`Modules`. For our extension, we will create the following files: -- ``Modules/cfoo/foomodule.h`` --- the shared prototypes for our mini-project. -- ``Modules/cfoo/foomodule.c`` --- the actual module's implementation. -- ``Modules/cfoo/helper.c`` --- helpers implementation. +- ``Modules/_foo/_foomodule.h`` --- the shared prototypes for our mini-project. +- ``Modules/_foo/_foomodule.c`` --- the actual module's implementation. +- ``Modules/_foo/helper.c`` --- helpers implementation. We deliberately named the mini-project directory and files with names distinct from the actual Python module to import (whether it is the pure Python module @@ -70,16 +70,16 @@ or its C implementation) to highlight the differences in configuration files. .. note:: - If ``Modules/cfoo/foomodule.c`` contains Argument Clinic directives, - ``make clinic`` creates the file ``Modules/cfoo/clinic/foomodule.c.h``. + If ``Modules/_foo/_foomodule.c`` contains Argument Clinic directives, + ``make clinic`` creates the file ``Modules/_foo/clinic/_foomodule.c.h``. The following code snippets illustrate the possible contents of the above files: .. code-block:: c - :caption: Modules/cfoo/foomodule.h + :caption: Modules/_foo/_foomodule.h - #ifndef CFOO_FOOMODULE_H - #define CFOO_FOOMODULE_H + #ifndef _FOO__FOOMODULE_H + #define _FOO__FOOMODULE_H #include "Python.h" @@ -95,18 +95,18 @@ The following code snippets illustrate the possible contents of the above files: return (foomodule_state *)state; } - /* Helper used in Modules/cfoo/foomodule.c - * but implemented in Modules/cfoo/helper.c. + /* Helper used in Modules/_foo/_foomodule.c + * but implemented in Modules/_foo/helper.c. */ extern PyObject *_Py_greet_fast(void); - #endif // CFOO_FOOMODULE_H + #endif // _FOO__FOOMODULE_H .. code-block:: c - :caption: Modules/cfoo/foomodule.c + :caption: Modules/_foo/_foomodule.c - #include "foomodule.h" - #include "clinic/foomodule.c.h" + #include "_foomodule.h" + #include "clinic/_foomodule.c.h" /* Functions for the module's state */ static int @@ -158,7 +158,7 @@ The following code snippets illustrate the possible contents of the above files: /* Exported module's data */ static PyMethodDef foomodule_methods[] = { - // macro available in 'clinic/foomodule.c.h' after running 'make clinic' + // macro available in 'clinic/_foomodule.c.h' after running 'make clinic' FOO_GREET_METHODDEF {NULL, NULL} }; @@ -191,14 +191,15 @@ The following code snippets illustrate the possible contents of the above files: .. tip:: - Recall that the ``PyInit_`` function must be suffixed by the same - module name as defined by :c:member:`PyModuleDef.m_name` (here, ``fastfoo``). - The other identifiers or functions do not have such naming requirements. + Recall that the ``PyInit_`` function must be suffixed by the *same* + module name as that of :c:member:`PyModuleDef.m_name` (here, ``fastfoo``). + The other identifiers or functions such as those used in Argument Clinic + inputs or as local variables do not have such naming requirements. .. code-block:: c - :caption: Modules/cfoo/helper.c + :caption: Modules/_foo/helper.c - #include "foomodule.h" + #include "_foomodule.h" PyObject *_Py_greet_fast(void) { return PyUnicode_FromString("Hello World!"); @@ -212,7 +213,7 @@ The following code snippets illustrate the possible contents of the above files: One could imagine having more ``.h`` files, or no ``helper.c`` file. Here, we wanted to illustrate a simple example without making it too trivial. If the extension module does not require additional files, it may directly be -placed in :cpy-file:`Modules` as ``Modules/foomodule.c``. +placed in :cpy-file:`Modules` as ``Modules/_foomodule.c`` for instance. Extension Modules Types ----------------------- @@ -225,19 +226,24 @@ Extension modules can be classified into the following types: .. seealso:: :data:`sys.builtin_module_names` --- names of built-in modules. -* A *dynamic* (or *shared*) extension module is built as a *dynamic* library - (``.so`` or ``.dll`` file) and is dynamically linked into the interpreter. +* A *shared* (or *dynamic*) extension module is built as a shared library + (``.so`` or ``.dll`` file) and is *dynamically* linked into the interpreter. In particular, the module's :attr:`__file__` attribute contains the path to the ``.so`` or ``.dll`` file. -Built-in extension modules are part of the interpreter, while dynamic extension +Built-in extension modules are part of the interpreter, while shared extension modules might be supplied or overridden externally. In particular, built-in extension modules do not need to have a pure Python implementation but shared extension modules should have one in case the shared library is not present on the system. +.. note:: + + Usually, accelerator modules are built as *shared* extension modules, + especially if they already have a pure Python implementation. + Make the CPython project compile -------------------------------- @@ -255,7 +261,7 @@ Updating :cpy-file:`configure.ac` AC_SUBST([SRCDIRS]) SRCDIRS="\ ... - Modules/cfoo \ + Modules/_foo \ ..." .. note:: @@ -263,8 +269,8 @@ Updating :cpy-file:`configure.ac` This step is only needed when adding new source directories to the CPython project. -* Find the section containing ``PY_STDLIB_MOD_SIMPLE`` usages and - add the following line: +* Find the section containing ``PY_STDLIB_MOD`` and ``PY_STDLIB_MOD_SIMPLE`` + usages and add the following line: .. code-block:: text :caption: :cpy-file:`configure.ac` @@ -272,7 +278,7 @@ Updating :cpy-file:`configure.ac` dnl always enabled extension modules ... - PY_STDLIB_MOD_SIMPLE([fastfoo], [-I\$(srcdir)/Modules/cfoo], []) + PY_STDLIB_MOD_SIMPLE([fastfoo], [-I\$(srcdir)/Modules/_foo], []) ... The ``PY_STDLIB_MOD_SIMPLE`` macro takes as arguments: @@ -281,6 +287,55 @@ Updating :cpy-file:`configure.ac` * the compiler flags (CFLAGS), and * the linker flags (LDFLAGS). + If the extension module may not be enabled or supported depending on the + host configuration. use ``PY_STDLIB_MOD`` which takes as arguments: + + * the module name as specified by :c:member:`PyModuleDef.m_name`, + * a boolean indicating whether the extension is **enabled** or not, + * a boolean indicating whether the extension is **supported** or not, + * the compiler flags (CFLAGS), and + * the linker flags (LDFLAGS). + + For instance, enabling the ``fastfoo`` extension on Linux systems, but + only providing support for 32-bit architecture is achieved as follows: + + .. code-block:: text + :caption: :cpy-file:`configure.ac` + :emphasize-lines: 2, 3 + + PY_STDLIB_MOD([fastfoo], + [test "$ac_sys_system" = "Linux"], + [test "$ARCH_RUN_32BIT" = "true"], + [-I\$(srcdir)/Modules/_foo], []) + + More generally, the status of the extension is determined as follows: + + +-----------+-----------------+----------+ + | Enabled | Supported | Status | + +===========+=================+==========+ + | true | true | yes | + +-----------+-----------------+----------+ + | true | false | missing | + +-----------+-----------------+----------+ + | false | true or false | disabled | + +-----------+-----------------+----------+ + + The extension status is ``n/a`` if the extension is marked unavailable + via the ``PY_STDLIB_MOD_SET_NA`` macro. To add an unavailable extension, + find the usage of ``PY_STDLIB_MOD_SET_NA`` in :cpy-file:`configure.ac` + and add the following line: + + .. code-block:: text + :caption: :cpy-file:`configure.ac` + :emphasize-lines: 4 + + dnl Modules that are not available on some platforms + AS_CASE([$ac_sys_system], + ... + [PLATFORM_NAME], [PY_STDLIB_MOD_SET_NA([fastfoo])], + ... + ) + Updating :cpy-file:`Makefile.pre.in` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -293,7 +348,7 @@ If needed, add the following line to the section for module dependencies: ########################################################################## # Module dependencies and platform-specific files ... - MODULE_FASTFOO_DEPS=$(srcdir)/Modules/cfoo/foomodule.h + MODULE_FASTFOO_DEPS=$(srcdir)/Modules/_foo/_foomodule.h ... Updating MSVC project files @@ -329,15 +384,15 @@ We describe the minimal steps for compiling on Windows using MSVC. ... - + ... ... - - + + ... @@ -350,8 +405,8 @@ We describe the minimal steps for compiling on Windows using MSVC. ... - - Modules\cfoo + + Modules\_foo ... @@ -359,11 +414,11 @@ We describe the minimal steps for compiling on Windows using MSVC. ... - - Modules\cfoo + + Modules\_foo - - Modules\cfoo + + Modules\_foo ... @@ -381,6 +436,11 @@ interpreter or not, we update :cpy-file:`Modules/Setup.bootstrap.in` or :cpy-file:`Modules/Setup.stdlib.in`. In the former case, the module is necessarily built as a built-in module. +.. tip:: + + For accelerator modules, :cpy-file:`Modules/Setup.stdlib.in` should be + preferred over :cpy-file:`Modules/Setup.bootstrap.in`. + For built-in extension modules, update :cpy-file:`Modules/Setup.bootstrap.in` by adding the following line after the ``*static*`` marker: @@ -390,7 +450,7 @@ by adding the following line after the ``*static*`` marker: *static* ... - fastfoo cfoo/foomodule.c cfoo/helper.c + fastfoo _foo/_foomodule.c _foo/helper.c ... For other extension modules, update :cpy-file:`Modules/Setup.stdlib.in` @@ -403,7 +463,7 @@ but before the ``*shared*`` marker: *@MODULE_BUILDTYPE@* ... - @MODULE_FASTFOO_TRUE@fastfoo cfoo/foomodule.c cfoo/helper.c + @MODULE_FASTFOO_TRUE@fastfoo _foo/_foomodule.c _foo/helper.c ... *shared* @@ -419,7 +479,7 @@ the ``*shared*`` marker: ... *shared* ... - @MODULE_FASTFOO_TRUE@fastfoo cfoo/foomodule.c cfoo/helper.c + @MODULE_FASTFOO_TRUE@fastfoo _foo/_foomodule.c _foo/helper.c Compile the CPython project --------------------------- From 4b7c7d8c5091b9107622ad1dae2c6b2117b68ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 17 Jul 2024 16:50:01 +0200 Subject: [PATCH 29/45] fix indents? --- developer-workflow/extension-modules.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 5bf4dee4f8..f224cc24b6 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -310,15 +310,15 @@ Updating :cpy-file:`configure.ac` More generally, the status of the extension is determined as follows: - +-----------+-----------------+----------+ - | Enabled | Supported | Status | - +===========+=================+==========+ - | true | true | yes | - +-----------+-----------------+----------+ - | true | false | missing | - +-----------+-----------------+----------+ - | false | true or false | disabled | - +-----------+-----------------+----------+ + +-----------+-----------------+----------+ + | Enabled | Supported | Status | + +===========+=================+==========+ + | true | true | yes | + +-----------+-----------------+----------+ + | true | false | missing | + +-----------+-----------------+----------+ + | false | true or false | disabled | + +-----------+-----------------+----------+ The extension status is ``n/a`` if the extension is marked unavailable via the ``PY_STDLIB_MOD_SET_NA`` macro. To add an unavailable extension, From 7abb6f1bf95dcc4a7a348bf3f50612fe9550d23f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 17 Jul 2024 16:52:31 +0200 Subject: [PATCH 30/45] add ref to clinic everywhere when needed --- developer-workflow/extension-modules.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index f224cc24b6..d7be24a4b2 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -70,8 +70,8 @@ or its C implementation) to highlight the differences in configuration files. .. note:: - If ``Modules/_foo/_foomodule.c`` contains Argument Clinic directives, - ``make clinic`` creates the file ``Modules/_foo/clinic/_foomodule.c.h``. + If ``Modules/_foo/_foomodule.c`` contains :ref:`Argument Clinic ` + directives, ``make clinic`` creates ``Modules/_foo/clinic/_foomodule.c.h``. The following code snippets illustrate the possible contents of the above files: @@ -193,8 +193,8 @@ The following code snippets illustrate the possible contents of the above files: Recall that the ``PyInit_`` function must be suffixed by the *same* module name as that of :c:member:`PyModuleDef.m_name` (here, ``fastfoo``). - The other identifiers or functions such as those used in Argument Clinic - inputs or as local variables do not have such naming requirements. + Other identifiers such as those used in :ref:`Argument Clinic ` + inputs do not have such naming requirements. .. code-block:: c :caption: Modules/_foo/helper.c From da9b58b48c1c40c0b86da11546aaa7438d6f4d9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 17 Jul 2024 16:53:28 +0200 Subject: [PATCH 31/45] fix typos --- developer-workflow/extension-modules.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index d7be24a4b2..bf843de7db 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -148,8 +148,9 @@ The following code snippets illustrate the possible contents of the above files: foo.greet -> object [clinic start generated code]*/ + static PyObject * - foo_greet(PyObject *module) + foo_greet_impl(PyObject *module) /*[clinic end generated code: output=... input=...]*/ { return _Py_greet_fast(); @@ -288,7 +289,8 @@ Updating :cpy-file:`configure.ac` * the linker flags (LDFLAGS). If the extension module may not be enabled or supported depending on the - host configuration. use ``PY_STDLIB_MOD`` which takes as arguments: + host configuration, use the ``PY_STDLIB_MOD`` macro instead, which takes + as arguments: * the module name as specified by :c:member:`PyModuleDef.m_name`, * a boolean indicating whether the extension is **enabled** or not, @@ -296,8 +298,8 @@ Updating :cpy-file:`configure.ac` * the compiler flags (CFLAGS), and * the linker flags (LDFLAGS). - For instance, enabling the ``fastfoo`` extension on Linux systems, but - only providing support for 32-bit architecture is achieved as follows: + For instance, enabling the ``fastfoo`` extension on Linux platforms, but + only providing support for 32-bit architecture, is achieved as follows: .. code-block:: text :caption: :cpy-file:`configure.ac` From 783e6db82e7b758a938e38c00c114971fe7e170e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:27:09 +0200 Subject: [PATCH 32/45] address encukou's review --- developer-workflow/extension-modules.rst | 39 ++++++++++++++++++------ 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index bf843de7db..0b097357c4 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -42,10 +42,25 @@ written in C. Ideally, we want to modify ``foo.py`` as follows: def greet(): return "Hello World!" -Some modules in the standard library are implemented both in C and in Python, -such as :mod:`decimal` or :mod:`itertools`, and the C implementation is expected -to improve performance when available (such modules are commonly referred -to as *accelerator modules*). In our example, we need to determine: +Some modules in the standard library, such as :mod:`datetime` or :mod:`pickle`, +have identical implementations in C and Python; the C implementation, when +available, is typically expected to improve performance (such modules are +commonly referred to as *accelerator modules*). + +Other modules mainly implemented in Python may import a C helper extension +providing implementation details (for instance, the :mod:`csv` module uses +the internal :mod:`!_csv` module defined in :cpy-file:`Modules/_csv.c`). + +.. note:: + + According to :pep:`399` guidelines, *new* modules must have a working + and tested implementation in pure Python, unless a special dispensation + is given. + + Please ask the :github:`Steering Council ` if + such dispensation is needed. + +In our example, we need to determine: - where to place the extension module source code in the CPython project tree; - which files to modify in order to compile the CPython project; @@ -208,8 +223,12 @@ The following code snippets illustrate the possible contents of the above files: .. tip:: - Do not forget that symbols exported by ``libpython`` must start - with ``Py`` or ``_Py``, which can be verified by ``make smelly``. + Functions or data that do not need to be shared across different C source + files should be declared ``static`` to avoid exporting the symbols from + ``libpython``. + + If symbols need to be exported, their names must start with ``Py`` or + ``_Py``. This can be verified by ``make smelly``. One could imagine having more ``.h`` files, or no ``helper.c`` file. Here, we wanted to illustrate a simple example without making it too trivial. If @@ -236,9 +255,11 @@ Extension modules can be classified into the following types: Built-in extension modules are part of the interpreter, while shared extension modules might be supplied or overridden externally. -In particular, built-in extension modules do not need to have a pure Python -implementation but shared extension modules should have one in case the shared -library is not present on the system. +New built-in extension modules could be considered exceptions to :pep:`399`, +but please ask the Steering Council for confirmation. Nevertheless, besides +respecting :pep:`399`, shared extension modules MUST provide a working and +tested Python implementation since the corresponding shared library might +not be present on the system. .. note:: From 8906ebdae1747cb0fa72d4609d7a5a7a18a637cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 18 Jul 2024 12:44:31 +0200 Subject: [PATCH 33/45] improve the page flow --- developer-workflow/extension-modules.rst | 169 +++++++++++------------ 1 file changed, 84 insertions(+), 85 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 0b097357c4..7881adf750 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -9,6 +9,15 @@ In this section, we are interested in extending the CPython project with an :term:`extension module`. We will not explain how to write the module in C but rather explain how to configure the project and make it compile. +Some modules in the standard library, such as :mod:`datetime` or :mod:`pickle`, +have identical implementations in C and Python; the C implementation, when +available, is expected to improve performance (such extension modules are +commonly referred to as *accelerator modules*). + +Other modules mainly implemented in Python may import a C helper extension +providing implementation details (for instance, the :mod:`csv` module uses +the internal :mod:`!_csv` module defined in :cpy-file:`Modules/_csv.c`). + For writing an extension module in C, we prefer to give you some links where you can read good documentation: @@ -17,6 +26,42 @@ where you can read good documentation: * :pep:`399` * https://pythonextensionpatterns.readthedocs.io/en/latest/ +Classifying Extension Modules +============================= + +Extension modules can be classified into two categories: + +* A *built-in* extension module is a module built and shipped with + the Python interpreter. A built-in module is *statically* linked + into the interpreter, thereby lacking a :attr:`__file__` attribute. + + .. seealso:: :data:`sys.builtin_module_names` --- names of built-in modules. + +* A *shared* (or *dynamic*) extension module is built as a shared library + (``.so`` or ``.dll`` file) and is *dynamically* linked into the interpreter. + + In particular, the module's :attr:`__file__` attribute contains the path + to the ``.so`` or ``.dll`` file. + +.. note:: + + Informally, built-in extension modules can be regarded as *required* + while shared extension modules are *optional* in the sense that they + might be supplied, overridden or disabled externally. + + Usually, accelerator modules are built as *shared* extension modules, + especially if they already have a pure Python implementation. + +According to :pep:`399`, *new* extension modules MUST provide a working and +tested pure Python implementation, unless a special dispensation is given. +Please ask the :github:`Steering Council ` if such +dispensation is needed. + +While new built-in extension modules could be considered exceptions to that +rule, shared extension modules must provide a Python implementation, not only +because of the guidelines, but also as a fallback if the corresponding shared +library is not be present on the system. + Adding an extension module to CPython ===================================== @@ -24,6 +69,7 @@ Assume that the standard library contains a pure Python module :mod:`!foo` together with the following :func:`!foo.greet` function: .. code-block:: python + :caption: Lib/foo.py def greet(): return "Hello World!" @@ -33,6 +79,7 @@ use its corresponding C implementation exposed in some :mod:`!fastfoo` module written in C. Ideally, we want to modify ``foo.py`` as follows: .. code-block:: python + :caption: Lib/foo.py try: # use the C implementation if possible @@ -42,24 +89,6 @@ written in C. Ideally, we want to modify ``foo.py`` as follows: def greet(): return "Hello World!" -Some modules in the standard library, such as :mod:`datetime` or :mod:`pickle`, -have identical implementations in C and Python; the C implementation, when -available, is typically expected to improve performance (such modules are -commonly referred to as *accelerator modules*). - -Other modules mainly implemented in Python may import a C helper extension -providing implementation details (for instance, the :mod:`csv` module uses -the internal :mod:`!_csv` module defined in :cpy-file:`Modules/_csv.c`). - -.. note:: - - According to :pep:`399` guidelines, *new* modules must have a working - and tested implementation in pure Python, unless a special dispensation - is given. - - Please ask the :github:`Steering Council ` if - such dispensation is needed. - In our example, we need to determine: - where to place the extension module source code in the CPython project tree; @@ -73,25 +102,24 @@ Usually, accelerator modules are added in the :cpy-file:`Modules` directory of the CPython project. If more than one file is needed for the extension module, it is more convenient to create a sub-directory in :cpy-file:`Modules`. -For our extension, we will create the following files: +For our extension module ``fastfoo``, we consider the following working tree: -- ``Modules/_foo/_foomodule.h`` --- the shared prototypes for our mini-project. -- ``Modules/_foo/_foomodule.c`` --- the actual module's implementation. -- ``Modules/_foo/helper.c`` --- helpers implementation. +- :ref:`Modules/_foo/_foomodule.h` --- the extension module shared prototypes. +- :ref:`Modules/_foo/_foomodule.c` --- the extension module implementation. +- :ref:`Modules/_foo/helper.c` --- the extension helpers implementation. -We deliberately named the mini-project directory and files with names distinct +We deliberately named the working tree directory and files with names distinct from the actual Python module to import (whether it is the pure Python module or its C implementation) to highlight the differences in configuration files. -.. note:: - - If ``Modules/_foo/_foomodule.c`` contains :ref:`Argument Clinic ` - directives, ``make clinic`` creates ``Modules/_foo/clinic/_foomodule.c.h``. - -The following code snippets illustrate the possible contents of the above files: +One could imagine having more ``.h`` files, or no ``helper.c`` file. Here, +we wanted to illustrate a simple example without making it too trivial. If +the extension module does not require additional files, it may directly be +placed in :cpy-file:`Modules` as ``Modules/_foomodule.c`` for instance. .. code-block:: c :caption: Modules/_foo/_foomodule.h + :name: Modules/_foo/_foomodule.h #ifndef _FOO__FOOMODULE_H #define _FOO__FOOMODULE_H @@ -119,11 +147,12 @@ The following code snippets illustrate the possible contents of the above files: .. code-block:: c :caption: Modules/_foo/_foomodule.c + :name: Modules/_foo/_foomodule.c #include "_foomodule.h" #include "clinic/_foomodule.c.h" - /* Functions for the module's state */ + /* Functions for the extension module's state */ static int foomodule_exec(PyObject *module) { @@ -174,7 +203,7 @@ The following code snippets illustrate the possible contents of the above files: /* Exported module's data */ static PyMethodDef foomodule_methods[] = { - // macro available in 'clinic/_foomodule.c.h' after running 'make clinic' + // macro in 'clinic/_foomodule.c.h' after running 'make clinic' FOO_GREET_METHODDEF {NULL, NULL} }; @@ -214,6 +243,7 @@ The following code snippets illustrate the possible contents of the above files: .. code-block:: c :caption: Modules/_foo/helper.c + :name: Modules/_foo/helper.c #include "_foomodule.h" @@ -224,56 +254,24 @@ The following code snippets illustrate the possible contents of the above files: .. tip:: Functions or data that do not need to be shared across different C source - files should be declared ``static`` to avoid exporting the symbols from + files should be declared ``static`` to avoid exporting their symbols from ``libpython``. If symbols need to be exported, their names must start with ``Py`` or ``_Py``. This can be verified by ``make smelly``. -One could imagine having more ``.h`` files, or no ``helper.c`` file. Here, -we wanted to illustrate a simple example without making it too trivial. If -the extension module does not require additional files, it may directly be -placed in :cpy-file:`Modules` as ``Modules/_foomodule.c`` for instance. - -Extension Modules Types ------------------------ - -Extension modules can be classified into the following types: - -* A *built-in* extension module is a module built and shipped with - the Python interpreter. A built-in module is *statically* linked - into the interpreter, thereby lacking a :attr:`__file__` attribute. - - .. seealso:: :data:`sys.builtin_module_names` --- names of built-in modules. - -* A *shared* (or *dynamic*) extension module is built as a shared library - (``.so`` or ``.dll`` file) and is *dynamically* linked into the interpreter. - - In particular, the module's :attr:`__file__` attribute contains the path - to the ``.so`` or ``.dll`` file. +Configuring the CPython project +------------------------------- -Built-in extension modules are part of the interpreter, while shared extension -modules might be supplied or overridden externally. - -New built-in extension modules could be considered exceptions to :pep:`399`, -but please ask the Steering Council for confirmation. Nevertheless, besides -respecting :pep:`399`, shared extension modules MUST provide a working and -tested Python implementation since the corresponding shared library might -not be present on the system. - -.. note:: - - Usually, accelerator modules are built as *shared* extension modules, - especially if they already have a pure Python implementation. - -Make the CPython project compile --------------------------------- - -Once we have our files, we need to update some configuration files. +Now that we have implemented our extension module, we need to update some +configuration files in order to compile the CPython project on different +platforms. Updating :cpy-file:`configure.ac` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. add section about configuration variable afterwards + * Locate the ``SRCDIRS`` variable and add the following line: .. code-block:: text @@ -344,9 +342,9 @@ Updating :cpy-file:`configure.ac` +-----------+-----------------+----------+ The extension status is ``n/a`` if the extension is marked unavailable - via the ``PY_STDLIB_MOD_SET_NA`` macro. To add an unavailable extension, - find the usage of ``PY_STDLIB_MOD_SET_NA`` in :cpy-file:`configure.ac` - and add the following line: + by the ``PY_STDLIB_MOD_SET_NA`` macro. To mark an extension as unavailable, + find the usages of ``PY_STDLIB_MOD_SET_NA`` in :cpy-file:`configure.ac` and + add the following line: .. code-block:: text :caption: :cpy-file:`configure.ac` @@ -456,8 +454,8 @@ Updating :cpy-file:`!Modules/Setup.{bootstrap,stdlib}.in` Depending on whether the extension module is required to get a functioning interpreter or not, we update :cpy-file:`Modules/Setup.bootstrap.in` or -:cpy-file:`Modules/Setup.stdlib.in`. In the former case, the module is -necessarily built as a built-in module. +:cpy-file:`Modules/Setup.stdlib.in`. In the former case, the extension +module is necessarily built as a built-in extension module. .. tip:: @@ -504,8 +502,8 @@ the ``*shared*`` marker: ... @MODULE_FASTFOO_TRUE@fastfoo _foo/_foomodule.c _foo/helper.c -Compile the CPython project ---------------------------- +Compiling the CPython project +----------------------------- Now that everything is in place, it remains to compile the project: @@ -527,12 +525,12 @@ Now that everything is in place, it remains to compile the project: invoking other scripts, such as :ref:`Argument Clinic `. Execute this rule if you do not know which files should be updated. -* ``regen-stdlib-module-names`` updates the standard module names, making +* ``make regen-stdlib-module-names`` updates the standard module names, making :mod:`!fastfoo` discoverable and importable via ``import fastfoo``. -* The final ``make`` step is generally not needed since ``make regen-all`` - and ``make regen-stdlib-module-names`` may completely rebuild the project, - but it could be needed in some specific cases. +* The final ``make`` step is generally not needed since the previous ``make`` + invokations may completely rebuild the project, but it could be needed in + some specific cases. Troubleshooting --------------- @@ -543,7 +541,8 @@ No rule to make target ``regen-configure`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This usually happens after running ``make distclean`` (which removes -the ``Makefile``). The solution is to regenerate :cpy-file:`configure`: +the ``Makefile``). The solution is to regenerate the :cpy-file:`configure` +script as follows: .. code-block:: shell @@ -551,8 +550,8 @@ the ``Makefile``). The solution is to regenerate :cpy-file:`configure`: make regen-configure # for updating 'configure' ./configure # for updating the Makefile -``make regen-configure`` does not work! -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +``make regen-configure`` and missing permissions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Since this rule requires `Docker `_ to be running, the following can be done on Linux platforms (``systemctl``-based): From 128c81cce9dc5c0f261d6d050afa9c6a8ea2f815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 18 Jul 2024 13:58:27 +0200 Subject: [PATCH 34/45] use sentence case (that's the reason why the previous title felt wrong to me!) --- developer-workflow/extension-modules.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 7881adf750..c8d6a9a893 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -26,7 +26,7 @@ where you can read good documentation: * :pep:`399` * https://pythonextensionpatterns.readthedocs.io/en/latest/ -Classifying Extension Modules +Classifying extension modules ============================= Extension modules can be classified into two categories: From 7d6c8d67d979fe6e4f69c2cdaad4010c3e161f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:23:33 +0200 Subject: [PATCH 35/45] add podman tip --- developer-workflow/extension-modules.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index c8d6a9a893..826dcbd485 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -573,3 +573,8 @@ Once the Docker service is running, check that you have an `Ubuntu 22.04 image docker images ubuntu:22.04 # check for the Docker image presence docker image pull ubuntu:22.04 # or pull the image if needed + +.. tip:: + + If the issue persists, you may try `podman `_. + The commands for listing or pulling an image are the same as ``docker``. From 01c25bc7780b66c3e210d20d7f682d26eb76e369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 18 Jul 2024 15:29:19 +0200 Subject: [PATCH 36/45] address rest of the review --- developer-workflow/extension-modules.rst | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 826dcbd485..ec0188d976 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -57,11 +57,6 @@ tested pure Python implementation, unless a special dispensation is given. Please ask the :github:`Steering Council ` if such dispensation is needed. -While new built-in extension modules could be considered exceptions to that -rule, shared extension modules must provide a Python implementation, not only -because of the guidelines, but also as a fallback if the corresponding shared -library is not be present on the system. - Adding an extension module to CPython ===================================== @@ -546,9 +541,17 @@ script as follows: .. code-block:: shell - ./configure # for creating a Makefile - make regen-configure # for updating 'configure' - ./configure # for updating the Makefile + ./configure # for creating the 'Makefile' file + make regen-configure # for updating the 'configure' script + ./configure # for updating the 'Makefile' file + +If missing, the :cpy-file:`configure` script can be regenerated +by executing :cpy-file:`Tools/build/regen-configure.sh`: + +.. code-block:: shell + + ./Tools/build/regen-configure.sh # create an up-to-date 'configure' + ./configure # create an up-to-date 'Makefile' ``make regen-configure`` and missing permissions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 56910fb692b114b10aebd957845171ee67219b2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 18 Jul 2024 17:47:28 +0200 Subject: [PATCH 37/45] address Alyssa's review --- developer-workflow/extension-modules.rst | 248 +++++++++++++---------- 1 file changed, 140 insertions(+), 108 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index ec0188d976..167de7ca60 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -70,24 +70,30 @@ together with the following :func:`!foo.greet` function: return "Hello World!" Instead of using the Python implementation of :func:`!foo.greet`, we want to -use its corresponding C implementation exposed in some :mod:`!fastfoo` module -written in C. Ideally, we want to modify ``foo.py`` as follows: +use its corresponding C implementation exposed in some :mod:`!_foo` module +written in C. Ideally, we want to modify :cpy-file:`!Lib/foo.py` as follows: .. code-block:: python :caption: Lib/foo.py try: # use the C implementation if possible - from fastfoo import greet + from _foo import greet except ImportError: # fallback to the pure Python implementation def greet(): return "Hello World!" -In our example, we need to determine: +.. note:: + + Accelerator modules should *never* be imported directly, whence the + convention is to mark them as private implementation details with the + underscore prefix (namely, :mod:`!_foo` in this example). + +In order to incorporate the accelerator module, we need to determine: -- where to place the extension module source code in the CPython project tree; -- which files to modify in order to compile the CPython project; +- where to place the extension module source code in the CPython project tree, +- which files to modify in order to compile the CPython project, and - which ``Makefile`` rules to invoke at the end. Updating the CPython project tree @@ -97,27 +103,28 @@ Usually, accelerator modules are added in the :cpy-file:`Modules` directory of the CPython project. If more than one file is needed for the extension module, it is more convenient to create a sub-directory in :cpy-file:`Modules`. -For our extension module ``fastfoo``, we consider the following working tree: +For our extension module :mod:`!_foo`, we consider the following working tree: -- :ref:`Modules/_foo/_foomodule.h` --- the extension module shared prototypes. - :ref:`Modules/_foo/_foomodule.c` --- the extension module implementation. -- :ref:`Modules/_foo/helper.c` --- the extension helpers implementation. +- :ref:`Modules/_foo/helper.h` --- the extension helpers declarations. +- :ref:`Modules/_foo/helper.c` --- the extension helpers implementations. -We deliberately named the working tree directory and files with names distinct -from the actual Python module to import (whether it is the pure Python module -or its C implementation) to highlight the differences in configuration files. +By convention, the source file containing the extension module implementation +is called ``module.c``, where ```` is the name of the module that +will be later imported (in our case :mod:`!_foo`). In addition, the directory +containing the implementation should also be named similarly. -One could imagine having more ``.h`` files, or no ``helper.c`` file. Here, +One could imagine having more files, or no helper files at all. Here, we wanted to illustrate a simple example without making it too trivial. If the extension module does not require additional files, it may directly be -placed in :cpy-file:`Modules` as ``Modules/_foomodule.c`` for instance. +placed in :cpy-file:`Modules` as ``Modules/_foomodule.c``. .. code-block:: c - :caption: Modules/_foo/_foomodule.h - :name: Modules/_foo/_foomodule.h + :caption: Modules/_foo/helper.h + :name: Modules/_foo/helper.h - #ifndef _FOO__FOOMODULE_H - #define _FOO__FOOMODULE_H + #ifndef _FOO_HELPER_H + #define _FOO_HELPER_H #include "Python.h" @@ -136,15 +143,36 @@ placed in :cpy-file:`Modules` as ``Modules/_foomodule.c`` for instance. /* Helper used in Modules/_foo/_foomodule.c * but implemented in Modules/_foo/helper.c. */ - extern PyObject *_Py_greet_fast(void); + extern PyObject * + _Py_greet_fast(void); + + #endif // _FOO_HELPER_H + +.. tip:: + + Functions or data that do not need to be shared across different C source + files should be declared ``static`` to avoid exporting their symbols from + ``libpython``. + + If symbols need to be exported, their names must start with ``Py`` or + ``_Py``. This can be verified by ``make smelly``. For more details, + please refer to the section on :ref:`Changing Python's C API `. + +.. code-block:: c + :caption: Modules/_foo/helper.c + :name: Modules/_foo/helper.c + + #include "_foomodule.h" - #endif // _FOO__FOOMODULE_H + PyObject *_Py_greet_fast(void) { + return PyUnicode_FromString("Hello World!"); + } .. code-block:: c :caption: Modules/_foo/_foomodule.c :name: Modules/_foo/_foomodule.c - #include "_foomodule.h" + #include "helper.h" #include "clinic/_foomodule.c.h" /* Functions for the extension module's state */ @@ -213,7 +241,7 @@ placed in :cpy-file:`Modules` as ``Modules/_foomodule.c`` for instance. static struct PyModuleDef foomodule = { PyModuleDef_HEAD_INIT, - .m_name = "fastfoo", // name to use in 'import' statements + .m_name = "_foo", .m_doc = "some doc", // or NULL if not needed .m_size = sizeof(foomodule_state), .m_methods = foomodule_methods, @@ -224,43 +252,84 @@ placed in :cpy-file:`Modules` as ``Modules/_foomodule.c`` for instance. }; PyMODINIT_FUNC - PyInit_fastfoo(void) + PyInit__foo(void) { return PyModuleDef_Init(&foomodule); } .. tip:: - Recall that the ``PyInit_`` function must be suffixed by the *same* - module name as that of :c:member:`PyModuleDef.m_name` (here, ``fastfoo``). + Recall that the ``PyInit_`` function must be suffixed by the + module name ```` used in import statements (here ``_foo``), + and which usually coincides with :c:member:`PyModuleDef.m_name`. + Other identifiers such as those used in :ref:`Argument Clinic ` inputs do not have such naming requirements. -.. code-block:: c - :caption: Modules/_foo/helper.c - :name: Modules/_foo/helper.c +Configuring the CPython project +------------------------------- - #include "_foomodule.h" +Now that we have implemented our extension module, we need to update some +configuration files in order to compile the CPython project on different +platforms. - PyObject *_Py_greet_fast(void) { - return PyUnicode_FromString("Hello World!"); - } +Updating :cpy-file:`!Modules/Setup.{bootstrap,stdlib}.in` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Depending on whether the extension module is required to get a functioning +interpreter or not, we update :cpy-file:`Modules/Setup.bootstrap.in` or +:cpy-file:`Modules/Setup.stdlib.in`. In the former case, the extension +module is necessarily built as a built-in extension module. .. tip:: - Functions or data that do not need to be shared across different C source - files should be declared ``static`` to avoid exporting their symbols from - ``libpython``. + For accelerator modules, :cpy-file:`Modules/Setup.stdlib.in` should be + preferred over :cpy-file:`Modules/Setup.bootstrap.in`. - If symbols need to be exported, their names must start with ``Py`` or - ``_Py``. This can be verified by ``make smelly``. +For built-in extension modules, update :cpy-file:`Modules/Setup.bootstrap.in` +by adding the following line after the ``*static*`` marker: -Configuring the CPython project -------------------------------- +.. code-block:: text + :caption: :cpy-file:`Modules/Setup.bootstrap.in` + :emphasize-lines: 3 -Now that we have implemented our extension module, we need to update some -configuration files in order to compile the CPython project on different -platforms. + *static* + ... + _foo _foo/_foomodule.c _foo/helper.c + ... + +The syntax is `` SOURCE [SOURCE ...]`` where ```` is the +name of the module used in :keyword:`import` statements. + +For other extension modules, update :cpy-file:`Modules/Setup.stdlib.in` +by adding the following line after the ``*@MODULE_BUILDTYPE@*`` marker +but before the ``*shared*`` marker: + +.. code-block:: text + :caption: :cpy-file:`Modules/Setup.stdlib.in` + :emphasize-lines: 3 + + *@MODULE_BUILDTYPE@* + ... + @MODULE__FOO_TRUE@_foo _foo/_foomodule.c _foo/helper.c + ... + *shared* + +The ``@MODULE__TRUE@`` marker expexts ```` to +be the upper-cased form of ````. In our case, ``_FOO`` and ``_foo`` +respectively. + +If the extension module must be built as a *shared* module, put the +``@MODULE__FOO_TRUE@_foo`` line after the ``*shared*`` marker: + +.. code-block:: text + :caption: :cpy-file:`Modules/Setup.stdlib.in` + :emphasize-lines: 4 + + ... + *shared* + ... + @MODULE__FOO_TRUE@_foo _foo/_foomodule.c _foo/helper.c Updating :cpy-file:`configure.ac` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -293,12 +362,12 @@ Updating :cpy-file:`configure.ac` dnl always enabled extension modules ... - PY_STDLIB_MOD_SIMPLE([fastfoo], [-I\$(srcdir)/Modules/_foo], []) + PY_STDLIB_MOD_SIMPLE([_foo], [-I\$(srcdir)/Modules/_foo], []) ... The ``PY_STDLIB_MOD_SIMPLE`` macro takes as arguments: - * the module name as specified by :c:member:`PyModuleDef.m_name`, + * the module name ```` used in :keyword:`import` statements, * the compiler flags (CFLAGS), and * the linker flags (LDFLAGS). @@ -306,20 +375,20 @@ Updating :cpy-file:`configure.ac` host configuration, use the ``PY_STDLIB_MOD`` macro instead, which takes as arguments: - * the module name as specified by :c:member:`PyModuleDef.m_name`, + * the module name ```` used in :keyword:`import` statements, * a boolean indicating whether the extension is **enabled** or not, * a boolean indicating whether the extension is **supported** or not, * the compiler flags (CFLAGS), and * the linker flags (LDFLAGS). - For instance, enabling the ``fastfoo`` extension on Linux platforms, but + For instance, enabling the :mod:`!_foo` extension on Linux platforms, but only providing support for 32-bit architecture, is achieved as follows: .. code-block:: text :caption: :cpy-file:`configure.ac` :emphasize-lines: 2, 3 - PY_STDLIB_MOD([fastfoo], + PY_STDLIB_MOD([_foo], [test "$ac_sys_system" = "Linux"], [test "$ARCH_RUN_32BIT" = "true"], [-I\$(srcdir)/Modules/_foo], []) @@ -348,10 +417,16 @@ Updating :cpy-file:`configure.ac` dnl Modules that are not available on some platforms AS_CASE([$ac_sys_system], ... - [PLATFORM_NAME], [PY_STDLIB_MOD_SET_NA([fastfoo])], + [PLATFORM_NAME], [PY_STDLIB_MOD_SET_NA([_foo])], ... ) +.. tip:: + + Consider reading the comments and configurations for existing modules + in :cpy-file:`configure.ac` for guidance on adding new external build + dependencies for extension modules that need them. + Updating :cpy-file:`Makefile.pre.in` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -364,9 +439,12 @@ If needed, add the following line to the section for module dependencies: ########################################################################## # Module dependencies and platform-specific files ... - MODULE_FASTFOO_DEPS=$(srcdir)/Modules/_foo/_foomodule.h + MODULE__FOO_DEPS=$(srcdir)/Modules/_foo/helper.h ... +The ``MODULE__DEPS`` variable follows the same naming +requirements as the ``@MODULE__TRUE@`` marker. + Updating MSVC project files ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -380,17 +458,21 @@ We describe the minimal steps for compiling on Windows using MSVC. ... // add the entry point prototype - extern PyObject* PyInit_fastfoo(void); + extern PyObject* PyInit__foo(void); ... // update the entry points table struct _inittab _PyImport_Inittab[] = { ... - {"fastfoo", PyInit_fastfoo}, + {"_foo", PyInit__foo}, ... {0, 0} }; ... + Each item in ``_PyImport_Inittab`` consists of the module name to import, + here :mod:`!_foo`, together with the corresponding ``PyInit_*`` function + correctly suffixed. + * Update :cpy-file:`PCbuild/pythoncore.vcxproj`: .. code-block:: xml @@ -400,7 +482,7 @@ We describe the minimal steps for compiling on Windows using MSVC. ... - + ... @@ -421,7 +503,7 @@ We describe the minimal steps for compiling on Windows using MSVC. ... - + Modules\_foo ... @@ -444,58 +526,6 @@ We describe the minimal steps for compiling on Windows using MSVC. Header files use ```` tags, whereas source files use ```` tags. -Updating :cpy-file:`!Modules/Setup.{bootstrap,stdlib}.in` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Depending on whether the extension module is required to get a functioning -interpreter or not, we update :cpy-file:`Modules/Setup.bootstrap.in` or -:cpy-file:`Modules/Setup.stdlib.in`. In the former case, the extension -module is necessarily built as a built-in extension module. - -.. tip:: - - For accelerator modules, :cpy-file:`Modules/Setup.stdlib.in` should be - preferred over :cpy-file:`Modules/Setup.bootstrap.in`. - -For built-in extension modules, update :cpy-file:`Modules/Setup.bootstrap.in` -by adding the following line after the ``*static*`` marker: - -.. code-block:: text - :caption: :cpy-file:`Modules/Setup.bootstrap.in` - :emphasize-lines: 3 - - *static* - ... - fastfoo _foo/_foomodule.c _foo/helper.c - ... - -For other extension modules, update :cpy-file:`Modules/Setup.stdlib.in` -by adding the following line after the ``*@MODULE_BUILDTYPE@*`` marker -but before the ``*shared*`` marker: - -.. code-block:: text - :caption: :cpy-file:`Modules/Setup.stdlib.in` - :emphasize-lines: 3 - - *@MODULE_BUILDTYPE@* - ... - @MODULE_FASTFOO_TRUE@fastfoo _foo/_foomodule.c _foo/helper.c - ... - *shared* - -The ``@MODULE__TRUE@`` marker expects ```` to be the -upper-cased module name ````. If the extension module must be built -as a *shared* module, put the ``@MODULE_FASTFOO_TRUE@fastfoo`` line after -the ``*shared*`` marker: - -.. code-block:: text - :caption: :cpy-file:`Modules/Setup.stdlib.in` - :emphasize-lines: 4 - - ... - *shared* - ... - @MODULE_FASTFOO_TRUE@fastfoo _foo/_foomodule.c _foo/helper.c Compiling the CPython project ----------------------------- @@ -512,7 +542,8 @@ Now that everything is in place, it remains to compile the project: .. tip:: - Use ``make -j12`` to speed-up compilation if you have enough CPU cores. + Use ``make -j`` to speed-up compilation by utilizing as many CPU cores + as possible or ``make -jN`` to allow at most *N* concurrent jobs. * ``make regen-configure`` updates the :cpy-file:`configure` script. @@ -521,7 +552,7 @@ Now that everything is in place, it remains to compile the project: Execute this rule if you do not know which files should be updated. * ``make regen-stdlib-module-names`` updates the standard module names, making - :mod:`!fastfoo` discoverable and importable via ``import fastfoo``. + :mod:`!_foo` discoverable and importable via ``import _foo``. * The final ``make`` step is generally not needed since the previous ``make`` invokations may completely rebuild the project, but it could be needed in @@ -530,7 +561,8 @@ Now that everything is in place, it remains to compile the project: Troubleshooting --------------- -This section addresses common issues that you may face when following this tutorial. +This section addresses common issues that you may face when following +this tutorial. No rule to make target ``regen-configure`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 3ee1ceab12bb6002461ddbf8f5421b0ae61c5855 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 18 Jul 2024 17:52:47 +0200 Subject: [PATCH 38/45] add details --- developer-workflow/extension-modules.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 167de7ca60..02738b2f0a 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -298,8 +298,9 @@ by adding the following line after the ``*static*`` marker: _foo _foo/_foomodule.c _foo/helper.c ... -The syntax is `` SOURCE [SOURCE ...]`` where ```` is the -name of the module used in :keyword:`import` statements. +The syntax is `` `` where ```` is the name of the +module used in :keyword:`import` statements and ```` is the list +of space-separated source files. For other extension modules, update :cpy-file:`Modules/Setup.stdlib.in` by adding the following line after the ``*@MODULE_BUILDTYPE@*`` marker @@ -315,9 +316,10 @@ but before the ``*shared*`` marker: ... *shared* -The ``@MODULE__TRUE@`` marker expexts ```` to -be the upper-cased form of ````. In our case, ``_FOO`` and ``_foo`` -respectively. +The ``@MODULE__TRUE@`` marker expects ```` to +be the upper-cased form of ````, where ```` has the same meaning +as before (in our case, ```` and ```` are ``_FOO`` and +``_foo`` respectively). The marker is followed by the list of source files. If the extension module must be built as a *shared* module, put the ``@MODULE__FOO_TRUE@_foo`` line after the ``*shared*`` marker: From 3d235f45687024ed30b713cde0e5e205eb95826b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 4 Aug 2024 11:36:08 +0200 Subject: [PATCH 39/45] address review - Add details on `Py_BUILD_CORE_*` macros - Add tips for `Py_LIMITED_API` --- developer-workflow/extension-modules.rst | 63 ++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 02738b2f0a..ea0ba0db8d 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -37,12 +37,19 @@ Extension modules can be classified into two categories: .. seealso:: :data:`sys.builtin_module_names` --- names of built-in modules. + Built-in modules are built with the :c:macro:`!Py_BUILD_CORE_BUILTIN` + macro defined. + * A *shared* (or *dynamic*) extension module is built as a shared library (``.so`` or ``.dll`` file) and is *dynamically* linked into the interpreter. In particular, the module's :attr:`__file__` attribute contains the path to the ``.so`` or ``.dll`` file. + Shared modules are built with the :c:macro:`!Py_BUILD_CORE_MODULE` + macro defined. Using the :c:macro:`!Py_BUILD_CORE_BUILTIN` macro + instead causes an :exc:`ImportError` when importing the module. + .. note:: Informally, built-in extension modules can be regarded as *required* @@ -615,3 +622,59 @@ Once the Docker service is running, check that you have an `Ubuntu 22.04 image If the issue persists, you may try `podman `_. The commands for listing or pulling an image are the same as ``docker``. + +Missing ``Py_BUILD_CORE`` define when using internal headers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default, the CPython :ref:`Stable ABI ` is exposed via +:code:`#include "Python.h"`. In some cases, this may be insufficient +and internal headers from :cpy-file:`Include/internal` are needed; +in particular, those headers require the :c:macro:`!Py_BUILD_CORE` +macro to be defined. + +To that end, one should define the :c:macro:`!Py_BUILD_CORE_BUILTIN` +or the :c:macro:`!Py_BUILD_CORE_MODULE` macro depending on whether the +extension module is built-in or shared. Using either of the two macros +implies :c:macro:`!Py_BUILD_CORE` and gives access to CPython internals: + +.. code-block:: c + :caption: Definition of :c:macro:`!Py_BUILD_CORE_BUILTIN` + + #ifndef Py_BUILD_CORE_MODULE + # define Py_BUILD_CORE_BUILTIN 1 + #endif + +.. code-block:: c + :caption: Definition of :c:macro:`!Py_BUILD_CORE_MODULE` + + #ifndef Py_BUILD_CORE_BUILTIN + # define Py_BUILD_CORE_MODULE 1 + #endif + +Tips +---- + +In this section, we give some tips for improving the quality of +extension modules meant to be included in the standard library. + +Restricting to the Limited API +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In order for non-CPython implementations to benefit from new extension modules, +it is recommended to use the :ref:`Limited API `. Instead of +exposing the entire Stable ABI, define the :c:macro:`Py_LIMITED_API` macro +*before* the :code:`#include "Python.h"` directive: + +.. code-block:: c + :caption: Using the 3.13 Limited API. + :emphasize-lines: 3, 6 + + #include "pyconfig.h" // Py_GIL_DISABLED + #ifndef Py_GIL_DISABLED + # define Py_LIMITED_API 0x030d0000 + #endif + + #include "Python.h" + +This makes the extension module non-CPython implementation-friendly by +removing the dependencies to CPython internals. From 7b0b23418ba1b2a7c67efaa84d2a95cd3586979e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 4 Aug 2024 11:36:55 +0200 Subject: [PATCH 40/45] Make it easier to update the required ubuntu version --- _extensions/ubuntu_version.py | 19 +++++++++++++++++++ conf.py | 3 +++ developer-workflow/extension-modules.rst | 11 +++++++---- 3 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 _extensions/ubuntu_version.py diff --git a/_extensions/ubuntu_version.py b/_extensions/ubuntu_version.py new file mode 100644 index 0000000000..75bb6df7fb --- /dev/null +++ b/_extensions/ubuntu_version.py @@ -0,0 +1,19 @@ +"""Sphinx extension to update the required Ubuntu version. + +The UBUNTU_VERSION must be synchronized with the Ubuntu version used by +https://github.com/python/cpython/blob/main/Tools/build/regen-configure.sh. +""" + +from sphinx.errors import ExtensionError + + +def replace_ubuntu_version(app, docname, source): + if (ubuntu_version := app.config.configure_ubuntu_version) is None: + raise ExtensionError('configure_ubuntu_version is not set in conf.py') + source[0] = source[0].replace('$CONFIGURE_UBUNTU_VERSION$', ubuntu_version) + + +def setup(app): + app.add_config_value('configure_ubuntu_version', None, 'env', types=(str,)) + app.connect('source-read', replace_ubuntu_version) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/conf.py b/conf.py index 6de594b235..6adc16d718 100644 --- a/conf.py +++ b/conf.py @@ -7,6 +7,7 @@ extensions = [ 'custom_roles', + 'ubuntu_version', 'notfound.extension', 'sphinx.ext.extlinks', 'sphinx.ext.intersphinx', @@ -197,3 +198,5 @@ copybutton_prompt_text = "$ " # https://sphinx-copybutton.readthedocs.io/en/latest/use.html#honor-line-continuation-characters-when-copying-multline-snippets copybutton_line_continuation_character = "\\" + +configure_ubuntu_version = '22.04' diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index ea0ba0db8d..1709cf7e7b 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -610,13 +610,16 @@ If Docker complains about missing permissions, this Stack Overflow post could be useful in solving the issue: `How to fix docker: permission denied `_. -Once the Docker service is running, check that you have an `Ubuntu 22.04 image -`_, or pull it if it is not case: +Once the Docker service is running, check that you have an `Ubuntu +$CONFIGURE_UBUNTU_VERSION$ image `_, +or pull it if it is not case: .. code-block:: shell - docker images ubuntu:22.04 # check for the Docker image presence - docker image pull ubuntu:22.04 # or pull the image if needed + # check for the Docker image presence + docker images ubuntu:$CONFIGURE_UBUNTU_VERSION$ + # pull the Docker image if needed + docker image pull ubuntu:$CONFIGURE_UBUNTU_VERSION$ .. tip:: From ec48fdba1a0d08db60a4cbc216069ab204585474 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 4 Aug 2024 12:34:51 +0200 Subject: [PATCH 41/45] fixup! --- conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conf.py b/conf.py index c6201db0d5..9c48c4350f 100644 --- a/conf.py +++ b/conf.py @@ -1,7 +1,6 @@ import time extensions = [ - 'custom_roles', 'ubuntu_version', 'notfound.extension', 'sphinx.ext.extlinks', From 1843e3ddd0fc369e4c9aa00e9844f6c3047c6759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 4 Aug 2024 12:36:08 +0200 Subject: [PATCH 42/45] fixup! --- conf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conf.py b/conf.py index 9c48c4350f..b273e72f38 100644 --- a/conf.py +++ b/conf.py @@ -1,5 +1,8 @@ +import sys import time +sys.path.insert(0, '_extensions') + extensions = [ 'ubuntu_version', 'notfound.extension', From 18c4d916f22c8fb903596c37043994dc460eea4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:36:50 +0200 Subject: [PATCH 43/45] improve comment --- _extensions/ubuntu_version.py | 11 ++++++++++- conf.py | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/_extensions/ubuntu_version.py b/_extensions/ubuntu_version.py index 75bb6df7fb..76c3a4ea6b 100644 --- a/_extensions/ubuntu_version.py +++ b/_extensions/ubuntu_version.py @@ -1,6 +1,10 @@ """Sphinx extension to update the required Ubuntu version. -The UBUNTU_VERSION must be synchronized with the Ubuntu version used by +The required Ubuntu version should be specified in conf.py by:: + + configure_ubuntu_version = "MAJOR.MINOR" # e.g., "22.04" + +The version must match the one used to regenerate the configure script in https://github.com/python/cpython/blob/main/Tools/build/regen-configure.sh. """ @@ -8,6 +12,11 @@ def replace_ubuntu_version(app, docname, source): + """Replace all occurrences of $CONFIGURE_UBUNTU_VERSION$. + + This is needed since RST replacement via ``|...|`` is not supported + in code-blocks directives. + """ if (ubuntu_version := app.config.configure_ubuntu_version) is None: raise ExtensionError('configure_ubuntu_version is not set in conf.py') source[0] = source[0].replace('$CONFIGURE_UBUNTU_VERSION$', ubuntu_version) diff --git a/conf.py b/conf.py index b273e72f38..71d0a221b0 100644 --- a/conf.py +++ b/conf.py @@ -199,4 +199,6 @@ # https://sphinx-copybutton.readthedocs.io/en/latest/use.html#honor-line-continuation-characters-when-copying-multline-snippets copybutton_line_continuation_character = "\\" +# Must be synchronized with the Ubuntu image version in +# https://github.com/python/cpython/blob/main/Tools/build/regen-configure.sh configure_ubuntu_version = '22.04' From 8224bbd63cec48c90ab929440400df1acb64b570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:41:34 +0200 Subject: [PATCH 44/45] use double quotes instead of single quotes --- _extensions/ubuntu_version.py | 8 ++++---- conf.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/_extensions/ubuntu_version.py b/_extensions/ubuntu_version.py index 76c3a4ea6b..1298f35d98 100644 --- a/_extensions/ubuntu_version.py +++ b/_extensions/ubuntu_version.py @@ -18,11 +18,11 @@ def replace_ubuntu_version(app, docname, source): in code-blocks directives. """ if (ubuntu_version := app.config.configure_ubuntu_version) is None: - raise ExtensionError('configure_ubuntu_version is not set in conf.py') - source[0] = source[0].replace('$CONFIGURE_UBUNTU_VERSION$', ubuntu_version) + raise ExtensionError("configure_ubuntu_version is not set in conf.py") + source[0] = source[0].replace("$CONFIGURE_UBUNTU_VERSION$", ubuntu_version) def setup(app): - app.add_config_value('configure_ubuntu_version', None, 'env', types=(str,)) - app.connect('source-read', replace_ubuntu_version) + app.add_config_value("configure_ubuntu_version", None, "env", types=(str,)) + app.connect("source-read", replace_ubuntu_version) return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/conf.py b/conf.py index 71d0a221b0..569245df00 100644 --- a/conf.py +++ b/conf.py @@ -201,4 +201,4 @@ # Must be synchronized with the Ubuntu image version in # https://github.com/python/cpython/blob/main/Tools/build/regen-configure.sh -configure_ubuntu_version = '22.04' +configure_ubuntu_version = "22.04" From 2f630532c42caeacdfaa43e033851a540ede998e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 11 Sep 2024 13:59:47 +0200 Subject: [PATCH 45/45] Address Carol's review. --- developer-workflow/extension-modules.rst | 68 +++++++++++------------- 1 file changed, 32 insertions(+), 36 deletions(-) diff --git a/developer-workflow/extension-modules.rst b/developer-workflow/extension-modules.rst index 1709cf7e7b..db391357ab 100644 --- a/developer-workflow/extension-modules.rst +++ b/developer-workflow/extension-modules.rst @@ -5,9 +5,15 @@ Standard library extension modules ================================== -In this section, we are interested in extending the CPython project with -an :term:`extension module`. We will not explain how to write the module -in C but rather explain how to configure the project and make it compile. +In this section, we explain how to configure and compile the CPython project +with a C :term:`extension module`. We will not explain how to write a C +extension module and prefer to give you some links where you can read good +documentation: + +* https://docs.python.org/dev/c-api/ +* https://docs.python.org/dev/extending/ +* :pep:`399` +* https://pythonextensionpatterns.readthedocs.io/en/latest/ Some modules in the standard library, such as :mod:`datetime` or :mod:`pickle`, have identical implementations in C and Python; the C implementation, when @@ -18,14 +24,6 @@ Other modules mainly implemented in Python may import a C helper extension providing implementation details (for instance, the :mod:`csv` module uses the internal :mod:`!_csv` module defined in :cpy-file:`Modules/_csv.c`). -For writing an extension module in C, we prefer to give you some links -where you can read good documentation: - -* https://docs.python.org/dev/c-api/ -* https://docs.python.org/dev/extending/ -* :pep:`399` -* https://pythonextensionpatterns.readthedocs.io/en/latest/ - Classifying extension modules ============================= @@ -60,15 +58,14 @@ Extension modules can be classified into two categories: especially if they already have a pure Python implementation. According to :pep:`399`, *new* extension modules MUST provide a working and -tested pure Python implementation, unless a special dispensation is given. -Please ask the :github:`Steering Council ` if such -dispensation is needed. +tested pure Python implementation, unless a special dispensation from +the :github:`Steering Council ` is given. Adding an extension module to CPython ===================================== Assume that the standard library contains a pure Python module :mod:`!foo` -together with the following :func:`!foo.greet` function: +with the following :func:`!foo.greet` function: .. code-block:: python :caption: Lib/foo.py @@ -77,8 +74,8 @@ together with the following :func:`!foo.greet` function: return "Hello World!" Instead of using the Python implementation of :func:`!foo.greet`, we want to -use its corresponding C implementation exposed in some :mod:`!_foo` module -written in C. Ideally, we want to modify :cpy-file:`!Lib/foo.py` as follows: +use its corresponding C extension implementation exposed in the :mod:`!_foo` +module. Ideally, we want to modify :cpy-file:`!Lib/foo.py` as follows: .. code-block:: python :caption: Lib/foo.py @@ -93,14 +90,14 @@ written in C. Ideally, we want to modify :cpy-file:`!Lib/foo.py` as follows: .. note:: - Accelerator modules should *never* be imported directly, whence the - convention is to mark them as private implementation details with the - underscore prefix (namely, :mod:`!_foo` in this example). + Accelerator modules should *never* be imported directly. The convention is + to mark them as private implementation details with the underscore prefix + (namely, :mod:`!_foo` in this example). In order to incorporate the accelerator module, we need to determine: -- where to place the extension module source code in the CPython project tree, -- which files to modify in order to compile the CPython project, and +- where to update the CPython project tree with the extension module source code, +- which files to modify to configure and compile the CPython project, and - which ``Makefile`` rules to invoke at the end. Updating the CPython project tree @@ -110,7 +107,10 @@ Usually, accelerator modules are added in the :cpy-file:`Modules` directory of the CPython project. If more than one file is needed for the extension module, it is more convenient to create a sub-directory in :cpy-file:`Modules`. -For our extension module :mod:`!_foo`, we consider the following working tree: +In the simplest example where the extension module consists of one file, it may +be placed in :cpy-file:`Modules` as ``Modules/_foomodule.c``. For a non-trivial +example of the extension module :mod:`!_foo`, we consider the following working +tree: - :ref:`Modules/_foo/_foomodule.c` --- the extension module implementation. - :ref:`Modules/_foo/helper.h` --- the extension helpers declarations. @@ -121,11 +121,6 @@ is called ``module.c``, where ```` is the name of the module that will be later imported (in our case :mod:`!_foo`). In addition, the directory containing the implementation should also be named similarly. -One could imagine having more files, or no helper files at all. Here, -we wanted to illustrate a simple example without making it too trivial. If -the extension module does not require additional files, it may directly be -placed in :cpy-file:`Modules` as ``Modules/_foomodule.c``. - .. code-block:: c :caption: Modules/_foo/helper.h :name: Modules/_foo/helper.h @@ -276,9 +271,9 @@ placed in :cpy-file:`Modules` as ``Modules/_foomodule.c``. Configuring the CPython project ------------------------------- -Now that we have implemented our extension module, we need to update some -configuration files in order to compile the CPython project on different -platforms. +Now that we have added our extension module to the CPython source tree, +we need to update some configuration files in order to compile the CPython +project on different platforms. Updating :cpy-file:`!Modules/Setup.{bootstrap,stdlib}.in` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -402,7 +397,8 @@ Updating :cpy-file:`configure.ac` [test "$ARCH_RUN_32BIT" = "true"], [-I\$(srcdir)/Modules/_foo], []) - More generally, the status of the extension is determined as follows: + More generally, the host's configuration status of the extension is + determined as follows: +-----------+-----------------+----------+ | Enabled | Supported | Status | @@ -479,8 +475,8 @@ We describe the minimal steps for compiling on Windows using MSVC. ... Each item in ``_PyImport_Inittab`` consists of the module name to import, - here :mod:`!_foo`, together with the corresponding ``PyInit_*`` function - correctly suffixed. + here :mod:`!_foo`, with the corresponding ``PyInit_*`` function correctly + suffixed. * Update :cpy-file:`PCbuild/pythoncore.vcxproj`: @@ -539,7 +535,7 @@ We describe the minimal steps for compiling on Windows using MSVC. Compiling the CPython project ----------------------------- -Now that everything is in place, it remains to compile the project: +Now that the configuration is in place, it remains to compile the project: .. code-block:: shell @@ -571,7 +567,7 @@ Troubleshooting --------------- This section addresses common issues that you may face when following -this tutorial. +this example of adding an extension module. No rule to make target ``regen-configure`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^