Merge branch 'main' into pythongh-115999-thread-local-bytecode

Author: mpage

Doc/Makefile

@@ -305,13 +305,15 @@ serve:
 
 # for development releases: always build
 .PHONY: autobuild-dev
+autobuild-dev: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short)
 autobuild-dev:
-	$(MAKE) dist-no-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1'
+	$(MAKE) dist-no-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION)
 
 # for HTML-only rebuilds
 .PHONY: autobuild-dev-html
+autobuild-dev-html: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short)
 autobuild-dev-html:
-	$(MAKE) dist-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1'
+	$(MAKE) dist-html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' DISTVERSION=$(DISTVERSION)
 
 # for stable releases: only build if not in pre-release stage (alpha, beta)
 # release candidate downloads are okay, since the stable tree can be in that stage

Doc/c-api/init.rst

@@ -430,7 +430,11 @@ Initializing and finalizing the interpreter
    Some memory allocated by extension modules may not be freed.  Some extensions may not
    work properly if their initialization routine is called more than once; this can
    happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx`
-   more than once.
+   more than once.  :c:func:`Py_FinalizeEx` must not be called recursively from
+   within itself.  Therefore, it must not be called by any code that may be run
+   as part of the interpreter shutdown process, such as :py:mod:`atexit`
+   handlers, object finalizers, or any code that may be run while flushing the
+   stdout and stderr files.
 
    .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx
 
@@ -960,6 +964,37 @@ thread, where the CPython global runtime was originally initialized.
 The only exception is if :c:func:`exec` will be called immediately
 after.
 
+.. _cautions-regarding-runtime-finalization:
+
+Cautions regarding runtime finalization
+---------------------------------------
+
+In the late stage of :term:`interpreter shutdown`, after attempting to wait for
+non-daemon threads to exit (though this can be interrupted by
+:class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime
+is marked as *finalizing*: :c:func:`_Py_IsFinalizing` and
+:func:`sys.is_finalizing` return true.  At this point, only the *finalization
+thread* that initiated finalization (typically the main thread) is allowed to
+acquire the :term:`GIL`.
+
+If any thread, other than the finalization thread, attempts to acquire the GIL
+during finalization, either explicitly via :c:func:`PyGILState_Ensure`,
+:c:macro:`Py_END_ALLOW_THREADS`, :c:func:`PyEval_AcquireThread`, or
+:c:func:`PyEval_AcquireLock`, or implicitly when the interpreter attempts to
+reacquire it after having yielded it, the thread enters **a permanently blocked
+state** where it remains until the program exits.  In most cases this is
+harmless, but this can result in deadlock if a later stage of finalization
+attempts to acquire a lock owned by the blocked thread, or otherwise waits on
+the blocked thread.
+
+Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++
+finalizations further up the call stack when such threads were forcibly exited
+here in CPython 3.13 and earlier. The CPython runtime GIL acquiring C APIs
+have never had any error reporting or handling expectations at GIL acquisition
+time that would've allowed for graceful exit from this situation. Changing that
+would require new stable C APIs and rewriting the majority of C code in the
+CPython ecosystem to use those with error handling.
+
 
 High-level API
 --------------
@@ -1033,11 +1068,14 @@ code, or when embedding the Python interpreter:
    ensues.
 
    .. note::
-      Calling this function from a thread when the runtime is finalizing
-      will terminate the thread, even if the thread was not created by Python.
-      You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to
-      check if the interpreter is in process of being finalized before calling
-      this function to avoid unwanted termination.
+      Calling this function from a thread when the runtime is finalizing will
+      hang the thread until the program exits, even if the thread was not
+      created by Python.  Refer to
+      :ref:`cautions-regarding-runtime-finalization` for more details.
+
+   .. versionchanged:: next
+      Hangs the current thread, rather than terminating it, if called while the
+      interpreter is finalizing.
 
 .. c:function:: PyThreadState* PyThreadState_Get()
 
@@ -1092,11 +1130,14 @@ with sub-interpreters:
    to call arbitrary Python code.  Failure is a fatal error.
 
    .. note::
-      Calling this function from a thread when the runtime is finalizing
-      will terminate the thread, even if the thread was not created by Python.
-      You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to
-      check if the interpreter is in process of being finalized before calling
-      this function to avoid unwanted termination.
+      Calling this function from a thread when the runtime is finalizing will
+      hang the thread until the program exits, even if the thread was not
+      created by Python.  Refer to
+      :ref:`cautions-regarding-runtime-finalization` for more details.
+
+   .. versionchanged:: next
+      Hangs the current thread, rather than terminating it, if called while the
+      interpreter is finalizing.
 
 .. c:function:: void PyGILState_Release(PyGILState_STATE)
 
@@ -1224,7 +1265,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
 .. c:function:: void PyThreadState_DeleteCurrent(void)
 
    Destroy the current thread state and release the global interpreter lock.
-   Like :c:func:`PyThreadState_Delete`, the global interpreter lock need not
+   Like :c:func:`PyThreadState_Delete`, the global interpreter lock must
    be held. The thread state must have been reset with a previous call
    to :c:func:`PyThreadState_Clear`.
 
@@ -1374,17 +1415,20 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
    If this thread already has the lock, deadlock ensues.
 
    .. note::
-      Calling this function from a thread when the runtime is finalizing
-      will terminate the thread, even if the thread was not created by Python.
-      You can use :c:func:`Py_IsFinalizing` or :func:`sys.is_finalizing` to
-      check if the interpreter is in process of being finalized before calling
-      this function to avoid unwanted termination.
+      Calling this function from a thread when the runtime is finalizing will
+      hang the thread until the program exits, even if the thread was not
+      created by Python.  Refer to
+      :ref:`cautions-regarding-runtime-finalization` for more details.
 
    .. versionchanged:: 3.8
       Updated to be consistent with :c:func:`PyEval_RestoreThread`,
       :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`,
       and terminate the current thread if called while the interpreter is finalizing.
 
+   .. versionchanged:: next
+      Hangs the current thread, rather than terminating it, if called while the
+      interpreter is finalizing.
+
    :c:func:`PyEval_RestoreThread` is a higher-level function which is always
    available (even when threads have not been initialized).
 

Doc/library/argparse.rst

@@ -30,7 +30,7 @@ Quick Links for ArgumentParser
 ========================= =========================================================================================================== ==================================================================================
 Name                      Description                                                                                                 Values
 ========================= =========================================================================================================== ==================================================================================
-prog_                     The name of the program                                                                                     Defaults to ``os.path.basename(sys.argv[0])``
+prog_                     The name of the program
 usage_                    The string describing the program usage
 description_              A brief description of what the program does
 epilog_                   Additional description of the program after the argument help
@@ -214,8 +214,8 @@ ArgumentParser objects
    as keyword arguments. Each parameter has its own more detailed description
    below, but in short they are:
 
-   * prog_ - The name of the program (default:
-     ``os.path.basename(sys.argv[0])``)
+   * prog_ - The name of the program (default: generated from the ``__main__``
+     module attributes and ``sys.argv[0]``)
 
    * usage_ - The string describing the program usage (default: generated from
      arguments added to parser)
@@ -268,10 +268,18 @@ The following sections describe how each of these are used.
 prog
 ^^^^
 
-By default, :class:`ArgumentParser` objects use the base name
-(see :func:`os.path.basename`) of ``sys.argv[0]`` to determine
-how to display the name of the program in help messages.  This default is almost
-always desirable because it will make the help messages match the name that was
+By default, :class:`ArgumentParser` calculates the name of the program
+to display in help messages depending on the way the Python inerpreter was run:
+
+* The :func:`base name <os.path.basename>` of ``sys.argv[0]`` if a file was
+  passed as argument.
+* The Python interpreter name followed by ``sys.argv[0]`` if a directory or
+  a zipfile was passed as argument.
+* The Python interpreter name followed by ``-m`` followed by the
+  module or package name if the :option:`-m` option was used.
+
+This default is almost
+always desirable because it will make the help messages match the string that was
 used to invoke the program on the command line.  For example, consider a file
 named ``myprogram.py`` with the following code::
 
@@ -281,7 +289,7 @@ named ``myprogram.py`` with the following code::
    args = parser.parse_args()
 
 The help for this program will display ``myprogram.py`` as the program name
-(regardless of where the program was invoked from):
+(regardless of where the program was invoked from) if it is run as a script:
 
 .. code-block:: shell-session
 
@@ -299,6 +307,17 @@ The help for this program will display ``myprogram.py`` as the program name
     -h, --help  show this help message and exit
     --foo FOO   foo help
 
+If it is executed via the :option:`-m` option, the help will display a corresponding command line:
+
+.. code-block:: shell-session
+
+   $ /usr/bin/python3 -m subdir.myprogram --help
+   usage: python3 -m subdir.myprogram [-h] [--foo FOO]
+
+   options:
+    -h, --help  show this help message and exit
+    --foo FOO   foo help
+
 To change this default behavior, another value can be supplied using the
 ``prog=`` argument to :class:`ArgumentParser`::
 
@@ -309,7 +328,8 @@ To change this default behavior, another value can be supplied using the
    options:
     -h, --help  show this help message and exit
 
-Note that the program name, whether determined from ``sys.argv[0]`` or from the
+Note that the program name, whether determined from ``sys.argv[0]``,
+from the ``__main__`` module attributes or from the
 ``prog=`` argument, is available to help messages using the ``%(prog)s`` format
 specifier.
 
@@ -324,6 +344,9 @@ specifier.
     -h, --help  show this help message and exit
     --foo FOO   foo of the myprogram program
 
+.. versionchanged:: 3.14
+   The default ``prog`` value now reflects how ``__main__`` was actually executed,
+   rather than always being ``os.path.basename(sys.argv[0])``.
 
 usage
 ^^^^^

Doc/library/dataclasses.rst

@@ -399,7 +399,7 @@ Module contents
    :func:`!astuple` raises :exc:`TypeError` if *obj* is not a dataclass
    instance.
 
-.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None)
+.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None, decorator=dataclass)
 
    Creates a new dataclass with name *cls_name*, fields as defined
    in *fields*, base classes as given in *bases*, and initialized
@@ -415,6 +415,11 @@ Module contents
    of the dataclass is set to that value.
    By default, it is set to the module name of the caller.
 
+   The *decorator* parameter is a callable that will be used to create the dataclass.
+   It should take the class object as a first argument and the same keyword arguments
+   as :func:`@dataclass <dataclass>`. By default, the :func:`@dataclass <dataclass>`
+   function is used.
+
    This function is not strictly required, because any Python
    mechanism for creating a new class with :attr:`!__annotations__` can
    then apply the :func:`@dataclass <dataclass>` function to convert that class to
@@ -438,6 +443,9 @@ Module contents
          def add_one(self):
              return self.x + 1
 
+   .. versionadded:: 3.14
+      Added the *decorator* parameter.
+
 .. function:: replace(obj, /, **changes)
 
    Creates a new object of the same type as *obj*, replacing

Doc/library/datetime.rst

@@ -295,6 +295,20 @@ Instance attributes (read-only):
 
    Between 0 and 86,399 inclusive.
 
+   .. caution::
+
+      It is a somewhat common bug for code to unintentionally use this attribute
+      when it is actually intended to get a :meth:`~timedelta.total_seconds`
+      value instead:
+
+      .. doctest::
+
+         >>> from datetime import timedelta
+         >>> duration = timedelta(seconds=11235813)
+         >>> duration.days, duration.seconds
+         (130, 3813)
+         >>> duration.total_seconds()
+         11235813.0
 
 .. attribute:: timedelta.microseconds
 
@@ -351,7 +365,7 @@ Supported operations:
 |                                | same value. (2)                               |
 +--------------------------------+-----------------------------------------------+
 | ``-t1``                        | Equivalent to ``timedelta(-t1.days,           |
-|                                | -t1.seconds*, -t1.microseconds)``,            |
+|                                | -t1.seconds, -t1.microseconds)``,             |
 |                                | and to ``t1 * -1``. (1)(4)                    |
 +--------------------------------+-----------------------------------------------+
 | ``abs(t)``                     | Equivalent to ``+t`` when ``t.days >= 0``,    |

Doc/library/functools.rst

@@ -347,8 +347,7 @@ The :mod:`functools` module defines the following functions:
 
       def partial(func, /, *args, **keywords):
           def newfunc(*more_args, **more_keywords):
-              keywords_union = {**keywords, **more_keywords}
-              return func(*args, *more_args, **keywords_union)
+              return func(*args, *more_args, **(keywords | more_keywords))
           newfunc.func = func
           newfunc.args = args
           newfunc.keywords = keywords

Doc/library/sys.monitoring.rst

@@ -50,16 +50,14 @@ Registering and using tools
    *tool_id* must be in the range 0 to 5 inclusive.
    Raises a :exc:`ValueError` if *tool_id* is in use.
 
-.. function:: free_tool_id(tool_id: int, /) -> None
+.. function:: clear_tool_id(tool_id: int, /) -> None
 
-   Should be called once a tool no longer requires *tool_id*.
+   Unregister all events and callback functions associated with *tool_id*.
 
-.. note::
+.. function:: free_tool_id(tool_id: int, /) -> None
 
-   :func:`free_tool_id` will not disable global or local events associated
-   with *tool_id*, nor will it unregister any callback functions. This
-   function is only intended to be used to notify the VM that the
-   particular *tool_id* is no longer in use.
+   Should be called once a tool no longer requires *tool_id*.
+   Will call :func:`clear_tool_id` before releasing *tool_id*.
 
 .. function:: get_tool(tool_id: int, /) -> str | None
 

Doc/tools/extensions/patchlevel.py

@@ -74,4 +74,8 @@ def get_version_info():
 
 
 if __name__ == "__main__":
-    print(format_version_info(get_header_version_info())[0])
+    short_ver, full_ver = format_version_info(get_header_version_info())
+    if sys.argv[1:2] == ["--short"]:
+        print(short_ver)
+    else:
+        print(full_ver)

Doc/using/windows.rst

@@ -23,8 +23,9 @@ available for application-local distributions.
 
 As specified in :pep:`11`, a Python release only supports a Windows platform
 while Microsoft considers the platform under extended support. This means that
-Python |version| supports Windows 8.1 and newer. If you require Windows 7
-support, please install Python 3.8.
+Python |version| supports Windows 10 and newer. If you require Windows 7
+support, please install Python 3.8. If you require Windows 8.1 support,
+please install Python 3.12.
 
 There are a number of different installers available for Windows, each with
 certain benefits and downsides.