Merge branch 'main' into pythongh-115999-thread-local-bytecode

Author: mpage

.github/CODEOWNERS

@@ -38,7 +38,6 @@ Python/compile.c              @markshannon @iritkatriel
 Python/assemble.c             @markshannon @iritkatriel
 Python/flowgraph.c            @markshannon @iritkatriel
 Python/instruction_sequence.c @iritkatriel
-Python/ast_opt.c              @isidentical
 Python/bytecodes.c            @markshannon
 Python/optimizer*.c           @markshannon
 Python/optimizer_analysis.c   @Fidget-Spinner
@@ -158,10 +157,12 @@ Include/internal/pycore_time.h  @pganssle @abalkin
 /Tools/cases_generator/        @markshannon
 
 # AST
-Python/ast.c                  @isidentical @JelleZijlstra
-Parser/asdl.py                @isidentical @JelleZijlstra
-Parser/asdl_c.py              @isidentical @JelleZijlstra
-Lib/ast.py                    @isidentical @JelleZijlstra
+Python/ast.c                  @isidentical @JelleZijlstra @eclips4
+Python/ast_opt.c              @isidentical @eclips4
+Parser/asdl.py                @isidentical @JelleZijlstra @eclips4
+Parser/asdl_c.py              @isidentical @JelleZijlstra @eclips4
+Lib/ast.py                    @isidentical @JelleZijlstra @eclips4
+Lib/test/test_ast/            @eclips4
 
 # Mock
 /Lib/unittest/mock.py         @cjw296

.github/workflows/build.yml

@@ -195,13 +195,14 @@ jobs:
 
   build_ubuntu_ssltests:
     name: 'Ubuntu SSL tests with OpenSSL'
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     strategy:
       fail-fast: false
       matrix:
+        os: [ubuntu-22.04]
         openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2]
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
@@ -231,7 +232,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
-        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+        key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
     - name: Install OpenSSL
       if: steps.cache-openssl.outputs.cache-hit != 'true'
       run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
@@ -410,7 +411,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
-        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+        key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
     - name: Install OpenSSL
       if: steps.cache-openssl.outputs.cache-hit != 'true'
       run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux

.github/workflows/jit.yml

@@ -110,17 +110,15 @@ jobs:
       - name: Native Windows
         if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
         run: |
-          choco upgrade llvm -y
-          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
+          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
           ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # No PGO or tests (yet):
       - name: Emulated Windows
         if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
         run: |
-          choco upgrade llvm -y
-          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
+          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
 
       - name: Native macOS
@@ -159,7 +157,7 @@ jobs:
           CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \
             CPP="$CC --preprocess" \
             HOSTRUNNER=qemu-${{ matrix.architecture }} \
-            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
+            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
           make all --jobs 4
           ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 

.github/workflows/reusable-ubuntu.yml

@@ -14,7 +14,11 @@ jobs:
   build_ubuntu_reusable:
     name: 'build and test'
     timeout-minutes: 60
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-22.04]
     env:
       FORCE_COLOR: 1
       OPENSSL_VER: 3.0.15
@@ -36,7 +40,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
-        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+        key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
     - name: Install OpenSSL
       if: steps.cache-openssl.outputs.cache-hit != 'true'
       run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.10
+    rev: v0.6.7
     hooks:
       - id: ruff
         name: Run Ruff (lint) on Doc/
@@ -10,6 +10,10 @@ repos:
         name: Run Ruff (lint) on Lib/test/
         args: [--exit-non-zero-on-fix]
         files: ^Lib/test/
+      - id: ruff
+        name: Run Ruff (lint) on Tools/build/check_warnings.py
+        args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml]
+        files: ^Tools/build/check_warnings.py
       - id: ruff
         name: Run Ruff (lint) on Argument Clinic
         args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml]
@@ -22,6 +26,11 @@ repos:
   - repo: https://github.com/psf/black-pre-commit-mirror
     rev: 24.8.0
     hooks:
+      - id: black
+        name: Run Black on Tools/build/check_warnings.py
+        files: ^Tools/build/check_warnings.py
+        language_version: python3.12
+        args: [--line-length=79]
       - id: black
         name: Run Black on Tools/jit/
         files: ^Tools/jit/

Android/README.md

@@ -1,13 +1,9 @@
 # Python for Android
 
 These instructions are only needed if you're planning to compile Python for
-Android yourself. Most users should *not* need to do this. If you're looking to
-use Python on Android, one of the following tools will provide a much more
-approachable user experience:
-
-* [Briefcase](https://briefcase.readthedocs.io), from the BeeWare project
-* [Buildozer](https://buildozer.readthedocs.io), from the Kivy project
-* [Chaquopy](https://chaquo.com/chaquopy/)
+Android yourself. Most users should *not* need to do this. Instead, use one of
+the tools listed in `Doc/using/android.rst`, which will provide a much easier
+experience.
 
 
 ## Prerequisites
@@ -89,10 +85,10 @@ The test suite can be run on Linux, macOS, or Windows:
 The test suite can usually be run on a device with 2 GB of RAM, but this is
 borderline, so you may need to increase it to 4 GB. As of Android
 Studio Koala, 2 GB is the default for all emulators, although the user interface
-may indicate otherwise. The effective setting is `hw.ramSize` in
-~/.android/avd/*.avd/hardware-qemu.ini, whereas Android Studio displays the
-value from config.ini. Changing the value in Android Studio will update both of
-these files.
+may indicate otherwise. Locate the emulator's directory under `~/.android/avd`,
+and find `hw.ramSize` in both config.ini and hardware-qemu.ini. Either set these
+manually to the same value, or use the Android Studio Device Manager, which will
+update both files.
 
 Before running the test suite, follow the instructions in the previous section
 to build the architecture you want to test. Then run the test script in one of
@@ -133,3 +129,8 @@ Every time you run `android.py test`, changes in pure-Python files in the
 repository's `Lib` directory will be picked up immediately. Changes in C files,
 and architecture-specific files such as sysconfigdata, will not take effect
 until you re-run `android.py make-host` or `build`.
+
+
+## Using in your own app
+
+See `Doc/using/android.rst`.

Doc/Makefile

@@ -152,7 +152,7 @@ htmlview: html
 	$(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))"
 
 .PHONY: htmllive
-htmllive: SPHINXBUILD = $(VENVDIR)/bin/sphinx-autobuild
+htmllive: SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-autobuild
 htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0
 htmllive: _ensure-sphinx-autobuild html
 
@@ -182,13 +182,26 @@ venv:
 		echo "The venv has been created in the $(VENVDIR) directory"; \
 	fi
 
+.PHONY: dist-no-html
+dist-no-html: dist-text dist-pdf dist-epub dist-texinfo
+
 .PHONY: dist
 dist:
 	rm -rf dist
 	mkdir -p dist
-
+	$(MAKE) dist-html
+	$(MAKE) dist-text
+	$(MAKE) dist-pdf
+	$(MAKE) dist-epub
+	$(MAKE) dist-texinfo
+
+.PHONY: dist-html
+dist-html:
 	# archive the HTML
 	@echo "Building HTML..."
+	mkdir -p dist
+	rm -rf build/html
+	find dist -name 'python-$(DISTVERSION)-docs-html*' -exec rm -rf {} \;
 	$(MAKE) html
 	cp -pPR build/html dist/python-$(DISTVERSION)-docs-html
 	tar -C dist -cf dist/python-$(DISTVERSION)-docs-html.tar python-$(DISTVERSION)-docs-html
@@ -198,8 +211,13 @@ dist:
 	rm dist/python-$(DISTVERSION)-docs-html.tar
 	@echo "Build finished and archived!"
 
+.PHONY: dist-text
+dist-text:
 	# archive the text build
 	@echo "Building text..."
+	mkdir -p dist
+	rm -rf build/text
+	find dist -name 'python-$(DISTVERSION)-docs-text*' -exec rm -rf {} \;
 	$(MAKE) text
 	cp -pPR build/text dist/python-$(DISTVERSION)-docs-text
 	tar -C dist -cf dist/python-$(DISTVERSION)-docs-text.tar python-$(DISTVERSION)-docs-text
@@ -209,9 +227,13 @@ dist:
 	rm dist/python-$(DISTVERSION)-docs-text.tar
 	@echo "Build finished and archived!"
 
+.PHONY: dist-pdf
+dist-pdf:
 	# archive the A4 latex
 	@echo "Building LaTeX (A4 paper)..."
+	mkdir -p dist
 	rm -rf build/latex
+	find dist -name 'python-$(DISTVERSION)-docs-pdf*' -exec rm -rf {} \;
 	$(MAKE) latex PAPER=a4
 	# remove zip & bz2 dependency on all-pdf,
 	# as otherwise the full latexmk process is run twice.
@@ -222,16 +244,24 @@ dist:
 	cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2
 	@echo "Build finished and archived!"
 
+.PHONY: dist-epub
+dist-epub:
 	# copy the epub build
 	@echo "Building EPUB..."
+	mkdir -p dist
 	rm -rf build/epub
+	rm -f dist/python-$(DISTVERSION)-docs.epub
 	$(MAKE) epub
 	cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub
 	@echo "Build finished and archived!"
 
+.PHONY: dist-texinfo
+dist-texinfo:
 	# archive the texinfo build
 	@echo "Building Texinfo..."
+	mkdir -p dist
 	rm -rf build/texinfo
+	find dist -name 'python-$(DISTVERSION)-docs-texinfo*' -exec rm -rf {} \;
 	$(MAKE) texinfo
 	$(MAKE) info --directory=build/texinfo
 	cp -pPR build/texinfo dist/python-$(DISTVERSION)-docs-texinfo

Doc/c-api/complex.rst

@@ -79,6 +79,8 @@ pointers.  This is consistent throughout the API.
    If *num* is null and *exp* is not a positive real number,
    this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`.
 
+   Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows.
+
 
 Complex Numbers as Python Objects
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

Doc/c-api/contextvars.rst

@@ -101,6 +101,52 @@ Context object management functions:
    current context for the current thread.  Returns ``0`` on success,
    and ``-1`` on error.
 
+.. c:function:: int PyContext_AddWatcher(PyContext_WatchCallback callback)
+
+   Register *callback* as a context object watcher for the current interpreter.
+   Return an ID which may be passed to :c:func:`PyContext_ClearWatcher`.
+   In case of error (e.g. no more watcher IDs available),
+   return ``-1`` and set an exception.
+
+   .. versionadded:: 3.14
+
+.. c:function:: int PyContext_ClearWatcher(int watcher_id)
+
+   Clear watcher identified by *watcher_id* previously returned from
+   :c:func:`PyContext_AddWatcher` for the current interpreter.
+   Return ``0`` on success, or ``-1`` and set an exception on error
+   (e.g. if the given *watcher_id* was never registered.)
+
+   .. versionadded:: 3.14
+
+.. c:type:: PyContextEvent
+
+   Enumeration of possible context object watcher events:
+   - ``Py_CONTEXT_EVENT_ENTER``
+   - ``Py_CONTEXT_EVENT_EXIT``
+
+   .. versionadded:: 3.14
+
+.. c:type:: int (*PyContext_WatchCallback)(PyContextEvent event, PyContext* ctx)
+
+   Type of a context object watcher callback function.
+   If *event* is ``Py_CONTEXT_EVENT_ENTER``, then the callback is invoked
+   after *ctx* has been set as the current context for the current thread.
+   Otherwise, the callback is invoked before the deactivation of *ctx* as the current context
+   and the restoration of the previous contex object for the current thread.
+
+   If the callback returns with an exception set, it must return ``-1``; this
+   exception will be printed as an unraisable exception using
+   :c:func:`PyErr_FormatUnraisable`. Otherwise it should return ``0``.
+
+   There may already be a pending exception set on entry to the callback. In
+   this case, the callback should return ``0`` with the same exception still
+   set. This means the callback may not call any other API that can set an
+   exception unless it saves and clears the exception state first, and restores
+   it before returning.
+
+   .. versionadded:: 3.14
+
 
 Context variable functions:
 

Doc/c-api/exceptions.rst

@@ -1004,6 +1004,7 @@ the variables:
    single: PyExc_OverflowError (C var)
    single: PyExc_PermissionError (C var)
    single: PyExc_ProcessLookupError (C var)
+   single: PyExc_PythonFinalizationError (C var)
    single: PyExc_RecursionError (C var)
    single: PyExc_ReferenceError (C var)
    single: PyExc_RuntimeError (C var)
@@ -1096,6 +1097,8 @@ the variables:
 +-----------------------------------------+---------------------------------+----------+
 | :c:data:`PyExc_ProcessLookupError`      | :exc:`ProcessLookupError`       |          |
 +-----------------------------------------+---------------------------------+----------+
+| :c:data:`PyExc_PythonFinalizationError` | :exc:`PythonFinalizationError`  |          |
++-----------------------------------------+---------------------------------+----------+
 | :c:data:`PyExc_RecursionError`          | :exc:`RecursionError`           |          |
 +-----------------------------------------+---------------------------------+----------+
 | :c:data:`PyExc_ReferenceError`          | :exc:`ReferenceError`           |          |

Doc/c-api/long.rst

@@ -509,6 +509,8 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
    Currently, ``-1`` corresponds to
    ``Py_ASNATIVEBYTES_NATIVE_ENDIAN | Py_ASNATIVEBYTES_UNSIGNED_BUFFER``.
 
+   .. c:namespace:: NULL
+
    ============================================= ======
    Flag                                          Value
    ============================================= ======

Doc/c-api/module.rst

@@ -421,6 +421,8 @@ The available slot types are:
 
    Specifies one of the following values:
 
+   .. c:namespace:: NULL
+
    .. c:macro:: Py_MOD_GIL_USED
 
       The module depends on the presence of the global interpreter lock (GIL),