From e81025e6d2e01e38c5a5b656af5739ac9df5ff55 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 7 May 2026 19:01:11 +0300 Subject: [PATCH 001/110] Post 3.15.0b1 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 32c85792d550c3..cdca931566577f 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -27,7 +27,7 @@ #define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.15.0b1" +#define PY_VERSION "3.15.0b1+" /*--end constants--*/ From 4caee143d2703f424ce66c3662a9f9a4fcdd0b54 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Thu, 7 May 2026 17:15:06 -0500 Subject: [PATCH 002/110] [3.15] Forward-port 'check-abi' CI job from 3.14 (GH-149517) Also add the python3.15.abi file as generated by the new job and remove the 'main branch only' entry from .gitignore. (adapted from commit 0eb2291a7e85062dba387dfecaee94858db8a0a9) --- .github/workflows/build.yml | 47 + .gitignore | 4 - Doc/data/python3.15.abi | 33491 ++++++++++++++++++++++++++++++++++ 3 files changed, 33538 insertions(+), 4 deletions(-) create mode 100644 Doc/data/python3.15.abi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1af3a0607f9ad2..12bf160178e3c7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,6 +49,53 @@ jobs: if: fromJSON(needs.build-context.outputs.run-docs) uses: ./.github/workflows/reusable-docs.yml + check-abi: + name: 'Check if the ABI has changed' + runs-on: ubuntu-22.04 # 24.04 causes spurious errors + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.x' + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + sudo apt-get install -yq --no-install-recommends abigail-tools + - name: Build CPython + env: + CFLAGS: -g3 -O0 + run: | + # Build Python with the libpython dynamic library + ./configure --enable-shared + make -j4 + - name: Check for changes in the ABI + id: check + run: | + if ! make check-abidump; then + echo "Generated ABI file is not up to date." + echo "Please add the release manager of this branch as a reviewer of this PR." + echo "" + echo "The up to date ABI file should be attached to this build as an artifact." + echo "" + echo "To learn more about this check: https://devguide.python.org/getting-started/setup-building/index.html#regenerate-the-abi-dump" + echo "" + exit 1 + fi + - name: Generate updated ABI files + if: ${{ failure() && steps.check.conclusion == 'failure' }} + run: | + make regen-abidump + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + name: Publish updated ABI files + if: ${{ failure() && steps.check.conclusion == 'failure' }} + with: + name: abi-data + path: ./Doc/data/*.abi + check-autoconf-regen: name: 'Check if Autoconf files are up to date' # Don't use ubuntu-latest but a specific version to make the job diff --git a/.gitignore b/.gitignore index 118eb5ee76e805..78b6d4efb0e109 100644 --- a/.gitignore +++ b/.gitignore @@ -177,7 +177,3 @@ Python/frozen_modules/MANIFEST # People's custom https://docs.anthropic.com/en/docs/claude-code/memory configs. /.claude/ CLAUDE.local.md - -#### main branch only stuff below this line, things to backport go above. #### -# main branch only: ABI files are not checked/maintained. -Doc/data/python*.abi diff --git a/Doc/data/python3.15.abi b/Doc/data/python3.15.abi new file mode 100644 index 00000000000000..04211b6e4e274a --- /dev/null +++ b/Doc/data/python3.15.abi @@ -0,0 +1,33491 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From b311ea869141acff7985ba3b7ae304ec04e39229 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 8 May 2026 00:27:34 +0200 Subject: [PATCH 003/110] [3.15] Improve error messages when the WASI SDK can't be found (GH-149519) (cherry picked from commit b142878db1e54149feba62b08df1236432793bf0) Co-authored-by: Brett Cannon --- Platforms/WASI/_build.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Platforms/WASI/_build.py b/Platforms/WASI/_build.py index 76d2853163baa9..c1a91a9c833b8e 100644 --- a/Platforms/WASI/_build.py +++ b/Platforms/WASI/_build.py @@ -222,10 +222,8 @@ def wasi_sdk(context): if wasi_sdk_path := context.wasi_sdk_path: if not wasi_sdk_path.exists(): raise ValueError( - "WASI SDK not found; " - "download from " - "https://github.com/WebAssembly/wasi-sdk and/or " - "specify via $WASI_SDK_PATH or --wasi-sdk" + "WASI SDK not found at " + f"{os.fsdecode(wasi_sdk_path)!r} (via --wasi-sdk)" ) return wasi_sdk_path @@ -237,7 +235,8 @@ def wasi_sdk(context): wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var) if not wasi_sdk_path.exists(): raise ValueError( - f"WASI SDK not found at $WASI_SDK_PATH ({wasi_sdk_path})" + f"WASI SDK not found at {os.fsdecode(wasi_sdk_path)!r} " + "(via $WASI_SDK_PATH)" ) else: opt_path = pathlib.Path("/opt") @@ -272,6 +271,14 @@ def wasi_sdk(context): f" Found WASI SDK {major_version}, " f"but WASI SDK {wasi_sdk_version} is the supported version", ) + elif not wasi_sdk_path: + raise ValueError( + f"WASI SDK {wasi_sdk_version} not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and install in " + f"{os.fsdecode(opt_path)!r} or specify the SDK via " + "$WASI_SDK_PATH or --wasi-sdk" + ) # Cache the result. context.wasi_sdk_path = wasi_sdk_path From 5cb915da44642ceabac7a2d24a7cc2c5620f65ab Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 8 May 2026 04:08:12 +0200 Subject: [PATCH 004/110] [3.15] gh-149481: skip `FOR_ITER` inline specialization for Python `__next__` (GH-149491) (#149523) gh-149481: skip `FOR_ITER` inline specialization for Python `__next__` (GH-149491) (cherry picked from commit 49918f5b0ceb1950c3222fd4fd6be872d2e15c6f) Co-authored-by: Neko Asakura Co-authored-by: Savannah Ostrowski Co-authored-by: Stan Ulbrych --- Include/internal/pycore_typeobject.h | 2 ++ Lib/test/test_capi/test_opt.py | 3 ++- Objects/typeobject.c | 6 ++++++ Python/optimizer_bytecodes.c | 4 +++- Python/optimizer_cases.c.h | 3 ++- 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 8d48cf6605ca7e..785b77d3e3be81 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -122,6 +122,8 @@ extern PyObject* _Py_BaseObject_RichCompare(PyObject* self, PyObject* other, int extern PyObject* _Py_slot_tp_getattro(PyObject *self, PyObject *name); extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); +extern int _PyType_HasSlotTpIternext(PyTypeObject *type); + extern PyTypeObject _PyBufferWrapper_Type; PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index d80fec9a8a0d2b..aaa5050208ced9 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -598,7 +598,8 @@ def testfunc(n, m): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_ITER_NEXT_INLINE", uops) + self.assertIn("_FOR_ITER_TIER_TWO", uops) + self.assertNotIn("_ITER_NEXT_INLINE", uops) @requires_specialization diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4f43747ba83fd9..9a18ca72516da7 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -11079,6 +11079,12 @@ slot_tp_iternext(PyObject *self) return vectorcall_method(&_Py_ID(__next__), stack, 1); } +int +_PyType_HasSlotTpIternext(PyTypeObject *type) +{ + return type->tp_iternext == slot_tp_iternext; +} + static PyObject * slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) { diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e10a096baa3318..39cc36ae79fead 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -2,6 +2,7 @@ #include "pycore_long.h" #include "pycore_opcode_utils.h" #include "pycore_optimizer.h" +#include "pycore_typeobject.h" #include "pycore_uops.h" #include "pycore_uop_ids.h" #include "internal/pycore_moduleobject.h" @@ -1459,7 +1460,8 @@ dummy_func(void) { type = sym_get_probable_type(iter); definite = false; } - if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL) { + if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL + && !_PyType_HasSlotTpIternext(type)) { PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); if (!definite) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 01ecb3790aa2cd..db3dcbb97b2645 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3706,7 +3706,8 @@ type = sym_get_probable_type(iter); definite = false; } - if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL) { + if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL + && !_PyType_HasSlotTpIternext(type)) { PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); if (!definite) { From 831dac8b518c9de262f84d3faca3443715902e07 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 8 May 2026 12:33:35 +0200 Subject: [PATCH 005/110] [3.15] gh-146445: Update CODEOWNERS for Android and iOS migration to Platforms directory (GH-149543) (#149545) gh-146445: Update CODEOWNERS for Android and iOS migration to Platforms directory (GH-149543) (cherry picked from commit 5b58fbc07c8173df98ce6d378ded1bc605997c3f) Co-authored-by: Malcolm Smith --- .github/CODEOWNERS | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index af904a567cfb7e..769d739a8057aa 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -156,7 +156,7 @@ Misc/libabigail.abignore @encukou # ---------------------------------------------------------------------------- # Android -Android/ @mhsmith @freakboy3742 +Platforms/Android/ @mhsmith @freakboy3742 Doc/using/android.rst @mhsmith @freakboy3742 Lib/_android_support.py @mhsmith @freakboy3742 Lib/test/test_android.py @mhsmith @freakboy3742 @@ -164,8 +164,7 @@ Lib/test/test_android.py @mhsmith @freakboy3742 # iOS Doc/using/ios.rst @freakboy3742 Lib/_ios_support.py @freakboy3742 -Apple/ @freakboy3742 -iOS/ @freakboy3742 +Platforms/Apple/ @freakboy3742 # macOS Mac/ @python/macos-team From 20e298bc37a48049244553862cc1918713494a23 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 8 May 2026 13:06:52 +0200 Subject: [PATCH 006/110] [3.15] docs: Clarify docs for error case of `PyDict_GetItemRef` (GH-149506) (#149546) docs: Clarify docs for error case of `PyDict_GetItemRef` (GH-149506) (cherry picked from commit 3565d31690d30a189933bce7b27d0bd2c6973f47) Co-authored-by: Nathan Goldbaum --- Doc/c-api/dict.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index a2a0d0d80657eb..556113a97bf772 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -151,7 +151,7 @@ Dictionary objects * If the key is present, set *\*result* to a new :term:`strong reference` to the value and return ``1``. * If the key is missing, set *\*result* to ``NULL`` and return ``0``. - * On error, raise an exception and return ``-1``. + * On error, raise an exception, set *\*result* to ``NULL`` and return ``-1``. The first argument can be a :class:`dict` or a :class:`frozendict`. From b922b42be7c9bedd3ecf6f34a560fa692793a530 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 8 May 2026 13:09:06 +0200 Subject: [PATCH 007/110] [3.15] gh-145176: Update CODEOWNERS for Emscripten migration to Platforms directory (GH-149544) (#149550) gh-145176: Update CODEOWNERS for Emscripten migration to Platforms directory (GH-149544) (cherry picked from commit 52a05e8da71abcc83df54e465d0a4df50785e910) Co-authored-by: Malcolm Smith --- .github/CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 769d739a8057aa..f4ffa24edca453 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -175,8 +175,8 @@ Lib/test/test__osx_support.py @python/macos-team Tools/wasm/README.md @brettcannon @freakboy3742 @emmatyping # WebAssembly (Emscripten) -Tools/wasm/config.site-wasm32-emscripten @freakboy3742 @emmatyping -Tools/wasm/emscripten @freakboy3742 @emmatyping +Platforms/emscripten @freakboy3742 @emmatyping +Tools/wasm/emscripten @freakboy3742 @emmatyping # WebAssembly (WASI) Platforms/WASI @brettcannon @emmatyping @savannahostrowski From f9e5975deb6b01c434eadff66f28bd53e0ca4c55 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 8 May 2026 13:48:04 +0200 Subject: [PATCH 008/110] [3.15] gh-149459: Fix segfault when `_LOAD_SPECIAL` guard deoptimizes (GH-149478) (#149552) gh-149459: Fix segfault when `_LOAD_SPECIAL` guard deoptimizes (GH-149478) (cherry picked from commit c341e341b25cec03d28d1b2c368bb871d76ca88b) Co-authored-by: Hai Zhu --- Lib/test/test_capi/test_opt.py | 14 ++++++++++++++ .../2026-05-07-03-18-59.gh-issue-149459.5fhAqP.rst | 1 + Python/optimizer_bytecodes.c | 11 ++++++++++- Python/optimizer_cases.c.h | 6 +++++- 4 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-07-03-18-59.gh-issue-149459.5fhAqP.rst diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index aaa5050208ced9..2f606c2c6eba2d 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -6138,6 +6138,20 @@ def __init__(self, x): C(0) if i else str(0) """)) + def test_load_special_type_guard_deopt(self): + script_helper.assert_python_ok("-s", "-c", textwrap.dedent(f""" + def f1(): + class Context: + def __enter__(self): ... + def __exit__(self, e, v, t): ... + + with Context(): + pass + + for _ in range({TIER2_THRESHOLD + 5}): + f1() + """), PYTHON_JIT="1") + def global_identity(x): return x diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-07-03-18-59.gh-issue-149459.5fhAqP.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-07-03-18-59.gh-issue-149459.5fhAqP.rst new file mode 100644 index 00000000000000..4cd0a148df3c70 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-07-03-18-59.gh-issue-149459.5fhAqP.rst @@ -0,0 +1 @@ +Fix a crash in the JIT optimizer when a specialized ``LOAD_SPECIAL`` guard deoptimized after inserting the synthetic ``NULL`` stack entry. diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 39cc36ae79fead..96dbaea5a5797e 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -2043,7 +2043,16 @@ dummy_func(void) { PyObject *name = _Py_SpecialMethods[oparg].name; PyObject *descr = _PyType_Lookup(type, name); if (descr != NULL && (Py_TYPE(descr)->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR)) { - ADD_OP(_GUARD_TYPE_VERSION, 0, type->tp_version_tag); + /* LOAD_SPECIAL expands to _RECORD_TOS_TYPE + _INSERT_NULL + + * _LOAD_SPECIAL. Insert _GUARD_TYPE_VERSION before the + * already-emitted _INSERT_NULL so deopt sees the original + * stack shape.*/ + _PyUOpInstruction *insert_null = uop_buffer_last(&ctx->out_buffer); + assert(insert_null->opcode == _INSERT_NULL); + assert(insert_null->target == this_instr->target); + REPLACE_OP(insert_null, _GUARD_TYPE_VERSION, 0, type->tp_version_tag); + ADD_OP(_INSERT_NULL, 0, 0); + bool immortal = _Py_IsImmortal(descr) || (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE); ADD_OP(immortal ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, 0, (uintptr_t)descr); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index db3dcbb97b2645..f336549d2ed244 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3896,7 +3896,11 @@ PyObject *name = _Py_SpecialMethods[oparg].name; PyObject *descr = _PyType_Lookup(type, name); if (descr != NULL && (Py_TYPE(descr)->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR)) { - ADD_OP(_GUARD_TYPE_VERSION, 0, type->tp_version_tag); + _PyUOpInstruction *insert_null = uop_buffer_last(&ctx->out_buffer); + assert(insert_null->opcode == _INSERT_NULL); + assert(insert_null->target == this_instr->target); + REPLACE_OP(insert_null, _GUARD_TYPE_VERSION, 0, type->tp_version_tag); + ADD_OP(_INSERT_NULL, 0, 0); bool immortal = _Py_IsImmortal(descr) || (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE); ADD_OP(immortal ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, 0, (uintptr_t)descr); From 333b7c54c01b2eb1aca9b4d0ffe35b45e833f18a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 8 May 2026 15:06:08 +0200 Subject: [PATCH 009/110] [3.15] Add Diego as author of PEP 831 (GH-149551) (#149561) --- Doc/whatsnew/3.15.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 9e2f789334ff02..0f7782ba1813d1 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -411,8 +411,8 @@ embedding applications, and native libraries. unwinding for the whole Python process. (Contributed by Pablo Galindo Salgado and Savannah Ostrowski in -:gh:`149201`; PEP 831 written by Pablo Galindo Salgado, Ken Jin, and -Savannah Ostrowski.) +:gh:`149201`; PEP 831 written by Pablo Galindo Salgado, Ken Jin, +Savannah Ostrowski, and Diego Russo.) .. seealso:: :pep:`831` for further details. From 54a187d3b3ae11e37ae04fd91f3ea0cdfb6fe410 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 8 May 2026 15:32:59 +0200 Subject: [PATCH 010/110] [3.15] Skip GNU backtrace test on Arm 32-bit (GH-149493) (#149562) --- Lib/test/test_frame_pointer_unwind.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Lib/test/test_frame_pointer_unwind.py b/Lib/test/test_frame_pointer_unwind.py index faa012c9c00d8f..1cf5083fd0fdcf 100644 --- a/Lib/test/test_frame_pointer_unwind.py +++ b/Lib/test/test_frame_pointer_unwind.py @@ -89,6 +89,21 @@ def _frame_pointers_expected(machine): return None +def _is_arm32_build(): + if sys.maxsize >= 2**32: + return False + + abi = " ".join( + value for value in ( + sysconfig.get_config_var("MULTIARCH"), + sysconfig.get_config_var("HOST_GNU_TYPE"), + sysconfig.get_config_var("SOABI"), + ) + if value + ).lower() + return "arm" in abi + + def _build_stack_and_unwind(unwinder): import operator @@ -295,6 +310,10 @@ def test_manual_unwind_respects_frame_pointers(self): @support.requires_gil_enabled("test requires the GIL enabled") @unittest.skipIf(support.is_wasi, "test not supported on WASI") @unittest.skipUnless(sys.platform == "linux", "GNU backtrace unwinding test requires Linux") +@unittest.skipIf( + _is_arm32_build(), + "GNU backtrace unwinding skipped on Arm 32-bit", +) class GnuBacktraceUnwindTests(unittest.TestCase): def setUp(self): From 0bdbc4810165bcbaefd16560d48e94a472557d23 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 8 May 2026 16:24:49 +0200 Subject: [PATCH 011/110] [3.15] Rename fp unwind test module to C stack unwind (GH-149563) (#149565) --- ...inter_unwind.py => test_c_stack_unwind.py} | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) rename Lib/test/{test_frame_pointer_unwind.py => test_c_stack_unwind.py} (92%) diff --git a/Lib/test/test_frame_pointer_unwind.py b/Lib/test/test_c_stack_unwind.py similarity index 92% rename from Lib/test/test_frame_pointer_unwind.py rename to Lib/test/test_c_stack_unwind.py index 1cf5083fd0fdcf..91bf44e463473d 100644 --- a/Lib/test/test_frame_pointer_unwind.py +++ b/Lib/test/test_c_stack_unwind.py @@ -1,3 +1,12 @@ +"""Test in-process C stack unwinders against Python and JIT frames. + +The tests build a recursive Python call stack, ask each _testinternalcapi +unwinder for return addresses, and classify those addresses as Python, JIT, or +other frames. The backends include CPython's manual stack-chain unwinder and +GNU backtrace(), so this module is about in-process C stack unwinding rather +than a single unwind mechanism. GDB integration tests live in test_gdb. +""" + import json import os import platform @@ -20,7 +29,7 @@ STACK_DEPTH = 10 -def _frame_pointers_expected(machine): +def _manual_unwind_expected(machine): _Py_WITH_FRAME_POINTERS = getattr( _testinternalcapi, "_Py_WITH_FRAME_POINTERS", @@ -195,7 +204,7 @@ def _annotate_unwind_after_executor_free(unwinder_name="gnu_backtrace_unwind"): def _run_unwind_helper(helper_name, unwinder_name, **env): code = ( - f"from test.test_frame_pointer_unwind import {helper_name}; " + f"from test.test_c_stack_unwind import {helper_name}; " f"print({helper_name}({unwinder_name!r}));" ) run_env = os.environ.copy() @@ -235,15 +244,17 @@ def _unwind_after_executor_free_result(unwinder_name, **env): @support.requires_gil_enabled("test requires the GIL enabled") @unittest.skipIf(support.is_wasi, "test not supported on WASI") -class FramePointerUnwindTests(unittest.TestCase): +class ManualStackUnwindTests(unittest.TestCase): def setUp(self): super().setUp() machine = platform.machine().lower() - expected = _frame_pointers_expected(machine) + expected = _manual_unwind_expected(machine) if expected is None: - self.skipTest(f"unsupported architecture for frame pointer check: {machine}") + self.skipTest( + f"unsupported architecture for manual stack unwind check: {machine}" + ) if expected == "crash": self.skipTest(f"test does crash on {machine}") @@ -251,12 +262,14 @@ def setUp(self): _testinternalcapi.manual_frame_pointer_unwind() except RuntimeError as exc: if "not supported" in str(exc): - self.skipTest("manual frame pointer unwinding not supported on this platform") + self.skipTest( + "manual stack unwinding not supported on this platform" + ) raise self.machine = machine - self.frame_pointers_expected = expected + self.manual_unwind_expected = expected - def test_manual_unwind_respects_frame_pointers(self): + def test_manual_unwind_finds_expected_frames(self): jit_available = hasattr(sys, "_jit") and sys._jit.is_available() envs = [({"PYTHON_JIT": "0"}, False)] if jit_available: @@ -268,7 +281,7 @@ def test_manual_unwind_respects_frame_pointers(self): jit_frames = result["jit_frames"] python_frames = result.get("python_frames", 0) jit_backend = result.get("jit_backend") - if self.frame_pointers_expected: + if self.manual_unwind_expected: self.assertGreaterEqual( python_frames, STACK_DEPTH, From bb060b82f6723110b399431843e7de7d209c3a1d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 8 May 2026 22:53:55 +0200 Subject: [PATCH 012/110] [3.15] gh-79638: Test other HTTP error codes besides 403 in test_robotparser (GH-149569) (GH-149580) Also, use urllib.request.urlcleanup() in NetworkTestCase. (cherry picked from commit 57ef2199503387617b8af3d719c74089fb70dbd4) Co-authored-by: Serhiy Storchaka --- Lib/test/test_robotparser.py | 83 +++++++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 19 deletions(-) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 3ea0ec66fbfbe9..cd1477037e94b7 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -646,26 +646,23 @@ def test_group_without_user_agent(self): ) class BaseLocalNetworkTestCase: - def setUp(self): + @classmethod + def setUpClass(cls): # clear _opener global variable - self.addCleanup(urllib.request.urlcleanup) + cls.addClassCleanup(urllib.request.urlcleanup) - self.server = HTTPServer((socket_helper.HOST, 0), self.RobotHandler) + cls.server = HTTPServer((socket_helper.HOST, 0), cls.RobotHandler) + cls.addClassCleanup(cls.server.server_close) - self.t = threading.Thread( + t = threading.Thread( name='HTTPServer serving', - target=self.server.serve_forever, + target=cls.server.serve_forever, # Short poll interval to make the test finish quickly. # Time between requests is short enough that we won't wake # up spuriously too many times. kwargs={'poll_interval':0.01}) - self.t.daemon = True # In case this function raises. - self.t.start() - - def tearDown(self): - self.server.shutdown() - self.t.join() - self.server.server_close() + cls.enterClassContext(threading_helper.start_threads([t])) + cls.addClassCleanup(cls.server.shutdown) SAMPLE_ROBOTS_TXT = b'''\ @@ -687,7 +684,6 @@ def do_GET(self): def log_message(self, format, *args): pass - @threading_helper.reap_threads def testRead(self): # Test that reading a weird robots.txt doesn't fail. addr = self.server.server_address @@ -702,31 +698,79 @@ def testRead(self): self.assertTrue(parser.can_fetch(agent, url + '/utf8/')) self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) self.assertFalse(parser.can_fetch(agent, url + '/utf8/%F0%9F%90%8D')) - self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) self.assertTrue(parser.can_fetch(agent, url + '/non-utf8/')) self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/%F0')) self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/\U0001f40d')) self.assertFalse(parser.can_fetch(agent, url + '/%2F[spam]/path')) -class PasswordProtectedSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase): +class HttpErrorsTestCase(BaseLocalNetworkTestCase, unittest.TestCase): class RobotHandler(BaseHTTPRequestHandler): def do_GET(self): - self.send_error(403, "Forbidden access") + self.send_error(self.server.return_code) def log_message(self, format, *args): pass - @threading_helper.reap_threads - def testPasswordProtectedSite(self): + def setUp(self): + # Make sure that a valid code is set in the test. + self.server.return_code = None + + def testUnauthorized(self): + self.server.return_code = 401 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/some/file.html')) + + def testForbidden(self): + self.server.return_code = 403 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/some/file.html')) + + def testNotFound(self): + self.server.return_code = 404 addr = self.server.server_address - url = 'http://' + socket_helper.HOST + ':' + str(addr[1]) + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertTrue(parser.can_fetch("*", robots_url)) + self.assertTrue(parser.can_fetch("*", url + '/path/file.html')) + + def testTeapot(self): + self.server.return_code = 418 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertTrue(parser.can_fetch("*", robots_url)) + self.assertTrue(parser.can_fetch("*", url + '/pot-1?milk-type=Cream')) + + def testServiceUnavailable(self): + self.server.return_code = 503 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' robots_url = url + "/robots.txt" parser = urllib.robotparser.RobotFileParser() parser.set_url(url) parser.read() self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/path/file.html')) @support.requires_working_socket() @@ -738,6 +782,7 @@ class NetworkTestCase(unittest.TestCase): @classmethod def setUpClass(cls): support.requires('network') + cls.addClassCleanup(urllib.request.urlcleanup) with socket_helper.transient_internet(cls.base_url): cls.parser = urllib.robotparser.RobotFileParser(cls.robots_txt) cls.parser.read() From 2b1eed460d598a30a5eab63f5f1f4d8ac8e43468 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 9 May 2026 02:28:21 +0200 Subject: [PATCH 013/110] [3.15] gh-149474: use `Py_fopen` in `Binary{Reader,Writer}` for audit hook and path-like support (GH-149524) (#149586) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-149474: use `Py_fopen` in `Binary{Reader,Writer}` for audit hook and path-like support (GH-149524) (cherry picked from commit 354ef336e4cd48cf0c02bc9a0c642adf5d543184) Co-authored-by: Maurycy Pawłowski-Wieroński --- Lib/test/audit-tests.py | 14 ++++++ .../test_binary_format.py | 30 +++++++++++++ ...-05-08-02-18-54.gh-issue-149474.ujQ-mu.rst | 3 ++ Modules/_remote_debugging/binary_io.h | 12 ++--- Modules/_remote_debugging/binary_io_reader.c | 45 +++++++------------ Modules/_remote_debugging/binary_io_writer.c | 18 ++------ Modules/_remote_debugging/clinic/module.c.h | 38 +++------------- Modules/_remote_debugging/module.c | 12 ++--- 8 files changed, 83 insertions(+), 89 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-05-08-02-18-54.gh-issue-149474.ujQ-mu.rst diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index a893932169a089..8be5bf8aa4f546 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -208,6 +208,16 @@ def rl(name): else: return None + try: + import _remote_debugging + except ImportError: + _remote_debugging = None + + def rd(name): + if _remote_debugging: + return getattr(_remote_debugging, name, None) + return None + # Try a range of "open" functions. # All of them should fail with TestHook(raise_on_events={"open"}) as hook: @@ -225,6 +235,8 @@ def rl(name): (rl("append_history_file"), 0, None), (rl("read_init_file"), testfn), (rl("read_init_file"), None), + (rd("BinaryWriter"), testfn, 1000, 0), + (rd("BinaryReader"), testfn), ]: if not fn: continue @@ -258,6 +270,8 @@ def rl(name): ("~/.history", "a") if rl("append_history_file") else None, (testfn, "r") if readline else None, ("", "r") if readline else None, + (testfn, "wb") if rd("BinaryWriter") else None, + (testfn, "rb") if rd("BinaryReader") else None, ] if i is not None ], diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py index 9cf706aa2dafee..1fbb4e2d6c6fbb 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py @@ -2,6 +2,7 @@ import json import os +import pathlib import random import struct import tempfile @@ -814,6 +815,35 @@ def test_invalid_file_path(self): with BinaryReader("/nonexistent/path/file.bin") as reader: reader.replay_samples(RawCollector()) + def test_path_arguments_round_trip(self): + """Reader and writer accept str, bytes or os.PathLike.""" + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + + for path_arg in (filename, os.fsencode(filename), pathlib.Path(filename)): + with self.subTest(path_type=type(path_arg).__name__): + writer = _remote_debugging.BinaryWriter(path_arg, 1000, 0) + writer.finalize() + reader = _remote_debugging.BinaryReader(path_arg) + info = reader.get_info() + reader.close() + self.assertEqual(info["sample_count"], 0) + + def test_rejects_non_pathlike(self): + """Reader and writer raise TypeError on non-path-like filenames.""" + with self.assertRaises(TypeError): + _remote_debugging.BinaryWriter(123, 1000, 0) + with self.assertRaises(TypeError): + _remote_debugging.BinaryReader(123) + + def test_invalid_path_error_preserves_pathlib(self): + """Missing path: OSError carries the original path object, not a string.""" + missing = pathlib.Path("/i/do/not/exist") + with self.assertRaises(FileNotFoundError) as cm: + _remote_debugging.BinaryReader(missing) + self.assertEqual(os.fspath(cm.exception.filename), os.fspath(missing)) + def test_writer_handles_empty_stack_first_sample(self): """BinaryWriter.write_sample tolerates an empty stack on a fresh thread. diff --git a/Misc/NEWS.d/next/Security/2026-05-08-02-18-54.gh-issue-149474.ujQ-mu.rst b/Misc/NEWS.d/next/Security/2026-05-08-02-18-54.gh-issue-149474.ujQ-mu.rst new file mode 100644 index 00000000000000..48e718b95ebe3a --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-08-02-18-54.gh-issue-149474.ujQ-mu.rst @@ -0,0 +1,3 @@ +Fix the binary writer in :mod:`profiling.sampling` not firing the audit +(:pep:`578`) when creating the output file. The writer and the reader now +accept any path-like object. Patch by Maurycy Pawłowski-Wieroński. diff --git a/Modules/_remote_debugging/binary_io.h b/Modules/_remote_debugging/binary_io.h index 87a54371c774f1..d4188335c0b6d0 100644 --- a/Modules/_remote_debugging/binary_io.h +++ b/Modules/_remote_debugging/binary_io.h @@ -253,7 +253,6 @@ typedef struct { /* Main binary writer structure */ typedef struct { FILE *fp; - char *filename; /* Write buffer for batched I/O */ uint8_t *write_buffer; @@ -311,10 +310,7 @@ typedef struct { /* Main binary reader structure */ typedef struct { - char *filename; - #if USE_MMAP - int fd; uint8_t *mapped_data; size_t mapped_size; #else @@ -522,7 +518,7 @@ grow_array_inplace(void **ptr_addr, size_t count, size_t *capacity, size_t elem_ * Create a new binary writer. * * Arguments: - * filename: Path to output file + * path: Path to output file * sample_interval_us: Sampling interval in microseconds * compression_type: COMPRESSION_NONE or COMPRESSION_ZSTD * start_time_us: Start timestamp in microseconds (from time.monotonic() * 1e6) @@ -531,7 +527,7 @@ grow_array_inplace(void **ptr_addr, size_t count, size_t *capacity, size_t elem_ * New BinaryWriter* on success, NULL on failure (PyErr set) */ BinaryWriter *binary_writer_create( - const char *filename, + PyObject *path, uint64_t sample_interval_us, int compression_type, uint64_t start_time_us @@ -583,12 +579,12 @@ void binary_writer_destroy(BinaryWriter *writer); * Open a binary file for reading. * * Arguments: - * filename: Path to input file + * path: Path to input file * * Returns: * New BinaryReader* on success, NULL on failure (PyErr set) */ -BinaryReader *binary_reader_open(const char *filename); +BinaryReader *binary_reader_open(PyObject *path); /* * Replay samples from binary file through a collector. diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index 551530b519952c..972b197cfbad86 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -358,7 +358,7 @@ reader_parse_frame_table(BinaryReader *reader, const uint8_t *data, size_t file_ } BinaryReader * -binary_reader_open(const char *filename) +binary_reader_open(PyObject *path) { BinaryReader *reader = PyMem_Calloc(1, sizeof(BinaryReader)); if (!reader) { @@ -366,29 +366,18 @@ binary_reader_open(const char *filename) return NULL; } -#if USE_MMAP - reader->fd = -1; /* Explicit initialization for cleanup safety */ -#endif - - reader->filename = PyMem_Malloc(strlen(filename) + 1); - if (!reader->filename) { - PyMem_Free(reader); - PyErr_NoMemory(); - return NULL; - } - strcpy(reader->filename, filename); - #if USE_MMAP /* Open with mmap on Unix */ - reader->fd = open(filename, O_RDONLY); - if (reader->fd < 0) { - PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename); + FILE *fp = Py_fopen(path, "rb"); + if (!fp) { goto error; } + int fd = fileno(fp); struct stat st; - if (fstat(reader->fd, &st) < 0) { + if (fstat(fd, &st) < 0) { PyErr_SetFromErrno(PyExc_IOError); + Py_fclose(fp); goto error; } reader->mapped_size = st.st_size; @@ -400,14 +389,15 @@ binary_reader_open(const char *filename) */ #ifdef __linux__ reader->mapped_data = mmap(NULL, reader->mapped_size, PROT_READ, - MAP_PRIVATE | MAP_POPULATE, reader->fd, 0); + MAP_PRIVATE | MAP_POPULATE, fd, 0); #else reader->mapped_data = mmap(NULL, reader->mapped_size, PROT_READ, - MAP_PRIVATE, reader->fd, 0); + MAP_PRIVATE, fd, 0); #endif if (reader->mapped_data == MAP_FAILED) { reader->mapped_data = NULL; PyErr_SetFromErrno(PyExc_IOError); + Py_fclose(fp); goto error; } @@ -428,19 +418,20 @@ binary_reader_open(const char *filename) /* Add file descriptor-level hints for better kernel I/O scheduling */ #if defined(__linux__) && defined(POSIX_FADV_SEQUENTIAL) - (void)posix_fadvise(reader->fd, 0, 0, POSIX_FADV_SEQUENTIAL); + (void)posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); if (reader->mapped_size > (64 * 1024 * 1024)) { - (void)posix_fadvise(reader->fd, 0, 0, POSIX_FADV_WILLNEED); + (void)posix_fadvise(fd, 0, 0, POSIX_FADV_WILLNEED); } #endif + (void)Py_fclose(fp); + uint8_t *data = reader->mapped_data; size_t file_size = reader->mapped_size; #else /* Use stdio on Windows */ - reader->fp = fopen(filename, "rb"); + reader->fp = Py_fopen(path, "rb"); if (!reader->fp) { - PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename); goto error; } @@ -1263,8 +1254,6 @@ binary_reader_close(BinaryReader *reader) return; } - PyMem_Free(reader->filename); - #if USE_MMAP if (reader->mapped_data) { munmap(reader->mapped_data, reader->mapped_size); @@ -1274,13 +1263,9 @@ binary_reader_close(BinaryReader *reader) /* Clear sample_data which may point into the now-unmapped region */ reader->sample_data = NULL; reader->sample_data_size = 0; - if (reader->fd >= 0) { - close(reader->fd); - reader->fd = -1; /* Mark as closed */ - } #else if (reader->fp) { - fclose(reader->fp); + Py_fclose(reader->fp); reader->fp = NULL; } if (reader->file_data) { diff --git a/Modules/_remote_debugging/binary_io_writer.c b/Modules/_remote_debugging/binary_io_writer.c index 4cfed7300ac5ab..c31ed7d746466f 100644 --- a/Modules/_remote_debugging/binary_io_writer.c +++ b/Modules/_remote_debugging/binary_io_writer.c @@ -717,7 +717,7 @@ write_sample_with_encoding(BinaryWriter *writer, ThreadEntry *entry, } BinaryWriter * -binary_writer_create(const char *filename, uint64_t sample_interval_us, int compression_type, +binary_writer_create(PyObject *path, uint64_t sample_interval_us, int compression_type, uint64_t start_time_us) { BinaryWriter *writer = PyMem_Calloc(1, sizeof(BinaryWriter)); @@ -726,14 +726,6 @@ binary_writer_create(const char *filename, uint64_t sample_interval_us, int comp return NULL; } - writer->filename = PyMem_Malloc(strlen(filename) + 1); - if (!writer->filename) { - PyMem_Free(writer); - PyErr_NoMemory(); - return NULL; - } - strcpy(writer->filename, filename); - writer->start_time_us = start_time_us; writer->sample_interval_us = sample_interval_us; writer->compression_type = compression_type; @@ -799,9 +791,8 @@ binary_writer_create(const char *filename, uint64_t sample_interval_us, int comp } } - writer->fp = fopen(filename, "wb"); + writer->fp = Py_fopen(path, "wb"); if (!writer->fp) { - PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename); goto error; } @@ -1193,7 +1184,7 @@ binary_writer_finalize(BinaryWriter *writer) return -1; } - if (fclose(writer->fp) != 0) { + if (Py_fclose(writer->fp) != 0) { writer->fp = NULL; PyErr_SetFromErrno(PyExc_IOError); return -1; @@ -1211,10 +1202,9 @@ binary_writer_destroy(BinaryWriter *writer) } if (writer->fp) { - fclose(writer->fp); + Py_fclose(writer->fp); } - PyMem_Free(writer->filename); PyMem_Free(writer->write_buffer); #ifdef HAVE_ZSTD diff --git a/Modules/_remote_debugging/clinic/module.c.h b/Modules/_remote_debugging/clinic/module.c.h index 1133db808efaec..d56622fb82ab56 100644 --- a/Modules/_remote_debugging/clinic/module.c.h +++ b/Modules/_remote_debugging/clinic/module.c.h @@ -688,7 +688,7 @@ PyDoc_STRVAR(_remote_debugging_BinaryWriter___init____doc__, static int _remote_debugging_BinaryWriter___init___impl(BinaryWriterObject *self, - const char *filename, + PyObject *filename, unsigned long long sample_interval_us, unsigned long long start_time_us, int compression); @@ -728,7 +728,7 @@ _remote_debugging_BinaryWriter___init__(PyObject *self, PyObject *args, PyObject PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 3; - const char *filename; + PyObject *filename; unsigned long long sample_interval_us; unsigned long long start_time_us; int compression = 0; @@ -738,19 +738,7 @@ _remote_debugging_BinaryWriter___init__(PyObject *self, PyObject *args, PyObject if (!fastargs) { goto exit; } - if (!PyUnicode_Check(fastargs[0])) { - _PyArg_BadArgument("BinaryWriter", "argument 'filename'", "str", fastargs[0]); - goto exit; - } - Py_ssize_t filename_length; - filename = PyUnicode_AsUTF8AndSize(fastargs[0], &filename_length); - if (filename == NULL) { - goto exit; - } - if (strlen(filename) != (size_t)filename_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } + filename = fastargs[0]; if (!_PyLong_UnsignedLongLong_Converter(fastargs[1], &sample_interval_us)) { goto exit; } @@ -1009,7 +997,7 @@ PyDoc_STRVAR(_remote_debugging_BinaryReader___init____doc__, static int _remote_debugging_BinaryReader___init___impl(BinaryReaderObject *self, - const char *filename); + PyObject *filename); static int _remote_debugging_BinaryReader___init__(PyObject *self, PyObject *args, PyObject *kwargs) @@ -1045,26 +1033,14 @@ _remote_debugging_BinaryReader___init__(PyObject *self, PyObject *args, PyObject PyObject *argsbuf[1]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); - const char *filename; + PyObject *filename; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } - if (!PyUnicode_Check(fastargs[0])) { - _PyArg_BadArgument("BinaryReader", "argument 'filename'", "str", fastargs[0]); - goto exit; - } - Py_ssize_t filename_length; - filename = PyUnicode_AsUTF8AndSize(fastargs[0], &filename_length); - if (filename == NULL) { - goto exit; - } - if (strlen(filename) != (size_t)filename_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } + filename = fastargs[0]; return_value = _remote_debugging_BinaryReader___init___impl((BinaryReaderObject *)self, filename); exit: @@ -1564,4 +1540,4 @@ _remote_debugging_get_gc_stats(PyObject *module, PyObject *const *args, Py_ssize exit: return return_value; } -/*[clinic end generated code: output=36674f4cb8a653f3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5e2a29746a0c5d65 input=a9049054013a1b77]*/ diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index 172f8711a2a2a0..efdd2e1a2d7b7a 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -1476,7 +1476,7 @@ class _remote_debugging.BinaryWriter "BinaryWriterObject *" "&PyBinaryWriter_Typ /*[clinic input] @permit_long_docstring_body _remote_debugging.BinaryWriter.__init__ - filename: str + filename: object sample_interval_us: unsigned_long_long start_time_us: unsigned_long_long * @@ -1495,11 +1495,11 @@ Use as a context manager or call finalize() when done. static int _remote_debugging_BinaryWriter___init___impl(BinaryWriterObject *self, - const char *filename, + PyObject *filename, unsigned long long sample_interval_us, unsigned long long start_time_us, int compression) -/*[clinic end generated code: output=014c0306f1bacf4b input=3bdf01c1cc2f5a1d]*/ +/*[clinic end generated code: output=00446656ea2e5986 input=b92f0c77ba4cd274]*/ { if (self->writer) { binary_writer_destroy(self->writer); @@ -1742,7 +1742,7 @@ class _remote_debugging.BinaryReader "BinaryReaderObject *" "&PyBinaryReader_Typ /*[clinic input] _remote_debugging.BinaryReader.__init__ - filename: str + filename: object High-performance binary reader for profiling data. @@ -1754,8 +1754,8 @@ Use as a context manager or call close() when done. static int _remote_debugging_BinaryReader___init___impl(BinaryReaderObject *self, - const char *filename) -/*[clinic end generated code: output=9699226f7ae052bb input=4201f9cc500ef2f6]*/ + PyObject *filename) +/*[clinic end generated code: output=f04b33ee5c5e6dbf input=9d7cbe8b4f1a97c9]*/ { if (self->reader) { binary_reader_close(self->reader); From 915b6c1572285f2bd469be177fe523eca52951f1 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 9 May 2026 10:27:29 +0200 Subject: [PATCH 014/110] [3.15] gh-149083: Convert `_initial_missing` for pure py `reduce` to `sentinel` (GH-149536) (#149592) gh-149083: Convert `_initial_missing` for pure py `reduce` to `sentinel` (GH-149536) (cherry picked from commit bc8cf07d8dbb4341955dc85d9b2bf273ec5852c7) Co-authored-by: sobolevn --- Lib/functools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/functools.py b/Lib/functools.py index cd374631f16792..e03a77f204b544 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -232,7 +232,7 @@ def __ge__(self, other): ### reduce() sequence to a single item ################################################################################ -_initial_missing = object() +_initial_missing = sentinel('_initial_missing') def reduce(function, sequence, initial=_initial_missing): """ From dc8c32e59d1ae4163b04b015f48dd54a6d6a11a2 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 9 May 2026 15:33:05 +0200 Subject: [PATCH 015/110] [3.15] gh-149430: Fix edge-cases in `profiling.sampling` outputs (GH-149431) (#149602) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-149430: Fix edge-cases in `profiling.sampling` outputs (GH-149431) The line highlights on the heatmap are driven by the URL hash and the `:target` selector. When clicking a caller/callee link for the line that was already selected, the hash doesn't change, so the browser keeps the existing target state and doesn't restart the animation. Due to this the highlight only works the first time. With this fix, line navigation goes through JavaScript. If the target URL already points to the current location, the highlight is replayed by clearing the animation, forcing style recalculation, and restoring it. The `baseline_self` variable isn't initialized for structural elided roots. This variable is accessed later unconditionally and leads to a crash. The child process ends up being invoked with `--diff_flamegraph` instead of the correct argument. (cherry picked from commit 9587726a3ebbcdb780e3f15c9e016e3a28c646e3) Co-authored-by: László Kiss Kollár --- .../sampling/_heatmap_assets/heatmap.js | 22 ++++++++- Lib/profiling/sampling/cli.py | 4 +- Lib/profiling/sampling/stack_collector.py | 2 + .../test_sampling_profiler/test_children.py | 33 +++++++++++++ .../test_sampling_profiler/test_collectors.py | 46 +++++++++++++++++++ 5 files changed, 104 insertions(+), 3 deletions(-) diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap.js b/Lib/profiling/sampling/_heatmap_assets/heatmap.js index 2da1103b82a52a..1f698779f3a46e 100644 --- a/Lib/profiling/sampling/_heatmap_assets/heatmap.js +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap.js @@ -84,7 +84,7 @@ function showNavigationMenu(button, items, title) { item.appendChild(funcDiv); item.appendChild(createElement('div', 'callee-menu-file', linkData.file)); - item.addEventListener('click', () => window.location.href = linkData.link); + item.addEventListener('click', () => navigateToLine(linkData.link)); menu.appendChild(item); }); @@ -105,7 +105,7 @@ function handleNavigationClick(button, e) { const navData = button.getAttribute('data-nav'); if (navData) { - window.location.href = JSON.parse(navData).link; + navigateToLine(JSON.parse(navData).link); return; } @@ -117,11 +117,29 @@ function handleNavigationClick(button, e) { } } +function restartLineHighlight(target) { + target.style.animation = 'none'; + // Force style recalculation so restoring the animation restarts it. + void target.offsetWidth; + target.style.animation = ''; +} + +function navigateToLine(link) { + const url = new URL(link, window.location.href); + + if (url.href === window.location.href) { + scrollToTargetLine(); + } else { + window.location.href = link; + } +} + function scrollToTargetLine() { if (!window.location.hash) return; const target = document.querySelector(window.location.hash); if (target) { target.scrollIntoView({ behavior: 'smooth', block: 'start' }); + restartLineHighlight(target); } } diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index 0648713edc52af..a5d9573ae6b6dd 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -167,7 +167,9 @@ def _build_child_profiler_args(args): child_args.extend(["--mode", mode]) # Format options (skip pstats as it's the default) - if args.format != "pstats": + if args.format == "diff_flamegraph": + child_args.extend(["--diff-flamegraph", args.diff_baseline]) + elif args.format != "pstats": child_args.append(f"--{args.format}") return child_args diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 04622a8c1e89ef..60df026ed76a6c 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -698,6 +698,8 @@ def _add_elided_metadata(self, node, baseline_stats, scale, path): func_key = self._extract_func_key(node, self._baseline_collector._string_table) current_path = path + (func_key,) if func_key else path + baseline_self = 0 + baseline_total = 0 if func_key and current_path in baseline_stats: baseline_data = baseline_stats[current_path] baseline_self = baseline_data["self"] * scale diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_children.py b/Lib/test/test_profiling/test_sampling_profiler/test_children.py index bb49faa890f348..e64d917eedde56 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_children.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_children.py @@ -109,6 +109,39 @@ def _wait_for_process_ready(proc, timeout): return proc.poll() is None +@unittest.skipIf( + _build_child_profiler_args is None, + "profiling.sampling.cli unavailable", +) +class TestChildProfilerArgBuilder(unittest.TestCase): + """Tests for child profiler CLI argument construction.""" + + def test_build_child_profiler_args_diff_flamegraph(self): + """Test child args use the real --diff-flamegraph flag.""" + args = argparse.Namespace( + sample_interval_usec=1000, + duration=None, + all_threads=False, + realtime_stats=False, + native=False, + gc=True, + opcodes=False, + async_aware=False, + mode="wall", + format="diff_flamegraph", + diff_baseline="baseline.bin", + ) + + child_args = _build_child_profiler_args(args) + + self.assertIn("--diff-flamegraph", child_args) + self.assertNotIn("--diff_flamegraph", child_args) + + flag_index = child_args.index("--diff-flamegraph") + self.assertGreater(len(child_args), flag_index + 1) + self.assertEqual(child_args[flag_index + 1], "baseline.bin") + + @requires_remote_subprocess_debugging() class TestGetChildPids(unittest.TestCase): """Tests for the get_child_pids function.""" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index b42e7aa579f40c..390a1479fdd297 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -18,6 +18,7 @@ ) from profiling.sampling.jsonl_collector import JsonlCollector from profiling.sampling.gecko_collector import GeckoCollector + from profiling.sampling.heatmap_collector import _TemplateLoader from profiling.sampling.collector import extract_lineno, normalize_location from profiling.sampling.opcode_utils import get_opcode_info, format_opcode from profiling.sampling.constants import ( @@ -82,6 +83,18 @@ def test_mock_frame_info_with_empty_and_unicode_values(self): self.assertEqual(frame.location.lineno, 999999) self.assertEqual(frame.funcname, long_funcname) + def test_heatmap_navigation_restarts_line_highlight(self): + """Test heatmap navigation can replay target line highlights.""" + loader = _TemplateLoader() + + self.assertIn(".code-line:target", loader.file_css) + self.assertIn("function restartLineHighlight(target)", loader.file_js) + self.assertIn("target.style.animation = 'none'", loader.file_js) + self.assertIn("void target.offsetWidth", loader.file_js) + self.assertIn("url.href === window.location.href", loader.file_js) + self.assertIn("navigateToLine(JSON.parse(navData).link)", loader.file_js) + self.assertIn("navigateToLine(linkData.link)", loader.file_js) + def test_pstats_collector_with_extreme_intervals_and_empty_data(self): """Test PstatsCollector handles zero/large intervals, empty frames, None thread IDs, and duplicate frames.""" # Test with zero interval @@ -1403,6 +1416,39 @@ def test_diff_flamegraph_elided_stacks(self): self.assertGreater(child["baseline"], 0) self.assertAlmostEqual(child["diff"], -child["baseline"]) + def test_diff_flamegraph_elided_top_level_root(self): + """Elided top-level roots do not crash metadata generation.""" + baseline_frames_1 = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 10, "kept_leaf"), + MockFrameInfo("file.py", 20, "kept_root"), + ]) + ]) + ] + baseline_frames_2 = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 30, "old_leaf"), + MockFrameInfo("file.py", 40, "old_root"), + ]) + ]) + ] + + diff = make_diff_collector_with_mock_baseline([ + baseline_frames_1, + baseline_frames_2, + ]) + diff.collect(baseline_frames_1) + + data = diff._convert_to_flamegraph_format() + elided = data["stats"]["elided_flamegraph"] + elided_strings = elided.get("strings", []) + children = elided.get("children", []) + + self.assertEqual(len(children), 1) + self.assertIn("old_root", resolve_name(children[0], elided_strings)) + def test_diff_flamegraph_function_matched_despite_line_change(self): """Functions match by (filename, funcname), ignoring lineno.""" baseline_frames = [ From 212e996cba9224ae8ba7cb03d018774e818fef33 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 9 May 2026 17:08:34 +0200 Subject: [PATCH 016/110] [3.15] gh-149388: Make asyncio `PipeHandle.close` idempotent (GH-149518) (#149605) gh-149388: Make asyncio `PipeHandle.close` idempotent (GH-149518) (cherry picked from commit 7241f2739c4bbdf4519238689e5e4ea9268b411e) Co-authored-by: Max Schmitt --- Lib/asyncio/windows_utils.py | 3 ++- Lib/test/test_asyncio/test_windows_utils.py | 24 +++++++++++++++++++ ...-05-07-21-58-17.gh-issue-149388.DDBPeA.rst | 1 + 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-07-21-58-17.gh-issue-149388.DDBPeA.rst diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py index acd49441131b04..d6393f0b1ffee5 100644 --- a/Lib/asyncio/windows_utils.py +++ b/Lib/asyncio/windows_utils.py @@ -111,8 +111,9 @@ def fileno(self): def close(self, *, CloseHandle=_winapi.CloseHandle): if self._handle is not None: - CloseHandle(self._handle) + handle = self._handle self._handle = None + CloseHandle(handle) def __del__(self, _warn=warnings.warn): if self._handle is not None: diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py index f9ee2f4f68150a..50969761347595 100644 --- a/Lib/test/test_asyncio/test_windows_utils.py +++ b/Lib/test/test_asyncio/test_windows_utils.py @@ -77,6 +77,30 @@ def test_pipe_handle(self): else: raise RuntimeError('expected ERROR_INVALID_HANDLE') + def test_pipe_handle_close_after_external_close(self): + # gh-149388: PipeHandle.close() must clear ``_handle`` before calling + # CloseHandle so that if CloseHandle raises on a stale handle the + # PipeHandle is still marked closed and __del__ / subsequent close() + # calls are silent no-ops. + h1, h2 = windows_utils.pipe(overlapped=(False, False)) + try: + p = windows_utils.PipeHandle(h1) + # Simulate an external close of the underlying handle (e.g. + # a finalizer race or a concurrent close on the same object). + _winapi.CloseHandle(p.handle) + # First close() still propagates the OSError from CloseHandle, + # but must clear ``_handle`` first. + with self.assertRaises(OSError): + p.close() + self.assertIsNone(p.handle) + # Second close() is a no-op. + p.close() + # __del__ through GC is also a silent no-op — no unraisable. + del p + support.gc_collect() + finally: + _winapi.CloseHandle(h2) + class PopenTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2026-05-07-21-58-17.gh-issue-149388.DDBPeA.rst b/Misc/NEWS.d/next/Library/2026-05-07-21-58-17.gh-issue-149388.DDBPeA.rst new file mode 100644 index 00000000000000..4a1c6f3f5b4e57 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-07-21-58-17.gh-issue-149388.DDBPeA.rst @@ -0,0 +1 @@ +Make :class:`!asyncio.windows_utils.PipeHandle` closing idempotent. From f488c7d45f5186be2bb52054b4dd3bc59a45fde5 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 9 May 2026 23:39:35 +0200 Subject: [PATCH 017/110] [3.15] Fix minor typos in unicode.rst (GH-149587) (#149620) Fix minor typos in unicode.rst (GH-149587) (cherry picked from commit 4e97ff3351f381a61b238bd8e805e4e8dd3ea5cf) Co-authored-by: Manoj K M --- Doc/c-api/unicode.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 059a7ef399ae0f..401c99ebeb0fec 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -762,7 +762,7 @@ APIs: The string must not have been “used” yet. See :c:func:`PyUnicode_New` for details. - Return the number of written character, or return ``-1`` and raise an + Return the number of written characters, or return ``-1`` and raise an exception on error. .. versionadded:: 3.3 @@ -1174,7 +1174,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. .. versionchanged:: 3.10 This function is a part of the :ref:`limited API `. @@ -1196,7 +1196,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. UTF-32 Codecs From 6ba3ea43a032ceedaf917fd0f1952afb238756eb Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 9 May 2026 23:47:21 +0200 Subject: [PATCH 018/110] [3.15] gh-139871: Fix 3.15 bytearray.take_bytes example (GH-149520) (#149622) gh-139871: Fix 3.15 bytearray.take_bytes example (GH-149520) Currently: ```python buffer = bytearray(b'abc\ndef') n = buffer.find(b'\n') data = bytes(buffer[:n + 1]) del buffer[:n + 1] assert data == b'abc' Traceback (most recent call last): File "", line 1, in assert data == b'abc' ^^^^^^^^^^^^^^ AssertionError ``` Adding in the `\n` makes the two match: ```python buffer = bytearray(b'abc\ndef') n = buffer.find(b'\n') data = bytes(buffer[:n + 1]) del buffer[:n + 1] assert data == b'abc\n' assert buffer == bytearray(b'def') buffer = bytearray(b'abc\ndef') n = buffer.find(b'\n') data = buffer.take_bytes(n + 1) assert data == b'abc\n' assert buffer == bytearray(b'def') ``` (cherry picked from commit cc5cf14ae0a3665ba9d192cc4152c0a46a9dab2f) Co-authored-by: Cody Maloney --- Doc/whatsnew/3.15.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 0f7782ba1813d1..fb0755e8ffec5b 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -798,7 +798,7 @@ Other language changes n = buffer.find(b'\n') data = bytes(buffer[:n + 1]) del buffer[:n + 1] - assert data == b'abc' + assert data == b'abc\n' assert buffer == bytearray(b'def') - .. code:: python From 46a54ea5b0e4658db6bc40706ac3d4d86734e599 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 10 May 2026 16:11:57 +0200 Subject: [PATCH 019/110] [3.15] gh-148441: Avoid integer overflow in Expat's CharacterDataHandler (GH-148904) (#149639) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-148441: Avoid integer overflow in Expat's CharacterDataHandler (GH-148904) (cherry picked from commit bc1be4f6174086b4a46e3fe656552f5bb4e6e7b2) Co-authored-by: ByteFlow Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_pyexpat.py | 14 ++++++++++++++ .../2026-04-23-12-50-15.gh-issue-148441.zvpCkR.rst | 4 ++++ Modules/pyexpat.c | 2 +- 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-04-23-12-50-15.gh-issue-148441.zvpCkR.rst diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index aaa91aca36e3c4..9a1620029c6da9 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -712,6 +712,20 @@ def test_change_size_2(self): parser.Parse(xml2, True) self.assertEqual(self.n, 4) + @support.requires_resource('cpu') + @support.requires_resource('walltime') + @support.bigmemtest(size=2**31, memuse=4, dry_run=False) + def test_large_character_data_does_not_crash(self): + # See https://github.com/python/cpython/issues/148441 + parser = expat.ParserCreate() + parser.buffer_text = True + parser.buffer_size = 2**31 - 1 # INT_MAX + N = 2049 * (1 << 20) - 3 # Character data greater than INT_MAX + self.assertGreater(N, parser.buffer_size) + parser.CharacterDataHandler = lambda text: None + xml_data = b"" + b"A" * N + b"" + self.assertEqual(parser.Parse(xml_data, True), 1) + class ElementDeclHandlerTest(unittest.TestCase): def test_trigger_leak(self): # Unfixed, this test would leak the memory of the so-called diff --git a/Misc/NEWS.d/next/Library/2026-04-23-12-50-15.gh-issue-148441.zvpCkR.rst b/Misc/NEWS.d/next/Library/2026-04-23-12-50-15.gh-issue-148441.zvpCkR.rst new file mode 100644 index 00000000000000..762815270e4d40 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-23-12-50-15.gh-issue-148441.zvpCkR.rst @@ -0,0 +1,4 @@ +:mod:`xml.parsers.expat`: prevent a crash in +:meth:`~xml.parsers.expat.xmlparser.CharacterDataHandler` +when the character data size exceeds the parser's +:attr:`buffer size `. diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 0f0afe17513ef1..c01f7babe74527 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -393,7 +393,7 @@ my_CharacterDataHandler(void *userData, const XML_Char *data, int len) if (self->buffer == NULL) call_character_handler(self, data, len); else { - if ((self->buffer_used + len) > self->buffer_size) { + if (len > (self->buffer_size - self->buffer_used)) { if (flush_character_buffer(self) < 0) return; /* handler might have changed; drop the rest on the floor From 1162834c0ee310f7497b4f9d18d17a998e26831c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 10 May 2026 23:11:04 +0200 Subject: [PATCH 020/110] [3.15] gh-139489: Add is_valid_text to xml.__all__ (GH-149641) (#149652) gh-139489: Add is_valid_text to xml.__all__ (GH-149641) (cherry picked from commit b45319e13273ee17e84e6b8c459f03b141518289) Co-authored-by: Jelle Zijlstra --- Lib/xml/__init__.py | 2 +- .../next/Library/2026-05-10-07-21-51.gh-issue-139489.rS7LTA.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-10-07-21-51.gh-issue-139489.rS7LTA.rst diff --git a/Lib/xml/__init__.py b/Lib/xml/__init__.py index 002d6d3e0e8267..ecfce1c6ae52cf 100644 --- a/Lib/xml/__init__.py +++ b/Lib/xml/__init__.py @@ -18,4 +18,4 @@ from .utils import * -__all__ = ["dom", "parsers", "sax", "etree", "is_valid_name"] +__all__ = ["dom", "parsers", "sax", "etree", "is_valid_name", "is_valid_text"] diff --git a/Misc/NEWS.d/next/Library/2026-05-10-07-21-51.gh-issue-139489.rS7LTA.rst b/Misc/NEWS.d/next/Library/2026-05-10-07-21-51.gh-issue-139489.rS7LTA.rst new file mode 100644 index 00000000000000..40fe7e9fd6a008 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-10-07-21-51.gh-issue-139489.rS7LTA.rst @@ -0,0 +1 @@ +Add :func:`xml.is_valid_text` to ``xml.__all__``. From 73c80cb859589ef9e5871c30b17c951a44a26e33 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 11 May 2026 01:32:35 +0200 Subject: [PATCH 021/110] [3.15] gh-149083: use sentinel to fix _functools.reduce() signature (GH-149591) (#149653) gh-149083: use sentinel to fix _functools.reduce() signature (GH-149591) (cherry picked from commit c6fd7de64ac7591a9708c14a34737eb9baf050bc) Co-authored-by: Sergey B Kirpichev --- Lib/inspect.py | 3 ++- Lib/test/test_inspect/test_inspect.py | 3 +-- Modules/_functoolsmodule.c | 4 ++-- Modules/clinic/_functoolsmodule.c.h | 5 +++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py index a96b3dc954ef0c..dc5a6e3be883bb 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2200,7 +2200,8 @@ def wrap_value(s): except NameError: raise ValueError - if isinstance(value, (str, int, float, bytes, bool, type(None))): + if isinstance(value, (str, int, float, bytes, bool, type(None), + sentinel)): return ast.Constant(value) raise ValueError diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 9028d42c617fb4..7351f97fd9a4b5 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -6255,8 +6255,7 @@ def test_faulthandler_module_has_signatures(self): self._test_module_has_signatures(faulthandler, unsupported_signature=unsupported_signature) def test_functools_module_has_signatures(self): - unsupported_signature = {"reduce"} - self._test_module_has_signatures(functools, unsupported_signature=unsupported_signature) + self._test_module_has_signatures(functools) def test_gc_module_has_signatures(self): import gc diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 19bdf3d47c2fad..c702eecc700ac8 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -1066,7 +1066,7 @@ _functools.reduce function as func: object iterable as seq: object / - initial as result: object = NULL + initial as result: object(c_default="NULL") = functools._initial_missing Apply a function of two arguments cumulatively to the items of an iterable, from left to right. @@ -1081,7 +1081,7 @@ calculates ((((1 + 2) + 3) + 4) + 5). static PyObject * _functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, PyObject *result) -/*[clinic end generated code: output=30d898fe1267c79d input=4ccfb74548ce5170]*/ +/*[clinic end generated code: output=30d898fe1267c79d input=5c9088c98ffe2793]*/ { PyObject *args, *it; diff --git a/Modules/clinic/_functoolsmodule.c.h b/Modules/clinic/_functoolsmodule.c.h index 23f66631085031..87cdef2ad3cff3 100644 --- a/Modules/clinic/_functoolsmodule.c.h +++ b/Modules/clinic/_functoolsmodule.c.h @@ -71,7 +71,8 @@ _functools_cmp_to_key(PyObject *module, PyObject *const *args, Py_ssize_t nargs, } PyDoc_STRVAR(_functools_reduce__doc__, -"reduce($module, function, iterable, /, initial=)\n" +"reduce($module, function, iterable, /,\n" +" initial=functools._initial_missing)\n" "--\n" "\n" "Apply a function of two arguments cumulatively to the items of an iterable, from left to right.\n" @@ -192,4 +193,4 @@ _functools__lru_cache_wrapper_cache_clear(PyObject *self, PyObject *Py_UNUSED(ig return return_value; } -/*[clinic end generated code: output=7f2abc718fcc35d5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ac9e26d0a5a23d40 input=a9049054013a1b77]*/ From 5cf47a248c35c375d610b87b2f72fd1ed454b558 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 11 May 2026 11:57:32 +0200 Subject: [PATCH 022/110] [3.15] gh-149486: tarfile.data_filter: validate written link target (GH-149487) (GH-149553) gh-149486: tarfile.data_filter: validate written link target (GH-149487) The data filter rewrote linknames with normpath() but ran the containment check against the un-normalised value, and computed a symlink's directory before stripping trailing slashes. Both let a crafted archive create links pointing outside the destination. Also reject link members that resolve to the destination directory itself, which could otherwise replace it with a symlink and redirect all subsequent members. (cherry picked from commit 578411982c16f753f4893532510099ef665117da) Co-authored-by: Gregory P. Smith --- Lib/tarfile.py | 16 ++-- Lib/test/test_tarfile.py | 87 ++++++++++++++++++- ...-05-03-21-00-00.gh-issue-149486.tarflt.rst | 5 ++ 3 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index d0e7dec5575047..1394a26f2096ff 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -830,16 +830,22 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) + # A link member that resolves to the destination directory itself + # would replace it with a (sym)link, redirecting the destination + # for all subsequent members. + if target_path == dest_path: + raise OutsideDestinationError(member, target_path) normalized = os.path.normpath(member.linkname) if normalized != member.linkname: new_attrs['linkname'] = normalized if member.issym(): - target_path = os.path.join(dest_path, - os.path.dirname(name), - member.linkname) + # The symlink is created at `name` with trailing separators + # stripped, so its target is relative to the directory + # containing that path. + link_dir = os.path.dirname(name.rstrip('/' + os.sep)) + target_path = os.path.join(dest_path, link_dir, normalized) else: - target_path = os.path.join(dest_path, - member.linkname) + target_path = os.path.join(dest_path, normalized) target_path = os.path.realpath(target_path, strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index e270cbb22e2d1a..192c948edc6056 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3911,10 +3911,19 @@ def test_parent_symlink(self): + "which is outside the destination") with self.check_context(arc.open(), 'data'): - self.expect_exception( - tarfile.LinkOutsideDestinationError, - """'parent' would link to ['"].*outerdir['"], """ - + "which is outside the destination") + if self.dotdot_resolves_early: + # 'current/../..' normalises to '..', which is rejected. + self.expect_exception( + tarfile.LinkOutsideDestinationError, + """'parent' would link to ['"].*outerdir['"], """ + + "which is outside the destination") + else: + # 'current/..' normalises to '.'; the rewritten link is + # created and 'parent/evil' lands harmlessly inside the + # destination. + self.expect_file('current', symlink_to='.') + self.expect_file('parent', symlink_to='.') + self.expect_file('evil') else: # No symlink support. The symlinks are ignored. @@ -4174,6 +4183,76 @@ def test_sly_relative2(self): + """['"].*moo['"], which is outside the """ + "destination") + @symlink_test + @os_helper.skip_unless_symlink + def test_normpath_realpath_mismatch(self): + # The link-target check must validate the value that will actually + # be written to disk (the normalised linkname), not the original. + # Here 'a' is a symlink to a deep nonexistent path, so realpath() + # of 'a/../../...' stays inside the destination while normpath() + # collapses 'a/..' lexically and escapes. + depth = len(self.destdir.parts) + 5 + deep = '/'.join(f'p{i}' for i in range(depth)) + sneaky = 'a/' + '../' * depth + 'flag' + for kind in 'symlink_to', 'hardlink_to': + with self.subTest(kind): + with ArchiveMaker() as arc: + arc.add('a', symlink_to=deep) + arc.add('escape', **{kind: sneaky}) + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.LinkOutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_symlink_trailing_slash(self): + # A trailing slash on a symlink member's name must not cause the + # link target to be resolved relative to the wrong directory. + with ArchiveMaker() as arc: + t = tarfile.TarInfo('x/') + t.type = tarfile.SYMTYPE + t.linkname = '..' + arc.tar_w.addfile(t) + arc.add('x/escaped', content='hi') + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.LinkOutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_link_at_destination(self): + # A link member whose name resolves to the destination directory + # itself must be rejected: otherwise the destination is replaced + # by a symlink and later members can be redirected through it. + for name in '', '.', './': + with ArchiveMaker() as arc: + t = tarfile.TarInfo(name) + t.type = tarfile.SYMTYPE + t.linkname = '.' + arc.tar_w.addfile(t) + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.OutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_empty_name_symlink_chain(self): + # Regression test for a chain of empty-named symlinks that + # incrementally redirects the destination outwards. + with ArchiveMaker() as arc: + for name, target in [('', ''), ('a/', '..'), + ('', 'dummy'), ('', 'a'), + ('b/', '..'), + ('', 'dummy'), ('', 'a/b')]: + t = tarfile.TarInfo(name) + t.type = tarfile.SYMTYPE + t.linkname = target + arc.tar_w.addfile(t) + arc.add('escaped', content='hi') + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.FilterError) + @symlink_test def test_deep_symlink(self): # Test that symlinks and hardlinks inside a directory diff --git a/Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst b/Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst new file mode 100644 index 00000000000000..7c69edb683cf80 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst @@ -0,0 +1,5 @@ +:func:`tarfile.data_filter` now validates link targets using the same +normalised value that is written to disk, strips trailing separators from +the member name when resolving a symlink's directory, and rejects link +members that would replace the destination directory itself. This closes +several path-traversal bypasses of the ``data`` extraction filter. From 4277df2421bc5b39ce5a133c032f8ef5428d3718 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 11 May 2026 13:44:11 +0200 Subject: [PATCH 023/110] [3.15] gh-146061: Clarify indent=None in json docs (GH-146095) (GH-149667) (cherry picked from commit 833dae7c1fdc556200cbfc3e76bad4d54628042c) Co-authored-by: Jonathan Dung --- Doc/library/json.rst | 2 +- Lib/json/__init__.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index b354e7ba534835..383ccad9df041b 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -211,7 +211,7 @@ Basic Usage a string (such as ``"\t"``) is used to indent each level. If zero, negative, or ``""`` (the empty string), only newlines are inserted. - If ``None`` (the default), the most compact representation is used. + If ``None`` (the default), no newlines are inserted. :type indent: int | str | None :param separators: diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index 251025efac14b8..94c177cafa0294 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -142,8 +142,8 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent - level of 0 will only insert newlines. ``None`` is the most compact - representation. + level of 0 will only insert newlines. ``None`` is the default and gives + a representation with no newlines inserted. If specified, ``separators`` should be an ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is @@ -206,8 +206,8 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent - level of 0 will only insert newlines. ``None`` is the most compact - representation. + level of 0 will only insert newlines. ``None`` is the default and gives + a representation with no newlines inserted. If specified, ``separators`` should be an ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is From 592a356fb5250c83c1dcd4df47d2ea9f7c58cca3 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 11 May 2026 14:02:28 +0200 Subject: [PATCH 024/110] [3.15] gh-149663: fix typo in `unittest` docs (GH-149670) (#149672) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-149663: fix typo in `unittest` docs (GH-149670) `hastattr` -> `hasattr` (cherry picked from commit 4956d2be9d5e555f2cf64faed9ef39e6a797c360) Co-authored-by: Árni Már Jónsson --- Doc/library/unittest.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index c54f3e2792c388..ff619f97923325 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1262,10 +1262,10 @@ Test cases | :meth:`assertNotEndsWith(a, b) | ``not a.endswith(b)`` | 3.14 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertHasAttr(a, b) | ``hastattr(a, b)`` | 3.14 | + | :meth:`assertHasAttr(a, b) | ``hasattr(a, b)`` | 3.14 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertNotHasAttr(a, b) | ``not hastattr(a, b)`` | 3.14 | + | :meth:`assertNotHasAttr(a, b) | ``not hasattr(a, b)`` | 3.14 | | ` | | | +---------------------------------------+--------------------------------+--------------+ From 8fdeb2dd3a2392669e7bbb75e3bd7142d15e00fb Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 11 May 2026 15:36:35 +0200 Subject: [PATCH 025/110] [3.15] gh-144957: Fix lazy imports + module __getattr__ (GH-149624) (#149678) gh-144957: Fix lazy imports + module __getattr__ (GH-149624) (cherry picked from commit 56171da3417bc14fded2f42033d72f63e1bf7cd9) Co-authored-by: Jelle Zijlstra --- Lib/test/test_lazy_import/__init__.py | 31 +++++++++++++++++++ .../data/module_with_getattr.py | 4 +++ .../test_lazy_import/data/pkg/__init__.py | 5 +++ ...-05-09-15-22-32.gh-issue-144957.u1F2aQ.rst | 2 ++ Objects/moduleobject.c | 19 ++++++++++++ 5 files changed, 61 insertions(+) create mode 100644 Lib/test/test_lazy_import/data/module_with_getattr.py create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-09-15-22-32.gh-issue-144957.u1F2aQ.rst diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index 1d1d2e00bd733f..ea534a8ee5b981 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -88,6 +88,26 @@ def test_basic_used(self): import test.test_lazy_import.data.basic_used self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + @support.requires_subprocess() + def test_from_import_with_module_getattr(self): + """Lazy from import should respect module-level __getattr__.""" + code = textwrap.dedent(""" + lazy from test.test_lazy_import.data.module_with_getattr import dynamic_attr + assert dynamic_attr == "from_getattr" + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_imported_module_getattr(self): + """Lazy from import should not shadow an imported module's __getattr__.""" + code = textwrap.dedent(""" + import test.test_lazy_import.data.module_with_getattr as mod + lazy from test.test_lazy_import.data.module_with_getattr import dynamic_attr + assert dynamic_attr == "from_getattr" + assert mod.dynamic_attr == "from_getattr" + """) + assert_python_ok("-c", code) + class GlobalLazyImportModeTests(unittest.TestCase): """Tests for sys.set_lazy_imports() global mode control.""" @@ -385,6 +405,17 @@ def test_lazy_import_pkg_cross_import(self): self.assertEqual(type(g["x"]), int) self.assertEqual(type(g["b"]), types.LazyImportType) + @support.requires_subprocess() + def test_package_from_import_with_module_getattr(self): + """Lazy from import should respect a package's __getattr__.""" + code = textwrap.dedent(""" + import test.test_lazy_import.data.pkg as pkg + lazy from test.test_lazy_import.data.pkg import dynamic_attr + assert dynamic_attr == "from_getattr" + assert pkg.dynamic_attr == "from_getattr" + """) + assert_python_ok("-c", code) + class DunderLazyImportTests(unittest.TestCase): """Tests for __lazy_import__ builtin function.""" diff --git a/Lib/test/test_lazy_import/data/module_with_getattr.py b/Lib/test/test_lazy_import/data/module_with_getattr.py new file mode 100644 index 00000000000000..2ac01a90d76e62 --- /dev/null +++ b/Lib/test/test_lazy_import/data/module_with_getattr.py @@ -0,0 +1,4 @@ +def __getattr__(name): + if name == "dynamic_attr": + return "from_getattr" + raise AttributeError(name) diff --git a/Lib/test/test_lazy_import/data/pkg/__init__.py b/Lib/test/test_lazy_import/data/pkg/__init__.py index 2d76abaa89f893..e526aab94131b8 100644 --- a/Lib/test/test_lazy_import/data/pkg/__init__.py +++ b/Lib/test/test_lazy_import/data/pkg/__init__.py @@ -1 +1,6 @@ x = 42 + +def __getattr__(name): + if name == "dynamic_attr": + return "from_getattr" + raise AttributeError(name) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-09-15-22-32.gh-issue-144957.u1F2aQ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-09-15-22-32.gh-issue-144957.u1F2aQ.rst new file mode 100644 index 00000000000000..3063f1a3c0e6d3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-09-15-22-32.gh-issue-144957.u1F2aQ.rst @@ -0,0 +1,2 @@ +Fix lazy ``from`` imports of module attributes provided by module-level +``__getattr__``. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index b7d2e5ffde4fe7..f7b83c1d111cde 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1307,6 +1307,25 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) attr = _PyObject_GenericGetAttrWithDict((PyObject *)m, name, NULL, suppress); if (attr) { if (PyLazyImport_CheckExact(attr)) { + // gh-144957: Module __getattr__ should get a chance to provide + // the attribute before resolving a lazy import placeholder. + if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { + Py_DECREF(attr); + return NULL; + } + if (getattr) { + PyObject *result = PyObject_CallOneArg(getattr, name); + Py_DECREF(getattr); + if (result != NULL) { + Py_DECREF(attr); + return result; + } + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + Py_DECREF(attr); + return NULL; + } + PyErr_Clear(); + } PyObject *new_value = _PyImport_LoadLazyImportTstate( PyThreadState_GET(), attr); if (new_value == NULL) { From b4f2e882032d71b2c60d74befdc2615d5596f2fe Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 11 May 2026 18:07:50 +0200 Subject: [PATCH 026/110] [3.15] gh-112821: Fix rlcompleter failures on objects with descriptors (GH-149577) (#149656) gh-112821: Fix rlcompleter failures on objects with descriptors (GH-149577) * gh-112821: Fix rlcompleter failures on objects with descriptors * Confirm no accesses (cherry picked from commit f23a1837d7156c4c478528321a423eae2b31e4bf) Co-authored-by: Michael Droettboom --- Lib/rlcompleter.py | 16 +++--- Lib/test/test_rlcompleter.py | 52 +++++++++++++++++++ ...-05-08-15-08-35.gh-issue-112821.t9T1YD.rst | 4 ++ 3 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-08-15-08-35.gh-issue-112821.t9T1YD.rst diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index e8cef29d00467f..6c6d9bb6b34244 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -179,14 +179,14 @@ def attr_matches(self, text): if (word[:n] == attr and not (noprefix and word[:n+1] == noprefix)): match = "%s.%s" % (expr, word) - if isinstance(getattr(type(thisobject), word, None), - property): - # bpo-44752: thisobject.word is a method decorated by - # `@property`. What follows applies a postfix if - # thisobject.word is callable, but know we know that - # this is not callable (because it is a property). - # Also, getattr(thisobject, word) will evaluate the - # property method, which is not desirable. + + class_attr = getattr(type(thisobject), word, None) + if isinstance( + class_attr, + (property, types.GetSetDescriptorType, types.MemberDescriptorType) + ) or (hasattr(class_attr, '__get__') and not callable(class_attr)): + # Avoid evaluating descriptors, which could run + # arbitrary code or raise exceptions. matches.append(match) continue diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index a8914953ce9eb4..e6d727d417b298 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import patch import builtins +import types import rlcompleter from test.support import MISSING_C_DOCSTRINGS @@ -135,6 +136,57 @@ def bar(self): self.assertEqual(completer.complete('f.b', 0), 'f.bar') self.assertFalse(f.property_called) + def test_released_memoryview_completion_works(self): + mv = memoryview(b"abc") + mv.release() + + self.assertIsInstance(type(mv).shape, types.GetSetDescriptorType) + self.assertIsInstance(type(mv).strides, types.GetSetDescriptorType) + + completer = rlcompleter.Completer(dict(mv=mv)) + matches = completer.attr_matches('mv.') + + # These are getset descriptors on memoryview and should be completed + # without evaluating the released-memoryview getters. + self.assertIn('mv.shape', matches) + self.assertIn('mv.strides', matches) + + def test_member_descriptor_not_evaluated(self): + class Foo: + __slots__ = ("boom",) + boom_accesses = 0 + + def __getattribute__(self, name): + if name == "boom": + type(self).boom_accesses += 1 + raise RuntimeError("boom access should be skipped") + return super().__getattribute__(name) + + self.assertIsInstance(Foo.boom, types.MemberDescriptorType) + + completer = rlcompleter.Completer(dict(f=Foo())) + matches = completer.attr_matches('f.') + self.assertIn('f.boom', matches) + self.assertEqual(Foo.boom_accesses, 0) + + def test_raising_descriptor_completion_works(self): + class ExplodingDescriptor: + def __init__(self): + self.instance_get_calls = 0 + + def __get__(self, obj, owner): + if obj is None: + return self + self.instance_get_calls += 1 + raise RuntimeError("descriptor getter exploded") + + class Foo: + boom = ExplodingDescriptor() + + completer = rlcompleter.Completer(dict(f=Foo())) + matches = completer.attr_matches('f.') + self.assertIn('f.boom', matches) + self.assertEqual(Foo.boom.instance_get_calls, 0) def test_uncreated_attr(self): # Attributes like properties and slots should be completed even when diff --git a/Misc/NEWS.d/next/Library/2026-05-08-15-08-35.gh-issue-112821.t9T1YD.rst b/Misc/NEWS.d/next/Library/2026-05-08-15-08-35.gh-issue-112821.t9T1YD.rst new file mode 100644 index 00000000000000..cfbcde81493e22 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-08-15-08-35.gh-issue-112821.t9T1YD.rst @@ -0,0 +1,4 @@ +In the REPL, autocompletion might run arbitrary code in the getter of a +descriptor. If that getter raised an exception, autocompletion would fail to +present any options for the entire object. Autocompletion now works as +expected for these objects. From 8297d50a63906757a92f86fc7f1eb5064c816b93 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 11 May 2026 18:23:39 +0000 Subject: [PATCH 027/110] [3.15] gh-145235: Make dict watcher API thread-safe for free-threaded builds (gh-145233) (#149690) In free-threaded builds, concurrent calls to PyDict_AddWatcher, PyDict_ClearWatcher, PyDict_Watch, and PyDict_Unwatch can race on the shared callback array and the per-dict watcher tags. This change adds a mutex to serialize watcher registration and removal, atomic operations for tag updates, and atomic acquire/release synchronization for callback dispatch in _PyDict_SendEvent. (cherry picked from commit 8a4895985f42282504d83b9bd0c77b129f95a5d5) Co-authored-by: Alper --- Include/internal/pycore_dict_state.h | 2 + .../internal/pycore_pyatomic_ft_wrappers.h | 2 + .../test_free_threading/test_dict_watcher.py | 89 +++++++++++++++++++ ...-02-25-13-37-10.gh-issue-145235.-1ySNR.rst | 3 + Modules/_testcapi/watchers.c | 26 +++--- Objects/dictobject.c | 42 ++++++--- Python/optimizer_analysis.c | 20 +++-- Python/pystate.c | 1 + Tools/c-analyzer/cpython/ignored.tsv | 1 + 9 files changed, 159 insertions(+), 27 deletions(-) create mode 100644 Lib/test/test_free_threading/test_dict_watcher.py create mode 100644 Misc/NEWS.d/next/C_API/2026-02-25-13-37-10.gh-issue-145235.-1ySNR.rst diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index 11932b8d1e1ab6..bb6fe262597559 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -13,6 +13,8 @@ extern "C" { struct _Py_dict_state { uint32_t next_keys_version; + PyMutex watcher_mutex; // Protects the watchers array (free-threaded builds) + _PyOnceFlag watcher_setup_once; // One-time optimizer watcher setup PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index fafdd728a8229a..d8ec306a0dae3f 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -138,6 +138,7 @@ extern "C" { #define FT_ATOMIC_ADD_SSIZE(value, new_value) \ (void)_Py_atomic_add_ssize(&value, new_value) #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) +#define FT_MUTEX_LOCK_FLAGS(lock, flags) PyMutex_LockFlags(lock, flags) #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) #else @@ -201,6 +202,7 @@ extern "C" { #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_ADD_SSIZE(value, new_value) (void)(value += new_value) #define FT_MUTEX_LOCK(lock) do {} while (0) +#define FT_MUTEX_LOCK_FLAGS(lock, flags) do {} while (0) #define FT_MUTEX_UNLOCK(lock) do {} while (0) #endif diff --git a/Lib/test/test_free_threading/test_dict_watcher.py b/Lib/test/test_free_threading/test_dict_watcher.py new file mode 100644 index 00000000000000..6a6843f9344f64 --- /dev/null +++ b/Lib/test/test_free_threading/test_dict_watcher.py @@ -0,0 +1,89 @@ +import unittest + +from test.support import import_helper, threading_helper + +_testcapi = import_helper.import_module("_testcapi") + +ITERS = 100 +NTHREADS = 20 + + +@threading_helper.requires_working_threading() +class TestDictWatcherThreadSafety(unittest.TestCase): + # Watcher kinds from _testcapi + EVENTS = 0 # appends dict events as strings to global event list + + def test_concurrent_add_clear_watchers(self): + """Race AddWatcher and ClearWatcher from multiple threads. + + Uses more threads than available watcher slots (5 user slots out + of DICT_MAX_WATCHERS=8). + """ + results = [] + + def worker(): + for _ in range(ITERS): + try: + wid = _testcapi.add_dict_watcher(self.EVENTS) + except RuntimeError: + continue # All slots taken + self.assertGreaterEqual(wid, 0) + results.append(wid) + _testcapi.clear_dict_watcher(wid) + + threading_helper.run_concurrently(worker, NTHREADS) + + # Verify at least some watchers were successfully added + self.assertGreater(len(results), 0) + + def test_concurrent_watch_unwatch(self): + """Race Watch and Unwatch on the same dict from multiple threads.""" + wid = _testcapi.add_dict_watcher(self.EVENTS) + dicts = [{} for _ in range(10)] + + def worker(): + for _ in range(ITERS): + for d in dicts: + _testcapi.watch_dict(wid, d) + for d in dicts: + _testcapi.unwatch_dict(wid, d) + + try: + threading_helper.run_concurrently(worker, NTHREADS) + + # Verify watching still works after concurrent watch/unwatch + _testcapi.watch_dict(wid, dicts[0]) + dicts[0]["key"] = "value" + events = _testcapi.get_dict_watcher_events() + self.assertIn("new:key:value", events) + finally: + _testcapi.clear_dict_watcher(wid) + + def test_concurrent_modify_watched_dict(self): + """Race dict mutations (triggering callbacks) with watch/unwatch.""" + wid = _testcapi.add_dict_watcher(self.EVENTS) + d = {} + _testcapi.watch_dict(wid, d) + + def mutator(): + for i in range(ITERS): + d[f"key_{i}"] = i + d.pop(f"key_{i}", None) + + def toggler(): + for i in range(ITERS): + _testcapi.watch_dict(wid, d) + d[f"toggler_{i}"] = i + _testcapi.unwatch_dict(wid, d) + + workers = [mutator, toggler] * (NTHREADS // 2) + try: + threading_helper.run_concurrently(workers) + events = _testcapi.get_dict_watcher_events() + self.assertGreater(len(events), 0) + finally: + _testcapi.clear_dict_watcher(wid) + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/C_API/2026-02-25-13-37-10.gh-issue-145235.-1ySNR.rst b/Misc/NEWS.d/next/C_API/2026-02-25-13-37-10.gh-issue-145235.-1ySNR.rst new file mode 100644 index 00000000000000..98a8c268735726 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-02-25-13-37-10.gh-issue-145235.-1ySNR.rst @@ -0,0 +1,3 @@ +Made :c:func:`PyDict_AddWatcher`, :c:func:`PyDict_ClearWatcher`, +:c:func:`PyDict_Watch`, and :c:func:`PyDict_Unwatch` thread-safe on the +:term:`free threaded ` build. diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index e0abf6b1845d8e..71cdc54009017a 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -9,6 +9,7 @@ #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_interp_structs.h" // CODE_MAX_WATCHERS #include "pycore_context.h" // CONTEXT_MAX_WATCHERS +#include "pycore_lock.h" // _PyOnceFlag /*[clinic input] module _testcapi @@ -18,6 +19,14 @@ module _testcapi // Test dict watching static PyObject *g_dict_watch_events = NULL; static int g_dict_watchers_installed = 0; +static _PyOnceFlag g_dict_watch_once = {0}; + +static int +_init_dict_watch_events(void *arg) +{ + g_dict_watch_events = PyList_New(0); + return g_dict_watch_events ? 0 : -1; +} static int dict_watch_callback(PyDict_WatchEvent event, @@ -106,13 +115,10 @@ add_dict_watcher(PyObject *self, PyObject *kind) if (watcher_id < 0) { return NULL; } - if (!g_dict_watchers_installed) { - assert(!g_dict_watch_events); - if (!(g_dict_watch_events = PyList_New(0))) { - return NULL; - } + if (_PyOnceFlag_CallOnce(&g_dict_watch_once, _init_dict_watch_events, NULL) < 0) { + return NULL; } - g_dict_watchers_installed++; + _Py_atomic_add_int(&g_dict_watchers_installed, 1); return PyLong_FromLong(watcher_id); } @@ -122,10 +128,8 @@ clear_dict_watcher(PyObject *self, PyObject *watcher_id) if (PyDict_ClearWatcher(PyLong_AsLong(watcher_id))) { return NULL; } - g_dict_watchers_installed--; - if (!g_dict_watchers_installed) { - assert(g_dict_watch_events); - Py_CLEAR(g_dict_watch_events); + if (_Py_atomic_add_int(&g_dict_watchers_installed, -1) == 1) { + PyList_Clear(g_dict_watch_events); } Py_RETURN_NONE; } @@ -164,7 +168,7 @@ _testcapi_unwatch_dict_impl(PyObject *module, int watcher_id, PyObject *dict) static PyObject * get_dict_watcher_events(PyObject *self, PyObject *Py_UNUSED(args)) { - if (!g_dict_watch_events) { + if (_Py_atomic_load_int(&g_dict_watchers_installed) <= 0) { PyErr_SetString(PyExc_RuntimeError, "no watchers active"); return NULL; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 42bc63acd9049c..09135e031e6fc7 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -8015,13 +8015,19 @@ validate_watcher_id(PyInterpreterState *interp, int watcher_id) PyErr_Format(PyExc_ValueError, "Invalid dict watcher ID %d", watcher_id); return -1; } - if (!interp->dict_state.watchers[watcher_id]) { + PyDict_WatchCallback cb = FT_ATOMIC_LOAD_PTR_RELAXED( + interp->dict_state.watchers[watcher_id]); + if (cb == NULL) { PyErr_Format(PyExc_ValueError, "No dict watcher set for ID %d", watcher_id); return -1; } return 0; } +// In free-threaded builds, Add/Clear serialize on watcher_mutex and publish +// callbacks with release stores. SendEvent reads them lock-free using +// acquire loads. + int PyDict_Watch(int watcher_id, PyObject* dict) { @@ -8033,7 +8039,8 @@ PyDict_Watch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - FT_ATOMIC_OR_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, (1LL << watcher_id)); + FT_ATOMIC_OR_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, + 1ULL << watcher_id); return 0; } @@ -8048,36 +8055,48 @@ PyDict_Unwatch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - FT_ATOMIC_AND_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, ~(1LL << watcher_id)); + FT_ATOMIC_AND_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, + ~(1ULL << watcher_id)); return 0; } int PyDict_AddWatcher(PyDict_WatchCallback callback) { + int watcher_id = -1; PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_MUTEX_LOCK_FLAGS(&interp->dict_state.watcher_mutex, + _Py_LOCK_DONT_DETACH); /* Some watchers are reserved for CPython, start at the first available one */ for (int i = FIRST_AVAILABLE_WATCHER; i < DICT_MAX_WATCHERS; i++) { if (!interp->dict_state.watchers[i]) { - interp->dict_state.watchers[i] = callback; - return i; + FT_ATOMIC_STORE_PTR_RELEASE(interp->dict_state.watchers[i], callback); + watcher_id = i; + goto done; } } - PyErr_SetString(PyExc_RuntimeError, "no more dict watcher IDs available"); - return -1; +done: + FT_MUTEX_UNLOCK(&interp->dict_state.watcher_mutex); + return watcher_id; } int PyDict_ClearWatcher(int watcher_id) { + int res = 0; PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_MUTEX_LOCK_FLAGS(&interp->dict_state.watcher_mutex, + _Py_LOCK_DONT_DETACH); if (validate_watcher_id(interp, watcher_id)) { - return -1; + res = -1; + goto done; } - interp->dict_state.watchers[watcher_id] = NULL; - return 0; + FT_ATOMIC_STORE_PTR_RELEASE(interp->dict_state.watchers[watcher_id], NULL); +done: + FT_MUTEX_UNLOCK(&interp->dict_state.watcher_mutex); + return res; } static const char * @@ -8102,7 +8121,8 @@ _PyDict_SendEvent(int watcher_bits, PyInterpreterState *interp = _PyInterpreterState_GET(); for (int i = 0; i < DICT_MAX_WATCHERS; i++) { if (watcher_bits & 1) { - PyDict_WatchCallback cb = interp->dict_state.watchers[i]; + PyDict_WatchCallback cb = FT_ATOMIC_LOAD_PTR_ACQUIRE( + interp->dict_state.watchers[i]); if (cb && (cb(event, (PyObject*)mp, key, value) < 0)) { // We don't want to resurrect the dict by potentially having an // unraisablehook keep a reference to it, so we don't pass the diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 1dc3a248f45f0c..e726dc0e6fd111 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -18,6 +18,7 @@ #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_uop_metadata.h" #include "pycore_long.h" @@ -127,7 +128,7 @@ static void increment_mutations(PyObject* dict) { assert(PyDict_CheckExact(dict)); PyDictObject *d = (PyDictObject *)dict; - FT_ATOMIC_ADD_UINT64(d->_ma_watcher_tag, (1 << DICT_MAX_WATCHERS)); + FT_ATOMIC_ADD_UINT64(d->_ma_watcher_tag, 1ULL << DICT_MAX_WATCHERS); } /* The first two dict watcher IDs are reserved for CPython, @@ -156,6 +157,17 @@ type_watcher_callback(PyTypeObject* type) return 0; } +static int +_setup_optimizer_watchers(void *Py_UNUSED(arg)) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_ATOMIC_STORE_PTR_RELEASE( + interp->dict_state.watchers[GLOBALS_WATCHER_ID], + globals_watcher_callback); + interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; + return 0; +} + static void watch_type(PyTypeObject *type, _PyBloomFilter *filter) { @@ -580,10 +592,8 @@ optimize_uops( // Make sure that watchers are set up PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { - interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback; - interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; - } + _PyOnceFlag_CallOnce(&interp->dict_state.watcher_setup_once, + _setup_optimizer_watchers, NULL); _Py_uop_abstractcontext_init(ctx, dependencies); _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, (PyCodeObject *)func->func_code, NULL, 0); diff --git a/Python/pystate.c b/Python/pystate.c index bf2616a49148a7..ff712019affbf9 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -320,6 +320,7 @@ _Py_COMP_DIAG_POP &(runtime)->allocators.mutex, \ &(runtime)->_main_interpreter.types.mutex, \ &(runtime)->_main_interpreter.code_state.mutex, \ + &(runtime)->_main_interpreter.dict_state.watcher_mutex, \ } static void diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 7af64ed017ba73..ddfb93a424c018 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -467,6 +467,7 @@ Modules/_testcapi/object.c - MyObject_dealloc_called - Modules/_testcapi/object.c - MyType - Modules/_testcapi/structmember.c - test_structmembersType_OldAPI - Modules/_testcapi/watchers.c - g_dict_watch_events - +Modules/_testcapi/watchers.c - g_dict_watch_once - Modules/_testcapi/watchers.c - g_dict_watchers_installed - Modules/_testcapi/watchers.c - g_type_modified_events - Modules/_testcapi/watchers.c - g_type_watchers_installed - From a8c420879e344a365ec4542107db5c4830ef2520 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 11 May 2026 11:25:40 -0700 Subject: [PATCH 028/110] =?UTF-8?q?[3.15]=20gh-149614=20-=20Restore=20deep?= =?UTF-8?q?copiability=20of=20argparse.ArgumentParser=E2=80=A6=20(#149693)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [3.15] gh-149614 - Restore deepcopiability of argparse.ArgumentParser instances (GH-149617) (cherry picked from commit fadd9bc14e43041c84bb8d06824990264fe1434a) Co-authored-by: David Ellis Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Lib/argparse.py | 2 + Lib/test/test_argparse.py | 55 +++++++++++++++++++ ...-05-09-21-02-08.gh-issue-149614.U4snj3.rst | 1 + 3 files changed, 58 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-05-09-21-02-08.gh-issue-149614.U4snj3.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 6d21823e652429..29e6ebb9634261 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -163,6 +163,8 @@ class _ColorlessTheme: def __getattr__(self, name): # _colorize's no_color themes are just all empty strings # by directly using empty strings the import is avoided + if name.startswith("_"): + raise AttributeError(name) return "" _colorless_theme = _ColorlessTheme() diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 4ea5b6f53a0426..1dc3f538f4ad8b 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -140,6 +140,48 @@ def test_parse_args(self): ) +class TestArgumentParserCopiable(unittest.TestCase): + def _get_parser(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo', type=int, default=42) + parser.add_argument('bar', nargs='?', default='baz') + return parser + + @force_not_colorized + def test_copiable(self): + import copy + parser = self._get_parser() + parser2 = copy.copy(parser) + ns = parser2.parse_args(['--foo', '123', 'quux']) + self.assertEqual(ns.foo, 123) + self.assertEqual(ns.bar, 'quux') + ns2 = parser2.parse_args([]) + self.assertEqual(ns2.foo, 42) + self.assertEqual(ns2.bar, 'baz') + + # Test shallow copy also gets new arguments + parser.add_argument("--extra") + ns3 = parser2.parse_args(["--extra", "bar"]) + self.assertEqual(ns3.extra, "bar") + + @force_not_colorized + def test_deepcopiable(self): + import copy + parser = self._get_parser() + parser2 = copy.deepcopy(parser) + ns = parser2.parse_args(['--foo', '123', 'quux']) + self.assertEqual(ns.foo, 123) + self.assertEqual(ns.bar, 'quux') + ns2 = parser2.parse_args([]) + self.assertEqual(ns2.foo, 42) + self.assertEqual(ns2.bar, 'baz') + + # Test deep copy does not get new arguments + parser.add_argument("--extra") + with self.assertRaises(argparse.ArgumentError): + parser2.parse_args(["--extra", "bar"]) + + class TestArgumentParserPickleable(unittest.TestCase): @force_not_colorized @@ -7863,12 +7905,25 @@ def fake_can_colorize(*, file=None): def test_fake_color_theme_matches_real(self): from argparse import _colorless_theme + + # Check the attributes match those of the 'real' theme _colorize_nocolor = _colorize.get_theme(force_no_color=True).argparse for k in _colorize_nocolor: self.assertEqual( getattr(_colorless_theme, k), getattr(_colorize_nocolor, k) ) + def test_fake_color_theme_raises(self): + from argparse import _colorless_theme + + # Make sure the _colorless_theme doesn't return empty strings + # for magic methods or private attributes + with self.assertRaises(AttributeError): + _colorless_theme.__unknown_dunder__ + + with self.assertRaises(AttributeError): + _colorless_theme._private_attribute + class TestModule(unittest.TestCase): def test_deprecated__version__(self): diff --git a/Misc/NEWS.d/next/Library/2026-05-09-21-02-08.gh-issue-149614.U4snj3.rst b/Misc/NEWS.d/next/Library/2026-05-09-21-02-08.gh-issue-149614.U4snj3.rst new file mode 100644 index 00000000000000..5169c6c203fc1b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-09-21-02-08.gh-issue-149614.U4snj3.rst @@ -0,0 +1 @@ +Fix a regression that broke the ability to deepcopy :class:`argparse.ArgumentParser` instances. From 9138bf2612730f8d6bca46e08f645072c7bf32fa Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 12 May 2026 04:54:10 +0200 Subject: [PATCH 029/110] [3.15] gh-148669: Clarify `__reduce__()` module lookup behavior (GH-148670) (#149703) gh-148669: Clarify `__reduce__()` module lookup behavior (GH-148670) (cherry picked from commit 54a5fd4126df74f7b84d1f8a6a36ef79803f66f9) Co-authored-by: Victorien <65306057+Viicos@users.noreply.github.com> --- Doc/library/pickle.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index f8975c2f4281d4..8eadc2cf2b1ef0 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -56,7 +56,7 @@ The :mod:`!pickle` module differs from :mod:`marshal` in several significant way * :mod:`marshal` cannot be used to serialize user-defined classes and their instances. :mod:`!pickle` can save and restore class instances transparently, however the class definition must be importable and live in the same module as - when the object was stored. + when the object was pickled. * The :mod:`marshal` serialization format is not guaranteed to be portable across Python versions. Because its primary job in life is to support @@ -693,7 +693,10 @@ or both. If a string is returned, the string should be interpreted as the name of a global variable. It should be the object's local name relative to its module; the pickle module searches the module namespace to determine the - object's module. This behaviour is typically useful for singletons. + object's module: for a given ``obj`` to be pickled, the ``__module__`` + attribute is looked up on ``obj`` directly, which falls back to a lookup + on the type of ``obj`` if no ``__module__`` instance attribute is set. + This behaviour is typically useful for singletons. When a tuple is returned, it must be between two and six items long. Optional items can either be omitted, or ``None`` can be provided as their From 45c431a55db2b05bb21b76cd95f6775cc13ce9fb Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 12 May 2026 11:09:52 +0200 Subject: [PATCH 030/110] [3.15] Update mypy to 2.1.0 (GH-149709) (#149711) Update mypy to 2.1.0 (GH-149709) (cherry picked from commit b546cc10f5c659344ce3cf49db6d9c92307ed1fc) Co-authored-by: sobolevn --- Lib/test/libregrtest/single.py | 2 +- Lib/tomllib/mypy.ini | 2 -- Tools/build/check_extension_modules.py | 2 +- Tools/build/mypy.ini | 2 -- Tools/requirements-dev.txt | 6 +++--- 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 958a915626ad24..d0759d2626989d 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -145,7 +145,7 @@ def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: # Storage of uncollectable GC objects (gc.garbage) -GC_GARBAGE = [] +GC_GARBAGE: list[object] = [] def _load_run_test(result: TestResult, runtests: RunTests) -> None: diff --git a/Lib/tomllib/mypy.ini b/Lib/tomllib/mypy.ini index 1761dce45562a6..f7eeffd575c1c7 100644 --- a/Lib/tomllib/mypy.ini +++ b/Lib/tomllib/mypy.ini @@ -12,6 +12,4 @@ pretty = True # Enable most stricter settings enable_error_code = ignore-without-code strict = True -strict_bytes = True -local_partial_types = True warn_unreachable = True diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index f23c1d5286f92a..c619a9a0c1c5a1 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -463,7 +463,7 @@ def get_location(self, modinfo: ModuleInfo) -> pathlib.Path | None: def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec) -> None: """Check that the module file is present and not empty""" if spec.loader is BuiltinImporter: # type: ignore[comparison-overlap] - return + return # type: ignore[unreachable] try: assert spec.origin is not None st = os.stat(spec.origin) diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index 5465e2d4b6171f..485c9314cf7001 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -24,8 +24,6 @@ python_version = 3.10 # ...And be strict: strict = True -strict_bytes = True -local_partial_types = True extra_checks = True enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined warn_unreachable = True diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index af5cbaa7689f33..46381ea58a1238 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,7 +1,7 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.19.1 +mypy==2.1.0 # needed for peg_generator: -types-psutil==7.2.2.20260130 -types-setuptools==82.0.0.20260210 +types-psutil==7.2.2.20260508 +types-setuptools==82.0.0.20260508 From d3b86111ef7913077394c40fff1eb8941b30be51 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 12 May 2026 15:57:02 +0200 Subject: [PATCH 031/110] [3.15] gh-149676: Fix hash(frozendict | frozendict) (GH-149675) (#149717) gh-149676: Fix hash(frozendict | frozendict) (GH-149675) Fix new_dict_impl() to properly initialize ma_hash on frozendict. (cherry picked from commit f5fb491341e566bbaf17d9bf3e4ec3af4a56bb3f) Co-authored-by: Thomas Kowalski Co-authored-by: Victor Stinner --- Lib/test/test_dict.py | 5 +++++ .../2026-05-11-14-48-56.gh-issue-149676.6aTrw1.rst | 1 + Objects/dictobject.c | 9 ++++++--- 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-14-48-56.gh-issue-149676.6aTrw1.rst diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index b2f4363b23e748..4efb066d4fd01c 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1868,6 +1868,11 @@ def test_merge(self): self.assertEqual(fd | {}, fd) self.assertEqual(frozendict() | fd, fd) + # gh-149676: Test hash(frozendict | frozendict) + a = frozendict({"a": 1}) + b = frozendict({"b": 2}) + self.assertEqual(hash(a | b), hash(frozendict({"a": 1, "b": 2}))) + def test_update(self): # test "a |= b" operator d = frozendict(x=1) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-14-48-56.gh-issue-149676.6aTrw1.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-14-48-56.gh-issue-149676.6aTrw1.rst new file mode 100644 index 00000000000000..96f407cf5ad25a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-14-48-56.gh-issue-149676.6aTrw1.rst @@ -0,0 +1 @@ +Fix ``frozendict | frozendict`` hash. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 09135e031e6fc7..b33a273dac3b95 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -900,7 +900,7 @@ free_values(PyDictValues *values, bool use_qsbr) static inline PyObject * new_dict_impl(PyDictObject *mp, PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, - int free_values_on_failure) + int free_values_on_failure, int frozendict) { assert(keys != NULL); if (mp == NULL) { @@ -915,6 +915,9 @@ new_dict_impl(PyDictObject *mp, PyDictKeysObject *keys, mp->ma_values = values; mp->ma_used = used; mp->_ma_watcher_tag = 0; + if (frozendict) { + ((PyFrozenDictObject *)mp)->ma_hash = -1; + } ASSERT_CONSISTENT(mp); _PyObject_GC_TRACK(mp); return (PyObject *)mp; @@ -931,7 +934,7 @@ new_dict(PyDictKeysObject *keys, PyDictValues *values, } assert(mp == NULL || Py_IS_TYPE(mp, &PyDict_Type)); - return new_dict_impl(mp, keys, values, used, free_values_on_failure); + return new_dict_impl(mp, keys, values, used, free_values_on_failure, 0); } /* Consumes a reference to the keys object */ @@ -940,7 +943,7 @@ new_frozendict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free_values_on_failure) { PyDictObject *mp = PyObject_GC_New(PyDictObject, &PyFrozenDict_Type); - return new_dict_impl(mp, keys, values, used, free_values_on_failure); + return new_dict_impl(mp, keys, values, used, free_values_on_failure, 1); } static PyObject * From 5f9d0d0b866d2d1275b2aefaf9f70b6333e03947 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 12 May 2026 18:09:41 +0200 Subject: [PATCH 032/110] [3.15] Remove myself from CODEOWNERS (GH-149727) (#149732) Remove myself from CODEOWNERS (GH-149727) (cherry picked from commit 058c12528d98954c44d6f92f2eea48b881c1967f) Co-authored-by: Berker Peksag --- .github/CODEOWNERS | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f4ffa24edca453..b4b3b1013e3095 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -586,10 +586,10 @@ Lib/test/test_string/test_templatelib.py @lysnikolaou @AA-Turner **/*sysconfig* @FFY00 # SQLite 3 -Doc/library/sqlite3.rst @berkerpeksag @erlend-aasland -Lib/sqlite3/ @berkerpeksag @erlend-aasland -Lib/test/test_sqlite3/ @berkerpeksag @erlend-aasland -Modules/_sqlite/ @berkerpeksag @erlend-aasland +Doc/library/sqlite3.rst @erlend-aasland +Lib/sqlite3/ @erlend-aasland +Lib/test/test_sqlite3/ @erlend-aasland +Modules/_sqlite/ @erlend-aasland # Subprocess Lib/subprocess.py @gpshead @@ -622,9 +622,6 @@ Modules/_typesmodule.c @AA-Turner Lib/unittest/mock.py @cjw296 Lib/test/test_unittest/testmock/ @cjw296 -# Urllib -**/*robotparser* @berkerpeksag - # Venv **/*venv* @vsajip @FFY00 From 6a6600569985e4006900bae16812f2d19c8ab97b Mon Sep 17 00:00:00 2001 From: deadlovelll <128279579+deadlovelll@users.noreply.github.com> Date: Tue, 12 May 2026 20:22:36 +0300 Subject: [PATCH 033/110] [3.15] gh-149694: Fix missing docstring on `asyncio.iscoroutinefunction` (#149696) --- Lib/asyncio/coroutines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index a51319cb72a6a9..6727065bbe323f 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -18,8 +18,8 @@ def _is_debug_mode(): def iscoroutinefunction(func): - import warnings """Return True if func is a decorated coroutine function.""" + import warnings warnings._deprecated("asyncio.iscoroutinefunction", f"{warnings._DEPRECATED_MSG}; " "use inspect.iscoroutinefunction() instead", From e2e9cea7692fdb0ac3e34fd38a25d9025035769f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 12 May 2026 19:58:31 +0200 Subject: [PATCH 034/110] [3.15] GH-149501: Fix compilation warning in `_YIELD_VALUE` uop (GH-149502) (#149737) GH-149501: Fix compilation warning in `_YIELD_VALUE` uop (GH-149502) (cherry picked from commit 1a79fd0ad650f0a0f21f653cc46a89bc1741d253) Co-authored-by: Sergey Miryanov --- Modules/_testinternalcapi/test_cases.c.h | 10 ++++++---- Python/bytecodes.c | 5 +++-- Python/executor_cases.c.h | 5 +++-- Python/generated_cases.c.h | 10 ++++++---- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 238e17bea303d3..a2506524f0bb6d 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -7946,8 +7946,9 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif @@ -13056,8 +13057,9 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3bd489122da9d4..f7487c7136962f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1867,8 +1867,9 @@ dummy_func( assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b6a2821db3007e..efa61d7de74e88 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -9346,8 +9346,9 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2623105656c90c..53e09a8f4523c7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7945,8 +7945,9 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif @@ -13053,8 +13054,9 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif From 564902ea8ae409d46a8ff9c06f3f9d58f754cf59 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 12 May 2026 22:10:24 +0200 Subject: [PATCH 035/110] [3.15] gh-139808: Add branch protections for aarch64 in asm_trampoline.S (GH-130864) (#149730) gh-139808: Add branch protections for aarch64 in asm_trampoline.S (GH-130864) Apply protection against ROP/JOP attacks for aarch64 on asm_trampoline.S. The BTI flag must be applied in assembler sources for this class of attacks to be mitigated on newer aarch64 processors. See also: https://sourceware.org/annobin/annobin.html/Test-branch-protection.html and https://community.arm.com/arm-community-blogs/b/architectures-and-processors-blog/posts/enabling-pac-and-bti-on-aarch64 (cherry picked from commit da8477b25c6124c961306d4d7cd5ec7dafda6be4) Co-authored-by: stratakis Co-authored-by: Victor Stinner --- ...-05-12-16-47-23.gh-issue-139808.iIs7_E.rst | 2 + Python/asm_trampoline.S | 4 ++ Python/asm_trampoline_aarch64.h | 56 +++++++++++++++++++ Python/jit_unwind.c | 13 +++++ 4 files changed, 75 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-12-16-47-23.gh-issue-139808.iIs7_E.rst create mode 100644 Python/asm_trampoline_aarch64.h diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-12-16-47-23.gh-issue-139808.iIs7_E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-12-16-47-23.gh-issue-139808.iIs7_E.rst new file mode 100644 index 00000000000000..3e9d930bf1de89 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-12-16-47-23.gh-issue-139808.iIs7_E.rst @@ -0,0 +1,2 @@ +Add branch protections for AArch64 (BTI/PAC) in assembly code used by +:option:`-X perf_jit <-X>` (Linux perf profiler integration). diff --git a/Python/asm_trampoline.S b/Python/asm_trampoline.S index 93adae3d99038f..9f3ca909ab7d85 100644 --- a/Python/asm_trampoline.S +++ b/Python/asm_trampoline.S @@ -1,3 +1,5 @@ +#include "asm_trampoline_aarch64.h" + .text #if defined(__APPLE__) .globl __Py_trampoline_func_start @@ -29,10 +31,12 @@ _Py_trampoline_func_start: #if defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) // ARM64 little endian, 64bit ABI // generate with aarch64-linux-gnu-gcc 12.1 + SIGN_LR stp x29, x30, [sp, -16]! mov x29, sp blr x3 ldp x29, x30, [sp], 16 + VERIFY_LR ret #endif #ifdef __riscv diff --git a/Python/asm_trampoline_aarch64.h b/Python/asm_trampoline_aarch64.h new file mode 100644 index 00000000000000..bc83aa460b6860 --- /dev/null +++ b/Python/asm_trampoline_aarch64.h @@ -0,0 +1,56 @@ +#ifndef ASM_TRAMPOLINE_AARCH_64_H_ +#define ASM_TRAMPOLINE_AARCH_64_H_ + +/* + * References: + * - https://developer.arm.com/documentation/101028/0012/5--Feature-test-macros + * - https://github.com/ARM-software/abi-aa/blob/main/aaelf64/aaelf64.rst + */ + +#if defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1 + #define BTI_J hint 36 /* bti j: for jumps, IE br instructions */ + #define BTI_C hint 34 /* bti c: for calls, IE bl instructions */ + #define GNU_PROPERTY_AARCH64_BTI 1 /* bit 0 GNU Notes is for BTI support */ +#else + #define BTI_J + #define BTI_C + #define GNU_PROPERTY_AARCH64_BTI 0 +#endif + +#if defined(__ARM_FEATURE_PAC_DEFAULT) + #if __ARM_FEATURE_PAC_DEFAULT & 1 + #define SIGN_LR hint 25 /* paciasp: sign with the A key */ + #define VERIFY_LR hint 29 /* autiasp: verify with the A key */ + #elif __ARM_FEATURE_PAC_DEFAULT & 2 + #define SIGN_LR hint 27 /* pacibsp: sign with the b key */ + #define VERIFY_LR hint 31 /* autibsp: verify with the b key */ + #endif + #define GNU_PROPERTY_AARCH64_POINTER_AUTH 2 /* bit 1 GNU Notes is for PAC support */ +#else + #define SIGN_LR BTI_C + #define VERIFY_LR + #define GNU_PROPERTY_AARCH64_POINTER_AUTH 0 +#endif + +#if defined(__ARM_FEATURE_GCS_DEFAULT) && __ARM_FEATURE_GCS_DEFAULT == 1 + #define GNU_PROPERTY_AARCH64_GCS 4 /* bit 2 GNU Notes is for GCS support */ +#else + #define GNU_PROPERTY_AARCH64_GCS 0 +#endif + +/* Add the BTI, PAC and GCS support to GNU Notes section */ +#if GNU_PROPERTY_AARCH64_BTI != 0 || GNU_PROPERTY_AARCH64_POINTER_AUTH != 0 || GNU_PROPERTY_AARCH64_GCS != 0 + .pushsection .note.gnu.property, "a"; /* Start a new allocatable section */ + .balign 8; /* align it on a byte boundry */ + .long 4; /* size of "GNU\0" */ + .long 0x10; /* size of descriptor */ + .long 0x5; /* NT_GNU_PROPERTY_TYPE_0 */ + .asciz "GNU"; + .long 0xc0000000; /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */ + .long 4; /* Four bytes of data */ + .long (GNU_PROPERTY_AARCH64_BTI|GNU_PROPERTY_AARCH64_POINTER_AUTH|GNU_PROPERTY_AARCH64_GCS); /* BTI, PAC or GCS is enabled */ + .long 0; /* padding for 8 byte alignment */ + .popsection; /* end the section */ +#endif + +#endif diff --git a/Python/jit_unwind.c b/Python/jit_unwind.c index 646106f0a9655c..0941ed593ff7d1 100644 --- a/Python/jit_unwind.c +++ b/Python/jit_unwind.c @@ -60,6 +60,9 @@ enum { DWRF_CFA_offset_extended_sf = 0x11, // Extended signed offset DWRF_CFA_advance_loc = 0x40, // Advance location counter DWRF_CFA_offset = 0x80, // Simple offset instruction +#if defined(__aarch64__) + DWRF_CFA_AARCH64_negate_ra_state = 0x2d, // Toggle return address signing state +#endif DWRF_CFA_restore = 0xc0 // Restore register }; @@ -562,6 +565,13 @@ static void elf_init_ehframe_perf(ELFObjectContext* ctx) { DWRF_UV(8); // New offset: SP + 8 #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) /* AArch64 calling convention unwinding rules */ +#if defined(__ARM_FEATURE_PAC_DEFAULT) || \ + (defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1) + DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past SIGN_LR (4 bytes) +#endif +#if defined(__ARM_FEATURE_PAC_DEFAULT) + DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // Saved LR is PAC-signed from here +#endif DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance by 1 instruction (4 bytes) DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 DWRF_UV(16); // Stack pointer moved by 16 bytes @@ -570,6 +580,9 @@ static void elf_init_ehframe_perf(ELFObjectContext* ctx) { DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // x30 (link register) saved DWRF_UV(1); // At CFA-8 (1 * 8 = 8 bytes from CFA) DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (12 bytes) +#if defined(__ARM_FEATURE_PAC_DEFAULT) + DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // LR is authenticated, no longer PAC-signed +#endif DWRF_U8(DWRF_CFA_def_cfa_register); // CFA = FP (x29) + 16 DWRF_UV(DWRF_REG_FP); DWRF_U8(DWRF_CFA_restore | DWRF_REG_RA); // Restore x30 - NO DWRF_UV() after this! From 670f982fbd6b8655e9d3ff832aac4bdbb0d89d34 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 12 May 2026 22:25:28 +0200 Subject: [PATCH 036/110] [3.15] gh-134837: Correct and improve base85 documentation for base64 and binascii modules (GH-145843) (GH-149742) (cherry picked from commit e667d62f114b54dcba17bdfad3835b9c91fda348) Co-authored-by: David Huggins-Daines --- Doc/library/base64.rst | 89 ++++++++++++++++++++++++------------- Doc/library/binascii.rst | 27 +++++++---- Lib/base64.py | 38 ++++++++++------ Modules/binascii.c | 13 +++--- Modules/clinic/binascii.c.h | 9 ++-- 5 files changed, 112 insertions(+), 64 deletions(-) diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index a722607b2c1f19..8af40a2f8a65e3 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -16,8 +16,10 @@ This module provides functions for encoding binary data to printable ASCII characters and decoding such encodings back to binary data. This includes the :ref:`encodings specified in ` -:rfc:`4648` (Base64, Base32 and Base16) -and the non-standard :ref:`Base85 encodings `. +:rfc:`4648` (Base64, Base32 and Base16), the :ref:`Base85 encoding +` specified in `PDF 2.0 +`_, and non-standard variants +of Base85 used elsewhere. There are two interfaces provided by this module. The modern interface supports encoding :term:`bytes-like objects ` to ASCII @@ -284,19 +286,28 @@ POST request. Base85 Encodings ----------------- -Base85 encoding is not formally specified but rather a de facto standard, -thus different systems perform the encoding differently. +Base85 encoding is a family of algorithms which represent four bytes +using five ASCII characters. Originally implemented in the Unix +``btoa(1)`` utility, a version of it was later adopted by Adobe in the +PostScript language and is standardized in PDF 2.0 (ISO 32000-2). +This version, in both its ``btoa`` and PDF variants, is implemented by +:func:`a85encode`. -The :func:`a85encode` and :func:`b85encode` functions in this module are two implementations of -the de facto standard. You should call the function with the Base85 -implementation used by the software you intend to work with. +A separate version, using a different output character set, was +defined as an April Fool's joke in :rfc:`1924` but is now used by Git +and other software. This version is implemented by :func:`b85encode`. -The two functions present in this module differ in how they handle the following: +Finally, a third version, using yet another output character set +designed for safe inclusion in programming language strings, is +defined by ZeroMQ and implemented here by :func:`z85encode`. -* Whether to include enclosing ``<~`` and ``~>`` markers -* Whether to include newline characters -* The set of ASCII characters used for encoding -* Handling of null bytes +The functions present in this module differ in how they handle the following: + +* Whether to include and expect enclosing ``<~`` and ``~>`` markers. +* Whether to fold the input into multiple lines. +* The set of ASCII characters used for encoding. +* Compact encodings of sequences of spaces and null bytes. +* The encoding of zero-padding bytes applied to the input. Refer to the documentation of the individual functions for more information. @@ -307,18 +318,22 @@ Refer to the documentation of the individual functions for more information. *foldspaces* is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Ascii85 encoding. + feature is not supported by the standard encoding used in PDF. If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. - Note that the ``btoa`` implementation always pads. + *pad* controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by ``btoa``, + producing an exact multiple of 5 bytes of output. This is not part + of the standard encoding used in PDF, as it does not preserve the + length of the data. - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. Note + that while ASCII85Decode streams in PDF documents *must* be + terminated with ``~>``, they *must not* use a leading ``<~``. .. versionadded:: 3.4 @@ -330,10 +345,12 @@ Refer to the documentation of the individual functions for more information. *foldspaces* is a flag that specifies whether the 'y' short sequence should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). - This feature is not supported by the "standard" Ascii85 encoding. + This feature is not supported by the standard Ascii85 encoding used in + PDF and PostScript. - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). + *adobe* controls whether the ``<~`` and ``~>`` markers are + present. While the leading ``<~`` is not required, the input must + end with ``~>``, or a :exc:`ValueError` is raised. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. @@ -356,8 +373,11 @@ Refer to the documentation of the individual functions for more information. Encode the :term:`bytes-like object` *b* using base85 (as used in e.g. git-style binary diffs) and return the encoded :class:`bytes`. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + The input is padded with ``b'\0'`` so its length is a multiple of 4 + bytes before encoding. If *pad* is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, and thus the length of the data may not be + preserved on decoding. If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. @@ -372,8 +392,7 @@ Refer to the documentation of the individual functions for more information. .. function:: b85decode(b, *, ignorechars=b'', canonical=False) Decode the base85-encoded :term:`bytes-like object` or ASCII string *b* and - return the decoded :class:`bytes`. Padding is implicitly removed, if - necessary. + return the decoded :class:`bytes`. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. @@ -392,11 +411,12 @@ Refer to the documentation of the individual functions for more information. .. function:: z85encode(s, pad=False, *, wrapcol=0) Encode the :term:`bytes-like object` *s* using Z85 (as used in ZeroMQ) - and return the encoded :class:`bytes`. See `Z85 specification - `_ for more information. + and return the encoded :class:`bytes`. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + The input is padded with ``b'\0'`` so its length is a multiple of 4 + bytes before encoding. If *pad* is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, as required by the ZeroMQ standard. If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. @@ -414,8 +434,7 @@ Refer to the documentation of the individual functions for more information. .. function:: z85decode(s, *, ignorechars=b'', canonical=False) Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and - return the decoded :class:`bytes`. See `Z85 specification - `_ for more information. + return the decoded :class:`bytes`. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. @@ -499,3 +518,11 @@ recommended to review the security section for any code deployed to production. Section 5.2, "Base64 Content-Transfer-Encoding," provides the definition of the base64 encoding. + `ISO 32000-2 Portable document format - Part 2: PDF 2.0 `_ + Section 7.4.3, "ASCII85Decode Filter," provides the definition + of the Ascii85 encoding used in PDF and PostScript, including + the output character set and the details of data length preservation + using zero-padding and partial output groups. + + `ZeroMQ RFC 32/Z85 `_ + The "Formal Specification" section provides the character set used in Z85. diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 8b4ba6ae9fb254..60afe9261d51fa 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -133,8 +133,11 @@ The :mod:`!binascii` module defines the following functions: should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is not supported by the "standard" Ascii85 encoding. - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. If + *adobe* is true, a leading ``<~`` is optionally accepted, while a + trailing ``~>`` is *required*, and :exc:`binascii.Error` is raised + if it is not found. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. @@ -164,12 +167,16 @@ The :mod:`!binascii` module defines the following functions: after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. - Note that the ``btoa`` implementation always pads. + If *pad* is true, the zero-padding applied to the end of the input + is fully retained in the output encoding, as done by ``btoa``, + producing an exact multiple of 5 bytes of output. This is not part + of the standard encoding used in PDF, as it does not preserve the + length of the data. - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. Note + that while ASCII85Decode streams in PDF documents *must* be + terminated with ``~>``, they *must not* use a leading ``<~``. .. versionadded:: 3.15 @@ -213,8 +220,10 @@ The :mod:`!binascii` module defines the following functions: after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + If *pad* is true, the zero-padding applied to the end of the input + is retained in the output, which will always be a multiple of 5 + bytes, and thus the length of the data may not be preserved on + decoding. .. versionadded:: 3.15 diff --git a/Lib/base64.py b/Lib/base64.py index 4b810e08569e5b..4a0e9d446edb0b 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -315,16 +315,20 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): foldspaces is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Adobe encoding. + feature is not supported by the standard encoding used in PDF. If wrapcol is non-zero, insert a newline (b'\\n') character after at most every wrapcol characters. - pad controls whether the input is padded to a multiple of 4 before - encoding. Note that the btoa implementation always pads. + pad controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by btoa, + producing an exact multiple of 5 bytes of output. + + adobe controls whether the encoded byte sequence is framed with <~ + and ~>, as in a PostScript base-85 string literal. Note that + while ASCII85Decode streams in PDF documents must be terminated + with ~>, they must not use a leading <~. - adobe controls whether the encoded byte sequence is framed with <~ and ~>, - which is used by the Adobe implementation. """ return binascii.b2a_ascii85(b, foldspaces=foldspaces, adobe=adobe, wrapcol=wrapcol, pad=pad) @@ -333,12 +337,14 @@ def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v', canonical=False): """Decode the Ascii85 encoded bytes-like object or ASCII string b. - foldspaces is a flag that specifies whether the 'y' short sequence should be - accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is - not supported by the "standard" Adobe encoding. + foldspaces is a flag that specifies whether the 'y' short sequence + should be accepted as shorthand for 4 consecutive spaces (ASCII + 0x20). This feature is not supported by the standard Ascii85 + encoding used in PDF and PostScript. - adobe controls whether the input sequence is in Adobe Ascii85 format (i.e. - is framed with <~ and ~>). + adobe controls whether the <~ and ~> markers are present. While + the leading <~ is not required, the input must end with ~>, or a + ValueError is raised. ignorechars should be a byte string containing characters to ignore from the input. This should only contain whitespace characters, and by default @@ -358,8 +364,10 @@ def b85encode(b, pad=False, *, wrapcol=0): If wrapcol is non-zero, insert a newline (b'\\n') character after at most every wrapcol characters. - If pad is true, the input is padded with b'\\0' so its length is a multiple of - 4 bytes before encoding. + The input is padded with b'\0' so its length is a multiple of 4 + bytes before encoding. If pad is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes. """ return binascii.b2a_base85(b, wrapcol=wrapcol, pad=pad) @@ -379,8 +387,10 @@ def z85encode(s, pad=False, *, wrapcol=0): If wrapcol is non-zero, insert a newline (b'\\n') character after at most every wrapcol characters. - If pad is true, the input is padded with b'\\0' so its length is a multiple of - 4 bytes before encoding. + The input is padded with b'\0' so its length is a multiple of + bytes before encoding. If pad is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, as required by the ZeroMQ standard. """ return binascii.b2a_base85(s, wrapcol=wrapcol, pad=pad, alphabet=binascii.Z85_ALPHABET) diff --git a/Modules/binascii.c b/Modules/binascii.c index 673dca6ee134bd..0e7af135a6f6ce 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -1057,7 +1057,8 @@ binascii.a2b_ascii85 foldspaces: bool = False Allow 'y' as a short form encoding four spaces. adobe: bool = False - Expect data to be wrapped in '<~' and '~>' as in Adobe Ascii85. + Expect data to be terminated with '~>' as in Adobe Ascii85, and + optionally accept leading '<~'. ignorechars: Py_buffer = b'' A byte string containing characters to ignore from the input. canonical: bool = False @@ -1069,7 +1070,7 @@ Decode Ascii85 data. static PyObject * binascii_a2b_ascii85_impl(PyObject *module, Py_buffer *data, int foldspaces, int adobe, Py_buffer *ignorechars, int canonical) -/*[clinic end generated code: output=09b35f1eac531357 input=dd050604ed30199e]*/ +/*[clinic end generated code: output=09b35f1eac531357 input=08eab2e53c62f1a8]*/ { const unsigned char *ascii_data = data->buf; Py_ssize_t ascii_len = data->len; @@ -1264,7 +1265,7 @@ binascii.b2a_ascii85 wrapcol: size_t = 0 Split result into lines of provided width. pad: bool = False - Pad input to a multiple of 4 before encoding. + Retain zero-padding bytes at end of output. adobe: bool = False Wrap result in '<~' and '~>' as in Adobe Ascii85. @@ -1274,7 +1275,7 @@ Ascii85-encode data. static PyObject * binascii_b2a_ascii85_impl(PyObject *module, Py_buffer *data, int foldspaces, size_t wrapcol, int pad, int adobe) -/*[clinic end generated code: output=5ce8fdee843073f4 input=791da754508c7d17]*/ +/*[clinic end generated code: output=5ce8fdee843073f4 input=a77e31d63517bf19]*/ { const unsigned char *bin_data = data->buf; Py_ssize_t bin_len = data->len; @@ -1539,7 +1540,7 @@ binascii.b2a_base85 / * pad: bool = False - Pad input to a multiple of 4 before encoding. + Retain zero-padding bytes at end of output. wrapcol: size_t = 0 alphabet: Py_buffer(c_default="{NULL, NULL}") = BASE85_ALPHABET @@ -1549,7 +1550,7 @@ Base85-code line of data. static PyObject * binascii_b2a_base85_impl(PyObject *module, Py_buffer *data, int pad, size_t wrapcol, Py_buffer *alphabet) -/*[clinic end generated code: output=98b962ed52c776a4 input=1b20b0bd6572691b]*/ +/*[clinic end generated code: output=98b962ed52c776a4 input=54886d05128d41a8]*/ { const unsigned char *bin_data = data->buf; Py_ssize_t bin_len = data->len; diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index ed695758ef998c..29fa9e87de87c7 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -372,7 +372,8 @@ PyDoc_STRVAR(binascii_a2b_ascii85__doc__, " foldspaces\n" " Allow \'y\' as a short form encoding four spaces.\n" " adobe\n" -" Expect data to be wrapped in \'<~\' and \'~>\' as in Adobe Ascii85.\n" +" Expect data to be terminated with \'~>\' as in Adobe Ascii85, and\n" +" optionally accept leading \'<~\'.\n" " ignorechars\n" " A byte string containing characters to ignore from the input.\n" " canonical\n" @@ -492,7 +493,7 @@ PyDoc_STRVAR(binascii_b2a_ascii85__doc__, " wrapcol\n" " Split result into lines of provided width.\n" " pad\n" -" Pad input to a multiple of 4 before encoding.\n" +" Retain zero-padding bytes at end of output.\n" " adobe\n" " Wrap result in \'<~\' and \'~>\' as in Adobe Ascii85."); @@ -709,7 +710,7 @@ PyDoc_STRVAR(binascii_b2a_base85__doc__, "Base85-code line of data.\n" "\n" " pad\n" -" Pad input to a multiple of 4 before encoding."); +" Retain zero-padding bytes at end of output."); #define BINASCII_B2A_BASE85_METHODDEF \ {"b2a_base85", _PyCFunction_CAST(binascii_b2a_base85), METH_FASTCALL|METH_KEYWORDS, binascii_b2a_base85__doc__}, @@ -1684,4 +1685,4 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -/*[clinic end generated code: output=b41544f39b0ef681 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=42dd48f323cbb118 input=a9049054013a1b77]*/ From e3fbcc3eac2527a0f1eda039bcbd4cc3eab708f9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 12 May 2026 23:25:21 +0200 Subject: [PATCH 037/110] [3.15] gh-149496: Fix MacOSTest.test_default regression when BROWSER env var is set (GH-149579) (#149745) gh-149496: Fix MacOSTest.test_default regression when BROWSER env var is set (GH-149579) gh-149496: Fix MacOSTest.test_default failing when BROWSER env var is set MacOSTest.test_default calls webbrowser.get() and asserts it returns a MacOS instance. When BROWSER is set in the environment (e.g. BROWSER=open, a common macOS workaround for the old osascript-based implementation), register_standard_browsers() registers a GenericBrowser as the preferred browser instead, causing the assertion to fail. This is a regression introduced in gh-137586, which added MacOSTest and moved test_default into it from MacOSXOSAScriptTest. MacOSXOSAScriptTest had an identical setUp() guard added in gh-131254 specifically to fix this same failure. The guard was not carried over to MacOSTest. Add setUp() to MacOSTest to unset BROWSER for the duration of each test, restoring the isolation that was already established as the correct pattern for macOS webbrowser tests. (cherry picked from commit 45c47d26c230086163ac1ef0aa9f955f794fb69c) Co-authored-by: Jeff Lyon <146767590+secengjeff@users.noreply.github.com> --- Lib/test/test_webbrowser.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 51d627d24c5a8a..82f14ca968f266 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -340,6 +340,10 @@ def close(self): @requires_subprocess() class MacOSTest(unittest.TestCase): + def setUp(self): + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env.unset("BROWSER") + def test_default(self): browser = webbrowser.get() self.assertIsInstance(browser, webbrowser.MacOS) From 8b73ce9ab86a0048a4e9a74ce71397b41f4da3f0 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 13 May 2026 02:30:22 +0200 Subject: [PATCH 038/110] [3.15] gh-149718: Aggregate same stack frames in Tachyon in some collectors (GH-149719) (#149747) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-149718: Aggregate same stack frames in Tachyon in some collectors (GH-149719) (cherry picked from commit 76f22853410d3ded872cbfe1430852cf8c048962) Co-authored-by: Maurycy Pawłowski-Wieroński --- Lib/profiling/sampling/collector.py | 2 + Lib/profiling/sampling/gecko_collector.py | 2 + Lib/profiling/sampling/heatmap_collector.py | 7 +- Lib/profiling/sampling/pstats_collector.py | 2 + Lib/profiling/sampling/sample.py | 33 +++++++- Lib/profiling/sampling/stack_collector.py | 2 + Lib/test/test_profiling/test_heatmap.py | 15 ++++ .../test_sampling_profiler/test_profiler.py | 79 ++++++++++++++++++- ...-05-12-13-03-45.gh-issue-149718.SaM1NJ.rst | 4 + 9 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-12-13-03-45.gh-issue-149718.SaM1NJ.rst diff --git a/Lib/profiling/sampling/collector.py b/Lib/profiling/sampling/collector.py index 81ec6344ebdea4..8e0f0c44c4f8f3 100644 --- a/Lib/profiling/sampling/collector.py +++ b/Lib/profiling/sampling/collector.py @@ -143,6 +143,8 @@ def iter_async_frames(awaited_info_list): class Collector(ABC): + aggregating = False + @abstractmethod def collect(self, stack_frames, timestamps_us=None): """Collect profiling data from stack frames. diff --git a/Lib/profiling/sampling/gecko_collector.py b/Lib/profiling/sampling/gecko_collector.py index 8986194268b3ce..54392af9500008 100644 --- a/Lib/profiling/sampling/gecko_collector.py +++ b/Lib/profiling/sampling/gecko_collector.py @@ -63,6 +63,8 @@ class GeckoCollector(Collector): + aggregating = True + def __init__(self, sample_interval_usec, *, skip_idle=False, opcodes=False): self.sample_interval_usec = sample_interval_usec self.skip_idle = skip_idle diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index 5c36d78f5535e7..6e650ec08f410b 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -452,7 +452,8 @@ def process_frames(self, frames, thread_id, weight=1): next_lineno = extract_lineno(next_frame[1]) self._record_call_relationship( (filename, lineno, funcname), - (next_frame[0], next_lineno, next_frame[2]) + (next_frame[0], next_lineno, next_frame[2]), + weight=weight, ) def _is_valid_frame(self, filename, lineno): @@ -561,7 +562,7 @@ def _get_bytecode_data_for_line(self, filename, lineno): result.sort(key=lambda x: (-x['samples'], x['opcode'])) return result - def _record_call_relationship(self, callee_frame, caller_frame): + def _record_call_relationship(self, callee_frame, caller_frame, weight=1): """Record caller/callee relationship between adjacent frames.""" callee_filename, callee_lineno, callee_funcname = callee_frame caller_filename, caller_lineno, caller_funcname = caller_frame @@ -587,7 +588,7 @@ def _record_call_relationship(self, callee_frame, caller_frame): # Count this call edge for path analysis edge_key = (caller_key, callee_key) - self.edge_samples[edge_key] += 1 + self.edge_samples[edge_key] += weight def export(self, output_path): """Export heatmap data as HTML files in a directory. diff --git a/Lib/profiling/sampling/pstats_collector.py b/Lib/profiling/sampling/pstats_collector.py index 50500296c15acc..43b1daf2a119d4 100644 --- a/Lib/profiling/sampling/pstats_collector.py +++ b/Lib/profiling/sampling/pstats_collector.py @@ -8,6 +8,8 @@ class PstatsCollector(Collector): + aggregating = True + def __init__(self, sample_interval_usec, *, skip_idle=False): self.result = collections.defaultdict( lambda: dict(total_rec_calls=0, direct_calls=0, cumulative_calls=0) diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 5bbe2483581333..b9e7e2625d09e4 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -47,6 +47,9 @@ def _pause_threads(unwinder, blocking): # If fewer samples are collected, we skip the TUI and just print a message MIN_SAMPLES_FOR_TUI = 200 +# Maximum number of consecutive identical samples to keep before flushing. +MAX_PENDING_SAMPLES = 8192 + class SampleProfiler: def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, native=False, gc=True, opcodes=False, skip_non_matching_threads=True, collect_stats=False, blocking=False): self.pid = pid @@ -109,6 +112,20 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): last_sample_time = start_time realtime_update_interval = 1.0 # Update every second last_realtime_update = start_time + aggregating = getattr(collector, 'aggregating', False) is True + prev_stack = None + pending_count = 0 + pending_timestamps = [] if aggregating else None + + def flush_pending(): + nonlocal pending_count, pending_timestamps + if pending_count == 0: + return + pending_count = 0 + ts = pending_timestamps + pending_timestamps = [] + collector.collect(prev_stack, timestamps_us=ts) + try: while duration_sec is None or running_time_sec < duration_sec: # Check if live collector wants to stop @@ -116,6 +133,7 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): break current_time = time.perf_counter() + current_time_us = int(current_time * 1_000_000) if next_time > current_time: sleep_time = (next_time - current_time) * 0.9 if sleep_time > 0.0001: @@ -125,13 +143,24 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): stack_frames = self._get_stack_trace( async_aware=async_aware ) - collector.collect(stack_frames) + if aggregating: + if stack_frames != prev_stack: + flush_pending() + prev_stack = stack_frames + pending_count += 1 + pending_timestamps.append(current_time_us) + if pending_count >= MAX_PENDING_SAMPLES: + flush_pending() + else: + collector.collect(stack_frames) except ProcessLookupError as e: running_time_sec = current_time - start_time break except (RuntimeError, UnicodeDecodeError, MemoryError, OSError): + flush_pending() collector.collect_failed_sample() errors += 1 + prev_stack = None except Exception as e: if not _is_process_running(self.pid): break @@ -163,6 +192,8 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): interrupted = True running_time_sec = time.perf_counter() - start_time print("Interrupted by user.") + finally: + flush_pending() # Clear real-time stats line if it was being displayed if self.realtime_stats and len(self.sample_intervals) > 0: diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 60df026ed76a6c..42281dc6454c83 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -16,6 +16,8 @@ class StackTraceCollector(Collector): + aggregating = True + def __init__(self, sample_interval_usec, *, skip_idle=False): self.sample_interval_usec = sample_interval_usec self.skip_idle = skip_idle diff --git a/Lib/test/test_profiling/test_heatmap.py b/Lib/test/test_profiling/test_heatmap.py index b2acb1cf577341..ee27fdd3fa3053 100644 --- a/Lib/test/test_profiling/test_heatmap.py +++ b/Lib/test/test_profiling/test_heatmap.py @@ -345,6 +345,21 @@ def test_process_frames_tracks_edge_samples(self): # Check that edge count is tracked self.assertGreater(len(collector.edge_samples), 0) + def test_process_frames_weight_applies_to_identical_samples(self): + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('callee.py', (5, 5, -1, -1), 'callee', None), + ('caller.py', (10, 10, -1, -1), 'caller', None), + ] + + collector.process_frames(frames, thread_id=1, weight=5) + + edge_key = (('caller.py', 10), ('callee.py', 5)) + self.assertEqual(collector.edge_samples[edge_key], 5) + self.assertEqual(collector.line_samples[('callee.py', 5)], 5) + self.assertEqual(collector.line_samples[('caller.py', 10)], 5) + def test_process_frames_handles_empty_frames(self): """Test that process_frames handles empty frame list.""" collector = HeatmapCollector(sample_interval_usec=100) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py index 68bc59a5414a05..2f5a5e27328659 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py @@ -198,8 +198,83 @@ def test_sample_profiler_sample_method_timing(self): self.assertIn("samples", result) # Verify collector was called multiple times - self.assertGreaterEqual(mock_collector.collect.call_count, 5) - self.assertLessEqual(mock_collector.collect.call_count, 11) + total_weight = sum( + len(c.kwargs.get("timestamps_us") or [None]) + for c in mock_collector.collect.call_args_list + ) + self.assertGreaterEqual(total_weight, 5) + self.assertLessEqual(total_weight, 11) + + def test_sample_profiler_does_not_buffer_non_aggregating_collectors(self): + """Test that non-aggregating collectors get each sample immediately.""" + + stack_frames = [mock.sentinel.stack_frames] + mock_collector = mock.MagicMock() + mock_collector.aggregating = False + + with self._patched_unwinder() as u: + u.instance.get_stack_trace.return_value = stack_frames + + manager = mock.Mock() + manager.attach_mock(u.instance.get_stack_trace, "unwind") + manager.attach_mock(mock_collector.collect, "collect") + + profiler = SampleProfiler( + pid=12345, sample_interval_usec=10000, all_threads=False + ) + + times = [0.0, 0.01, 0.011, 0.02, 0.03] + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.025) + + self.assertEqual( + manager.mock_calls, + [ + mock.call.unwind(), + mock.call.collect(stack_frames), + mock.call.unwind(), + mock.call.collect(stack_frames), + ], + ) + + def test_sample_profiler_flushes_aggregated_batches_at_limit(self): + """Test that aggregating collectors flush after MAX_PENDING_SAMPLES samples.""" + + stack_frames = [mock.sentinel.stack_frames] + mock_collector = mock.MagicMock() + mock_collector.aggregating = True + + with self._patched_unwinder() as u: + u.instance.get_stack_trace.return_value = stack_frames + + profiler = SampleProfiler( + pid=12345, sample_interval_usec=10000, all_threads=False + ) + + times = [ + 0.0, + 0.01, 0.011, + 0.02, 0.021, + 0.03, 0.031, + 0.04, 0.041, + 0.05, 0.051, + ] + with mock.patch("profiling.sampling.sample.MAX_PENDING_SAMPLES", 2): + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.045) + + batches = [ + (c.args[0], len(c.kwargs["timestamps_us"])) + for c in mock_collector.collect.call_args_list + ] + self.assertEqual( + batches, + [(stack_frames, 2), (stack_frames, 2), (stack_frames, 1)], + ) def test_sample_profiler_error_handling(self): """Test that the sample method handles errors gracefully.""" diff --git a/Misc/NEWS.d/next/Library/2026-05-12-13-03-45.gh-issue-149718.SaM1NJ.rst b/Misc/NEWS.d/next/Library/2026-05-12-13-03-45.gh-issue-149718.SaM1NJ.rst new file mode 100644 index 00000000000000..25344e5a90f022 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-12-13-03-45.gh-issue-149718.SaM1NJ.rst @@ -0,0 +1,4 @@ +Coalesce consecutive identical stack frames in Tachyon, so aggregating +collectors (pstats, collapsed, flamegraph, gecko) receive one collect. +Improves sample rate 3x, error rate and missed rate drop by 70%. Patch by +Maurycy Pawłowski-Wieroński. From bc20c2b14604f934d598835e6528211785a4b703 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 13 May 2026 05:08:49 +0200 Subject: [PATCH 039/110] [3.15] gh-149574: Document that is_typeddict, is_protocol, is_dataclass, isclass return False for generic aliases (GH-149604) (#149750) gh-149574: Document that is_typeddict, is_protocol, is_dataclass, isclass return False for generic aliases (GH-149604) (cherry picked from commit a4e51c8dac9fdd49ae26ff8c6cd3c808fd8ba15e) Co-authored-by: Jelle Zijlstra --- Doc/library/dataclasses.rst | 3 ++- Doc/library/inspect.rst | 3 +++ Doc/library/stdtypes.rst | 3 ++- Doc/library/typing.rst | 30 +++++++++++++++++++++++++++--- 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 0bce3e5b762b8b..a09c28ad979158 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -498,7 +498,8 @@ Module contents .. function:: is_dataclass(obj) Return ``True`` if its parameter is a dataclass (including subclasses of a - dataclass) or an instance of one, otherwise return ``False``. + dataclass, but not including :ref:`generic aliases `) + or an instance of one, otherwise return ``False``. If you need to know if a class is an instance of a dataclass (and not a dataclass itself), then add a further check for ``not diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 8713765b8aebfb..48ae9147587c64 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -416,6 +416,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a class, whether built-in or created in Python code. + This function returns ``False`` for :ref:`generic aliases ` of classes, + such as ``list[int]``. + .. function:: ismethod(object) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 3d943566be34ff..e3bd1a46891adc 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -5858,7 +5858,8 @@ type and the :class:`bytes` data type: ``GenericAlias`` objects are instances of the class :class:`types.GenericAlias`, which can also be used to create ``GenericAlias`` -objects directly. +objects directly. Specializations of user-defined :ref:`generic classes ` +may not be instances of :class:`types.GenericAlias`, but they provide similar functionality. .. describe:: T[X, Y, ...] diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index dca51b8014da5a..71b395c80166cc 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -3633,14 +3633,27 @@ Introspection helpers Determine if a type is a :class:`Protocol`. - For example:: + For example: + + .. testcode:: class P(Protocol): def a(self) -> str: ... b: int - is_protocol(P) # => True - is_protocol(int) # => False + assert is_protocol(P) + assert not is_protocol(int) + + This function only returns true for ``Protocol`` classes, not for + :ref:`generic aliases ` of them: + + .. testcode:: + + class GenericP[T](Protocol): + def a(self) -> T: ... + b: int + + assert not is_protocol(GenericP[int]) .. versionadded:: 3.13 @@ -3663,6 +3676,17 @@ Introspection helpers # not a typed dict itself assert not is_typeddict(TypedDict) + This function only returns true for ``TypedDict`` classes, not for + :ref:`generic aliases ` of them: + + .. testcode:: + + class GenericFilm[T](TypedDict): + title: str + year: T + + assert not is_typeddict(GenericFilm[int]) + .. versionadded:: 3.10 .. class:: ForwardRef From 15a597e9ba0dd79bf953e181251788782fff6164 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 13 May 2026 05:36:32 +0200 Subject: [PATCH 040/110] [3.15] gh-149642: Fix interaction between exec and lazy_imports=all (GH-149643) (#149749) gh-149642: Fix interaction between exec and lazy_imports=all (GH-149643) (cherry picked from commit 4087ff859958abc897711b501bb66dc308890ba5) Co-authored-by: Jelle Zijlstra --- Lib/test/test_lazy_import/__init__.py | 53 +++++++++++++++++++ ...-05-10-07-42-36.gh-issue-149642.6ZksML.rst | 2 + Python/ceval.c | 16 ++++-- 3 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-07-42-36.gh-issue-149642.6ZksML.rst diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index ea534a8ee5b981..5d770eeae07a15 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -301,6 +301,15 @@ def f(): f() self.assertIn("only allowed at module level", str(cm.exception)) + def test_lazy_import_exec_in_class(self): + """lazy import via exec() inside a class should raise SyntaxError.""" + # exec() inside a class body also has non-module-level locals. + with self.assertRaises(SyntaxError) as cm: + class C: + exec("lazy import json") + + self.assertIn("only allowed at module level", str(cm.exception)) + @support.requires_subprocess() def test_lazy_import_exec_at_module_level(self): """lazy import via exec() at module level should work.""" @@ -352,6 +361,50 @@ def test_eager_import_func(self): f = test.test_lazy_import.data.eager_import_func.f self.assertEqual(type(f()), type(sys)) + def test_exec_import_func(self): + """Implicit lazy imports via exec() inside functions should be eager.""" + sys.set_lazy_imports("all") + + def f(): + exec("import test.test_lazy_import.data.basic2") + + f() + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_exec_import_func_with_lazy_modules(self): + """__lazy_modules__ should not make exec() imports lazy inside functions.""" + globals()["__lazy_modules__"] = ["test.test_lazy_import.data.basic2"] + try: + def f(): + exec("import test.test_lazy_import.data.basic2") + + f() + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + finally: + del globals()["__lazy_modules__"] + + def test_exec_import_class(self): + """Implicit lazy imports via exec() inside classes should be eager.""" + sys.set_lazy_imports("all") + + class C: + exec("import test.test_lazy_import.data.basic2") + + self.assertIsNotNone(C) + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_exec_import_class_with_lazy_modules(self): + """__lazy_modules__ should not make exec() imports lazy inside classes.""" + globals()["__lazy_modules__"] = ["test.test_lazy_import.data.basic2"] + try: + class C: + exec("import test.test_lazy_import.data.basic2") + + self.assertIsNotNone(C) + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + finally: + del globals()["__lazy_modules__"] + class WithStatementTests(unittest.TestCase): """Tests for lazy imports in with statement context.""" diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-07-42-36.gh-issue-149642.6ZksML.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-07-42-36.gh-issue-149642.6ZksML.rst new file mode 100644 index 00000000000000..815a084db69d8d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-07-42-36.gh-issue-149642.6ZksML.rst @@ -0,0 +1,2 @@ +Allow imports inside ``exec()`` calls within functions under +``PYTHON_LAZY_IMPORTS=all``. diff --git a/Python/ceval.c b/Python/ceval.c index 060e948e6b01c9..a080ae42b93766 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3059,25 +3059,35 @@ check_lazy_import_compatibility(PyThreadState *tstate, PyObject *globals, return res; } +static int +is_lazy_import_module_level(void) +{ + _PyInterpreterFrame *frame = _PyEval_GetFrame(); + return frame != NULL && frame->f_globals == frame->f_locals; +} + PyObject * _PyEval_LazyImportName(PyThreadState *tstate, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level, int lazy) { PyObject *res = NULL; + PyImport_LazyImportsMode mode = PyImport_GetLazyImportsMode(); // Check if global policy overrides the local syntax - switch (PyImport_GetLazyImportsMode()) { + switch (mode) { case PyImport_LAZY_NONE: lazy = 0; break; case PyImport_LAZY_ALL: - lazy = 1; + if (!lazy) { + lazy = is_lazy_import_module_level(); + } break; case PyImport_LAZY_NORMAL: break; } - if (!lazy && PyImport_GetLazyImportsMode() != PyImport_LAZY_NONE) { + if (!lazy && mode != PyImport_LAZY_NONE && is_lazy_import_module_level()) { // See if __lazy_modules__ forces this to be lazy. lazy = check_lazy_import_compatibility(tstate, globals, name, level); if (lazy < 0) { From 4e369c1deaea5843d08f83c0dadcb4f2cc4de3a9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 13 May 2026 11:28:27 +0200 Subject: [PATCH 041/110] [3.15] gh-149701: Fully silence potential `hash -r` error (GH-149702) (GH-149757) (cherry picked from commit cd6096887e22cdb6d6365ad0eb5b0ffac50d4791) --- Lib/test/test_venv.py | 20 +++++++++++++++++++ Lib/venv/scripts/common/activate | 4 ++-- ...-05-12-06-24-54.gh-issue-149701.8v9RTm.rst | 1 + 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-12-06-24-54.gh-issue-149701.8v9RTm.rst diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 78461abcd69f33..a42787f261bfe8 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -656,6 +656,26 @@ def test_deactivate_with_strict_bash_opts(self): self.assertEqual(out, "".encode()) self.assertEqual(err, "".encode()) + # gh-149701: Test exit code is zero even when hashing is disabled + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') + def test_deactivate_with_strict_bash_opts_and_hashing_disabled(self): + bash = shutil.which("bash") + if bash is None: + self.skipTest("bash required for this test") + rmtree(self.env_dir) + builder = venv.EnvBuilder(clear=True) + builder.create(self.env_dir) + activate = os.path.join(self.env_dir, self.bindir, "activate") + test_script = os.path.join(self.env_dir, "test_hash_disabled.sh") + with open(test_script, "w") as f: + f.write("set -euo pipefail\n" + "set +h\n" # disable hashing + f"source {activate}\n" + "deactivate") + out, err = check_output([bash, test_script]) + self.assertEqual(out, "".encode()) + self.assertEqual(err, "".encode()) + @unittest.skipUnless(sys.platform == 'darwin', 'only relevant on macOS') def test_macos_env(self): diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 70673a265d41f8..241a8650bda33a 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -17,7 +17,7 @@ deactivate () { # Call hash to forget past locations. Without forgetting # past locations the $PATH changes we made may not be respected. # See "man bash" for more details. hash is usually a builtin of your shell - hash -r 2> /dev/null + hash -r 2> /dev/null || true if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then PS1="${_OLD_VIRTUAL_PS1:-}" @@ -73,4 +73,4 @@ fi # Call hash to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected -hash -r 2> /dev/null +hash -r 2> /dev/null || true diff --git a/Misc/NEWS.d/next/Library/2026-05-12-06-24-54.gh-issue-149701.8v9RTm.rst b/Misc/NEWS.d/next/Library/2026-05-12-06-24-54.gh-issue-149701.8v9RTm.rst new file mode 100644 index 00000000000000..676d788cbce62a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-12-06-24-54.gh-issue-149701.8v9RTm.rst @@ -0,0 +1 @@ +Fix bad return code from Lib/venv/bin/activate if hashing is disabled From a5f77a13fdce49bcc6699ab4e7ec8b88e1d261af Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 13 May 2026 12:39:38 +0200 Subject: [PATCH 042/110] [3.15] gh-148829: Add PySentinel_CheckExact() (GH-149725) (#149766) gh-148829: Add PySentinel_CheckExact() (GH-149725) (cherry picked from commit 94df62542cdf1c9eb082abab1534cbd1fd425062) Co-authored-by: scoder --- Doc/c-api/sentinel.rst | 16 ++++++++++++++-- Include/cpython/sentinelobject.h | 5 ++++- Lib/test/test_capi/test_object.py | 2 ++ ...026-05-12-16-47-21.gh-issue-149725.HZLBTZ.rst | 2 ++ Modules/_testcapi/object.c | 7 +++++++ 5 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-05-12-16-47-21.gh-issue-149725.HZLBTZ.rst diff --git a/Doc/c-api/sentinel.rst b/Doc/c-api/sentinel.rst index 89e0a28bf3b835..937cae18e86f50 100644 --- a/Doc/c-api/sentinel.rst +++ b/Doc/c-api/sentinel.rst @@ -14,8 +14,20 @@ Sentinel objects .. c:function:: int PySentinel_Check(PyObject *o) - Return true if *o* is a :class:`sentinel` object. The :class:`sentinel` type - does not allow subclasses, so this check is exact. + Return true if *o* is a :class:`sentinel` object or a subtype. + The :class:`sentinel` type does not currently allow subclasses, + so this check is exact. + Future Python versions may choose to allow subtyping. + This function always succeeds. + + .. versionadded:: 3.15 + +.. c:function:: int PySentinel_CheckExact(PyObject *o) + + Return true if *o* is a :class:`sentinel` object, but not a subtype. + The :class:`sentinel` type does not currently allow subclasses. + Future Python versions may choose to allow subtyping. + This function always succeeds. .. versionadded:: 3.15 diff --git a/Include/cpython/sentinelobject.h b/Include/cpython/sentinelobject.h index 0b6ff0f17e6f8c..8d5b1886ce5436 100644 --- a/Include/cpython/sentinelobject.h +++ b/Include/cpython/sentinelobject.h @@ -9,7 +9,10 @@ extern "C" { PyAPI_DATA(PyTypeObject) PySentinel_Type; -#define PySentinel_Check(op) Py_IS_TYPE((op), &PySentinel_Type) +#define PySentinel_CheckExact(op) Py_IS_TYPE((op), &PySentinel_Type) + +/* Alias as long as subclasses are not allowed. */ +#define PySentinel_Check(op) PySentinel_CheckExact(op) PyAPI_FUNC(PyObject *) PySentinel_New( const char *name, diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py index 635deaa73f7efa..e6fd068dc20d8d 100644 --- a/Lib/test/test_capi/test_object.py +++ b/Lib/test/test_capi/test_object.py @@ -71,6 +71,8 @@ def test_pysentinel_new(self): self.assertIs(type(marker), sentinel) self.assertTrue(_testcapi.pysentinel_check(marker)) self.assertFalse(_testcapi.pysentinel_check(object())) + self.assertTrue(_testcapi.pysentinel_checkexact(marker)) + self.assertFalse(_testcapi.pysentinel_checkexact(object())) self.assertEqual(marker.__name__, "CAPI_SENTINEL") self.assertEqual(marker.__module__, __name__) self.assertEqual(repr(marker), "CAPI_SENTINEL") diff --git a/Misc/NEWS.d/next/C_API/2026-05-12-16-47-21.gh-issue-149725.HZLBTZ.rst b/Misc/NEWS.d/next/C_API/2026-05-12-16-47-21.gh-issue-149725.HZLBTZ.rst new file mode 100644 index 00000000000000..97721430edbd69 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-05-12-16-47-21.gh-issue-149725.HZLBTZ.rst @@ -0,0 +1,2 @@ +Add :c:func:`PySentinel_CheckExact` for exact :class:`sentinel` type tests +to accompany the existing :c:func:`PySentinel_Check`. diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 6e5c8dcbb725fa..c62dc1144df688 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -572,6 +572,12 @@ pysentinel_check(PyObject *self, PyObject *obj) return PyBool_FromLong(PySentinel_Check(obj)); } +static PyObject * +pysentinel_checkexact(PyObject *self, PyObject *obj) +{ + return PyBool_FromLong(PySentinel_CheckExact(obj)); +} + static PyMethodDef test_methods[] = { {"call_pyobject_print", call_pyobject_print, METH_VARARGS}, @@ -604,6 +610,7 @@ static PyMethodDef test_methods[] = { {"pyobject_dump", pyobject_dump, METH_VARARGS}, {"pysentinel_new", pysentinel_new, METH_VARARGS}, {"pysentinel_check", pysentinel_check, METH_O}, + {"pysentinel_checkexact", pysentinel_checkexact, METH_O}, {NULL}, }; From 37f3deb571c02eccd8edc1457fcfc0eeeac909ce Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 13 May 2026 12:54:10 +0200 Subject: [PATCH 043/110] [3.15] bpo-45509: Check gzip headers for corrupted fields (GH-29028) (GH-149769) Check the header checksum it the HCRC field is present. (cherry picked from commit dd94457893a1dd2c99c2405e197f54a7692cbe09) Co-authored-by: Ruben Vorderman --- Lib/gzip.py | 51 ++++++++++++++----- Lib/test/test_gzip.py | 29 +++++++++++ .../2021-10-18-13-46-55.bpo-45509.Upwb60.rst | 1 + 3 files changed, 67 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-10-18-13-46-55.bpo-45509.Upwb60.rst diff --git a/Lib/gzip.py b/Lib/gzip.py index 971063aa24f871..a89ebf806c8572 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -484,40 +484,63 @@ def _read_exact(fp, n): return data +def _read_until_null(fp, append_to): + '''Read until the first encountered null byte in fp. + Append to given byte array object''' + while True: + s = fp.read(1) + append_to += s + if not s or s == b'\000': + break + + def _read_gzip_header(fp): '''Read a gzip header from `fp` and progress to the end of the header. Returns last mtime if header was present or None otherwise. ''' magic = fp.read(2) - if magic == b'': + if not magic: return None if magic != b'\037\213': raise BadGzipFile('Not a gzipped file (%r)' % magic) - - (method, flag, last_mtime) = struct.unpack(" Date: Wed, 13 May 2026 13:10:30 +0200 Subject: [PATCH 044/110] [3.15] gh-148821: Add more strict tests for XML encodings (GH-149765) (GH-149770) Exclude encodings like 'utf-8-sig', 'iso2022-jp' and 'hz' from the list of supported encodings. (cherry picked from commit fa2afa64d9467fb7362672ed603d29d8e246d240) Co-authored-by: Serhiy Storchaka --- Lib/test/test_pyexpat.py | 76 +++++++++++++++++++++++++++++++++++++- Lib/test/test_xml_etree.py | 27 ++++++++------ 2 files changed, 89 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 9a1620029c6da9..4fe2e02326f04f 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -227,8 +227,7 @@ def _verify_parse_output(self, operations): "Character data: '\xb5'", "End element: 'root'", ] - for operation, expected_operation in zip(operations, expected_operations): - self.assertEqual(operation, expected_operation) + self.assertEqual(operations, expected_operations) def test_parse_bytes(self): out = self.Outputter() @@ -276,6 +275,79 @@ def test_parse_again(self): self.assertEqual(expat.ErrorString(cm.exception.code), expat.errors.XML_ERROR_FINISHED) + @support.subTests('encoding', [ + 'utf-8', 'utf-16', 'utf-16be', 'utf-16le', + 'iso8859-1', 'iso8859-2', 'iso8859-3', 'iso8859-4', 'iso8859-5', + 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', + 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', + 'cp437', 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', + 'cp855', 'cp856', 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', + 'cp863', 'cp865', 'cp866', 'cp869', 'cp874', 'cp1006', 'cp1125', + 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255', + 'cp1256', 'cp1257', 'cp1258', + 'mac-cyrillic', 'mac-greek', 'mac-iceland', 'mac-latin2', + 'mac-roman', 'mac-turkish', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'ptcp154', + ]) + def test_supported_ecodings(self, encoding): + out = self.Outputter() + parser = expat.ParserCreate() + self._hookup_callbacks(parser, out) + c = 'éπя\u05d0\u060c€'.encode(encoding, 'ignore').decode(encoding)[0] + data = (f'\n' + f'{c}').encode(encoding) + parser.Parse(data, True) + self.assertEqual(out.out, [ + ('XML declaration', ('1.0', encoding, -1)), + "Start element: 'root' {}", + f'Character data: {c!r}', + "End element: 'root'", + ]) + + @support.subTests('encoding', [ + 'UTF-8', 'utf-8', 'utf-16', 'utf-16le', 'utf-16be', + 'koi8-u', 'cp1125', 'cp1251', 'iso8859-5', 'mac-cyrillic', + ]) + def test_supported_ecodings2(self, encoding): + out = self.Outputter() + parser = expat.ParserCreate() + self._hookup_callbacks(parser, out) + data = (f'\n' + '' + '<корінь атрибут="значення">зміст').encode(encoding) + parser.Parse(data, True) + self.assertEqual(out.out, [ + ('XML declaration', ('1.0', encoding, -1)), + "Comment: ' коментар '", + "Start element: 'корінь' {'атрибут': 'значення'}", + "Character data: 'зміст'", + "End element: 'корінь'", + ]) + + @support.subTests('encoding', [ + 'UTF-7', + "Big5-HKSCS", "Big5", + "cp932", "cp949", "cp950", + "EUC_JIS-2004", "EUC_JISX0213", "EUC-JP", "EUC-KR", + "GB18030", "GB2312", "GBK", + "ISO-2022-KR", + "johab", + "Shift_JIS", "Shift_JIS-2004", "Shift_JISX0213", + ]) + def test_unsupportes_ecodings(self, encoding): + parser = expat.ParserCreate() + data = (f'\n' + '').encode(encoding) + with self.assertRaises(ValueError): + parser.Parse(data, True) + + def test_unknown_ecoding(self): + parser = expat.ParserCreate() + data = b'\n' + with self.assertRaises(LookupError): + parser.Parse(data, True) + + class NamespaceSeparatorTest(unittest.TestCase): def test_legal(self): # Tests that make sure we get errors when the namespace_separator value diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 8f3efe9fc90794..3a41ea97a2e0a2 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1009,12 +1009,12 @@ def check(encoding, body=''): check("cp437", '\u221a') check("mac-roman", '\u02da') - def xml(encoding): - return "" % encoding - def bxml(encoding): - return xml(encoding).encode(encoding) + def xml(encoding, body=''): + return "%s" % (encoding, body) + def bxml(encoding, body=''): + return xml(encoding, body).encode(encoding) supported_encodings = [ - 'ascii', 'utf-8', 'utf-8-sig', 'utf-16', 'utf-16be', 'utf-16le', + 'utf-8', 'utf-16', 'utf-16be', 'utf-16le', 'iso8859-1', 'iso8859-2', 'iso8859-3', 'iso8859-4', 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', @@ -1025,13 +1025,14 @@ def bxml(encoding): 'cp1256', 'cp1257', 'cp1258', 'mac-cyrillic', 'mac-greek', 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', - 'iso2022-jp', 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', - 'iso2022-jp-3', 'iso2022-jp-ext', - 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', - 'hz', 'ptcp154', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'ptcp154', ] for encoding in supported_encodings: - self.assertEqual(ET.tostring(ET.XML(bxml(encoding))), b'') + with self.subTest(encoding=encoding): + self.assertEqual(ET.tostring(ET.XML(bxml(encoding))), b'') + c = 'éπя\u05d0\u060c€'.encode(encoding, 'ignore').decode(encoding)[0] + self.assertEqual(ET.tostring(ET.XML(bxml(encoding, c))), + ('&#%d;' % ord(c)).encode()) unsupported_ascii_compatible_encodings = [ 'big5', 'big5hkscs', @@ -1043,14 +1044,16 @@ def bxml(encoding): 'utf-7', ] for encoding in unsupported_ascii_compatible_encodings: - self.assertRaises(ValueError, ET.XML, bxml(encoding)) + with self.subTest(encoding=encoding): + self.assertRaises(ValueError, ET.XML, bxml(encoding)) unsupported_ascii_incompatible_encodings = [ 'cp037', 'cp424', 'cp500', 'cp864', 'cp875', 'cp1026', 'cp1140', 'utf_32', 'utf_32_be', 'utf_32_le', ] for encoding in unsupported_ascii_incompatible_encodings: - self.assertRaises(ET.ParseError, ET.XML, bxml(encoding)) + with self.subTest(encoding=encoding): + self.assertRaises(ET.ParseError, ET.XML, bxml(encoding)) self.assertRaises(ValueError, ET.XML, xml('undefined').encode('ascii')) self.assertRaises(LookupError, ET.XML, xml('xxx').encode('ascii')) From b37e91f6cfc6bd3f51a0f020cdf9368b24489087 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 13 May 2026 13:33:31 +0200 Subject: [PATCH 045/110] [3.15] Run mypy with four worker processes and uv (GH-149726) (#149773) Run mypy with four worker processes and uv (GH-149726) (cherry picked from commit 6304eb1f5b93f682bff558befe4a7b9585f4601e) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/mypy.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 490c32ecfc9a62..d748b6ff63e68a 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -69,12 +69,11 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: python-version: "3.15" - allow-prereleases: true - cache: pip - cache-dependency-path: Tools/requirements-dev.txt - - run: pip install -r Tools/requirements-dev.txt + activate-environment: true + cache-dependency-glob: Tools/requirements-dev.txt + - run: uv pip install -r Tools/requirements-dev.txt - run: python3 Misc/mypy/make_symlinks.py --symlink - - run: mypy --config-file ${{ matrix.target }}/mypy.ini + - run: mypy --num-workers 4 --config-file ${{ matrix.target }}/mypy.ini From bb7a539da2fdd81724ecf8d899aa369a247709b9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 13 May 2026 17:36:27 +0200 Subject: [PATCH 046/110] [3.15] gh-149776: Skip UDP Lite tests if it's not supported (GH-149777) (#149780) gh-149776: Skip UDP Lite tests if it's not supported (GH-149777) Fix test_socket on Linux kernel 7.1 and newer: skip UDP Lite tests if it's not supported. (cherry picked from commit 3cfc249e11a132dc69624150843779aa96c72b2b) Co-authored-by: Victor Stinner --- Lib/test/test_socket.py | 24 +++++++++++++++---- ...-05-13-14-53-23.gh-issue-149776.orqgsn.rst | 2 ++ 2 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2026-05-13-14-53-23.gh-issue-149776.orqgsn.rst diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 9e03069494345b..47830d0e9645ef 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -205,6 +205,25 @@ def _have_socket_hyperv(): return True +def _have_udp_lite(): + if not hasattr(socket, "IPPROTO_UDPLITE"): + return False + # Older Android versions block UDPLITE with SELinux. + if support.is_android and platform.android_ver().api_level < 29: + return False + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDPLITE) + except OSError as exc: + # Linux 7.1 removed UDP Lite support + if exc.errno == errno.EPROTONOSUPPORT: + return False + raise + sock.close() + + return True + + @contextlib.contextmanager def socket_setdefaulttimeout(timeout): old_timeout = socket.getdefaulttimeout() @@ -247,10 +266,7 @@ def downgrade_malformed_data_warning(): HAVE_SOCKET_VSOCK = _have_socket_vsock() -# Older Android versions block UDPLITE with SELinux. -HAVE_SOCKET_UDPLITE = ( - hasattr(socket, "IPPROTO_UDPLITE") - and not (support.is_android and platform.android_ver().api_level < 29)) +HAVE_SOCKET_UDPLITE = _have_udp_lite() HAVE_SOCKET_BLUETOOTH = _have_socket_bluetooth() diff --git a/Misc/NEWS.d/next/Tests/2026-05-13-14-53-23.gh-issue-149776.orqgsn.rst b/Misc/NEWS.d/next/Tests/2026-05-13-14-53-23.gh-issue-149776.orqgsn.rst new file mode 100644 index 00000000000000..e86a9130ff9bfb --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-05-13-14-53-23.gh-issue-149776.orqgsn.rst @@ -0,0 +1,2 @@ +Fix test_socket on Linux kernel 7.1 and newer: skip UDP Lite tests if it's +not supported. Patch by Victor Stinner. From 166c56b31162924ddbb594b67169004882dd1c07 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 13 May 2026 20:09:33 +0200 Subject: [PATCH 047/110] [3.15] gh-148906: fix performance scaling of descriptors on free-threading (GH-148915) (#149798) gh-148906: fix performance scaling of descriptors on free-threading (GH-148915) (cherry picked from commit 94bca40ff09c20f6168d6a27e3aa42bf8a8077b8) Co-authored-by: Kumar Aditya --- Objects/typeobject.c | 36 ++++++++++++++++++-------- Tools/ftscalingbench/ftscalingbench.py | 17 ++++++++++++ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 9a18ca72516da7..7cca137f74be58 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4841,6 +4841,18 @@ type_new_set_attrs(const type_new_ctx *ctx, PyTypeObject *type) if (type_new_set_classdictcell(dict) < 0) { return -1; } + +#ifdef Py_GIL_DISABLED + // enable deferred reference counting on functions and descriptors + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(dict, &pos, &key, &value)) { + if (PyFunction_Check(value) || Py_TYPE(value)->tp_descr_get != NULL) { + PyUnstable_Object_EnableDeferredRefcount(value); + } + } +#endif + return 0; } @@ -6746,12 +6758,11 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_MANAGED_DICT)); #ifdef Py_GIL_DISABLED - // gh-139103: Enable deferred refcounting for functions assigned - // to type objects. This is important for `dataclass.__init__`, - // which is generated dynamically. - if (value != NULL && - PyFunction_Check(value) && - !_PyObject_HasDeferredRefcount(value)) + // gh-139103: Enable deferred refcounting for functions and descriptors + // assigned to type objects. This is important for `dataclass.__init__`, + // which is generated dynamically, and for descriptor scaling on + // free-threaded builds. + if (value != NULL && (PyFunction_Check(value) || Py_TYPE(value)->tp_descr_get != NULL)) { PyUnstable_Object_EnableDeferredRefcount(value); } @@ -11089,10 +11100,12 @@ static PyObject * slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) { PyTypeObject *tp = Py_TYPE(self); - PyObject *get; - - get = _PyType_LookupRef(tp, &_Py_ID(__get__)); - if (get == NULL) { + PyThreadState *tstate = _PyThreadState_GET(); + _PyCStackRef cref; + _PyThreadState_PushCStackRef(tstate, &cref); + _PyType_LookupStackRefAndVersion(tp, &_Py_ID(__get__), &cref.ref); + if (PyStackRef_IsNull(cref.ref)) { + _PyThreadState_PopCStackRef(tstate, &cref); #ifndef Py_GIL_DISABLED /* Avoid further slowdowns */ if (tp->tp_descr_get == slot_tp_descr_get) @@ -11104,9 +11117,10 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) obj = Py_None; if (type == NULL) type = Py_None; + PyObject *get = PyStackRef_AsPyObjectBorrow(cref.ref); PyObject *stack[3] = {self, obj, type}; PyObject *res = PyObject_Vectorcall(get, stack, 3, NULL); - Py_DECREF(get); + _PyThreadState_PopCStackRef(tstate, &cref); return res; } diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py index 60f43b99c0f69d..c8a914c22a9e13 100644 --- a/Tools/ftscalingbench/ftscalingbench.py +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -279,6 +279,23 @@ def staticmethod_call(): for _ in range(1000 * WORK_SCALE): obj.my_staticmethod() + +class MyDescriptor: + def __get__(self, obj, objtype=None): + return 42 + + def __set__(self, obj, value): + pass + +class MyClassWithDescriptor: + attr = MyDescriptor() + +@register_benchmark +def descriptor(): + obj = MyClassWithDescriptor() + for _ in range(1000 * WORK_SCALE): + obj.attr + @register_benchmark def deepcopy(): x = {'list': [1, 2], 'tuple': (1, None)} From fb3500466d1fa0f79f7db3b06ce966e79707e952 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 13 May 2026 22:26:42 +0200 Subject: [PATCH 048/110] [3.15] gh-128110: Fix rfc2047 whitespace handling in email parser address headers (GH-130749) (#149787) RFC 2047 Section 6.2 requires that "any 'linear-white-space' that separates a pair of adjacent 'encoded-word's is ignored." The modern header value parser correctly implements that for unstructured headers, but had missed a case in structured headers. This could cause a parsed address header to include extraneous spaces in a display-name. Switch to @bitdancer's fix from review feedback. Recharacterize space between ews as fws after parsing in get_phrase. RDM: This fix is dependent on the fact that "subsequent" atoms will never have leading whitespace because that's been consumed already. I don't think it's worth adding extra code for the possibility of leading whitespace because the parser won't produce it. It's a bit of parser fragility in the face of code changes, but I think that's a minor concern given the parser design (which is that it consumes whitespace greedily) (cherry picked from commit 7a4c6dfb8839eb05fb87baf70364680e45001dd4) Co-authored-by: Mike Edmunds Co-authored-by: R David Murray --- Lib/email/_header_value_parser.py | 10 +++ .../test_email/test__header_value_parser.py | 88 +++++++++++++++++++ ...-03-01-13-36-02.gh-issue-128110.9wx_G0.rst | 5 ++ 3 files changed, 103 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-03-01-13-36-02.gh-issue-128110.9wx_G0.rst diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 9873958f5c2790..792072ab9f6128 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -1461,6 +1461,16 @@ def get_phrase(value): else: try: token, value = get_word(value) + if (token[0].token_type == 'encoded-word' + and phrase + and phrase[-1].token_type == 'atom' + and len(phrase[-1]) > 1 + and phrase[-1][-2].token_type == 'encoded-word' + and phrase[-1][-1].token_type == 'cfws' + and not phrase[-1][-1].comments + ): + # linear ws between ews needs special handing... + phrase[-1][-1] = EWWhiteSpaceTerminal(phrase[-1], 'fws') except errors.HeaderParseError: if value[0] in CFWS_LEADER: token, value = get_cfws(value) diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index aded44e85ee336..9d9fe418ee4d06 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -1060,6 +1060,78 @@ def get_phrase_cfws_only_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_phrase(' (foo) ') + def test_get_phrase_adjacent_ew(self): + # "'linear-white-space' that separates a pair of adjacent + # 'encoded-word's is ignored" (rfc2047 section 6.2) + self._test_get_x(parser.get_phrase, '=?ascii?q?Joi?= \t =?ascii?q?ned?=', 'Joined', 'Joined', [], '') + + def test_get_phrase_adjacent_ew_different_encodings(self): + self._test_get_x( + parser.get_phrase, + '=?utf-8?q?B=C3=A9r?= =?iso-8859-1?q?=E9nice?=', 'Bérénice', 'Bérénice', [], '' + ) + + def test_get_phrase_adjacent_ew_encoded_spaces(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Encoded?= =?ascii?q?_spaces_?= =?ascii?q?preserved?=', + 'Encoded spaces preserved', + 'Encoded spaces preserved', + [], + '' + ) + + def test_get_phrase_adjacent_ew_comment_is_not_linear_white_space(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Comment?= (is not) =?ascii?q?linear-white-space?=', + 'Comment (is not) linear-white-space', + 'Comment linear-white-space', + [], + '', + comments=['is not'], + ) + + def test_get_phrase_adjacent_ew_no_error_on_defects(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Def?= =?ascii?q?ect still joins?=', + 'Defect still joins', + 'Defect still joins', + [errors.InvalidHeaderDefect], # whitespace inside encoded word + '' + ) + + def test_get_phrase_adjacent_ew_ignore_non_ew(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?No?= =?join?= for non-ew', + 'No =?join?= for non-ew', + 'No =?join?= for non-ew', + [], + '' + ) + + def test_get_phrase_adjacent_ew_ignore_invalid_ew(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?No?= =?ascii?rot13?wbva= for invalid ew', + 'No =?ascii?rot13?wbva= for invalid ew', + 'No =?ascii?rot13?wbva= for invalid ew', + [], + '' + ) + + def test_get_phrase_adjacent_ew_missing_space(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Joi?==?ascii?q?ned?=', + 'Joined', + 'Joined', + [errors.InvalidHeaderDefect], # missing trailing whitespace + '' + ) + # get_local_part def test_get_local_part_simple(self): @@ -2387,6 +2459,22 @@ def test_get_address_rfc2047_display_name(self): self.assertEqual(address[0].token_type, 'mailbox') + def test_get_address_rfc2047_display_name_adjacent_ews(self): + address = self._test_get_x(parser.get_address, + '=?utf-8?q?B=C3=A9r?= =?utf-8?q?=C3=A9nice?= ', + 'Bérénice ', + 'Bérénice ', + [], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 1) + self.assertEqual(address.mailboxes, + address.all_mailboxes) + self.assertEqual(address.mailboxes[0].display_name, + 'Bérénice') + self.assertEqual(address[0].token_type, + 'mailbox') + def test_get_address_empty_group(self): address = self._test_get_x(parser.get_address, 'Monty Python:;', diff --git a/Misc/NEWS.d/next/Library/2025-03-01-13-36-02.gh-issue-128110.9wx_G0.rst b/Misc/NEWS.d/next/Library/2025-03-01-13-36-02.gh-issue-128110.9wx_G0.rst new file mode 100644 index 00000000000000..b08b1886cff9cf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-01-13-36-02.gh-issue-128110.9wx_G0.rst @@ -0,0 +1,5 @@ +Fix bug in the parsing of :mod:`email` address headers that could result in +extraneous spaces in the decoded text when using a modern email policy. +Space between pairs of adjacent :rfc:`2047` encoded-words is now ignored, per +section 6.2 (and consistent with existing parsing of unstructured +headers like *Subject*). From 63a4007d25160cf200c8710a641412f4de328482 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 13 May 2026 23:29:08 +0200 Subject: [PATCH 049/110] [3.15] gh-149685: Use the _Py prefix for private C macros (GH-149686) (GH-149790) (cherry picked from commit 125f26358ac7ecab98095fa85490e5465bdad698) Co-authored-by: Petr Viktorin --- Include/cpython/sentinelobject.h | 6 +++--- Include/cpython/sliceobject.h | 2 +- Include/cpython/structseq.h | 2 +- Include/internal/pycore_jit_unwind.h | 2 +- Include/internal/pycore_mmap.h | 4 ++-- Include/sliceobject.h | 4 ++-- Include/structseq.h | 4 ++-- Python/pystrhex.c | 6 +++--- configure | 6 +++--- configure.ac | 6 +++--- pyconfig.h.in | 22 +++++++++++----------- 11 files changed, 32 insertions(+), 32 deletions(-) diff --git a/Include/cpython/sentinelobject.h b/Include/cpython/sentinelobject.h index 8d5b1886ce5436..15643ef966af86 100644 --- a/Include/cpython/sentinelobject.h +++ b/Include/cpython/sentinelobject.h @@ -1,8 +1,8 @@ /* Sentinel object interface */ #ifndef Py_LIMITED_API -#ifndef Py_SENTINELOBJECT_H -#define Py_SENTINELOBJECT_H +#ifndef _Py_SENTINELOBJECT_H +#define _Py_SENTINELOBJECT_H #ifdef __cplusplus extern "C" { #endif @@ -21,5 +21,5 @@ PyAPI_FUNC(PyObject *) PySentinel_New( #ifdef __cplusplus } #endif -#endif /* !Py_SENTINELOBJECT_H */ +#endif /* !_Py_SENTINELOBJECT_H */ #endif /* !Py_LIMITED_API */ diff --git a/Include/cpython/sliceobject.h b/Include/cpython/sliceobject.h index 4c3ea1facebc4e..137206eff15b33 100644 --- a/Include/cpython/sliceobject.h +++ b/Include/cpython/sliceobject.h @@ -1,4 +1,4 @@ -#ifndef Py_CPYTHON_SLICEOBJECT_H +#ifndef _Py_CPYTHON_SLICEOBJECT_H # error "this header file must not be included directly" #endif diff --git a/Include/cpython/structseq.h b/Include/cpython/structseq.h index 328fbe86143b02..83a1abcd6f3b34 100644 --- a/Include/cpython/structseq.h +++ b/Include/cpython/structseq.h @@ -1,4 +1,4 @@ -#ifndef Py_CPYTHON_STRUCTSEQ_H +#ifndef _Py_CPYTHON_STRUCTSEQ_H # error "this header file must not be included directly" #endif diff --git a/Include/internal/pycore_jit_unwind.h b/Include/internal/pycore_jit_unwind.h index 508caee97c43ab..7099b88812ce7b 100644 --- a/Include/internal/pycore_jit_unwind.h +++ b/Include/internal/pycore_jit_unwind.h @@ -11,7 +11,7 @@ #if defined(_Py_JIT) && defined(__linux__) && defined(__ELF__) # define PY_HAVE_JIT_GDB_UNWIND # if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ - defined(HAVE_LIBGCC_EH_FRAME_REGISTRATION) + defined(_Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION) # define PY_HAVE_JIT_GNU_BACKTRACE_UNWIND # endif #endif diff --git a/Include/internal/pycore_mmap.h b/Include/internal/pycore_mmap.h index 897816db01077f..c117cbd16283da 100644 --- a/Include/internal/pycore_mmap.h +++ b/Include/internal/pycore_mmap.h @@ -11,12 +11,12 @@ extern "C" { #include "pycore_pystate.h" -#if defined(HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) +#if defined(_Py_HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) # include # include #endif -#if defined(HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) +#if defined(_Py_HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) static inline int _PyAnnotateMemoryMap(void *addr, size_t size, const char *name) { diff --git a/Include/sliceobject.h b/Include/sliceobject.h index 00c70a6e911b41..9d6a16da95fe2f 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -45,9 +45,9 @@ PyAPI_FUNC(Py_ssize_t) PySlice_AdjustIndices(Py_ssize_t length, #endif #ifndef Py_LIMITED_API -# define Py_CPYTHON_SLICEOBJECT_H +# define _Py_CPYTHON_SLICEOBJECT_H # include "cpython/sliceobject.h" -# undef Py_CPYTHON_SLICEOBJECT_H +# undef _Py_CPYTHON_SLICEOBJECT_H #endif #ifdef __cplusplus diff --git a/Include/structseq.h b/Include/structseq.h index e52d6188030af9..e5da785f13d46b 100644 --- a/Include/structseq.h +++ b/Include/structseq.h @@ -29,9 +29,9 @@ PyAPI_FUNC(void) PyStructSequence_SetItem(PyObject*, Py_ssize_t, PyObject*); PyAPI_FUNC(PyObject*) PyStructSequence_GetItem(PyObject*, Py_ssize_t); #ifndef Py_LIMITED_API -# define Py_CPYTHON_STRUCTSEQ_H +# define _Py_CPYTHON_STRUCTSEQ_H # include "cpython/structseq.h" -# undef Py_CPYTHON_STRUCTSEQ_H +# undef _Py_CPYTHON_STRUCTSEQ_H #endif #ifdef __cplusplus diff --git a/Python/pystrhex.c b/Python/pystrhex.c index 645bb013581288..8fb1fa36f85e73 100644 --- a/Python/pystrhex.c +++ b/Python/pystrhex.c @@ -36,7 +36,7 @@ _Py_hexlify_scalar(const unsigned char *src, Py_UCS1 *dst, Py_ssize_t len) adds a ton of complication. Who ever really hexes huge data? The 16-64 byte boosts align nicely with md5 - sha512 hexdigests. */ -#ifdef HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR +#ifdef _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR /* 128-bit vector of 16 unsigned bytes */ typedef unsigned char v16u8 __attribute__((vector_size(16))); @@ -110,7 +110,7 @@ _Py_hexlify_simd(const unsigned char *src, Py_UCS1 *dst, Py_ssize_t len) _Py_hexlify_scalar(src + i, dst, len - i); } -#endif /* HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR */ +#endif /* _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR */ static PyObject * _Py_strhex_impl(const char* argbuf, Py_ssize_t arglen, @@ -191,7 +191,7 @@ _Py_strhex_impl(const char* argbuf, Py_ssize_t arglen, unsigned char c; if (bytes_per_sep_group == 0) { -#ifdef HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR +#ifdef _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR if (arglen >= 16) { _Py_hexlify_simd((const unsigned char *)argbuf, retbuf, arglen); } diff --git a/configure b/configure index cff7dfbfba8b9a..e9672d74c52a36 100755 --- a/configure +++ b/configure @@ -14619,7 +14619,7 @@ if test "x$ac_cv_have_libgcc_eh_frame_registration" = xyes then : -printf "%s\n" "#define HAVE_LIBGCC_EH_FRAME_REGISTRATION 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION 1" >>confdefs.h fi @@ -19483,7 +19483,7 @@ if test "x$ac_cv_efficient_builtin_shufflevector" = xyes then : -printf "%s\n" "#define HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR 1" >>confdefs.h fi @@ -24582,7 +24582,7 @@ printf "%s\n" "#define HAVE_DECL_PR_SET_VMA_ANON_NAME $ac_have_decl" >>confdefs. if test $ac_have_decl = 1 then : -printf "%s\n" "#define HAVE_PR_SET_VMA_ANON_NAME 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_PR_SET_VMA_ANON_NAME 1" >>confdefs.h fi diff --git a/configure.ac b/configure.ac index ac3269ab765c0d..7f97db73ad807d 100644 --- a/configure.ac +++ b/configure.ac @@ -3863,7 +3863,7 @@ __deregister_frame(0); [ac_cv_have_libgcc_eh_frame_registration=no]) ]) AS_VAR_IF([ac_cv_have_libgcc_eh_frame_registration], [yes], [ - AC_DEFINE([HAVE_LIBGCC_EH_FRAME_REGISTRATION], [1], + AC_DEFINE([_Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION], [1], [Define to 1 if libgcc __register_frame and __deregister_frame are linkable.]) ]) @@ -5163,7 +5163,7 @@ AC_LINK_IFELSE([ ]) AS_VAR_IF([ac_cv_efficient_builtin_shufflevector], [yes], [ - AC_DEFINE([HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR], [1], + AC_DEFINE([_Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR], [1], [Define if compiler supports __builtin_shufflevector with 128-bit vectors AND the target architecture has native SIMD (not just API availability)]) @@ -5788,7 +5788,7 @@ AC_CHECK_DECLS([UT_NAMESIZE], # musl libc redefines struct prctl_mm_map and conflicts with linux/prctl.h AS_IF([test "$ac_cv_libc" != musl], [ AC_CHECK_DECLS([PR_SET_VMA_ANON_NAME], - [AC_DEFINE([HAVE_PR_SET_VMA_ANON_NAME], [1], + [AC_DEFINE([_Py_HAVE_PR_SET_VMA_ANON_NAME], [1], [Define if you have the 'PR_SET_VMA_ANON_NAME' constant.])], [], [@%:@include diff --git a/pyconfig.h.in b/pyconfig.h.in index ad372255445d13..7ef83fcd0b9e0b 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -320,10 +320,6 @@ /* Define to 1 if you have the header file. */ #undef HAVE_EDITLINE_READLINE_H -/* Define if compiler supports __builtin_shufflevector with 128-bit vectors - AND the target architecture has native SIMD (not just API availability) */ -#undef HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR - /* Define to 1 if you have the header file. */ #undef HAVE_ENDIAN_H @@ -701,10 +697,6 @@ /* Define to 1 if you have the 'dld' library (-ldld). */ #undef HAVE_LIBDLD -/* Define to 1 if libgcc __register_frame and __deregister_frame are linkable. - */ -#undef HAVE_LIBGCC_EH_FRAME_REGISTRATION - /* Define to 1 if you have the 'ieee' library (-lieee). */ #undef HAVE_LIBIEEE @@ -1007,9 +999,6 @@ /* Define if your compiler supports function prototype */ #undef HAVE_PROTOTYPES -/* Define if you have the 'PR_SET_VMA_ANON_NAME' constant. */ -#undef HAVE_PR_SET_VMA_ANON_NAME - /* Define to 1 if you have the 'pthread_condattr_setclock' function. */ #undef HAVE_PTHREAD_CONDATTR_SETCLOCK @@ -2067,6 +2056,17 @@ /* HACL* library can compile SIMD256 implementations */ #undef _Py_HACL_CAN_COMPILE_VEC256 +/* Define if compiler supports __builtin_shufflevector with 128-bit vectors + AND the target architecture has native SIMD (not just API availability) */ +#undef _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR + +/* Define to 1 if libgcc __register_frame and __deregister_frame are linkable. + */ +#undef _Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION + +/* Define if you have the 'PR_SET_VMA_ANON_NAME' constant. */ +#undef _Py_HAVE_PR_SET_VMA_ANON_NAME + /* Define to 1 if the machine stack grows down (default); 0 if it grows up. */ #undef _Py_STACK_GROWS_DOWN From b6503057b2ac5aa8b976965dc6ccf16bbd13b50d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 14 May 2026 00:00:29 +0200 Subject: [PATCH 050/110] [3.15] gh-140550: Update xxlimited with 3.15 limited API (GH-142827) (GH-149785) (cherry picked from commit fa81cd976ad88e022839a37501d75d8716e22e3b) Co-authored-by: Petr Viktorin --- Lib/test/test_xxlimited.py | 119 +++-- Modules/Setup | 1 + Modules/Setup.stdlib.in | 1 + Modules/xxlimited.c | 419 +++++++++++---- Modules/xxlimited_35.c | 2 +- Modules/xxlimited_3_13.c | 499 ++++++++++++++++++ PC/layout/main.py | 3 +- PCbuild/pcbuild.proj | 1 + PCbuild/readme.txt | 5 +- PCbuild/xxlimited_3_13.vcxproj | 111 ++++ PCbuild/xxlimited_3_13.vcxproj.filters | 13 + Tools/build/generate_stdlib_module_names.py | 1 + Tools/c-analyzer/c_parser/preprocessor/gcc.py | 1 + Tools/c-analyzer/cpython/_analyzer.py | 1 + configure | 47 ++ configure.ac | 2 + 16 files changed, 1062 insertions(+), 164 deletions(-) create mode 100644 Modules/xxlimited_3_13.c create mode 100644 PCbuild/xxlimited_3_13.vcxproj create mode 100644 PCbuild/xxlimited_3_13.vcxproj.filters diff --git a/Lib/test/test_xxlimited.py b/Lib/test/test_xxlimited.py index b52e78bc4fb7e0..c6e9dc375d9a67 100644 --- a/Lib/test/test_xxlimited.py +++ b/Lib/test/test_xxlimited.py @@ -1,19 +1,39 @@ import unittest from test.support import import_helper -import types xxlimited = import_helper.import_module('xxlimited') -xxlimited_35 = import_helper.import_module('xxlimited_35') - -class CommonTests: - module: types.ModuleType - - def test_xxo_new(self): - xxo = self.module.Xxo() - - def test_xxo_attributes(self): - xxo = self.module.Xxo() +# if import of xxlimited succeeded, the other ones should be importable. +import xxlimited_3_13 +import xxlimited_35 + +MODULES = { + (3, 15): xxlimited, + (3, 13): xxlimited_3_13, + (3, 5): xxlimited_35, +} + +def test_with_xxlimited_modules(since=None, until=None): + def _decorator(func): + def _wrapper(self, *args, **kwargs): + for version, module in MODULES.items(): + if since and version < since: + continue + if until and version >= until: + continue + with self.subTest(version=version): + func(self, module, *args, **kwargs) + return _wrapper + return _decorator + +class XXLimitedTests(unittest.TestCase): + @test_with_xxlimited_modules() + def test_xxo_new(self, module): + xxo = module.Xxo() + + @test_with_xxlimited_modules() + def test_xxo_attributes(self, module): + xxo = module.Xxo() with self.assertRaises(AttributeError): xxo.foo with self.assertRaises(AttributeError): @@ -26,40 +46,61 @@ def test_xxo_attributes(self): with self.assertRaises(AttributeError): xxo.foo - def test_foo(self): + @test_with_xxlimited_modules() + def test_foo(self, module): # the foo function adds 2 numbers - self.assertEqual(self.module.foo(1, 2), 3) + self.assertEqual(module.foo(1, 2), 3) - def test_str(self): - self.assertIsSubclass(self.module.Str, str) - self.assertIsNot(self.module.Str, str) + @test_with_xxlimited_modules() + def test_str(self, module): + self.assertIsSubclass(module.Str, str) + self.assertIsNot(module.Str, str) - custom_string = self.module.Str("abcd") + custom_string = module.Str("abcd") self.assertEqual(custom_string, "abcd") self.assertEqual(custom_string.upper(), "ABCD") - def test_new(self): - xxo = self.module.new() + @test_with_xxlimited_modules() + def test_new(self, module): + xxo = module.new() self.assertEqual(xxo.demo("abc"), "abc") - -class TestXXLimited(CommonTests, unittest.TestCase): - module = xxlimited - - def test_xxo_demo(self): - xxo = self.module.Xxo() - other = self.module.Xxo() + @test_with_xxlimited_modules() + def test_xxo_demo(self, module): + xxo = module.Xxo() self.assertEqual(xxo.demo("abc"), "abc") + self.assertEqual(xxo.demo(0), None) + self.assertEqual(xxo.__module__, module.__name__) + with self.assertRaises(TypeError): + module.Xxo('arg') + with self.assertRaises(TypeError): + module.Xxo(kwarg='arg') + + @test_with_xxlimited_modules(since=(3, 13)) + def test_xxo_demo_extra(self, module): + xxo = module.Xxo() + other = module.Xxo() self.assertEqual(xxo.demo(xxo), xxo) self.assertEqual(xxo.demo(other), other) - self.assertEqual(xxo.demo(0), None) - def test_error(self): - with self.assertRaises(self.module.Error): - raise self.module.Error - - def test_buffer(self): - xxo = self.module.Xxo() + @test_with_xxlimited_modules(since=(3, 15)) + def test_xxo_subclass(self, module): + class Sub(module.Xxo): + pass + sub = Sub() + sub.a = 123 + self.assertEqual(sub.a, 123) + with self.assertRaisesRegex(AttributeError, "cannot set 'reserved'"): + sub.reserved = 123 + + @test_with_xxlimited_modules(since=(3, 13)) + def test_error(self, module): + with self.assertRaises(module.Error): + raise module.Error + + @test_with_xxlimited_modules(since=(3, 13)) + def test_buffer(self, module): + xxo = module.Xxo() self.assertEqual(xxo.x_exports, 0) b1 = memoryview(xxo) self.assertEqual(xxo.x_exports, 1) @@ -69,21 +110,13 @@ def test_buffer(self): self.assertEqual(b1[0], 1) self.assertEqual(b2[0], 1) - -class TestXXLimited35(CommonTests, unittest.TestCase): - module = xxlimited_35 - - def test_xxo_demo(self): - xxo = self.module.Xxo() - other = self.module.Xxo() - self.assertEqual(xxo.demo("abc"), "abc") - self.assertEqual(xxo.demo(0), None) - + @test_with_xxlimited_modules(until=(3, 5)) def test_roj(self): # the roj function always fails with self.assertRaises(SystemError): self.module.roj(0) + @test_with_xxlimited_modules(until=(3, 5)) def test_null(self): null1 = self.module.Null() null2 = self.module.Null() diff --git a/Modules/Setup b/Modules/Setup index 33737c21cb4066..e97a78e628693d 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -273,6 +273,7 @@ PYTHONPATH=$(COREPYTHONPATH) #xx xxmodule.c #xxlimited xxlimited.c #xxlimited_35 xxlimited_35.c +#xxlimited_3_13 xxlimited_3_13.c #xxsubtype xxsubtype.c # Testing diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 19765bc313555b..5f8b0cf482472d 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -190,6 +190,7 @@ # Limited API template modules; must be built as shared modules. @MODULE_XXLIMITED_TRUE@xxlimited xxlimited.c @MODULE_XXLIMITED_35_TRUE@xxlimited_35 xxlimited_35.c +@MODULE_XXLIMITED_3_13_TRUE@xxlimited_3_13 xxlimited_3_13.c # for performance diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 09c8d9487f5426..96454ee5e83eab 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -11,7 +11,13 @@ other files, you'll have to create a file "foobarobject.h"; see floatobject.h for an example. - This module roughly corresponds to:: + This module uses Limited API 3.15. + See ``xxlimited_3_13.c`` if you want to support older CPython versions. + + This module roughly corresponds to the following. + (All underscore-prefixed attributes are not accessible from Python.) + + :: class Xxo: """A class that explicitly stores attributes in an internal dict @@ -27,6 +33,8 @@ return self._x_attr[name] def __setattr__(self, name, value): + if name == "reserved": + raise AttributeError("cannot set 'reserved'") self._x_attr[name] = value def __delattr__(self, name): @@ -64,11 +72,13 @@ pass */ -// Need limited C API version 3.13 for Py_mod_gil -#include "pyconfig.h" // Py_GIL_DISABLED -#ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030d0000 -#endif +// Target both flavors of the Stable ABI. +// Both are set to version 3.15, which adds PyModExport +// (When using a build tool, check if it has an option to set these +// so they do not need to be defined in the source.) +#define Py_LIMITED_API 0x030f0000 // abi3 (GIL-enabled builds) +#define Py_TARGET_ABI3T 0x030f0000 // abi3t (free-threaded builds) + #include "Python.h" #include @@ -77,43 +87,135 @@ // Module state typedef struct { - PyObject *Xxo_Type; // Xxo class + PyTypeObject *Xxo_Type; // Xxo class PyObject *Error_Type; // Error class } xx_state; -/* Xxo objects */ +/* Xxo objects. + * + * A non-trivial extension type, intentionally showing a number of features + * that aren't easy to implement in the Limited API. + */ + +// Forward declaration +static PyType_Spec Xxo_Type_spec; + +// Get the module state (xx_state*) from a given type object 'type', which +// must be a subclass of Xxo (the type we're defining). +// This is complicated by the fact that the Xxo type is dynamically allocated, +// and there may be several such types in a given Python process -- for +// example, in different subinterpreters, or through loading this +// extension module several times. +// So, we don't have a "global" pointer to the type, or to the module, etc.; +// instead we search based on `Xxo_Type_spec` (which is static, immutable, +// and process-global). +// +// When possible, it's better to avoid `PyType_GetBaseByToken` -- for an +// example, see the `demo` method (Xxo_demo C function), which uses a +// "defining class". But, in many cases it's the best solution. +static xx_state * +Xxo_state_from_type(PyTypeObject *type) +{ + PyTypeObject *base; + // Search all superclasses of 'type' for one that was defined using + // "Xxo_Type_spec". That must be our 'Xxo' class. + if (PyType_GetBaseByToken(type, &Xxo_Type_spec, &base) < 0) { + return NULL; + } + if (base == NULL) { + PyErr_SetString(PyExc_TypeError, "need Xxo subclass"); + return NULL; + } + // From this type, get the associated module. That must be the + // relevant `xxlimited` module. + xx_state *state = PyType_GetModuleState(base); + Py_DECREF(base); + return state; +} -// Instance state +// Structure for data needed by the XxoObject type. +// Since the object may be shared across threads, access to the fields +// usually needs to be synchronized (using Py_BEGIN_CRITICAL_SECTION). typedef struct { - PyObject_HEAD - PyObject *x_attr; /* Attributes dictionary. - * May be NULL, which acts as an - * empty dict. - */ - char x_buffer[BUFSIZE]; /* buffer for Py_buffer */ - Py_ssize_t x_exports; /* how many buffer are exported */ -} XxoObject; - -#define XxoObject_CAST(op) ((XxoObject *)(op)) -// TODO: full support for type-checking was added in 3.14 (Py_tp_token) -// #define XxoObject_Check(v) Py_IS_TYPE(v, Xxo_Type) - -static XxoObject * -newXxoObject(PyObject *module) + PyObject *x_attr; /* Attributes dictionary. + * May be NULL, which acts as an + * empty dict. + */ + Py_ssize_t x_exports; /* how many buffers are exported */ + char x_buffer[BUFSIZE]; /* buffer for Py_buffer (for simplicity, + * this is constant, so does not need + * synchronization) + */ +} XxoObject_Data; + +// Get the `XxoObject_Data` structure for a given instance of our type. +static XxoObject_Data * +Xxo_get_data(PyObject *self) { - xx_state *state = PyModule_GetState(module); + xx_state *state = Xxo_state_from_type(Py_TYPE(self)); + if (!state) { + return NULL; + } + XxoObject_Data *data = PyObject_GetTypeData(self, state->Xxo_Type); + return data; +} + +// A variant of Xxo_get_data to be used in the tp_traverse handler. +// This function cannot have side effects (including reference count +// manipulation, creating objects, and raising exceptions), and must not +// call API functions that might have side effects. +// See: https://docs.python.org/3.15/c-api/gcsupport.html#traversal +static XxoObject_Data * +Xxo_get_data_DuringGC(PyObject *self) +{ + PyTypeObject *base; + PyType_GetBaseByToken_DuringGC(Py_TYPE(self), &Xxo_Type_spec, &base); + if (base == NULL) { + return NULL; + } + xx_state *state = PyType_GetModuleState_DuringGC(base); if (state == NULL) { return NULL; } - XxoObject *self; - self = PyObject_GC_New(XxoObject, (PyTypeObject*)state->Xxo_Type); + XxoObject_Data *data = PyObject_GetTypeData_DuringGC(self, state->Xxo_Type); + return data; +} + +// Xxo initialization +// This is the implementation of Xxo.__new__ +static PyObject * +Xxo_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + // Validate that we did not get any arguments. + if ((args != NULL && PyObject_Length(args)) + || (kwargs != NULL && PyObject_Length(kwargs))) + { + PyErr_SetString(PyExc_TypeError, "Xxo.__new__() takes no arguments"); + return NULL; + } + // Create an instance of *type* (which may be a subclass) + allocfunc alloc = PyType_GetSlot(type, Py_tp_alloc); + PyObject *self = alloc(type, 0); if (self == NULL) { return NULL; } - self->x_attr = NULL; - memset(self->x_buffer, 0, BUFSIZE); - self->x_exports = 0; + + // Initialize the C members on the instance. + // This is only included for the sake of example. The default alloc + // function zeroes instance memory; we don't need to do it again. + // Note that we during initialization (and finalization), we hold the only + // reference to the object, so we don't need to synchronize with + // other threads. + XxoObject_Data *xxo_data = Xxo_get_data(self); + if (xxo_data == NULL) { + Py_DECREF(self); + return NULL; + } + + xxo_data->x_attr = NULL; + memset(xxo_data->x_buffer, 0, BUFSIZE); + xxo_data->x_exports = 0; return self; } @@ -125,45 +227,63 @@ newXxoObject(PyObject *module) // traverse: Visit all references from an object, including its type static int -Xxo_traverse(PyObject *op, visitproc visit, void *arg) +Xxo_traverse(PyObject *self, visitproc visit, void *arg) { // Visit the type - Py_VISIT(Py_TYPE(op)); + Py_VISIT(Py_TYPE(self)); // Visit the attribute dict - XxoObject *self = XxoObject_CAST(op); - Py_VISIT(self->x_attr); + XxoObject_Data *data = Xxo_get_data_DuringGC(self); + if (data == NULL) { + return 0; + } + Py_VISIT(data->x_attr); return 0; } // clear: drop references in order to break all reference cycles static int -Xxo_clear(PyObject *op) +Xxo_clear(PyObject *self) { - XxoObject *self = XxoObject_CAST(op); - Py_CLEAR(self->x_attr); + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return 0; + } + Py_CLEAR(data->x_attr); return 0; } // finalize: like clear, but should leave the object in a consistent state. // Equivalent to `__del__` in Python. static void -Xxo_finalize(PyObject *op) +Xxo_finalize(PyObject *self) { - XxoObject *self = XxoObject_CAST(op); - Py_CLEAR(self->x_attr); + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return; + } + Py_CLEAR(data->x_attr); } // dealloc: drop all remaining references and free memory static void Xxo_dealloc(PyObject *self) { + // This function must preserve currently raised exception, if any. + PyObject *exc = PyErr_GetRaisedException(); + PyObject_GC_UnTrack(self); Xxo_finalize(self); + PyTypeObject *tp = Py_TYPE(self); freefunc free = PyType_GetSlot(tp, Py_tp_free); free(self); Py_DECREF(tp); + + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(NULL); + } + PyErr_SetRaisedException(exc); } @@ -171,11 +291,20 @@ Xxo_dealloc(PyObject *self) // Get an attribute. static PyObject * -Xxo_getattro(PyObject *op, PyObject *name) +Xxo_getattro(PyObject *self, PyObject *name) { - XxoObject *self = XxoObject_CAST(op); - if (self->x_attr != NULL) { - PyObject *v = PyDict_GetItemWithError(self->x_attr, name); + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return 0; + } + + PyObject *x_attr; + Py_BEGIN_CRITICAL_SECTION(self); + x_attr = data->x_attr; + Py_END_CRITICAL_SECTION(); + + if (x_attr != NULL) { + PyObject *v = PyDict_GetItemWithError(x_attr, name); if (v != NULL) { return Py_NewRef(v); } @@ -185,24 +314,42 @@ Xxo_getattro(PyObject *op, PyObject *name) } // Fall back to generic implementation (this handles special attributes, // raising AttributeError, etc.) - return PyObject_GenericGetAttr(op, name); + return PyObject_GenericGetAttr(self, name); } // Set or delete an attribute. static int -Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) +Xxo_setattro(PyObject *self, PyObject *name, PyObject *v) { - XxoObject *self = XxoObject_CAST(op); - if (self->x_attr == NULL) { + // filter a specific attribute name + if (PyUnicode_Check(name) && PyUnicode_EqualToUTF8(name, "reserved")) { + PyErr_Format(PyExc_AttributeError, "cannot set %R", name); + return -1; + } + + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return -1; + } + + // If the attribute dict is not created yet, make one. + // This needs to be protected by a critical section to avoid another thread + // creating a duplicate dict. + PyObject *x_attr; + Py_BEGIN_CRITICAL_SECTION(self); + x_attr = data->x_attr; + if (x_attr == NULL) { // prepare the attribute dict - self->x_attr = PyDict_New(); - if (self->x_attr == NULL) { - return -1; - } + data->x_attr = x_attr = PyDict_New(); } + Py_END_CRITICAL_SECTION(); + if (x_attr == NULL) { + return -1; + } + if (v == NULL) { // delete an attribute - int rv = PyDict_DelItem(self->x_attr, name); + int rv = PyDict_DelItem(x_attr, name); if (rv < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_SetString(PyExc_AttributeError, "delete non-existing Xxo attribute"); @@ -212,7 +359,7 @@ Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) } else { // set an attribute - return PyDict_SetItem(self->x_attr, name, v); + return PyDict_SetItem(x_attr, name, v); } } @@ -221,7 +368,7 @@ Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) */ static PyObject * -Xxo_demo(PyObject *op, PyTypeObject *defining_class, +Xxo_demo(PyObject *self, PyTypeObject *defining_class, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (kwnames != NULL && PyObject_Length(kwnames)) { @@ -260,30 +407,49 @@ static PyMethodDef Xxo_methods[] = { */ static int -Xxo_getbuffer(PyObject *op, Py_buffer *view, int flags) +Xxo_getbuffer(PyObject *self, Py_buffer *view, int flags) { - XxoObject *self = XxoObject_CAST(op); - int res = PyBuffer_FillInfo(view, op, - (void *)self->x_buffer, BUFSIZE, + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return -1; + } + int res = PyBuffer_FillInfo(view, self, + (void *)data->x_buffer, BUFSIZE, 0, flags); if (res == 0) { - self->x_exports++; + Py_BEGIN_CRITICAL_SECTION(self); + data->x_exports++; + Py_END_CRITICAL_SECTION(); } return res; } static void -Xxo_releasebuffer(PyObject *op, Py_buffer *Py_UNUSED(view)) +Xxo_releasebuffer(PyObject *self, Py_buffer *Py_UNUSED(view)) { - XxoObject *self = XxoObject_CAST(op); - self->x_exports--; + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return; + } + Py_BEGIN_CRITICAL_SECTION(self); + data->x_exports--; + Py_END_CRITICAL_SECTION(); } static PyObject * -Xxo_get_x_exports(PyObject *op, void *Py_UNUSED(closure)) +Xxo_get_x_exports(PyObject *self, void *Py_UNUSED(closure)) { - XxoObject *self = XxoObject_CAST(op); - return PyLong_FromSsize_t(self->x_exports); + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return NULL; + } + Py_ssize_t result; + + Py_BEGIN_CRITICAL_SECTION(self); + result = data->x_exports; + Py_END_CRITICAL_SECTION(); + + return PyLong_FromSsize_t(result); } /* Xxo type definition */ @@ -299,6 +465,7 @@ static PyGetSetDef Xxo_getsetlist[] = { static PyType_Slot Xxo_Type_slots[] = { {Py_tp_doc, (char *)Xxo_doc}, + {Py_tp_new, Xxo_new}, {Py_tp_traverse, Xxo_traverse}, {Py_tp_clear, Xxo_clear}, {Py_tp_finalize, Xxo_finalize}, @@ -309,13 +476,14 @@ static PyType_Slot Xxo_Type_slots[] = { {Py_bf_getbuffer, Xxo_getbuffer}, {Py_bf_releasebuffer, Xxo_releasebuffer}, {Py_tp_getset, Xxo_getsetlist}, + {Py_tp_token, Py_TP_USE_SPEC}, {0, 0}, /* sentinel */ }; static PyType_Spec Xxo_Type_spec = { .name = "xxlimited.Xxo", - .basicsize = sizeof(XxoObject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .basicsize = -(Py_ssize_t)sizeof(XxoObject_Data), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, .slots = Xxo_Type_slots, }; @@ -354,17 +522,17 @@ xx_foo(PyObject *module, PyObject *args) } -/* Function of no arguments returning new Xxo object */ +/* Function of no arguments returning new Xxo object. + * Note that a function exposed to Python with METH_NOARGS requires an unused + * second argument, so we cannot use newXxoObject directly. + */ static PyObject * xx_new(PyObject *module, PyObject *Py_UNUSED(unused)) { - XxoObject *rv; + xx_state *state = PyModule_GetState(module); - rv = newXxoObject(module); - if (rv == NULL) - return NULL; - return (PyObject *)rv; + return Xxo_new(state->Xxo_Type, NULL, NULL); } @@ -398,11 +566,12 @@ xx_modexec(PyObject *m) return -1; } - state->Xxo_Type = PyType_FromModuleAndSpec(m, &Xxo_Type_spec, NULL); + state->Xxo_Type = (PyTypeObject*)PyType_FromModuleAndSpec( + m, &Xxo_Type_spec, NULL); if (state->Xxo_Type == NULL) { return -1; } - if (PyModule_AddType(m, (PyTypeObject*)state->Xxo_Type) < 0) { + if (PyModule_AddType(m, state->Xxo_Type) < 0) { return -1; } @@ -410,12 +579,13 @@ xx_modexec(PyObject *m) // added to the module dict. // It does not inherit from "object" (PyObject_Type), but from "str" // (PyUnincode_Type). - PyObject *Str_Type = PyType_FromModuleAndSpec( + PyTypeObject *Str_Type = (PyTypeObject*)PyType_FromModuleAndSpec( m, &Str_Type_spec, (PyObject *)&PyUnicode_Type); if (Str_Type == NULL) { return -1; } - if (PyModule_AddType(m, (PyTypeObject*)Str_Type) < 0) { + if (PyModule_AddType(m, Str_Type) < 0) { + Py_DECREF(Str_Type); return -1; } Py_DECREF(Str_Type); @@ -423,29 +593,6 @@ xx_modexec(PyObject *m) return 0; } -static PyModuleDef_Slot xx_slots[] = { - - /* exec function to initialize the module (called as part of import - * after the object was added to sys.modules) - */ - {Py_mod_exec, xx_modexec}, - - /* Signal that this module supports being loaded in multiple interpreters - * with separate GILs (global interpreter locks). - * See "Isolating Extension Modules" on how to prepare a module for this: - * https://docs.python.org/3/howto/isolating-extensions.html - */ - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - - /* Signal that this module does not rely on the GIL for its own needs. - * Without this slot, free-threaded builds of CPython will enable - * the GIL when this module is loaded. - */ - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - - {0, NULL} -}; - // Module finalization: modules that hold references in their module state // need to implement the fullowing GC hooks. They're similar to the ones for // types (see "Xxo finalization"). @@ -453,7 +600,10 @@ static PyModuleDef_Slot xx_slots[] = { static int xx_traverse(PyObject *module, visitproc visit, void *arg) { - xx_state *state = PyModule_GetState(module); + xx_state *state = PyModule_GetState_DuringGC(module); + if (state == NULL) { + return 0; + } Py_VISIT(state->Xxo_Type); Py_VISIT(state->Error_Type); return 0; @@ -463,6 +613,9 @@ static int xx_clear(PyObject *module) { xx_state *state = PyModule_GetState(module); + if (state == NULL) { + return 0; + } Py_CLEAR(state->Xxo_Type); Py_CLEAR(state->Error_Type); return 0; @@ -473,27 +626,59 @@ xx_free(void *module) { // allow xx_modexec to omit calling xx_clear on error (void)xx_clear((PyObject *)module); + + xx_state *state = PyModule_GetState(module); + if (state == NULL) { + return; + } } -static struct PyModuleDef xxmodule = { - PyModuleDef_HEAD_INIT, - .m_name = "xxlimited", - .m_doc = module_doc, - .m_size = sizeof(xx_state), - .m_methods = xx_methods, - .m_slots = xx_slots, - .m_traverse = xx_traverse, - .m_clear = xx_clear, - .m_free = xx_free, +// Information that CPython uses to prevent loading incompatible extenstions +PyABIInfo_VAR(abi_info); + +static PySlot xx_slots[] = { + /* Basic metadata */ + PySlot_STATIC_DATA(Py_mod_name, "xxlimited"), + PySlot_STATIC_DATA(Py_mod_doc, (void*)module_doc), + PySlot_DATA(Py_mod_abi, &abi_info), + + /* The method table */ + PySlot_STATIC_DATA(Py_mod_methods, xx_methods), + + /* exec function to initialize the module (called as part of import + * after the object was added to sys.modules) + */ + PySlot_FUNC(Py_mod_exec, xx_modexec), + + /* Module state and associated functions */ + PySlot_SIZE(Py_mod_state_size, sizeof(xx_state)), + PySlot_FUNC(Py_mod_state_traverse, xx_traverse), + PySlot_FUNC(Py_mod_state_clear, xx_clear), + PySlot_FUNC(Py_mod_state_free, xx_free), + + /* Signal that this module supports being loaded in multiple interpreters + * with separate GILs (global interpreter locks). + * See "Isolating Extension Modules" on how to prepare a module for this: + * https://docs.python.org/3/howto/isolating-extensions.html + */ + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + + /* Signal that this module does not rely on the GIL for its own needs. + * Without this slot, free-threaded builds of CPython will enable + * the GIL when this module is loaded. + */ + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + + PySlot_END }; -/* Export function for the module. *Must* be called PyInit_xx; usually it is - * the only non-`static` object in a module definition. +/* Export function for the module. *Must* be called PyModExport_xx; usually + * it is the only non-`static` object in a module definition. */ -PyMODINIT_FUNC -PyInit_xxlimited(void) +PyMODEXPORT_FUNC +PyModExport_xxlimited(void) { - return PyModuleDef_Init(&xxmodule); + return xx_slots; } diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index b0a682ac4e6bb6..9ef0eac9a924e6 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -305,7 +305,7 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, #ifdef Py_GIL_DISABLED - // These definitions are in the limited API, but not until 3.13. + // In a free-threaded build, we don't use Limited API. {Py_mod_gil, Py_MOD_GIL_NOT_USED}, #endif {0, NULL} diff --git a/Modules/xxlimited_3_13.c b/Modules/xxlimited_3_13.c new file mode 100644 index 00000000000000..4f100f9150fc2a --- /dev/null +++ b/Modules/xxlimited_3_13.c @@ -0,0 +1,499 @@ +/* Use this file as a template to start implementing a module that + also declares object types. All occurrences of 'Xxo' should be changed + to something reasonable for your objects. After that, all other + occurrences of 'xx' should be changed to something reasonable for your + module. If your module is named foo your source file should be named + foo.c or foomodule.c. + + You will probably want to delete all references to 'x_attr' and add + your own types of attributes instead. Maybe you want to name your + local variables other than 'self'. If your object type is needed in + other files, you'll have to create a file "foobarobject.h"; see + floatobject.h for an example. + + This module roughly corresponds to:: + + class Xxo: + """A class that explicitly stores attributes in an internal dict + (to simulate custom attribute handling). + """ + + def __init__(self): + # In the C class, "_x_attr" is not accessible from Python code + self._x_attr = {} + self._x_exports = 0 + + def __getattr__(self, name): + return self._x_attr[name] + + def __setattr__(self, name, value): + self._x_attr[name] = value + + def __delattr__(self, name): + del self._x_attr[name] + + @property + def x_exports(self): + """Return the number of times an internal buffer is exported.""" + # Each Xxo instance has a 10-byte buffer that can be + # accessed via the buffer interface (e.g. `memoryview`). + return self._x_exports + + def demo(o, /): + if isinstance(o, str): + return o + elif isinstance(o, Xxo): + return o + else: + raise Error('argument must be str or Xxo') + + class Error(Exception): + """Exception raised by the xxlimited module""" + + def foo(i: int, j: int, /): + """Return the sum of i and j.""" + # Unlike this pseudocode, the C function will *only* work with + # integers and perform C long int arithmetic + return i + j + + def new(): + return Xxo() + + def Str(str): + # A trivial subclass of a built-in type + pass + */ + +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + +#include "Python.h" +#include + +#define BUFSIZE 10 + +// Module state +typedef struct { + PyObject *Xxo_Type; // Xxo class + PyObject *Error_Type; // Error class +} xx_state; + + +/* Xxo objects */ + +// Instance state +typedef struct { + PyObject_HEAD + PyObject *x_attr; /* Attributes dictionary. + * May be NULL, which acts as an + * empty dict. + */ + char x_buffer[BUFSIZE]; /* buffer for Py_buffer */ + Py_ssize_t x_exports; /* how many buffer are exported */ +} XxoObject; + +#define XxoObject_CAST(op) ((XxoObject *)(op)) +// TODO: full support for type-checking was added in 3.14 (Py_tp_token) +// #define XxoObject_Check(v) Py_IS_TYPE(v, Xxo_Type) + +static XxoObject * +newXxoObject(PyObject *module) +{ + xx_state *state = PyModule_GetState(module); + if (state == NULL) { + return NULL; + } + XxoObject *self; + self = PyObject_GC_New(XxoObject, (PyTypeObject*)state->Xxo_Type); + if (self == NULL) { + return NULL; + } + self->x_attr = NULL; + memset(self->x_buffer, 0, BUFSIZE); + self->x_exports = 0; + return self; +} + +/* Xxo finalization. + * + * Types that store references to other PyObjects generally need to implement + * the GC slots: traverse, clear, dealloc, and (optionally) finalize. + */ + +// traverse: Visit all references from an object, including its type +static int +Xxo_traverse(PyObject *op, visitproc visit, void *arg) +{ + // Visit the type + Py_VISIT(Py_TYPE(op)); + + // Visit the attribute dict + XxoObject *self = XxoObject_CAST(op); + Py_VISIT(self->x_attr); + return 0; +} + +// clear: drop references in order to break all reference cycles +static int +Xxo_clear(PyObject *op) +{ + XxoObject *self = XxoObject_CAST(op); + Py_CLEAR(self->x_attr); + return 0; +} + +// finalize: like clear, but should leave the object in a consistent state. +// Equivalent to `__del__` in Python. +static void +Xxo_finalize(PyObject *op) +{ + XxoObject *self = XxoObject_CAST(op); + Py_CLEAR(self->x_attr); +} + +// dealloc: drop all remaining references and free memory +static void +Xxo_dealloc(PyObject *self) +{ + PyObject_GC_UnTrack(self); + Xxo_finalize(self); + PyTypeObject *tp = Py_TYPE(self); + freefunc free = PyType_GetSlot(tp, Py_tp_free); + free(self); + Py_DECREF(tp); +} + + +/* Xxo attribute handling */ + +// Get an attribute. +static PyObject * +Xxo_getattro(PyObject *op, PyObject *name) +{ + XxoObject *self = XxoObject_CAST(op); + if (self->x_attr != NULL) { + PyObject *v = PyDict_GetItemWithError(self->x_attr, name); + if (v != NULL) { + return Py_NewRef(v); + } + else if (PyErr_Occurred()) { + return NULL; + } + } + // Fall back to generic implementation (this handles special attributes, + // raising AttributeError, etc.) + return PyObject_GenericGetAttr(op, name); +} + +// Set or delete an attribute. +static int +Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) +{ + XxoObject *self = XxoObject_CAST(op); + if (self->x_attr == NULL) { + // prepare the attribute dict + self->x_attr = PyDict_New(); + if (self->x_attr == NULL) { + return -1; + } + } + if (v == NULL) { + // delete an attribute + int rv = PyDict_DelItem(self->x_attr, name); + if (rv < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_SetString(PyExc_AttributeError, + "delete non-existing Xxo attribute"); + return -1; + } + return rv; + } + else { + // set an attribute + return PyDict_SetItem(self->x_attr, name, v); + } +} + +/* Xxo methods: C functions plus a PyMethodDef array that lists them and + * specifies metadata. + */ + +static PyObject * +Xxo_demo(PyObject *op, PyTypeObject *defining_class, + PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (kwnames != NULL && PyObject_Length(kwnames)) { + PyErr_SetString(PyExc_TypeError, "demo() takes no keyword arguments"); + return NULL; + } + if (nargs != 1) { + PyErr_SetString(PyExc_TypeError, "demo() takes exactly 1 argument"); + return NULL; + } + + PyObject *o = args[0]; + + /* Test if the argument is "str" */ + if (PyUnicode_Check(o)) { + return Py_NewRef(o); + } + + /* test if the argument is of the Xxo class */ + if (PyObject_TypeCheck(o, defining_class)) { + return Py_NewRef(o); + } + + return Py_NewRef(Py_None); +} + +static PyMethodDef Xxo_methods[] = { + {"demo", _PyCFunction_CAST(Xxo_demo), + METH_METHOD | METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("demo(o) -> o")}, + {NULL, NULL} /* sentinel */ +}; + +/* Xxo buffer interface: C functions later referenced from PyType_Slot array. + * Other interfaces (e.g. for sequence-like or number-like types) are defined + * similarly. + */ + +static int +Xxo_getbuffer(PyObject *op, Py_buffer *view, int flags) +{ + XxoObject *self = XxoObject_CAST(op); + int res = PyBuffer_FillInfo(view, op, + (void *)self->x_buffer, BUFSIZE, + 0, flags); + if (res == 0) { + self->x_exports++; + } + return res; +} + +static void +Xxo_releasebuffer(PyObject *op, Py_buffer *Py_UNUSED(view)) +{ + XxoObject *self = XxoObject_CAST(op); + self->x_exports--; +} + +static PyObject * +Xxo_get_x_exports(PyObject *op, void *Py_UNUSED(closure)) +{ + XxoObject *self = XxoObject_CAST(op); + return PyLong_FromSsize_t(self->x_exports); +} + +/* Xxo type definition */ + +PyDoc_STRVAR(Xxo_doc, + "A class that explicitly stores attributes in an internal dict"); + +static PyGetSetDef Xxo_getsetlist[] = { + {"x_exports", Xxo_get_x_exports, NULL, NULL}, + {NULL}, +}; + + +static PyType_Slot Xxo_Type_slots[] = { + {Py_tp_doc, (char *)Xxo_doc}, + {Py_tp_traverse, Xxo_traverse}, + {Py_tp_clear, Xxo_clear}, + {Py_tp_finalize, Xxo_finalize}, + {Py_tp_dealloc, Xxo_dealloc}, + {Py_tp_getattro, Xxo_getattro}, + {Py_tp_setattro, Xxo_setattro}, + {Py_tp_methods, Xxo_methods}, + {Py_bf_getbuffer, Xxo_getbuffer}, + {Py_bf_releasebuffer, Xxo_releasebuffer}, + {Py_tp_getset, Xxo_getsetlist}, + {0, 0}, /* sentinel */ +}; + +static PyType_Spec Xxo_Type_spec = { + .name = "xxlimited_3_13.Xxo", + .basicsize = sizeof(XxoObject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .slots = Xxo_Type_slots, +}; + + +/* Str type definition*/ + +static PyType_Slot Str_Type_slots[] = { + // slots array intentionally kept empty + {0, 0}, /* sentinel */ +}; + +static PyType_Spec Str_Type_spec = { + .name = "xxlimited_3_13.Str", + .basicsize = 0, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = Str_Type_slots, +}; + + +/* Function of two integers returning integer (with C "long int" arithmetic) */ + +PyDoc_STRVAR(xx_foo_doc, +"foo(i,j)\n\ +\n\ +Return the sum of i and j."); + +static PyObject * +xx_foo(PyObject *module, PyObject *args) +{ + long i, j; + long res; + if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) + return NULL; + res = i+j; /* XXX Do something here */ + return PyLong_FromLong(res); +} + + +/* Function of no arguments returning new Xxo object */ + +static PyObject * +xx_new(PyObject *module, PyObject *Py_UNUSED(unused)) +{ + XxoObject *rv; + + rv = newXxoObject(module); + if (rv == NULL) + return NULL; + return (PyObject *)rv; +} + + + +/* List of functions defined in the module */ + +static PyMethodDef xx_methods[] = { + {"foo", xx_foo, METH_VARARGS, + xx_foo_doc}, + {"new", xx_new, METH_NOARGS, + PyDoc_STR("new() -> new Xx object")}, + {NULL, NULL} /* sentinel */ +}; + + +/* The module itself */ + +PyDoc_STRVAR(module_doc, +"This is a template module just for instruction."); + +static int +xx_modexec(PyObject *m) +{ + xx_state *state = PyModule_GetState(m); + + state->Error_Type = PyErr_NewException("xxlimited_3_13.Error", NULL, NULL); + if (state->Error_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject*)state->Error_Type) < 0) { + return -1; + } + + state->Xxo_Type = PyType_FromModuleAndSpec(m, &Xxo_Type_spec, NULL); + if (state->Xxo_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject*)state->Xxo_Type) < 0) { + return -1; + } + + // Add the Str type. It is not needed from C code, so it is only + // added to the module dict. + // It does not inherit from "object" (PyObject_Type), but from "str" + // (PyUnincode_Type). + PyObject *Str_Type = PyType_FromModuleAndSpec( + m, &Str_Type_spec, (PyObject *)&PyUnicode_Type); + if (Str_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject*)Str_Type) < 0) { + return -1; + } + Py_DECREF(Str_Type); + + return 0; +} + +static PyModuleDef_Slot xx_slots[] = { + + /* exec function to initialize the module (called as part of import + * after the object was added to sys.modules) + */ + {Py_mod_exec, xx_modexec}, + + /* Signal that this module supports being loaded in multiple interpreters + * with separate GILs (global interpreter locks). + * See "Isolating Extension Modules" on how to prepare a module for this: + * https://docs.python.org/3/howto/isolating-extensions.html + */ + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + + /* Signal that this module does not rely on the GIL for its own needs. + * Without this slot, free-threaded builds of CPython will enable + * the GIL when this module is loaded. + */ + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + + {0, NULL} +}; + +// Module finalization: modules that hold references in their module state +// need to implement the fullowing GC hooks. They're similar to the ones for +// types (see "Xxo finalization"). + +static int +xx_traverse(PyObject *module, visitproc visit, void *arg) +{ + xx_state *state = PyModule_GetState(module); + Py_VISIT(state->Xxo_Type); + Py_VISIT(state->Error_Type); + return 0; +} + +static int +xx_clear(PyObject *module) +{ + xx_state *state = PyModule_GetState(module); + Py_CLEAR(state->Xxo_Type); + Py_CLEAR(state->Error_Type); + return 0; +} + +static void +xx_free(void *module) +{ + // allow xx_modexec to omit calling xx_clear on error + (void)xx_clear((PyObject *)module); +} + +static struct PyModuleDef xxmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "xxlimited_3_13", + .m_doc = module_doc, + .m_size = sizeof(xx_state), + .m_methods = xx_methods, + .m_slots = xx_slots, + .m_traverse = xx_traverse, + .m_clear = xx_clear, + .m_free = xx_free, +}; + + +/* Export function for the module. *Must* be called PyInit_xx; usually it is + * the only non-`static` object in a module definition. + */ + +PyMODINIT_FUNC +PyInit_xxlimited_3_13(void) +{ + return PyModuleDef_Init(&xxmodule); +} diff --git a/PC/layout/main.py b/PC/layout/main.py index 3566b8bd873874..3a62ea91420c9e 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -32,7 +32,8 @@ from .support.pymanager import * from .support.nuspec import * -TEST_PYDS_ONLY = FileStemSet("xxlimited", "xxlimited_35", "_ctypes_test", "_test*") +TEST_PYDS_ONLY = FileStemSet("xxlimited", "xxlimited_3_13", "xxlimited_35", + "_ctypes_test", "_test*") TEST_DLLS_ONLY = set() TEST_DIRS_ONLY = FileNameSet("test", "tests") diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index bb7d8042176d8f..9d077bbd3f0ba2 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -84,6 +84,7 @@ + false diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index c291b7f86325f2..14aac0b0dc84b6 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -168,8 +168,9 @@ xxlimited builds an example module that makes use of the PEP 384 Stable ABI, see Modules\xxlimited.c xxlimited_35 - ditto for testing the Python 3.5 stable ABI, see - Modules\xxlimited_35.c +xxlimited_3_13 + ditto for testing older Limited API, see + Modules\xxlimited_*.c The following sub-projects are for individual modules of the standard library which are implemented in C; each one builds a DLL (renamed to diff --git a/PCbuild/xxlimited_3_13.vcxproj b/PCbuild/xxlimited_3_13.vcxproj new file mode 100644 index 00000000000000..7a9760fd43121e --- /dev/null +++ b/PCbuild/xxlimited_3_13.vcxproj @@ -0,0 +1,111 @@ + + + + + Debug + ARM + + + Debug + ARM64 + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + ARM + + + PGInstrument + ARM64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + ARM + + + PGUpdate + ARM64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + ARM + + + Release + ARM64 + + + Release + Win32 + + + Release + x64 + + + + {fb868ea7-f93a-4d9b-be78-ca4e9ba14fff} + xxlimited_3_13 + Win32Proj + + + + + DynamicLibrary + NotSet + false + + + + $(PyStdlibPydExt) + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + wsock32.lib;%(AdditionalDependencies) + + + + + + + + {885d4898-d08d-4091-9c40-c700cfe3fc5a} + + + + + + diff --git a/PCbuild/xxlimited_3_13.vcxproj.filters b/PCbuild/xxlimited_3_13.vcxproj.filters new file mode 100644 index 00000000000000..3dfb7800edc441 --- /dev/null +++ b/PCbuild/xxlimited_3_13.vcxproj.filters @@ -0,0 +1,13 @@ + + + + + {5be27194-6530-452d-8d86-3767b991fa83} + + + + + Source Files + + + diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index bda72539640611..f8828a56b4c7da 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -42,6 +42,7 @@ 'test', 'xxlimited', 'xxlimited_35', + 'xxlimited_3_13', 'xxsubtype', } diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 4a55a1a24ee1be..92134bc1321e1b 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -11,6 +11,7 @@ '_testclinic_limited.c', 'xxlimited.c', 'xxlimited_35.c', + 'xxlimited_3_13.c', )) # C files in the fhe following directories must not be built with diff --git a/Tools/c-analyzer/cpython/_analyzer.py b/Tools/c-analyzer/cpython/_analyzer.py index 43ed552fcf75d9..404a81af11e39f 100644 --- a/Tools/c-analyzer/cpython/_analyzer.py +++ b/Tools/c-analyzer/cpython/_analyzer.py @@ -77,6 +77,7 @@ 'PyStructSequence_Field[]', 'PyStructSequence_Desc', 'PyABIInfo', + 'PySlot[]', } # XXX We should normalize all cases to a single name, diff --git a/configure b/configure index e9672d74c52a36..8979ec294bcdf0 100755 --- a/configure +++ b/configure @@ -647,6 +647,8 @@ MODULE_BLOCK JIT_SHIM_BUILD_O JIT_SHIM_O JIT_STENCILS_H +MODULE_XXLIMITED_3_13_FALSE +MODULE_XXLIMITED_3_13_TRUE MODULE_XXLIMITED_35_FALSE MODULE_XXLIMITED_35_TRUE MODULE_XXLIMITED_FALSE @@ -31944,6 +31946,7 @@ case $ac_sys_system in #( py_cv_module_termios=n/a py_cv_module_xxlimited=n/a py_cv_module_xxlimited_35=n/a + py_cv_module_xxlimited_3_13=n/a py_cv_module_=n/a ;; #( @@ -35005,6 +35008,46 @@ fi printf "%s\n" "$py_cv_module_xxlimited_35" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module xxlimited_3_13" >&5 +printf %s "checking for stdlib extension module xxlimited_3_13... " >&6; } + if test "$py_cv_module_xxlimited_3_13" != "n/a" +then : + + if test "$TEST_MODULES" = yes +then : + if test "$ac_cv_func_dlopen" = yes +then : + py_cv_module_xxlimited_3_13=yes +else case e in #( + e) py_cv_module_xxlimited_3_13=missing ;; +esac +fi +else case e in #( + e) py_cv_module_xxlimited_3_13=disabled ;; +esac +fi + +fi + as_fn_append MODULE_BLOCK "MODULE_XXLIMITED_3_13_STATE=$py_cv_module_xxlimited_3_13$as_nl" + if test "x$py_cv_module_xxlimited_3_13" = xyes +then : + + + + +fi + if test "$py_cv_module_xxlimited_3_13" = yes; then + MODULE_XXLIMITED_3_13_TRUE= + MODULE_XXLIMITED_3_13_FALSE='#' +else + MODULE_XXLIMITED_3_13_TRUE='#' + MODULE_XXLIMITED_3_13_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module_xxlimited_3_13" >&5 +printf "%s\n" "$py_cv_module_xxlimited_3_13" >&6; } + + # Determine JIT stencils header files based on target platform JIT_STENCILS_H="" JIT_SHIM_O="" @@ -35518,6 +35561,10 @@ if test -z "${MODULE_XXLIMITED_35_TRUE}" && test -z "${MODULE_XXLIMITED_35_FALSE as_fn_error $? "conditional \"MODULE_XXLIMITED_35\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE_XXLIMITED_3_13_TRUE}" && test -z "${MODULE_XXLIMITED_3_13_FALSE}"; then + as_fn_error $? "conditional \"MODULE_XXLIMITED_3_13\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi : "${CONFIG_STATUS=./config.status}" ac_write_fail=0 diff --git a/configure.ac b/configure.ac index 7f97db73ad807d..842e010f20cb74 100644 --- a/configure.ac +++ b/configure.ac @@ -8047,6 +8047,7 @@ AS_CASE([$ac_sys_system], [termios], [xxlimited], [xxlimited_35], + [xxlimited_3_13], ) ], [PY_STDLIB_MOD_SET_NA([_scproxy])] @@ -8438,6 +8439,7 @@ dnl Limited API template modules. dnl Emscripten does not support shared libraries yet. PY_STDLIB_MOD([xxlimited], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([xxlimited_35], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited_3_13], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) # Determine JIT stencils header files based on target platform JIT_STENCILS_H="" From 45fc9acb8cffd4009aedb8fdf2fce2add89f38bb Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Wed, 13 May 2026 15:45:39 -0700 Subject: [PATCH 051/110] [3.15] gh-149504: Fix re-entrancy bug when .pth/.start file invokes site.add sitedir() (#149659) (#149799) * gh-149504: Fix re-entrancy bug when .pth/.start file invokes site.addsitedir() (#149659) * Add re-entrant tests for gh-149504 * Add end-to-end integration test coverage This ensures that future whitebox internal test changes do not regress the public surface semantics. * Implement a state class to process .pth and .start files By using this state class and managing implicit and explicit batching, we make it structurally impossible to get bitten by re-entrant site startup processing. Fixes #149504 (cherry picked from commit b162307d7f216e87976d76c9b8f4a932961cb2d4) * Add myself back to CODEOWNERS --- .github/CODEOWNERS | 6 +- Lib/site.py | 500 ++++++++----- Lib/test/test_site.py | 665 +++++++++++------- ...-05-10-23-51-23.gh-issue-149504.pDSCbn.rst | 5 + 4 files changed, 753 insertions(+), 423 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-10-23-51-23.gh-issue-149504.pDSCbn.rst diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b4b3b1013e3095..bdf134254121e5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -573,9 +573,9 @@ Lib/shutil.py @giampaolo Lib/test/test_shutil.py @giampaolo # Site -Lib/site.py @FFY00 -Lib/test/test_site.py @FFY00 -Doc/library/site.rst @FFY00 +Lib/site.py @FFY00 @warsaw +Lib/test/test_site.py @FFY00 @warsaw +Doc/library/site.rst @FFY00 @warsaw # string.templatelib Doc/library/string.templatelib.rst @lysnikolaou @AA-Turner diff --git a/Lib/site.py b/Lib/site.py index 52dd9648734c3e..64e8192a9ac81a 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -154,13 +154,37 @@ def _init_pathinfo(): return d -# Accumulated entry points from .start files across all site-packages -# directories. Execution is deferred until all paths in .pth files have been -# appended to sys.path. Map the .pth/.start file the data is found in to the -# data. -_pending_entrypoints = {} -_pending_syspaths = {} -_pending_importexecs = {} +# PEP 829 implementation notes. +# +# Startup information (.pth and .start file information) can be processed in +# implicit or explicit batches. Implicit batches are handled by the site.py +# machinery automatically, while explicit batches are driven by user code and +# processed on boundaries defined by that code. +# +# addsitedir() calls which use the default defer_processing_start_files=False +# are self-contained: they create a per-call _StartupState, populate it from +# the site directory's .pth/.start files, run process() on it, and then throw +# the state away. This is implicit batching and in that case the +# _startup_state global variable stays None. +# +# main() needs different semantics: it accumulates state across multiple +# addsitedir() calls (user-site plus all global site-packages) so that +# every sys.path extension is visible *before* any startup code (.pth +# import lines and .start entry points) runs. Callers opt into this by +# passing defer_processing_start_files=True, which preserves the _StartupState +# into the global _startup_state. Subsequent addsitedir() calls (with +# or without defer_processing_start_files=True) then write into that +# same shared state, and a later process_startup_files() call flushes +# all the state and resets the global to None. +# +# Here's the CRITICAL reentrancy invariant: process_startup_files() must clear +# the global _startup_state *before* calling state.process(), so that any +# reentrant site.addsitedir() calls reached from an exec'd .pth import line or +# a .start entry point falls into the per-call branch and gets its own fresh +# state. Otherwise the recursive addsitedir() would mutate the very dicts +# that the outer state.process() is iterating. This is the bug reported in +# gh-149504. +_startup_state = None def _read_pthstart_file(sitedir, name, suffix): @@ -194,13 +218,13 @@ def _read_pthstart_file(sitedir, name, suffix): return None, filename try: - # Accept BOM markers in .start and .pth files as we do in source files (Windows PowerShell - # 5.1 makes it hard to emit UTF-8 files without a BOM). + # Accept BOM markers in .start and .pth files as we do in source files + # (Windows PowerShell 5.1 makes it hard to emit UTF-8 files without a BOM). content = raw_content.decode("utf-8-sig") except UnicodeDecodeError: _trace(f"Cannot read {filename!r} as UTF-8.") - # For .pth files only, and then only until Python 3.20, fallback to locale encoding for - # backward compatibility. + # For .pth files only, and then only until Python 3.20, fall back to + # locale encoding for backward compatibility. _warn_future_us( ".pth files decoded to locale encoding as a fallback", remove=(3, 20) @@ -214,153 +238,221 @@ def _read_pthstart_file(sitedir, name, suffix): return content.splitlines(), filename -def _read_pth_file(sitedir, name, known_paths): - """Parse a .pth file, accumulating sys.path extensions and import lines. - - Errors on individual lines do not abort processing of the rest of the - file (PEP 829). - """ - lines, filename = _read_pthstart_file(sitedir, name, ".pth") - if lines is None: - return - - for n, line in enumerate(lines, 1): - line = line.strip() - if not line or line.startswith("#"): - continue - - # In Python 3.18 and 3.19, `import` lines are silently ignored. In - # Python 3.20 and beyond, issue a warning when `import` lines in .pth - # files are detected. - if line.startswith(("import ", "import\t")): - _warn_future_us( - "import lines in .pth files are silently ignored", - remove=(3, 18) - ) - _warn_future_us( - "import lines in .pth files are noisily ignored", - remove=(3, 20) - ) - _pending_importexecs.setdefault(filename, []).append(line) - continue - - try: - dir_, dircase = makepath(sitedir, line) - except Exception as exc: - _trace(f"Error in {filename!r}, line {n:d}: {line!r}", exc) - continue - - if dircase in known_paths: - _trace(f"In {filename!r}, line {n:d}: " - f"skipping duplicate sys.path entry: {dir_}") - else: - _pending_syspaths.setdefault(filename, []).append(dir_) - known_paths.add(dircase) - +class _StartupState: + """Per-batch accumulator for .pth and .start file processing. -def _read_start_file(sitedir, name): - """Parse a .start file for a list of entry point strings.""" - lines, filename = _read_pthstart_file(sitedir, name, ".start") - if lines is None: - return + A _StartupState collects sys.path extensions, deprecated .pth import + lines, and .start entry points read from one or more site-packages + directories. Calling process() applies them in PEP 829 order: paths + are added to sys.path first, then import lines from .pth files (skipping + any with a matching .start), then entry points from .start files. - # PEP 829: the *presence* of a matching .start file disables `import` - # line processing in the matched .pth file, regardless of whether the - # .start file produced any entry points. Register the filename as a - # key now so an empty (or comment-only) .start file still suppresses. - entrypoints = _pending_entrypoints.setdefault(filename, []) + State lives entirely on the instance; there is no module-level pending + state. This is what makes the module reentrancy-safe: a site.addsitedir() + call reached recursively from an exec'd import line or a .start entry + point operates on a different _StartupState than the one being processed + by the outer call. - for n, line in enumerate(lines, 1): - line = line.strip() - if not line or line.startswith("#"): - continue - # Syntax validation is deferred to entry-point execution time, - # where pkgutil.resolve_name(strict=True) enforces the - # pkg.mod:callable form. - entrypoints.append(line) - - -def _extend_syspath(): - # We've already filtered out duplicates, either in the existing sys.path - # or in all the .pth files we've seen. We've also abspath/normpath'd all - # the entries, so all that's left to do is to ensure that the path exists. - for filename, dirs in _pending_syspaths.items(): - for dir_ in dirs: - if os.path.exists(dir_): - _trace(f"Extending sys.path with {dir_} from {filename}") - sys.path.append(dir_) - else: - _print_error( - f"In {filename}: {dir_} does not exist; " - f"skipping sys.path append") - - -def _exec_imports(): - # For all the `import` lines we've seen in .pth files, exec() them in - # order. However, if they come from a file with a matching .start, then - # we ignore these import lines. For the ones we do process, print a - # warning but only when -v was given. - for filename, imports in _pending_importexecs.items(): - name, dot, pth = filename.rpartition(".") - assert dot == "." and pth == "pth", f"Bad startup filename: {filename}" - - if f"{name}.start" in _pending_entrypoints: - # Skip import lines in favor of entry points. - continue + The internal data is intentionally private; the public methods + (read_pth_file, read_start_file, process) are the only supported write + APIs. + """ + __slots__ = ('_syspaths', '_importexecs', '_entrypoints') + + def __init__(self): + # All three dicts map "" -> list + # of items collected from that file. Mapping by filename lets us + # cross-reference a .pth and its matching .start (PEP 829 import + # suppression rule) and lets _print_error report the source file + # when an entry fails. + self._syspaths = {} + self._importexecs = {} + self._entrypoints = {} + + def read_pth_file(self, sitedir, name, known_paths): + """Parse a .pth file, accumulating sys.path extensions and import lines. + + Errors on individual lines do not abort processing of the rest of + the file (PEP 829). ``known_paths`` is the per-batch dedup + ledger: any path already in it is skipped, and newly accepted + paths are added to it so that subsequent .pth files in the same + batch don't add them more than once. + """ + lines, filename = _read_pthstart_file(sitedir, name, ".pth") + if lines is None: + return + + for n, line in enumerate(lines, 1): + line = line.strip() + if not line or line.startswith("#"): + continue - _trace( - f"import lines in {filename} are deprecated, " - f"use entry points in a {name}.start file instead." - ) + # In Python 3.18 and 3.19, `import` lines are silently + # ignored. In Python 3.20 and beyond, issue a warning when + # `import` lines in .pth files are detected. + if line.startswith(("import ", "import\t")): + _warn_future_us( + "import lines in .pth files are silently ignored", + remove=(3, 18), + ) + _warn_future_us( + "import lines in .pth files are noisily ignored", + remove=(3, 20), + ) + self._importexecs.setdefault(filename, []).append(line) + continue - for line in imports: try: - _trace(f"Exec'ing from {filename}: {line}") - exec(line) + dir_, dircase = makepath(sitedir, line) except Exception as exc: - _print_error( - f"Error in import line from {filename}: {line}", exc) - - -def _execute_start_entrypoints(): - """Execute all accumulated .start file entry points. + _trace(f"Error in {filename!r}, line {n:d}: {line!r}", exc) + continue - Called after all site-packages directories have been processed so that - sys.path is fully populated before any entry point code runs. Uses - pkgutil.resolve_name(strict=True) which both validates the strict - pkg.mod:callable form and resolves the entry point in one step. - """ - for filename, entrypoints in _pending_entrypoints.items(): - for entrypoint in entrypoints: - try: - _trace(f"Executing entry point: {entrypoint} from {filename}") - callable_ = pkgutil.resolve_name(entrypoint, strict=True) - except ValueError as exc: - _print_error( - f"Invalid entry point syntax in {filename}: " - f"{entrypoint!r}", exc) + # PEP 829 dedup: skip paths already seen in this batch. See + # _startup_state docstring above for batch lifetimes. + if dircase in known_paths: + _trace( + f"In {filename!r}, line {n:d}: " + f"skipping duplicate sys.path entry: {dir_}" + ) + else: + self._syspaths.setdefault(filename, []).append(dir_) + known_paths.add(dircase) + + def read_start_file(self, sitedir, name): + """Parse a .start file for a list of entry point strings.""" + lines, filename = _read_pthstart_file(sitedir, name, ".start") + if lines is None: + return + + # PEP 829: the *presence* of a matching .start file disables `import` + # line processing in the matched .pth file, regardless of whether this + # .start file contains any entry points. Register the filename as a + # key now so an empty (or comment-only) .start file still suppresses. + entrypoints = self._entrypoints.setdefault(filename, []) + + for n, line in enumerate(lines, 1): + line = line.strip() + if not line or line.startswith("#"): continue - except Exception as exc: - _print_error( - f"Error resolving entry point {entrypoint} " - f"from {filename}", exc) + # Syntax validation is deferred to entry point execution + # time, where pkgutil.resolve_name(strict=True) enforces the + # pkg.mod:callable form. + entrypoints.append(line) + + def process(self): + """Apply accumulated state in PEP 829 order. + + Phase order matters: all .pth path extensions are applied to + sys.path *before* any import line or .start entry point runs, so + that an entry point may live in a module reachable only via a + .pth-extended path. + """ + self._extend_syspath() + self._exec_imports() + self._execute_start_entrypoints() + + def _extend_syspath(self): + # Duplicates have already been filtered (in existing sys.path or + # across .pth files via known_paths), and entries are already + # abspath/normpath'd, so all that remains is to confirm the path + # exists. + for filename, dirs in self._syspaths.items(): + for dir_ in dirs: + if os.path.exists(dir_): + _trace(f"Extending sys.path with {dir_} from {filename}") + sys.path.append(dir_) + else: + _print_error( + f"In {filename}: {dir_} does not exist; " + f"skipping sys.path append" + ) + + def _exec_imports(self): + # For each `import` line we've seen in a .pth file, exec() it in + # order, unless the .pth has a matching .start file in this same + # batch. In that case, PEP 829 says the import lines are + # suppressed in favor of the .start's entry points. + for filename, imports in self._importexecs.items(): + # Given "/path/to/foo.pth", check whether "/path/to/foo.start" was + # registered in this same batch. + name, dot, pth = filename.rpartition(".") + assert dot == "." and pth == "pth", ( + f"Bad startup filename: {filename}" + ) + if f"{name}.start" in self._entrypoints: + _trace( + f"import lines in {filename} are suppressed " + f"due to matching {name}.start file." + ) continue - try: - callable_() - except Exception as exc: - _print_error( - f"Error in entry point {entrypoint} from {filename}", - exc) + + _trace( + f"import lines in {filename} are deprecated, " + f"use entry points in a {name}.start file instead." + ) + for line in imports: + try: + _trace(f"Exec'ing from {filename}: {line}") + exec(line) + except Exception as exc: + _print_error( + f"Error in import line from {filename}: {line}", + exc, + ) + + def _execute_start_entrypoints(self): + # Resolve each entry point string to a callable via + # pkgutil.resolve_name(strict=True), which both validates the + # required pkg.mod:callable form and performs the import in one + # step, then call it with no arguments. + for filename, entrypoints in self._entrypoints.items(): + for entrypoint in entrypoints: + try: + _trace( + f"Executing entry point: {entrypoint} from {filename}" + ) + callable_ = pkgutil.resolve_name(entrypoint, strict=True) + except ValueError as exc: + _print_error( + f"Invalid entry point syntax in {filename}: " + f"{entrypoint!r}", + exc, + ) + except Exception as exc: + _print_error( + f"Error resolving entry point {entrypoint} " + f"from {filename}", + exc, + ) + else: + try: + callable_() + except Exception as exc: + _print_error( + f"Error in entry point {entrypoint} from {filename}", + exc, + ) def process_startup_files(): - """Flush all pending sys.path and entry points.""" - _extend_syspath() - _exec_imports() - _execute_start_entrypoints() - _pending_syspaths.clear() - _pending_importexecs.clear() - _pending_entrypoints.clear() + """Flush any pending startup-file state accumulated during a batch. + + Used by main() (and any external caller that drove addsitedir() with + defer_processing_start_files=True) to apply the accumulated paths + and run the deferred import lines / entry points. + + Reentrancy: the active batch state is detached from _startup_state + *before* state.process() runs. This way, if an exec'd import line + or .start entry point itself calls site.addsitedir(), that call + creates its own per-call _StartupState rather than mutating the dicts + being iterated here. See gh-149504. + """ + global _startup_state + if _startup_state is None: + return + state, _startup_state = _startup_state, None + state.process() def addpackage(sitedir, name, known_paths): @@ -370,16 +462,26 @@ def addpackage(sitedir, name, known_paths): reset = True else: reset = False - _read_pth_file(sitedir, name, known_paths) - process_startup_files() - if reset: - known_paths = None - return known_paths + + # If a batch is already in progress (for example, main() is still + # accumulating sitedirs), participate in the batch by writing into the + # shared _startup_state and letting the eventual process_startup_files() + # flush it. Otherwise this is a standalone call, so create a unique + # per-call state, populate it, and process it before returning. + if _startup_state is None: + state = _StartupState() + state.read_pth_file(sitedir, name, known_paths) + state.process() + else: + _startup_state.read_pth_file(sitedir, name, known_paths) + + return None if reset else known_paths def addsitedir(sitedir, known_paths=None, *, defer_processing_start_files=False): """Add 'sitedir' argument to sys.path if missing and handle startup files.""" + global _startup_state _trace(f"Adding directory: {sitedir!r}") if known_paths is None: known_paths = _init_pathinfo() @@ -387,44 +489,74 @@ def addsitedir(sitedir, known_paths=None, *, defer_processing_start_files=False) else: reset = False sitedir, sitedircase = makepath(sitedir) - if not sitedircase in known_paths: - sys.path.append(sitedir) # Add path component - known_paths.add(sitedircase) - try: - names = os.listdir(sitedir) - except OSError: - return - # The following phases are defined by PEP 829. - # Phases 1-3: Read .pth files, accumulating paths and import lines. - pth_names = sorted( - name for name in names - if name.endswith(".pth") and not name.startswith(".") - ) - for name in pth_names: - _read_pth_file(sitedir, name, known_paths) - - # Phases 6-7: Discover .start files and accumulate their entry points. - # Import lines from .pth files with a matching .start file are discarded - # at flush time by _exec_imports(). - start_names = sorted( - name for name in names - if name.endswith(".start") and not name.startswith(".") - ) - for name in start_names: - _read_start_file(sitedir, name) + # If the normcase'd new sitedir isn't already known, append it to + # sys.path, keep a record of it, and process all .pth and .start files + # found in that directory. If the new sitedir is known, be sure not + # to process all of those more than once! gh-75723 + if sitedircase not in known_paths: + sys.path.append(sitedir) + known_paths.add(sitedircase) - # Generally, when addsitedir() is called explicitly, we'll want to process - # all the startup file data immediately. However, when called through - # main(), we'll want to batch up all the startup file processing. main() - # will set this flag to True to defer processing. - if not defer_processing_start_files: - process_startup_files() + try: + names = os.listdir(sitedir) + except OSError: + return None if reset else known_paths + + # Pick the _StartupState we'll write into. There are three cases: + # + # 1. A batch is already active (_startup_state is set, e.g. because + # main() previously called us with + # defer_processing_start_files=True). Participate in this batch by + # sharing the same state. Don't flush the state since the batch's + # eventual process_startup_files() will do that. + # + # 2. There is no active batch but the caller passed + # defer_processing_start_files=True. Preserve a fresh + # _StartupState into the global _startup_state so that subsequent + # addsitedir() calls participate in this batch, and so that the + # caller's later process_startup_files() finds it. + # + # 3. This is a standalone call (there is no active batch and + # defer_processing_start_files=False). Create a unique per-call + # state, populate it, process it, and then clear it. Per-call + # state is what makes reentrant addsitedir() safe; a recursive call + # from inside process() lands here too and gets its own independent + # state. + + if _startup_state is not None: + state = _startup_state + flush_now = False + elif defer_processing_start_files: + state = _startup_state = _StartupState() + flush_now = False + else: + state = _StartupState() + flush_now = True + + # The following phases are defined by PEP 829. + # Phases 1-3: Read .pth files, accumulating paths and import lines. + pth_names = sorted( + name for name in names + if name.endswith(".pth") and not name.startswith(".") + ) + for name in pth_names: + state.read_pth_file(sitedir, name, known_paths) + + # Phases 6-7: Discover .start files and accumulate their entry points. + # Import lines from .pth files with a matching .start file are + # discarded at flush time by _StartupState._exec_imports(). + start_names = sorted( + name for name in names + if name.endswith(".start") and not name.startswith(".") + ) + for name in start_names: + state.read_start_file(sitedir, name) - if reset: - known_paths = None + if flush_now: + state.process() - return known_paths + return None if reset else known_paths def check_enableusersite(): diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index ac69e2cbdbbe54..0e6f352f49cd38 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -196,8 +196,9 @@ def test_addsitedir_explicit_flush(self): pth_file.cleanup(prep=True) with pth_file.create(): # Pass defer_processing_start_files=True to prevent flushing. - site.addsitedir(pth_file.base_dir, set(), - defer_processing_start_files=True) + site.addsitedir( + pth_file.base_dir, set(), + defer_processing_start_files=True) self.assertNotIn(pth_file.imported, sys.modules) site.process_startup_files() self.pth_file_tests(pth_file) @@ -423,15 +424,14 @@ def create(self): Used as a context manager: self.cleanup() is called on exit. """ - FILE = open(self.file_path, 'w') - try: - print("#import @bad module name", file=FILE) - print("\n", file=FILE) - print("import %s" % self.imported, file=FILE) - print(self.good_dirname, file=FILE) - print(self.bad_dirname, file=FILE) - finally: - FILE.close() + with open(self.file_path, 'w') as fp: + print(f"""\ +#import @bad module name +import {self.imported} +{self.good_dirname} +{self.bad_dirname} +""", file=fp) + os.mkdir(self.good_dir_path) try: yield self @@ -915,18 +915,16 @@ class StartFileTests(unittest.TestCase): def setUp(self): self.enterContext(import_helper.DirsOnSysPath()) self.tmpdir = self.sitedir = self.enterContext(os_helper.temp_dir()) - # Save and clear all pending dicts. - self.saved_entrypoints = site._pending_entrypoints.copy() - self.saved_syspaths = site._pending_syspaths.copy() - self.saved_importexecs = site._pending_importexecs.copy() - site._pending_entrypoints.clear() - site._pending_syspaths.clear() - site._pending_importexecs.clear() + # Each test gets its own _StartupState to drive the parser and + # processor methods directly. Defensively clear any _startup_state + # that a prior test may have left set via defer_processing_start_files + # without a corresponding process_startup_files() flush. + self.state = site._StartupState() + site._startup_state = None + self.addCleanup(self._reset_startup_state) - def tearDown(self): - site._pending_entrypoints = self.saved_entrypoints.copy() - site._pending_syspaths = self.saved_syspaths.copy() - site._pending_importexecs = self.saved_importexecs.copy() + def _reset_startup_state(self): + site._startup_state = None def _make_start(self, content, name='testpkg'): """Write a .start file and return its basename.""" @@ -944,10 +942,32 @@ def _make_pth(self, content, name='testpkg'): f.write(content) return basename + def _make_mod(self, contents, name='mod', *, package=False, on_path=False): + """Write an importable module (or package), returning its parent dir.""" + extdir = os.path.join(self.sitedir, 'extdir') + os.makedirs(extdir, exist_ok=True) + + # Put the code in a package's dunder-init or flat module. + if package: + pkgdir = os.path.join(extdir, name) + os.mkdir(pkgdir) + modpath = os.path.join(pkgdir, '__init__.py') + else: + modpath = os.path.join(extdir, f'{name}.py') + + with open(modpath, 'w') as fp: + fp.write(contents) + + self.addCleanup(sys.modules.pop, name, None) + if on_path: + # Don't worry, DirsOnSysPath() in setUp() will clean this up. + sys.path.insert(0, extdir) + return extdir + def _all_entrypoints(self): - """Flatten _pending_entrypoints dict into a list of (filename, entry) tuples.""" + """Flatten state._entrypoints into a list of (filename, entry) tuples.""" result = [] - for filename, entries in site._pending_entrypoints.items(): + for filename, entries in self.state._entrypoints.items(): for entry in entries: result.append((filename, entry)) return result @@ -955,28 +975,42 @@ def _all_entrypoints(self): def _just_entrypoints(self): return [entry for filename, entry in self._all_entrypoints()] - # --- _read_start_file tests --- + # There are two classes of tests here. Tests that start with `test_impl_` + # know details about the implementation and they access non-public methods + # and data structures to perform focused functional tests. + # + # Tests that start with `test_addsitedir_` are end-to-end tests that ensure + # integration semantics and functionality as a caller of the public + # surfaces would see. + + # --- _StartupState.read_start_file tests --- - def test_read_start_file_basic(self): + def test_impl_read_start_file_basic(self): self._make_start("os.path:join\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], ['os.path:join']) + self.assertEqual( + self.state._entrypoints[fullname], ['os.path:join'] + ) - def test_read_start_file_multiple_entries(self): + def test_impl_read_start_file_multiple_entries(self): self._make_start("os.path:join\nos.path:exists\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], - ['os.path:join', 'os.path:exists']) + self.assertEqual( + self.state._entrypoints[fullname], + ['os.path:join', 'os.path:exists'], + ) - def test_read_start_file_comments_and_blanks(self): + def test_impl_read_start_file_comments_and_blanks(self): self._make_start("# a comment\n\nos.path:join\n \n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], ['os.path:join']) + self.assertEqual( + self.state._entrypoints[fullname], ['os.path:join'] + ) - def test_read_start_file_accepts_all_non_blank_lines(self): + def test_impl_read_start_file_accepts_all_non_blank_lines(self): # Syntax validation is deferred to entry-point execution time # (where pkgutil.resolve_name(strict=True) enforces the strict # pkg.mod:callable form), so parsing accepts every non-blank, @@ -989,9 +1023,9 @@ def test_read_start_file_accepts_all_non_blank_lines(self): "os.path:join\n" # valid ) self._make_start(content, name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], [ + self.assertEqual(self.state._entrypoints[fullname], [ 'os.path', 'pkg.mod:', ':callable', @@ -999,155 +1033,169 @@ def test_read_start_file_accepts_all_non_blank_lines(self): 'os.path:join', ]) - def test_read_start_file_empty(self): + def test_impl_read_start_file_empty(self): # PEP 829: an empty .start file is still registered as present - # (with an empty entry-point list) so that it suppresses `import` + # (with an empty entry point list) so that it suppresses `import` # lines in any matching .pth file. self._make_start("", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {fullname: []}) + self.assertEqual(self.state._entrypoints, {fullname: []}) - def test_read_start_file_comments_only(self): + def test_impl_read_start_file_comments_only(self): # As with an empty file, a comments-only .start file is registered # as present so it can suppress matching .pth `import` lines. self._make_start("# just a comment\n# another\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {fullname: []}) + self.assertEqual(self.state._entrypoints, {fullname: []}) - def test_read_start_file_nonexistent(self): + def test_impl_read_start_file_nonexistent(self): with captured_stderr(): - site._read_start_file(self.tmpdir, 'nonexistent.start') - self.assertEqual(site._pending_entrypoints, {}) + self.state.read_start_file(self.tmpdir, 'nonexistent.start') + self.assertEqual(self.state._entrypoints, {}) @unittest.skipUnless(hasattr(os, 'chflags'), 'test needs os.chflags()') - def test_read_start_file_hidden_flags(self): + def test_impl_read_start_file_hidden_flags(self): self._make_start("os.path:join\n", name='foo') filepath = os.path.join(self.tmpdir, 'foo.start') st = os.stat(filepath) os.chflags(filepath, st.st_flags | stat.UF_HIDDEN) - site._read_start_file(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {}) + self.state.read_start_file(self.sitedir, 'foo.start') + self.assertEqual(self.state._entrypoints, {}) - def test_read_start_file_duplicates_not_deduplicated(self): + def test_impl_one_start_file_with_duplicates_not_deduplicated(self): # PEP 829: duplicate entry points are NOT deduplicated. self._make_start("os.path:join\nos.path:join\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], - ['os.path:join', 'os.path:join']) + self.assertEqual( + self.state._entrypoints[fullname], + ['os.path:join', 'os.path:join'], + ) + + def test_impl_two_start_files_with_duplicates_not_deduplicated(self): + self._make_start("os.path:join", name="foo") + self._make_start("os.path:join", name="bar") + self.state.read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'bar.start') + self.assertEqual( + self._just_entrypoints(), + ['os.path:join', 'os.path:join'], + ) - def test_read_start_file_accepts_utf8_bom(self): + def test_impl_read_start_file_accepts_utf8_bom(self): # PEP 829: .start files MUST be utf-8-sig (UTF-8 with optional BOM). filepath = os.path.join(self.tmpdir, 'foo.start') with open(filepath, 'wb') as f: f.write(b'\xef\xbb\xbf' + b'os.path:join\n') - site._read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') self.assertEqual( - site._pending_entrypoints[fullname], ['os.path:join']) + self.state._entrypoints[fullname], ['os.path:join'] + ) - def test_read_start_file_invalid_utf8_silently_skipped(self): - # PEP 829: .start files MUST be utf-8-sig. Unlike .pth, there is - # no locale-encoding fallback -- a .start file that is not valid + def test_impl_read_start_file_invalid_utf8_silently_skipped(self): + # PEP 829: .start files MUST be utf-8-sig. Unlike .pth files, there + # is no locale-encoding fallback. A .start file that is not valid # UTF-8 is silently skipped, with no key registered in - # _pending_entrypoints and no output to stderr (parsing errors - # are reported only under -v). + # state._entrypoints and no output to stderr (parsing errors are + # reported only under -v). filepath = os.path.join(self.tmpdir, 'foo.start') with open(filepath, 'wb') as f: # Bare continuation byte -- invalid as a UTF-8 start byte. f.write(b'\x80\x80\x80\n') with captured_stderr() as err: - site._read_start_file(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {}) + self.state.read_start_file(self.sitedir, 'foo.start') + self.assertEqual(self.state._entrypoints, {}) self.assertEqual(err.getvalue(), "") - def test_two_start_files_with_duplicates_not_deduplicated(self): - self._make_start("os.path:join", name="foo") - self._make_start("os.path:join", name="bar") - site._read_start_file(self.sitedir, 'foo.start') - site._read_start_file(self.sitedir, 'bar.start') - self.assertEqual(self._just_entrypoints(), - ['os.path:join', 'os.path:join']) + # --- _StartupState.read_pth_file tests --- - # --- _read_pth_file tests --- - - def test_read_pth_file_paths(self): + def test_impl_read_pth_file_paths(self): subdir = os.path.join(self.sitedir, 'mylib') os.mkdir(subdir) self._make_pth("mylib\n", name='foo') - site._read_pth_file(self.sitedir, 'foo.pth', set()) + self.state.read_pth_file(self.sitedir, 'foo.pth', set()) fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths[fullname]) + self.assertIn(subdir, self.state._syspaths[fullname]) - def test_read_pth_file_imports_collected(self): + def test_impl_read_pth_file_imports_collected(self): self._make_pth("import sys\n", name='foo') - site._read_pth_file(self.sitedir, 'foo.pth', set()) + self.state.read_pth_file(self.sitedir, 'foo.pth', set()) fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertEqual(site._pending_importexecs[fullname], ['import sys']) + self.assertEqual( + self.state._importexecs[fullname], ['import sys'] + ) - def test_read_pth_file_comments_and_blanks(self): + def test_impl_read_pth_file_comments_and_blanks(self): self._make_pth("# comment\n\n \n", name='foo') - site._read_pth_file(self.sitedir, 'foo.pth', set()) - self.assertEqual(site._pending_syspaths, {}) - self.assertEqual(site._pending_importexecs, {}) + self.state.read_pth_file(self.sitedir, 'foo.pth', set()) + self.assertEqual(self.state._syspaths, {}) + self.assertEqual(self.state._importexecs, {}) - def test_read_pth_file_deduplication(self): + def test_impl_read_pth_file_deduplication(self): subdir = os.path.join(self.sitedir, 'mylib') os.mkdir(subdir) + # An accumulator acts as a deduplication ledger. known_paths = set() self._make_pth("mylib\n", name='a') self._make_pth("mylib\n", name='b') - site._read_pth_file(self.sitedir, 'a.pth', known_paths) - site._read_pth_file(self.sitedir, 'b.pth', known_paths) - # Only one entry across both files. + self.state.read_pth_file(self.sitedir, 'a.pth', known_paths) + self.state.read_pth_file(self.sitedir, 'b.pth', known_paths) + # There is only one entry across both files. all_dirs = [] - for dirs in site._pending_syspaths.values(): + for dirs in self.state._syspaths.values(): all_dirs.extend(dirs) self.assertEqual(all_dirs, [subdir]) - def test_read_pth_file_bad_line_continues(self): - # PEP 829: errors on individual lines don't abort the file. + def test_impl_read_pth_file_bad_line_continues(self): + # PEP 829: errors on individual lines don't abort processing the file. subdir = os.path.join(self.sitedir, 'goodpath') os.mkdir(subdir) self._make_pth("abc\x00def\ngoodpath\n", name='foo') with captured_stderr(): - site._read_pth_file(self.sitedir, 'foo.pth', set()) + self.state.read_pth_file(self.sitedir, 'foo.pth', set()) fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) + self.assertIn(subdir, self.state._syspaths.get(fullname, [])) def _flags_with_verbose(self, verbose): # Build a sys.flags clone with verbose overridden but every # other field preserved, so unrelated reads like # sys.flags.optimize during io.open_code() continue to work. - attrs = {name: getattr(sys.flags, name) - for name in sys.flags.__match_args__} + attrs = { + name: getattr(sys.flags, name) + for name in sys.flags.__match_args__ + } attrs['verbose'] = verbose return SimpleNamespace(**attrs) - def test_read_pth_file_parse_error_silent_by_default(self): + def test_impl_read_pth_file_parse_error_silent_by_default(self): # PEP 829: parse-time errors are silent unless -v is given. - # Force the error path by making makepath() raise. + # Force the error path by making makepath() raise an exception. self._make_pth("badline\n", name='foo') - with mock.patch('site.makepath', side_effect=ValueError("boom")), \ - mock.patch('sys.flags', self._flags_with_verbose(False)), \ - captured_stderr() as err: - site._read_pth_file(self.sitedir, 'foo.pth', set()) + with ( + mock.patch('site.makepath', side_effect=ValueError("boom")), + mock.patch('sys.flags', self._flags_with_verbose(False)), + captured_stderr() as err, + ): + self.state.read_pth_file(self.sitedir, 'foo.pth', set()) self.assertEqual(err.getvalue(), "") - def test_read_pth_file_parse_error_reported_under_verbose(self): + def test_impl_read_pth_file_parse_error_reported_under_verbose(self): # PEP 829: parse-time errors are reported when -v is given. self._make_pth("badline\n", name='foo') - with mock.patch('site.makepath', side_effect=ValueError("boom")), \ - mock.patch('sys.flags', self._flags_with_verbose(True)), \ - captured_stderr() as err: - site._read_pth_file(self.sitedir, 'foo.pth', set()) + with ( + mock.patch('site.makepath', side_effect=ValueError("boom")), + mock.patch('sys.flags', self._flags_with_verbose(True)), + captured_stderr() as err, + ): + self.state.read_pth_file(self.sitedir, 'foo.pth', set()) out = err.getvalue() self.assertIn('Error in', out) self.assertIn('foo.pth', out) - def test_read_pth_file_locale_fallback(self): + def test_impl_read_pth_file_locale_fallback(self): # PEP 829: .pth files that fail UTF-8 decoding fall back to the # locale encoding for backward compatibility (deprecated in # 3.15, to be removed in 3.20). Mock locale.getencoding() so @@ -1158,186 +1206,236 @@ def test_read_pth_file_locale_fallback(self): # \xe9 is invalid UTF-8 but valid in latin-1. with open(filepath, 'wb') as f: f.write(b'# caf\xe9 comment\nmylib\n') - with mock.patch('locale.getencoding', return_value='latin-1'), \ - captured_stderr(): - site._read_pth_file(self.sitedir, 'foo.pth', set()) + with ( + mock.patch('locale.getencoding', return_value='latin-1'), + captured_stderr(), + ): + self.state.read_pth_file(self.sitedir, 'foo.pth', set()) fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) + self.assertIn(subdir, self.state._syspaths.get(fullname, [])) - # --- _execute_start_entrypoints tests --- + # --- _StartupState._execute_start_entrypoints tests --- - def test_execute_entrypoints_with_callable(self): - # Entrypoint with callable is invoked. - mod_dir = os.path.join(self.sitedir, 'epmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ + def test_impl_execute_entrypoints_with_callable(self): + # An entry point with a callable. + self._make_mod("""\ called = False def startup(): global called called = True -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'epmod', None) +""", name='epmod', package=True, on_path=True) fullname = os.path.join(self.sitedir, 'epmod.start') - site._pending_entrypoints[fullname] = ['epmod:startup'] - site._execute_start_entrypoints() + self.state._entrypoints[fullname] = ['epmod:startup'] + self.state._execute_start_entrypoints() import epmod self.assertTrue(epmod.called) - def test_execute_entrypoints_import_error(self): - # Import error prints traceback but continues. + def test_impl_execute_entrypoints_import_error(self): + # Import errors print a traceback and continue. fullname = os.path.join(self.sitedir, 'bad.start') - site._pending_entrypoints[fullname] = [ - 'nosuchmodule_xyz:func', 'os.path:join'] + self.state._entrypoints[fullname] = [ + 'nosuchmodule_xyz:func', 'os.path:join', + ] with captured_stderr() as err: - site._execute_start_entrypoints() + self.state._execute_start_entrypoints() self.assertIn('nosuchmodule_xyz', err.getvalue()) # os.path:join should still have been called (no exception for it) - def test_execute_entrypoints_strict_syntax_rejection(self): - # PEP 829: only the strict pkg.mod:callable form is valid. - # At entry-point execution, pkgutil.resolve_name(strict=True) - # raises ValueError for invalid syntax; the invalid entry is - # reported and execution continues with the next one. + def test_impl_execute_entrypoints_strict_syntax_rejection(self): + # PEP 829: only the strict pkg.mod:callable form is valid. At entry + # point execution time, pkgutil.resolve_name(strict=True) raises a + # ValueError for the invalid syntax. The invalid entry is reported + # and execution continues with the next one. fullname = os.path.join(self.sitedir, 'bad.start') - site._pending_entrypoints[fullname] = [ + self.state._entrypoints[fullname] = [ 'os.path', # no colon 'pkg.mod:', # empty callable ':callable', # empty module 'pkg.mod:callable:extra', # multiple colons ] with captured_stderr() as err: - site._execute_start_entrypoints() + self.state._execute_start_entrypoints() out = err.getvalue() self.assertIn('Invalid entry point syntax', out) - for bad in ('os.path', 'pkg.mod:', ':callable', - 'pkg.mod:callable:extra'): + for bad in ( + 'os.path', + 'pkg.mod:', + ':callable', + 'pkg.mod:callable:extra', + ): self.assertIn(bad, out) - def test_execute_entrypoints_callable_error(self): - # Callable that raises prints traceback but continues. - mod_dir = os.path.join(self.sitedir, 'badmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ + def test_impl_execute_entrypoints_callable_error(self): + # A callable that errors prints a traceback but continues. + self._make_mod("""\ def fail(): raise RuntimeError("boom") -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'badmod', None) +""", name='badmod', package=True, on_path=True) fullname = os.path.join(self.sitedir, 'badmod.start') - site._pending_entrypoints[fullname] = ['badmod:fail'] + self.state._entrypoints[fullname] = ['badmod:fail'] with captured_stderr() as err: - site._execute_start_entrypoints() + self.state._execute_start_entrypoints() self.assertIn('RuntimeError', err.getvalue()) self.assertIn('boom', err.getvalue()) - def test_execute_entrypoints_duplicates_called_twice(self): + def test_impl_execute_entrypoints_duplicates_called_twice(self): # PEP 829: duplicate entry points execute multiple times. - mod_dir = os.path.join(self.sitedir, 'countmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ + self._make_mod("""\ call_count = 0 def bump(): global call_count call_count += 1 -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'countmod', None) +""", name='countmod', package=False, on_path=True) fullname = os.path.join(self.sitedir, 'countmod.start') - site._pending_entrypoints[fullname] = [ - 'countmod:bump', 'countmod:bump'] - site._execute_start_entrypoints() + self.state._entrypoints[fullname] = [ + 'countmod:bump', 'countmod:bump', + ] + self.state._execute_start_entrypoints() import countmod self.assertEqual(countmod.call_count, 2) - # --- _exec_imports tests --- + # --- _StartupState._exec_imports tests --- - def test_exec_imports_suppressed_by_matching_start(self): + def test_impl_exec_imports_suppressed_by_matching_start(self): # Import lines from foo.pth are suppressed when foo.start exists. + self._make_mod("""\ +call_count = 0 +def bump(): + global call_count + call_count += 1 +""", name='countmod', package=False, on_path=True) pth_fullname = os.path.join(self.sitedir, 'foo.pth') start_fullname = os.path.join(self.sitedir, 'foo.start') - site._pending_importexecs[pth_fullname] = ['import sys'] - site._pending_entrypoints[start_fullname] = ['os.path:join'] - # Should not exec the import line; no error expected. - site._exec_imports() + self.state._importexecs[pth_fullname] = ['import countmod; countmod.bump()'] + self.state._entrypoints[start_fullname] = ['os.path:join'] + self.state._exec_imports() + import countmod + self.assertEqual(countmod.call_count, 0) - def test_exec_imports_not_suppressed_by_different_start(self): + def test_impl_exec_imports_not_suppressed_by_different_start(self): # Import lines from foo.pth are NOT suppressed by bar.start. + self._make_mod("""\ +call_count = 0 +def bump(): + global call_count + call_count += 1 +""", name='countmod', package=False, on_path=True) pth_fullname = os.path.join(self.sitedir, 'foo.pth') start_fullname = os.path.join(self.sitedir, 'bar.start') - site._pending_importexecs[pth_fullname] = ['import sys'] - site._pending_entrypoints[start_fullname] = ['os.path:join'] - # Should execute the import line without error. - site._exec_imports() + self.state._importexecs[pth_fullname] = ['import countmod; countmod.bump()'] + self.state._entrypoints[start_fullname] = ['os.path:join'] + self.state._exec_imports() + import countmod + self.assertEqual(countmod.call_count, 1) - def test_exec_imports_suppressed_by_empty_matching_start(self): + def test_impl_exec_imports_suppressed_by_empty_matching_start(self): self._make_start("", name='foo') self._make_pth("import epmod; epmod.startup()", name='foo') - mod_dir = os.path.join(self.sitedir, 'epmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ + self._make_mod("""\ called = False def startup(): global called called = True -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'epmod', None) - site._read_pth_file(self.sitedir, 'foo.pth', set()) - site._read_start_file(self.sitedir, 'foo.start') - site._exec_imports() +""", name='epmod', package=True, on_path=True) + self.state.read_pth_file(self.sitedir, 'foo.pth', set()) + self.state.read_start_file(self.sitedir, 'foo.start') + self.state._exec_imports() import epmod self.assertFalse(epmod.called) - # --- _extend_syspath tests --- + # --- _StartupState._extend_syspath tests --- - def test_extend_syspath_existing_dir(self): + def test_impl_extend_syspath_existing_dir(self): subdir = os.path.join(self.sitedir, 'extlib') os.mkdir(subdir) - site._pending_syspaths['test.pth'] = [subdir] - site._extend_syspath() + self.state._syspaths['test.pth'] = [subdir] + self.state._extend_syspath() self.assertIn(subdir, sys.path) - def test_extend_syspath_nonexistent_dir(self): - nosuch = os.path.join(self.sitedir, 'nosuchdir') - site._pending_syspaths['test.pth'] = [nosuch] + def test_impl_extend_syspath_nonexistent_dir(self): + nonesuch = os.path.join(self.sitedir, 'nosuchdir') + self.state._syspaths['test.pth'] = [nonesuch] with captured_stderr() as err: - site._extend_syspath() - self.assertNotIn(nosuch, sys.path) + self.state._extend_syspath() + self.assertNotIn(nonesuch, sys.path) self.assertIn('does not exist', err.getvalue()) # --- addsitedir integration tests --- + def test_addsitedir_pth_import_skipped_when_matching_start_exists(self): + # PEP 829: an empty .start file disables the matching .pth's import + # lines, even when the .start has no entry points of its own. + self._make_mod("flag = False\n", name='suppressed', on_path=True) + self._make_start("", name='foo') + self._make_pth( + "import suppressed; suppressed.flag = True\n", + name='foo') + site.addsitedir(self.sitedir, set()) + import suppressed + self.assertFalse( + suppressed.flag, + "import line in foo.pth should be suppressed by foo.start") + + def test_addsitedir_dotfile_start_entrypoint_not_executed(self): + # .start files starting with '.' are skipped, so their entry + # points must not run. + self._make_mod("""\ +called = False +def hook(): + global called + called = True +""", + name='dotted', on_path=True) + self._make_start("dotted:hook\n", name='.hidden') + site.addsitedir(self.sitedir, set()) + import dotted + self.assertFalse(dotted.called) + + def test_addsitedir_dedups_paths_across_pth_files(self): + # PEP 829: when multiple .pth files reference the same path within + # a single addsitedir() invocation, the path is appended to + # sys.path exactly once. + subdir = os.path.join(self.sitedir, 'shared') + os.mkdir(subdir) + self._make_pth("shared\n", name='a') + self._make_pth("shared\n", name='b') + before = sys.path.count(subdir) + site.addsitedir(self.sitedir, set()) + self.assertEqual(sys.path.count(subdir), before + 1) + def test_addsitedir_discovers_start_files(self): # addsitedir() should discover .start files and accumulate entries. + # With defer_processing_start_files=True the preserved state lives on + # site._startup_state and isn't flushed until the caller invokes + # process_startup_files(). self._make_start("os.path:join\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + site.addsitedir( + self.sitedir, set(), + defer_processing_start_files=True, + ) fullname = os.path.join(self.sitedir, 'foo.start') - self.assertIn('os.path:join', site._pending_entrypoints[fullname]) + self.assertIn( + 'os.path:join', site._startup_state._entrypoints[fullname] + ) - def test_addsitedir_start_suppresses_pth_imports(self): + def test_impl_exec_imports_skips_when_matching_start(self): # When foo.start exists, import lines in foo.pth are skipped - # at flush time by _exec_imports(). + # at flush time by _StartupState._exec_imports(). self._make_start("os.path:join\n", name='foo') self._make_pth("import sys\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + site.addsitedir( + self.sitedir, set(), + defer_processing_start_files=True, + ) pth_fullname = os.path.join(self.sitedir, 'foo.pth') start_fullname = os.path.join(self.sitedir, 'foo.start') # Import line was collected... - self.assertIn('import sys', - site._pending_importexecs.get(pth_fullname, [])) + self.assertIn( + 'import sys', + site._startup_state._importexecs.get(pth_fullname, []), + ) # ...but _exec_imports() will skip it because foo.start exists. - site._exec_imports() + site._startup_state._exec_imports() def test_addsitedir_pth_paths_still_work_with_start(self): # Path lines in .pth files still work even when a .start file exists. @@ -1345,17 +1443,26 @@ def test_addsitedir_pth_paths_still_work_with_start(self): os.mkdir(subdir) self._make_start("os.path:join\n", name='foo') self._make_pth("mylib\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + site.addsitedir( + self.sitedir, set(), + defer_processing_start_files=True, + ) fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) + self.assertIn( + subdir, site._startup_state._syspaths.get(fullname, []) + ) def test_addsitedir_start_alphabetical_order(self): # Multiple .start files are discovered alphabetically. + # _all_entrypoints() reads from self.state, so swap in the + # preserved batch state for the duration of the assertion. self._make_start("os.path:join\n", name='zzz') self._make_start("os.path:exists\n", name='aaa') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + site.addsitedir( + self.sitedir, set(), + defer_processing_start_files=True, + ) + self.state = site._startup_state all_entries = self._all_entrypoints() entries = [entry for _, entry in all_entries] idx_a = entries.index('os.path:exists') @@ -1370,49 +1477,65 @@ def test_addsitedir_pth_before_start(self): os.mkdir(subdir) self._make_pth("mylib\n", name='foo') self._make_start("os.path:join\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + site.addsitedir( + self.sitedir, set(), + defer_processing_start_files=True, + ) # Both should be collected. pth_fullname = os.path.join(self.sitedir, 'foo.pth') start_fullname = os.path.join(self.sitedir, 'foo.start') - self.assertIn(subdir, site._pending_syspaths.get(pth_fullname, [])) - self.assertIn('os.path:join', - site._pending_entrypoints.get(start_fullname, [])) + self.assertIn( + subdir, site._startup_state._syspaths.get(pth_fullname, []) + ) + self.assertIn( + 'os.path:join', + site._startup_state._entrypoints.get(start_fullname, []), + ) - def test_addsitedir_dotfile_start_ignored(self): + def test_impl_addsitedir_skips_dotfile_start(self): # .start files starting with '.' are skipped. Defer flushing so - # the assertion against _pending_entrypoints is meaningful; - # otherwise process_startup_files() would clear the dict - # regardless of whether the dotfile was picked up. + # the preserved batch state stays inspectable on + # site._startup_state; otherwise process_startup_files() would + # detach and consume it regardless of whether the dotfile was + # picked up. self._make_start("os.path:join\n", name='.hidden') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) - self.assertEqual(site._pending_entrypoints, {}) + site.addsitedir( + self.sitedir, set(), + defer_processing_start_files=True, + ) + self.assertEqual(site._startup_state._entrypoints, {}) def test_addsitedir_standalone_flushes(self): - # When called with known_paths=None (standalone), addsitedir - # flushes immediately so the caller sees the effect. + # When called with defer_processing_start_files=False (the + # default), addsitedir creates a per-call _StartupState and + # processes it before returning, so the caller sees the effect + # immediately. No batch state is left behind on + # site._startup_state. subdir = os.path.join(self.sitedir, 'flushlib') os.mkdir(subdir) self._make_pth("flushlib\n", name='foo') site.addsitedir(self.sitedir) # known_paths=None self.assertIn(subdir, sys.path) - # Pending dicts should be cleared after flush. - self.assertEqual(site._pending_syspaths, {}) + self.assertIsNone(site._startup_state) def test_addsitedir_defer_does_not_flush(self): # With defer_processing_start_files=True, addsitedir accumulates # pending state but does not flush; sys.path is updated only when - # process_startup_files() is called explicitly. + # process_startup_files() is called explicitly. The accumulated + # state lives on the lazily-promoted site._startup_state. subdir = os.path.join(self.sitedir, 'acclib') os.mkdir(subdir) self._make_pth("acclib\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + site.addsitedir( + self.sitedir, set(), + defer_processing_start_files=True, + ) # Path is pending, not yet on sys.path. self.assertNotIn(subdir, sys.path) fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) + self.assertIn( + subdir, site._startup_state._syspaths.get(fullname, []) + ) def test_pth_path_is_available_to_start_entrypoint(self): # Core PEP 829 invariant: all .pth path extensions are applied to @@ -1420,18 +1543,12 @@ def test_pth_path_is_available_to_start_entrypoint(self): # point may live in a module reachable only via a .pth-extended # path. If the flush phases were inverted, resolving the entry # point would fail with ModuleNotFoundError. - extdir = os.path.join(self.sitedir, 'extdir') - os.mkdir(extdir) - modpath = os.path.join(extdir, 'mod.py') - with open(modpath, 'w') as f: - f.write("""\ + extdir = self._make_mod("""\ called = False def hook(): global called called = True """) - self.addCleanup(sys.modules.pop, 'mod', None) - # extdir is not on sys.path; only the .pth file makes it so. self.assertNotIn(extdir, sys.path) self._make_pth("extdir\n", name='extlib') @@ -1447,6 +1564,82 @@ def hook(): "entry point did not run; .pth path was likely not applied " "before .start entry-point execution") + # --- bugs --- + + # gh-75723 + def test_addsitdir_idempotent_pth(self): + # Adding the same sitedir twice with a known_paths, should not + # process .pth files twice. + extdir = self._make_mod("""\ +_pth_count = 0 +""") + self._make_pth(f"""\ +{extdir} +import mod; mod._pth_count += 1 +""") + dirs = set() + dirs = site.addsitedir(self.sitedir, dirs) + dirs = site.addsitedir(self.sitedir, dirs) + import mod + self.assertEqual(mod._pth_count, 1) + + def test_addsitdir_idempotent_start(self): + # Adding the same sitedir twice with a known_paths, should not + # process .pth files twice. + extdir = self._make_mod("""\ +_pth_count = 0 +def increment(): + global _pth_count + _pth_count += 1 +""") + self._make_pth(f"""\ +{extdir} +""") + self._make_start("""\ +mod:increment +""") + dirs = set() + dirs = site.addsitedir(self.sitedir, dirs) + dirs = site.addsitedir(self.sitedir, dirs) + import mod + self.assertEqual(mod._pth_count, 1) + + # gh-149504 + def test_reentrant_addsitedir_pth(self): + # An import line in a .pth file that calls site.addsitedir() + # must not crash or re-execute outer entries while the outer + # call is still processing its pending startup state. + overlay = self.enterContext(os_helper.temp_dir()) + overlay_pth = os.path.join(overlay, 'overlay.pth') + pkgdir = self.enterContext(os_helper.temp_dir()) + with open(overlay_pth, 'w', encoding='utf-8') as fp: + print(pkgdir, file=fp) + self._make_pth(f"import site; site.addsitedir({overlay!r})\n") + site.addsitedir(self.sitedir, set()) + self.assertIn(overlay, sys.path) + self.assertIn(pkgdir, sys.path) + + # gh-149504 + def test_reentrant_addsitedir_start(self): + # As above, but the re-entry happens from a .start entry point + # instead of a .pth import line. The entry point execution + # phase is vulnerable to the same class of bug. + overlay = self.enterContext(os_helper.temp_dir()) + overlay_pth = os.path.join(overlay, 'overlay.pth') + pkgdir = self.enterContext(os_helper.temp_dir()) + with open(overlay_pth, 'w', encoding='utf-8') as fp: + print(pkgdir, file=fp) + self._make_mod(f"""\ +import site +def bootstrap(): + site.addsitedir({overlay!r}) +""", + name='reenter_helper', on_path=True) + self._make_start("reenter_helper:bootstrap\n") + site.addsitedir(self.sitedir, set()) + self.assertIn(overlay, sys.path) + self.assertIn(pkgdir, sys.path) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2026-05-10-23-51-23.gh-issue-149504.pDSCbn.rst b/Misc/NEWS.d/next/Library/2026-05-10-23-51-23.gh-issue-149504.pDSCbn.rst new file mode 100644 index 00000000000000..88bf268123bbec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-10-23-51-23.gh-issue-149504.pDSCbn.rst @@ -0,0 +1,5 @@ +Fix :func:`site.addsitedir` to allow re-entrant calls from within startup +files. Previously, a ``.pth`` file containing an ``import`` line that +called :func:`site.addsitedir` (or a ``.start`` entry point doing the same) +could crash with ``RuntimeError: dictionary changed size during iteration`` +during site initialization, breaking tools such as ``uv run --with``. From dc7cad2f5db834aefb5ecabebc6f25bbb898381b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 14 May 2026 01:24:29 +0200 Subject: [PATCH 052/110] [3.15] gh-149231: tomllib: Limit the number of parts in a key (GH-149233) (GH-149677) (cherry picked from commit bc7c102f3462a9f014f3ac2546acfb471b2a7eae) Co-authored-by: Petr Viktorin Co-authored-by: Stan Ulbrych --- .../next/Library/2026-05-01-16-45-31.gh-issue-149231.x2nBEE.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2026-05-01-16-45-31.gh-issue-149231.x2nBEE.rst diff --git a/Misc/NEWS.d/next/Library/2026-05-01-16-45-31.gh-issue-149231.x2nBEE.rst b/Misc/NEWS.d/next/Library/2026-05-01-16-45-31.gh-issue-149231.x2nBEE.rst new file mode 100644 index 00000000000000..c265b54db8bed4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-01-16-45-31.gh-issue-149231.x2nBEE.rst @@ -0,0 +1 @@ +In :mod:`tomllib`, the number of parts in TOML keys is now limited. From 894ec10b56d93743e082ac9569abf896e082a919 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 14 May 2026 01:34:59 +0200 Subject: [PATCH 053/110] [3.15] Fix incorrect sentence in stable.rst (GH-149684) (GH-149814) (cherry picked from commit 374f9d3f5e70d2204d88ab123f29825d71537de2) Co-authored-by: Manoj K M --- Doc/c-api/stable.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 0ff066680b8c73..13e5d5c96135c0 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -114,7 +114,7 @@ versions of Python. All functions in Stable ABI are present as functions in Python's shared library, not solely as macros. -This makes them usable are usable from languages that don't use the C +This makes them usable in languages that don't use the C preprocessor, including Python's :py:mod:`ctypes`. From b5d508d40b5c87baa23f9f1f8200db8e00280e5b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 14 May 2026 10:52:51 +0200 Subject: [PATCH 054/110] [3.15] gh-149698: Update bundled expat to 2.8.1 (GH-149699) (#149812) (cherry picked from commit f1a47e79fb7081d3cde6364530bfa98240ebbe4c) Co-authored-by: Stan Ulbrych --- ...-05-11-21-15-07.gh-issue-149698.OudOcW.rst | 2 ++ Misc/sbom.spdx.json | 16 ++++----- Modules/expat/expat.h | 2 +- Modules/expat/refresh.sh | 6 ++-- Modules/expat/xmlparse.c | 36 +++++++++++++++---- 5 files changed, 44 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-05-11-21-15-07.gh-issue-149698.OudOcW.rst diff --git a/Misc/NEWS.d/next/Security/2026-05-11-21-15-07.gh-issue-149698.OudOcW.rst b/Misc/NEWS.d/next/Security/2026-05-11-21-15-07.gh-issue-149698.OudOcW.rst new file mode 100644 index 00000000000000..3c8671b9a5adc4 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-11-21-15-07.gh-issue-149698.OudOcW.rst @@ -0,0 +1,2 @@ +Update bundled `libexpat `_ to version 2.8.1 +for the fix for :cve:`2026-45186`. diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index aaeffd58e799ed..1eca892fb12ace 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "5343adc95840915b022b1d4524d0acb66b369ba2" + "checksumValue": "58101ef0951568acadd3117033bef084fea24cc1" }, { "algorithm": "SHA256", - "checksumValue": "1ec3bad08b6864c2c479e1fd941038c2dcd24c6d9a16400f4da54912d95aa321" + "checksumValue": "52d756026bf09befdb211c453e2009a646d6c6b519e6885e971b2550396619fb" } ], "fileName": "Modules/expat/expat.h" @@ -174,11 +174,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "cb0af01558ec7b6474d2bd0c9386380c82618e8f" + "checksumValue": "1dad2ab196cdbe37572674c465bd9187fdbe4495" }, { "algorithm": "SHA256", - "checksumValue": "6745a6b8cdd7344d4bd8f27f605363ed746e57ff02d4ebce3eb1806579cd030f" + "checksumValue": "740137e670d2f3b7269364ffb6f60064e6560091850c5d6f2c3bb1b8ca6e3dd1" } ], "fileName": "Modules/expat/xmlparse.c" @@ -1730,14 +1730,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "c7cec5f60ea3a42e7780781c6745255c19aa3dbfeeae58646b7132f88dc24780" + "checksumValue": "a52eb72108be160e190b5cafa5bba8663f1313f2013e26060d1c18e26e31067b" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_0/expat-2.8.0.tar.gz", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_1/expat-2.8.1.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.1:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1745,7 +1745,7 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.8.0" + "versionInfo": "2.8.1" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 79c609f19aa4cf..ec3f58544cb00d 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -1094,7 +1094,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ # define XML_MAJOR_VERSION 2 # define XML_MINOR_VERSION 8 -# define XML_MICRO_VERSION 0 +# define XML_MICRO_VERSION 1 # ifdef __cplusplus } diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh index 774e0b89d94c0e..fa3692f9379510 100755 --- a/Modules/expat/refresh.sh +++ b/Modules/expat/refresh.sh @@ -12,9 +12,9 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. These values are used for verifying the SBOM, too. -expected_libexpat_tag="R_2_8_0" -expected_libexpat_version="2.8.0" -expected_libexpat_sha256="c7cec5f60ea3a42e7780781c6745255c19aa3dbfeeae58646b7132f88dc24780" +expected_libexpat_tag="R_2_8_1" +expected_libexpat_version="2.8.1" +expected_libexpat_sha256="a52eb72108be160e190b5cafa5bba8663f1313f2013e26060d1c18e26e31067b" expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" cd ${expat_dir} diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index e6842f3f0bf750..95d346758563ab 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* a5d18f6a50f536615ac1c70304f87d94f99cc85a86b502188952440610ccf0f8 (2.8.0+) +/* 75ef4224f81c052e9e5aeea2ac7de75357d2169ff9908e39edc08b9dc3052513 (2.8.1+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -387,6 +387,7 @@ typedef struct { int nDefaultAtts; int allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; + HASH_TABLE defaultAttsNames; } ELEMENT_TYPE; typedef struct { @@ -3769,6 +3770,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sizeof(ELEMENT_TYPE)); if (! elementType) return XML_ERROR_NO_MEMORY; + if (! elementType->defaultAttsNames.parser) + hashTableInit(&(elementType->defaultAttsNames), parser); if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } @@ -7102,10 +7105,10 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, if (value || isId) { /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ - int i; - for (i = 0; i < type->nDefaultAtts; i++) - if (attId == type->defaultAtts[i].id) - return 1; + NAMED *const nameFound + = (NAMED *)lookup(parser, &(type->defaultAttsNames), attId->name, 0); + if (nameFound) + return 1; if (isId && ! type->idAtt && ! attId->xmlns) type->idAtt = attId; } @@ -7152,6 +7155,12 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, att->isCdata = isCdata; if (! isCdata) attId->maybeTokenized = XML_TRUE; + + NAMED *const nameAddedOrFound = (NAMED *)lookup( + parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); + if (! nameAddedOrFound) + return 0; + type->nDefaultAtts += 1; return 1; } @@ -7477,6 +7486,7 @@ dtdReset(DTD *p, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; + hashTableDestroy(&(e->defaultAttsNames)); if (e->allocDefaultAtts != 0) FREE(parser, e->defaultAtts); } @@ -7518,6 +7528,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; + hashTableDestroy(&(e->defaultAttsNames)); if (e->allocDefaultAtts != 0) FREE(parser, e->defaultAtts); } @@ -7611,6 +7622,10 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, sizeof(ELEMENT_TYPE)); if (! newE) return 0; + + if (! newE->defaultAttsNames.parser) + hashTableInit(&(newE->defaultAttsNames), parser); + if (oldE->nDefaultAtts) { /* Detect and prevent integer overflow. * The preprocessor guard addresses the "always false" warning @@ -7635,8 +7650,9 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, newE->prefix = (PREFIX *)lookup(oldParser, &(newDtd->prefixes), oldE->prefix->name, 0); for (i = 0; i < newE->nDefaultAtts; i++) { + const XML_Char *const attributeName = oldE->defaultAtts[i].id->name; newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup( - oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0); + oldParser, &(newDtd->attributeIds), attributeName, 0); newE->defaultAtts[i].isCdata = oldE->defaultAtts[i].isCdata; if (oldE->defaultAtts[i].value) { newE->defaultAtts[i].value @@ -7645,6 +7661,12 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, return 0; } else newE->defaultAtts[i].value = NULL; + + NAMED *const nameAddedOrFound = (NAMED *)lookup( + parser, &(newE->defaultAttsNames), attributeName, sizeof(NAMED)); + if (! nameAddedOrFound) { + return 0; + } } } @@ -8391,6 +8413,8 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, sizeof(ELEMENT_TYPE)); if (! ret) return NULL; + if (! ret->defaultAttsNames.parser) + hashTableInit(&(ret->defaultAttsNames), getRootParserOf(parser, NULL)); if (ret->name != name) poolDiscard(&dtd->pool); else { From 6302a8f17a6113a3ef0cbb3afad35632a7be9e02 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 14 May 2026 12:37:11 +0200 Subject: [PATCH 055/110] [3.15] gh-148821: Add more tests for invalid XML encodings (GH-149820) (GH-149821) (cherry picked from commit c6f7368157ecf9f2cdd537d8b6fad6e011bce344) Co-authored-by: Serhiy Storchaka --- Lib/test/test_pyexpat.py | 48 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 4fe2e02326f04f..10dca684accee3 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -289,7 +289,7 @@ def test_parse_again(self): 'mac-roman', 'mac-turkish', 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'ptcp154', ]) - def test_supported_ecodings(self, encoding): + def test_supported_encodings(self, encoding): out = self.Outputter() parser = expat.ParserCreate() self._hookup_callbacks(parser, out) @@ -308,7 +308,7 @@ def test_supported_ecodings(self, encoding): 'UTF-8', 'utf-8', 'utf-16', 'utf-16le', 'utf-16be', 'koi8-u', 'cp1125', 'cp1251', 'iso8859-5', 'mac-cyrillic', ]) - def test_supported_ecodings2(self, encoding): + def test_supported_encodings2(self, encoding): out = self.Outputter() parser = expat.ParserCreate() self._hookup_callbacks(parser, out) @@ -334,14 +334,54 @@ def test_supported_ecodings2(self, encoding): "johab", "Shift_JIS", "Shift_JIS-2004", "Shift_JISX0213", ]) - def test_unsupportes_ecodings(self, encoding): + def test_unsupported_encodings(self, encoding): parser = expat.ParserCreate() data = (f'\n' '').encode(encoding) with self.assertRaises(ValueError): parser.Parse(data, True) - def test_unknown_ecoding(self): + parser = expat.ParserCreate() + data = (f'\n' + '').encode() + with self.assertRaises(ValueError): + parser.Parse(data, True) + + @support.subTests('encoding', [ + 'cp037', 'cp273', 'cp424', 'cp500', 'cp864', 'cp875', + 'cp1026', 'cp1140', + 'mac_arabic', 'mac_farsi', + ]) + def test_incompatible_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'\n' + '').encode(encoding) + with self.assertRaises(expat.ExpatError): + parser.Parse(data, True) + + parser = expat.ParserCreate() + data = (f'\n' + '').encode() + with self.assertRaisesRegex(expat.ExpatError, 'unknown encoding'): + parser.Parse(data, True) + + @support.subTests('encoding', [ + 'hex_codec', 'rot_13', + ]) + def test_non_text_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'\n' + '').encode() + with self.assertRaises(LookupError): + parser.Parse(data, True) + + def test_undefined_encoding(self): + parser = expat.ParserCreate() + data = b'\n' + with self.assertRaises(UnicodeError): + parser.Parse(data, True) + + def test_unknown_encoding(self): parser = expat.ParserCreate() data = b'\n' with self.assertRaises(LookupError): From 451f06b02ef86c0c218249694f763758ae09d414 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 14 May 2026 19:09:05 +0200 Subject: [PATCH 056/110] [3.15] Link to existing rules in compound_stmts.rst (GH-149811) (GH-149836) Link to existing rules in compound_stmts.rst (GH-149811) In gh-138418, `!` was added to links to rules that don't exist in the docs, in order to silence broken link warnings. However, productionlist doesn't parse the `!`, which ends up in the rendered documentation. (It's possible that gh-127835 broke the `!` support.) Replace the names with ones that appear in docs: - `star_named_expression` in the grammar corresponds to `flexible_expression` in the docs - `star_named_expressions` in the grammar corresponds to `flexible_expression_list` in the docs - `named_expression` in the grammar corresponds to `assignment_expression` in the docs Having two sets of names isn't great of course. Consolidating them is tracked in (subissues of) gh-127833. (cherry picked from commit c37529293d1e05081cb4e8668162c76583b88007) Co-authored-by: Petr Viktorin --- Doc/reference/compound_stmts.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 72e1cad3bbd892..a819c41d834aa7 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -618,8 +618,8 @@ The match statement is used for pattern matching. Syntax: .. productionlist:: python-grammar match_stmt: 'match' `subject_expr` ":" NEWLINE INDENT `case_block`+ DEDENT - subject_expr: `!star_named_expression` "," `!star_named_expressions`? - : | `!named_expression` + subject_expr: `flexible_expression` "," [`flexible_expression_list` [',']] + : | `assignment_expression` case_block: 'case' `patterns` [`guard`] ":" `!block` .. note:: @@ -709,7 +709,7 @@ Guards .. index:: ! guard .. productionlist:: python-grammar - guard: "if" `!named_expression` + guard: "if" `assignment_expression` A ``guard`` (which is part of the ``case``) must succeed for code inside the ``case`` block to execute. It takes the form: :keyword:`if` followed by an From ff6f921ffadba51ed350588457347a6d9fc2c986 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 14 May 2026 19:16:23 +0200 Subject: [PATCH 057/110] [3.15] gh-149231: Revert extra NEWS entry (GH-149840) gh-149231: Revert extra NEWS entry This reverts commit dc7cad2f5db834aefb5ecabebc6f25bbb898381b. --- .../next/Library/2026-05-01-16-45-31.gh-issue-149231.x2nBEE.rst | 1 - 1 file changed, 1 deletion(-) delete mode 100644 Misc/NEWS.d/next/Library/2026-05-01-16-45-31.gh-issue-149231.x2nBEE.rst diff --git a/Misc/NEWS.d/next/Library/2026-05-01-16-45-31.gh-issue-149231.x2nBEE.rst b/Misc/NEWS.d/next/Library/2026-05-01-16-45-31.gh-issue-149231.x2nBEE.rst deleted file mode 100644 index c265b54db8bed4..00000000000000 --- a/Misc/NEWS.d/next/Library/2026-05-01-16-45-31.gh-issue-149231.x2nBEE.rst +++ /dev/null @@ -1 +0,0 @@ -In :mod:`tomllib`, the number of parts in TOML keys is now limited. From 21909e898e6cc444c2c860edf6bc6b9c3a86abb2 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 14 May 2026 20:49:29 +0200 Subject: [PATCH 058/110] [3.15] gh-124111: Update Windows build to use Tcl/Tk 9.0.3 (GH-149842) (cherry picked from commit c62c3710dc795a60c3c3dc8e2aeeeb16c06da197) Co-authored-by: Zachary Ware --- Lib/test/test_tcl.py | 12 +++++++++-- ...-05-06-21-36-53.gh-issue-124111.m4OBX8.rst | 1 + Misc/externals.spdx.json | 20 +++++++++---------- PCbuild/get_externals.bat | 6 +++--- PCbuild/readme.txt | 2 +- PCbuild/tcltk.props | 6 ++++-- Tools/msi/tcltk/tcltk_files.wxs | 11 ++++++---- Tools/msi/testrelease.bat | 2 -- 8 files changed, 36 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2026-05-06-21-36-53.gh-issue-124111.m4OBX8.rst diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 47450d3fd5976f..81a5477b496b5c 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -54,7 +54,11 @@ def test_eval_null_in_result(self): def test_eval_surrogates_in_result(self): tcl = self.interp - self.assertEqual(tcl.eval(r'set a "<\ud83d\udcbb>"'), '<\U0001f4bb>') + result = tcl.eval(r'set a "<\ud83d\udcbb>"') + if sys.platform == 'win32': + self.assertEqual('<\ud83d\udcbb>', result) + else: + self.assertEqual('<\U0001f4bb>', result) def testEvalException(self): tcl = self.interp @@ -289,7 +293,11 @@ def test_evalfile_surrogates_in_result(self): set b "<\\ud83d\\udcbb>" """) tcl.evalfile(filename) - self.assertEqual(tcl.eval('set b'), '<\U0001f4bb>') + result = tcl.eval('set b') + if sys.platform == 'win32': + self.assertEqual('<\ud83d\udcbb>', result) + else: + self.assertEqual('<\U0001f4bb>', result) def testEvalFileException(self): tcl = self.interp diff --git a/Misc/NEWS.d/next/Windows/2026-05-06-21-36-53.gh-issue-124111.m4OBX8.rst b/Misc/NEWS.d/next/Windows/2026-05-06-21-36-53.gh-issue-124111.m4OBX8.rst new file mode 100644 index 00000000000000..9a57536f1dc96b --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-05-06-21-36-53.gh-issue-124111.m4OBX8.rst @@ -0,0 +1 @@ +Updated Windows builds to use Tcl/Tk 9.0.3. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 593fa01bf25ed1..9a571fba732ab4 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -108,46 +108,46 @@ "versionInfo": "3.50.4.0" }, { - "SPDXID": "SPDXRef-PACKAGE-tcl-core", + "SPDXID": "SPDXRef-PACKAGE-tcl", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "4c23f0dd3efcbe6f3a22c503a68d147617bb30c4f5290f1eb3eaacf0b460440b" + "checksumValue": "7a1d1f3a2b8f4484a9c2a027a157963c18f85a81785e85fcb5d1e3df6b6a4fd4" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.15.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-9.0.3.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.15.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:9.0.3.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", - "name": "tcl-core", + "name": "tcl", "primaryPackagePurpose": "SOURCE", - "versionInfo": "8.6.15.0" + "versionInfo": "9.0.3.0" }, { "SPDXID": "SPDXRef-PACKAGE-tk", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "0ae56d39bca92865f338529557a1e56d110594184b6dc5a91339c5675751e264" + "checksumValue": "54fb59df12c489c6264f5b7d3d7444b150d1e3d6561fd59cdb11483440cec000" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.15.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-9.0.3.1.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.15.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:9.0.3.1:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "tk", "primaryPackagePurpose": "SOURCE", - "versionInfo": "8.6.15.0" + "versionInfo": "9.0.3.1" }, { "SPDXID": "SPDXRef-PACKAGE-xz", diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 405285b65dd270..368bc489bfa968 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -57,8 +57,8 @@ if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.5.6 set libraries=%libraries% mpdecimal-4.0.0 set libraries=%libraries% sqlite-3.50.4.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.15.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-9.0.3.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-9.0.3.1 set libraries=%libraries% xz-5.8.1.1 set libraries=%libraries% zlib-ng-2.2.4 set libraries=%libraries% zstd-1.5.7 @@ -80,7 +80,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.5.6 -if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 +if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-9.0.3.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-21.1.4.0 diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 14aac0b0dc84b6..6aecbfff182dcb 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -247,7 +247,7 @@ _sqlite3 https://www.sqlite.org/ _tkinter - Wraps version 8.6.15 of the Tk windowing system, which is downloaded + Wraps version 9.0.3 of the Tk windowing system, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index a1da1155b881fd..28e8c0db4d1eaf 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -2,7 +2,7 @@ - 8.6.15.0 + 9.0.3.0 $(TclVersion) $([System.Version]::Parse($(TclVersion)).Major) $([System.Version]::Parse($(TclVersion)).Minor) @@ -12,7 +12,9 @@ $([System.Version]::Parse($(TkVersion)).Minor) $([System.Version]::Parse($(TkVersion)).Build) $([System.Version]::Parse($(TkVersion)).Revision) - $(ExternalsDir)tcl-core-$(TclVersion)\ + + $(ExternalsDir)tcl-core-$(TclVersion)\ + $(ExternalsDir)tcl-$(TclVersion)\ $(ExternalsDir)tk-$(TkVersion)\ $(ExternalsDir)tcltk-$(TclVersion)\$(ArchName)\ t diff --git a/Tools/msi/tcltk/tcltk_files.wxs b/Tools/msi/tcltk/tcltk_files.wxs index 5dad7c98d4f048..7c7784741d9178 100644 --- a/Tools/msi/tcltk/tcltk_files.wxs +++ b/Tools/msi/tcltk/tcltk_files.wxs @@ -10,11 +10,14 @@ - - + + - - + + + + + diff --git a/Tools/msi/testrelease.bat b/Tools/msi/testrelease.bat index 02bcca943cf79b..db98f690151196 100644 --- a/Tools/msi/testrelease.bat +++ b/Tools/msi/testrelease.bat @@ -88,9 +88,7 @@ exit /B 0 ) @if not errorlevel 1 ( @echo Testing Tcl/tk - @set TCL_LIBRARY=%~2\Python\tcl\tcl8.6 "%~2\Python\python.exe" -m test -uall -v test_ttk_guionly test_tk test_idle > "%~2\tcltk.txt" 2>&1 - @set TCL_LIBRARY= ) @set EXITCODE=%ERRORLEVEL% From ed27363ddf340577afd5b7b67e604489f5ee7aa3 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 15 May 2026 00:38:11 +0200 Subject: [PATCH 059/110] [3.15] gh-149144: Use decodeURIComponent() for UTF-8 support in js_output() (GH-149157) (GH-149846) gh-149144: Use decodeURIComponent() for UTF-8 support in js_output() (GH-149157) (cherry picked from commit 461b1d96313de02992d284c1782be9aff24586c9) Co-authored-by: Seth Larson --- Lib/http/cookies.py | 7 +++---- Lib/test/test_http_cookies.py | 27 ++++++++++++++------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 800a2c18e3fa41..9a6f01dfb5e69a 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -391,21 +391,20 @@ def output(self, attrs=None, header="Set-Cookie:"): def __repr__(self): return '<%s: %s>' % (self.__class__.__name__, self.OutputString()) - def _js_output(self, attrs=None): """Internal implementation without deprecation warning.""" - import base64 + import urllib.parse # Print javascript output_string = self.OutputString(attrs) if _has_control_character(output_string): raise CookieError("Control characters are not allowed in cookies") # Base64-encode value to avoid template # injection in cookie values. - output_encoded = base64.b64encode(output_string.encode('utf-8')).decode("ascii") + output_encoded = urllib.parse.quote(output_string, safe='', encoding='utf-8') return """ """ % (output_encoded,) diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index cde268e3241850..d1df2ec42f0d14 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -1,11 +1,11 @@ # Simple test suite for http/cookies.py -import base64 import copy import unittest import doctest from http import cookies import pickle from test import support +import urllib.parse class CookieTests(unittest.TestCase): @@ -152,21 +152,21 @@ def test_load(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - cookie_encoded = base64.b64encode(b'Customer="WILE_E_COYOTE"; Path=/acme; Version=1').decode('ascii') + cookie_encoded = urllib.parse.quote('Customer="WILE_E_COYOTE"; Path=/acme; Version=1', safe='', encoding='utf-8') with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): self.assertEqual(C.js_output(), fr""" """) - cookie_encoded = base64.b64encode(b'Customer="WILE_E_COYOTE"; Path=/acme').decode('ascii') + cookie_encoded = urllib.parse.quote('Customer="WILE_E_COYOTE"; Path=/acme', safe='', encoding='utf-8') with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): self.assertEqual(C.js_output(['path']), fr""" """) @@ -271,21 +271,21 @@ def test_quoted_meta(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - expected_encoded_cookie = base64.b64encode(b'Customer=\"WILE_E_COYOTE\"; Path=/acme; Version=1').decode('ascii') + expected_encoded_cookie = urllib.parse.quote('Customer=\"WILE_E_COYOTE\"; Path=/acme; Version=1', safe='', encoding='utf-8') with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): self.assertEqual(C.js_output(), fr""" """) - expected_encoded_cookie = base64.b64encode(b'Customer=\"WILE_E_COYOTE\"; Path=/acme').decode('ascii') + expected_encoded_cookie = urllib.parse.quote('Customer=\"WILE_E_COYOTE\"; Path=/acme', safe='', encoding='utf-8') with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): self.assertEqual(C.js_output(['path']), fr""" """) @@ -376,13 +376,14 @@ def test_setter(self): self.assertEqual( M.output(), "Set-Cookie: %s=%s; Path=/foo" % (i, "%s_coded_val" % i)) - expected_encoded_cookie = base64.b64encode( - ("%s=%s; Path=/foo" % (i, "%s_coded_val" % i)).encode("ascii") - ).decode('ascii') + expected_encoded_cookie = urllib.parse.quote( + "%s=%s; Path=/foo" % (i, "%s_coded_val" % i), + safe='', encoding='utf-8', + ) expected_js_output = """ """ % (expected_encoded_cookie,) From 168a3c85be3f34dcd4175311a0bac558dcaa5ef7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 15 May 2026 09:24:43 +0200 Subject: [PATCH 060/110] [3.15] gh-149763: Improve availablity docs in `select.rst` (GH-149764) (#149854) gh-149763: Improve availablity docs in `select.rst` (GH-149764) (cherry picked from commit 7e98debdf4bfcf1c3f592c9424bc654117c2723e) Co-authored-by: sobolevn --- Doc/library/select.rst | 260 +++++++++++++++++++++-------------------- 1 file changed, 135 insertions(+), 125 deletions(-) diff --git a/Doc/library/select.rst b/Doc/library/select.rst index 09563af14d018a..6400005871746a 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -37,7 +37,7 @@ The module defines the following: .. function:: devpoll() - (Only supported on Solaris and derivatives.) Returns a ``/dev/poll`` + Returns a ``/dev/poll`` polling object; see section :ref:`devpoll-objects` below for the methods supported by devpoll objects. @@ -54,9 +54,11 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: Solaris. + .. function:: epoll(sizehint=-1, flags=0) - (Only supported on Linux 2.5.44 and newer.) Return an edge polling object, + Return an edge polling object, which can be used as Edge or Level Triggered interface for I/O events. @@ -94,18 +96,22 @@ The module defines the following: When CPython is built, this function may be disabled using :option:`--disable-epoll`. + .. availability:: Linux >= 2.5.44. + .. function:: poll() - (Not supported by all operating systems.) Returns a polling object, which + Returns a polling object, which supports registering and unregistering file descriptors, and then polling them for I/O events; see section :ref:`poll-objects` below for the methods supported by polling objects. + .. availability:: Unix. + .. function:: kqueue() - (Only supported on BSD.) Returns a kernel queue object; see section + Returns a kernel queue object; see section :ref:`kqueue-objects` below for the methods supported by kqueue objects. The new file descriptor is :ref:`non-inheritable `. @@ -113,12 +119,16 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: BSD, macOS. + .. function:: kevent(ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0) - (Only supported on BSD.) Returns a kernel event object; see section + Returns a kernel event object; see section :ref:`kevent-objects` below for the methods supported by kevent objects. + .. availability:: BSD, macOS. + .. function:: select(rlist, wlist, xlist, timeout=None) @@ -190,7 +200,7 @@ The module defines the following: .. _devpoll-objects: -``/dev/poll`` Polling Objects +``/dev/poll`` polling objects ----------------------------- Solaris and derivatives have ``/dev/poll``. While :c:func:`!select` is @@ -285,52 +295,52 @@ object. .. _epoll-objects: -Edge and Level Trigger Polling (epoll) Objects +Edge and level trigger polling (epoll) objects ---------------------------------------------- https://linux.die.net/man/4/epoll - *eventmask* - - +-------------------------+-----------------------------------------------+ - | Constant | Meaning | - +=========================+===============================================+ - | :const:`EPOLLIN` | Available for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLOUT` | Available for write | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLPRI` | Urgent data for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLERR` | Error condition happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLHUP` | Hang up happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | - | | Level Trigger behavior | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | - | | pulled out, the fd is internally disabled | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | - | | associated fd has an event. The default (if | - | | this flag is not set) is to wake all epoll | - | | objects polling on a fd. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | - | | down writing half of connection. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDBAND` | Priority data band can be read. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRBAND` | Priority data may be written. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLMSG` | Ignored. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | - +-------------------------+-----------------------------------------------+ + The *eventmask* is a bit mask using the following constants: + + +-------------------------+------------------------------------------------+ + | Constant | Meaning | + +=========================+================================================+ + | :const:`EPOLLIN` | Available for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLOUT` | Available for write. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLPRI` | Urgent data for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLERR` | Error condition happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLHUP` | Hang up happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | + | | Level Trigger behavior. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | + | | pulled out, the fd is internally disabled. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | + | | associated fd has an event. The default (if | + | | this flag is not set) is to wake all epoll | + | | objects polling on an fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | + | | down writing half of connection. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDBAND` | Priority data band can be read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT`. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRBAND` | Priority data may be written. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLMSG` | Ignored. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | + +-------------------------+------------------------------------------------+ .. versionadded:: 3.6 :const:`EPOLLEXCLUSIVE` was added. It's only supported by Linux Kernel 4.5 @@ -362,12 +372,12 @@ Edge and Level Trigger Polling (epoll) Objects .. method:: epoll.register(fd[, eventmask]) - Register a fd descriptor with the epoll object. + Register a file descriptor *fd* with the epoll object. .. method:: epoll.modify(fd, eventmask) - Modify a registered file descriptor. + Modify a registered file descriptor *fd*. .. method:: epoll.unregister(fd) @@ -396,7 +406,7 @@ Edge and Level Trigger Polling (epoll) Objects .. _poll-objects: -Polling Objects +Polling objects --------------- The :c:func:`!poll` system call, supported on most Unix systems, provides better @@ -421,24 +431,24 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w :const:`POLLPRI`, and :const:`POLLOUT`, described in the table below. If not specified, the default value used will check for all 3 types of events. - +-------------------+------------------------------------------+ - | Constant | Meaning | - +===================+==========================================+ - | :const:`POLLIN` | There is data to read | - +-------------------+------------------------------------------+ - | :const:`POLLPRI` | There is urgent data to read | - +-------------------+------------------------------------------+ - | :const:`POLLOUT` | Ready for output: writing will not block | - +-------------------+------------------------------------------+ - | :const:`POLLERR` | Error condition of some sort | - +-------------------+------------------------------------------+ - | :const:`POLLHUP` | Hung up | - +-------------------+------------------------------------------+ - | :const:`POLLRDHUP`| Stream socket peer closed connection, or | - | | shut down writing half of connection | - +-------------------+------------------------------------------+ - | :const:`POLLNVAL` | Invalid request: descriptor not open | - +-------------------+------------------------------------------+ + +-------------------+-------------------------------------------+ + | Constant | Meaning | + +===================+===========================================+ + | :const:`POLLIN` | There is data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLPRI` | There is urgent data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLOUT` | Ready for output: writing will not block. | + +-------------------+-------------------------------------------+ + | :const:`POLLERR` | Error condition of some sort. | + +-------------------+-------------------------------------------+ + | :const:`POLLHUP` | Hung up. | + +-------------------+-------------------------------------------+ + | :const:`POLLRDHUP`| Stream socket peer closed connection, or | + | | shut down writing half of connection. | + +-------------------+-------------------------------------------+ + | :const:`POLLNVAL` | Invalid request: descriptor not open. | + +-------------------+-------------------------------------------+ Registering a file descriptor that's already registered is not an error, and has the same effect as registering the descriptor exactly once. @@ -489,7 +499,7 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w .. _kqueue-objects: -Kqueue Objects +Kqueue objects -------------- .. method:: kqueue.close() @@ -533,7 +543,7 @@ Kqueue Objects .. _kevent-objects: -Kevent Objects +Kevent objects -------------- https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 @@ -553,66 +563,66 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 | Constant | Meaning | +===========================+=============================================+ | :const:`KQ_FILTER_READ` | Takes a descriptor and returns whenever | - | | there is data available to read | + | | there is data available to read. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_WRITE` | Takes a descriptor and returns whenever | - | | there is data available to write | + | | there is data available to write. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_AIO` | AIO requests | + | :const:`KQ_FILTER_AIO` | AIO requests. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_VNODE` | Returns when one or more of the requested | - | | events watched in *fflag* occurs | + | | events watched in *fflag* occurs. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_PROC` | Watch for events on a process id | + | :const:`KQ_FILTER_PROC` | Watch for events on a process ID. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_NETDEV` | Watch for events on a network device | - | | [not available on macOS] | + | | (not available on macOS). | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_SIGNAL` | Returns whenever the watched signal is | - | | delivered to the process | + | | delivered to the process. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer | + | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer. | +---------------------------+---------------------------------------------+ .. attribute:: kevent.flags Filter action. - +---------------------------+---------------------------------------------+ - | Constant | Meaning | - +===========================+=============================================+ - | :const:`KQ_EV_ADD` | Adds or modifies an event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DELETE` | Removes an event from the queue | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ENABLE` | Permits control() to return the event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DISABLE` | Disables event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_SYSFLAGS` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_FLAG1` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_EOF` | Filter specific EOF condition | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ERROR` | See return values | - +---------------------------+---------------------------------------------+ + +---------------------------+----------------------------------------------+ + | Constant | Meaning | + +===========================+==============================================+ + | :const:`KQ_EV_ADD` | Adds or modifies an event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DELETE` | Removes an event from the queue. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ENABLE` | Permits control() to return the event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DISABLE` | Disables event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_SYSFLAGS` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_FLAG1` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_EOF` | Filter-specific EOF condition. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ERROR` | See return values. | + +---------------------------+----------------------------------------------+ .. attribute:: kevent.fflags - Filter specific flags. + Filter-specific flags. :const:`KQ_FILTER_READ` and :const:`KQ_FILTER_WRITE` filter flags: +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LOWAT` | low water mark of a socket buffer | + | :const:`KQ_NOTE_LOWAT` | Low water mark of a socket buffer. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_VNODE` filter flags: @@ -620,19 +630,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_DELETE` | *unlink()* was called | + | :const:`KQ_NOTE_DELETE` | *unlink()* was called. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_WRITE` | a write occurred | + | :const:`KQ_NOTE_WRITE` | A write occurred. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXTEND` | the file was extended | + | :const:`KQ_NOTE_EXTEND` | The file was extended. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_ATTRIB` | an attribute was changed | + | :const:`KQ_NOTE_ATTRIB` | An attribute was changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINK` | the link count has changed | + | :const:`KQ_NOTE_LINK` | The link count has changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_RENAME` | the file was renamed | + | :const:`KQ_NOTE_RENAME` | The file was renamed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_REVOKE` | access to the file was revoked | + | :const:`KQ_NOTE_REVOKE` | Access to the file was revoked. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_PROC` filter flags: @@ -640,22 +650,22 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_EXIT` | the process has exited | + | :const:`KQ_NOTE_EXIT` | The process has exited. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_FORK` | the process has called *fork()* | + | :const:`KQ_NOTE_FORK` | The process has called *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXEC` | the process has executed a new process | + | :const:`KQ_NOTE_EXEC` | The process has executed a new process. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PCTRLMASK` | internal filter flag | + | :const:`KQ_NOTE_PCTRLMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PDATAMASK` | internal filter flag | + | :const:`KQ_NOTE_PDATAMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACK` | follow a process across *fork()* | + | :const:`KQ_NOTE_TRACK` | Follow a process across *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_CHILD` | returned on the child process for | - | | *NOTE_TRACK* | + | :const:`KQ_NOTE_CHILD` | Returned on the child process for | + | | *NOTE_TRACK*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACKERR` | unable to attach to a child | + | :const:`KQ_NOTE_TRACKERR` | Unable to attach to a child. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_NETDEV` filter flags (not available on macOS): @@ -663,19 +673,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LINKUP` | link is up | + | :const:`KQ_NOTE_LINKUP` | Link is up. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKDOWN` | link is down | + | :const:`KQ_NOTE_LINKDOWN` | Link is down. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKINV` | link state is invalid | + | :const:`KQ_NOTE_LINKINV` | Link state is invalid. | +----------------------------+--------------------------------------------+ .. attribute:: kevent.data - Filter specific data. + Filter-specific data. .. attribute:: kevent.udata - User defined value. + User-defined value. From 5dadc64673ce875ebfb24163907777dae0f6ca06 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 15 May 2026 12:50:45 +0200 Subject: [PATCH 061/110] [3.15] gh-87451: Apply CVE-2021-4189 PASV fix to ftplib.ftpcp() (GH-149648) (#149792) gh-87451: Apply CVE-2021-4189 PASV fix to ftplib.ftpcp() (GH-149648) ftpcp() called parse227() directly and passed the source server's self-reported PASV IPv4 address to the target server's PORT command, bypassing the CVE-2021-4189 fix that was applied only to FTP.makepasv(). A malicious source FTP server could use this to redirect the target server's data connection to an arbitrary host:port (SSRF). ftpcp() now uses the source server's actual peer address, honoring the existing trust_server_pasv_ipv4_address opt-out, the same as makepasv(). Thanks to Qi Ding at Aurascape AI for the report. (GHSA-w8c5-q2xf-gf7c) (cherry picked from commit eac4fe3b2c77693790a5ef7dfab127c1fee81bf9) Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> --- Lib/ftplib.py | 11 +++++- Lib/test/test_ftplib.py | 36 ++++++++++++++++++- ...6-05-10-18-05-32.gh-issue-87451.XkKB6M.rst | 6 ++++ 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 640acc64f620cc..2f092d50f31782 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -883,7 +883,16 @@ def ftpcp(source, sourcename, target, targetname = '', type = 'I'): type = 'TYPE ' + type source.voidcmd(type) target.voidcmd(type) - sourcehost, sourceport = parse227(source.sendcmd('PASV')) + # Don't trust the IPv4 address the source server advertises in its PASV + # reply: a malicious source could otherwise point the target's data + # connection at an arbitrary host (SSRF). A caller that needs the old + # behavior can set trust_server_pasv_ipv4_address on the source FTP + # object. See FTP.makepasv(), which applies the same rule. + untrusted_host, sourceport = parse227(source.sendcmd('PASV')) + if source.trust_server_pasv_ipv4_address: + sourcehost = untrusted_host + else: + sourcehost = source.sock.getpeername()[0] target.sendport(sourcehost, sourceport) # RFC 959: the user must "listen" [...] BEFORE sending the # transfer request. diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index c864d401f9ed67..f1eff9430f7351 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -16,7 +16,7 @@ except ImportError: ssl = None -from unittest import TestCase, skipUnless +from unittest import mock, TestCase, skipUnless from test import support from test.support import requires_subprocess from test.support import threading_helper @@ -1145,6 +1145,40 @@ def testTimeoutDirectAccess(self): ftp.close() +class TestFtpcpSecurity(TestCase): + """ftpcp() must not trust the host a source server advertises in PASV. + + A malicious source server can otherwise redirect the target server's + data connection to an arbitrary host:port (SSRF), so ftpcp() uses the + source server's actual peer address instead, the same as FTP.makepasv(). + """ + + def _make_pair(self, *, advertised_host, real_host, trust=False): + source = mock.Mock(spec=ftplib.FTP) + source.trust_server_pasv_ipv4_address = trust + source.sock.getpeername.return_value = (real_host, 21) + # PASV replies give the host as comma-separated octets, not dotted. + advertised = advertised_host.replace('.', ',') + source.sendcmd.side_effect = lambda cmd: ( + f'227 Entering Passive Mode ({advertised},1,2).' + if cmd == 'PASV' else '150 ok') + target = mock.Mock(spec=ftplib.FTP) + target.sendcmd.return_value = '150 ok' + return source, target + + def test_ftpcp_ignores_untrusted_pasv_host(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7') + ftplib.ftpcp(source, 'a', target, 'b') + target.sendport.assert_called_once_with('198.51.100.7', 258) + + def test_ftpcp_trust_server_pasv_ipv4_address(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7', trust=True) + ftplib.ftpcp(source, 'a', target, 'b') + target.sendport.assert_called_once_with('10.0.0.5', 258) + + class MiscTestCase(TestCase): def test__all__(self): not_exported = { diff --git a/Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst b/Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst new file mode 100644 index 00000000000000..21a79c3e0e7db7 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst @@ -0,0 +1,6 @@ +The :mod:`ftplib` module's undocumented ``ftpcp`` function no longer trusts +the IPv4 address value returned from the source server in response to the +``PASV`` command by default, completing the fix for CVE-2021-4189. As with +:class:`ftplib.FTP`, the former behavior can be re-enabled by setting the +``trust_server_pasv_ipv4_address`` attribute on the source :class:`ftplib.FTP` +instance to ``True``. Thanks to Qi Deng at Aurascape AI for the report. From 8d32ae75d45ff0c9fd848b78a95bbec882d7ca3a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 15 May 2026 13:36:21 +0200 Subject: [PATCH 062/110] [3.15] gh-149707: Fix compiler warning in _ctypes_test on strchr() (#149791) (#149865) gh-149707: Fix compiler warning in _ctypes_test on strchr() (#149791) Change my_strchr() return type to "const char*" (add "const"). Fix the compiler warning: Modules/_ctypes/_ctypes_test.c: In function 'my_strchr': Modules/_ctypes/_ctypes_test.c:451:12: warning: return discards 'const' qualifier from pointer target type [-Wdiscarded-qualifiers] 451 | return strchr(s, c); | ^~~~~~ When using C23, strchr(text, ch) return type is "const char*" if text type is "const char*". (cherry picked from commit 5465b69255890650df99debb8256e0a7bc68138b) --- Modules/_ctypes/_ctypes_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index a0c9d8b70fee46..991ff0d675c2f1 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -446,7 +446,7 @@ EXPORT(char *)my_strtok(char *token, const char *delim) return strtok(token, delim); } -EXPORT(char *)my_strchr(const char *s, int c) +EXPORT(const char *) my_strchr(const char *s, int c) { return strchr(s, c); } From cc624f74ba6d4a2b9fc39330c851269b02f8d9c9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 15 May 2026 13:52:55 +0200 Subject: [PATCH 063/110] [3.15] gh-148675: Use a string for ctypes cparam tag (GH-149778) (#149869) gh-148675: Use a string for ctypes cparam tag (GH-149778) (cherry picked from commit 3ecca22567249ae44bf4369fbdb4d6d056701405) Co-authored-by: Victor Stinner --- Lib/test/test_ctypes/test_parameters.py | 8 +++++ Modules/_ctypes/_ctypes.c | 32 +++++++++--------- Modules/_ctypes/callproc.c | 45 +++++++++++++------------ Modules/_ctypes/ctypes.h | 4 +-- 4 files changed, 50 insertions(+), 39 deletions(-) diff --git a/Lib/test/test_ctypes/test_parameters.py b/Lib/test/test_ctypes/test_parameters.py index 46f8ff93efa915..6dadb7b410d703 100644 --- a/Lib/test/test_ctypes/test_parameters.py +++ b/Lib/test/test_ctypes/test_parameters.py @@ -1,6 +1,7 @@ import sys import unittest import test.support +import ctypes from ctypes import (CDLL, PyDLL, ArgumentError, Structure, Array, Union, _Pointer, _SimpleCData, _CFuncPtr, @@ -247,6 +248,13 @@ def test_parameter_repr(self): self.assertRegex(repr(c_char_p.from_param(b'hihi')), r"^$") self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^$") self.assertRegex(repr(c_void_p.from_param(0x12)), r"^$") + if hasattr(ctypes, 'c_double_complex'): + self.assertRegex(repr(ctypes.c_double_complex.from_param(0)), + r"^$") + self.assertRegex(repr(ctypes.c_float_complex.from_param(0)), + r"^$") + self.assertRegex(repr(ctypes.c_longdouble_complex.from_param(0)), + r"^$") @test.support.cpython_only def test_from_param_result_refcount(self): diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 98ac821c525a64..09eae97dd21a36 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -708,7 +708,7 @@ StructUnionType_paramfunc(ctypes_state *st, CDataObject *self) } assert(stginfo); /* Cannot be NULL for structure/union instances */ - parg->tag = 'V'; + parg->tag = "V"; parg->pffi_type = &stginfo->ffi_type_pointer; parg->value.p = ptr; parg->size = self->b_size; @@ -1282,7 +1282,7 @@ PyCPointerType_paramfunc(ctypes_state *st, CDataObject *self) if (parg == NULL) return NULL; - parg->tag = 'P'; + parg->tag = "P"; parg->pffi_type = &ffi_type_pointer; parg->obj = Py_NewRef(self); parg->value.p = *(void **)self->b_ptr; @@ -1703,7 +1703,7 @@ PyCArrayType_paramfunc(ctypes_state *st, CDataObject *self) PyCArgObject *p = PyCArgObject_new(st); if (p == NULL) return NULL; - p->tag = 'P'; + p->tag = "P"; p->pffi_type = &ffi_type_pointer; p->value.p = (char *)self->b_ptr; p->obj = Py_NewRef(self); @@ -1909,7 +1909,7 @@ c_wchar_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'Z'; + parg->tag = "Z"; parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj == NULL) { Py_DECREF(parg); @@ -1998,7 +1998,7 @@ c_char_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'z'; + parg->tag = "z"; parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj == NULL) { Py_DECREF(parg); @@ -2092,7 +2092,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'P'; + parg->tag = "P"; parg->obj = fd->setfunc(&parg->value, value, sizeof(void*)); if (parg->obj == NULL) { Py_DECREF(parg); @@ -2110,7 +2110,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'z'; + parg->tag = "z"; parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj == NULL) { Py_DECREF(parg); @@ -2127,7 +2127,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'Z'; + parg->tag = "Z"; parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj == NULL) { Py_DECREF(parg); @@ -2152,7 +2152,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (PyCArg_CheckExact(st, value)) { /* byref(c_xxx()) */ PyCArgObject *a = (PyCArgObject *)value; - if (a->tag == 'P') { + if (strcmp(a->tag, "P") == 0) { return Py_NewRef(value); } } @@ -2165,7 +2165,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'P'; + parg->tag = "P"; Py_INCREF(value); // Function pointers don't change their contents, no need to lock parg->value.p = *(void **)func->b_ptr; @@ -2191,7 +2191,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'Z'; + parg->tag = "Z"; parg->obj = Py_NewRef(value); /* Remember: b_ptr points to where the pointer is stored! */ Py_BEGIN_CRITICAL_SECTION(value); @@ -2332,7 +2332,8 @@ PyCSimpleType_paramfunc(ctypes_state *st, CDataObject *self) if (parg == NULL) return NULL; - parg->tag = fmt[0]; + assert(strcmp(fd->code, fmt) == 0); + parg->tag = fd->code; parg->pffi_type = fd->pffi_type; parg->obj = Py_NewRef(self); memcpy(&parg->value, self->b_ptr, self->b_size); @@ -2578,7 +2579,8 @@ PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, if (parg == NULL) return NULL; - parg->tag = fmt[0]; + assert(strcmp(fd->code, fmt) == 0); + parg->tag = fd->code; parg->pffi_type = fd->pffi_type; parg->obj = fd->setfunc(&parg->value, value, info->size); if (parg->obj) @@ -2832,7 +2834,7 @@ PyCFuncPtrType_paramfunc(ctypes_state *st, CDataObject *self) if (parg == NULL) return NULL; - parg->tag = 'P'; + parg->tag = "P"; parg->pffi_type = &ffi_type_pointer; parg->obj = Py_NewRef(self); parg->value.p = *(void **)self->b_ptr; @@ -4303,7 +4305,7 @@ _byref(ctypes_state *st, PyObject *obj) return NULL; } - parg->tag = 'P'; + parg->tag = "P"; parg->pffi_type = &ffi_type_pointer; parg->obj = obj; parg->value.p = ((CDataObject *)obj)->b_ptr; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index e208e27c5dbed4..e453cfeec9cc8c 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -468,7 +468,7 @@ PyCArgObject_new(ctypes_state *st) if (p == NULL) return NULL; p->pffi_type = NULL; - p->tag = '\0'; + p->tag = ""; p->obj = NULL; memset(&p->value, 0, sizeof(p->value)); PyObject_GC_Track(p); @@ -512,45 +512,50 @@ static PyObject * PyCArg_repr(PyObject *op) { PyCArgObject *self = _PyCArgObject_CAST(op); - switch(self->tag) { + + if (strlen(self->tag) != 1) { + goto generic; + } + + switch(self->tag[0]) { case 'b': case 'B': - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->tag, self->value.b); case 'h': case 'H': - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->tag, self->value.h); case 'i': case 'I': - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->tag, self->value.i); case 'l': case 'L': - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->tag, self->value.l); case 'q': case 'Q': - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->tag, self->value.q); case 'd': case 'f': { - PyObject *f = PyFloat_FromDouble((self->tag == 'f') ? self->value.f : self->value.d); + PyObject *f = PyFloat_FromDouble((strcmp(self->tag, "f") == 0) ? self->value.f : self->value.d); if (f == NULL) { return NULL; } - PyObject *result = PyUnicode_FromFormat("", self->tag, f); + PyObject *result = PyUnicode_FromFormat("", self->tag, f); Py_DECREF(f); return result; } case 'c': if (is_literal_char((unsigned char)self->value.c)) { - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->tag, self->value.c); } else { - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->tag, (unsigned char)self->value.c); } @@ -561,20 +566,16 @@ PyCArg_repr(PyObject *op) case 'z': case 'Z': case 'P': - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->tag, self->value.p); - break; default: - if (is_literal_char((unsigned char)self->tag)) { - return PyUnicode_FromFormat("", - (unsigned char)self->tag, (void *)self); - } - else { - return PyUnicode_FromFormat("", - (unsigned char)self->tag, (void *)self); - } + break; } + +generic: + return PyUnicode_FromFormat("", + self->tag, (void *)self); } static PyMemberDef PyCArgType_members[] = { @@ -1807,7 +1808,7 @@ _ctypes_byref_impl(PyObject *module, PyObject *obj, Py_ssize_t offset) if (parg == NULL) return NULL; - parg->tag = 'P'; + parg->tag = "P"; parg->pffi_type = &ffi_type_pointer; parg->obj = Py_NewRef(obj); parg->value.p = (char *)((CDataObject *)obj)->b_ptr + offset; diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 7b6b7f08582251..248559aa364a19 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -494,7 +494,7 @@ PyObject *_ctypes_callproc(ctypes_state *st, struct tagPyCArgObject { PyObject_HEAD ffi_type *pffi_type; - char tag; + const char *tag; union { char c; char b; @@ -511,7 +511,7 @@ struct tagPyCArgObject { long double G[2]; } value; PyObject *obj; - Py_ssize_t size; /* for the 'V' tag */ + Py_ssize_t size; /* for the "V" tag */ }; #define _PyCArgObject_CAST(op) ((PyCArgObject *)(op)) From 176d0f51cf23e2059e89e15b0004a37121b97e1b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 15 May 2026 14:08:15 +0200 Subject: [PATCH 064/110] [3.15] gh-149801: Add IANA registered names and aliases with leading zeros (GH-149804) (GH-149870) Like IBM00858, CP00858, IBM01140, CP01140. (cherry picked from commit 20438866aefc2e63949d8bb85d8f8e55633fd977) Co-authored-by: Serhiy Storchaka --- Doc/library/codecs.rst | 4 ++-- Lib/encodings/aliases.py | 8 ++++++++ .../2026-05-13-23-18-39.gh-issue-149801.S_FfGr.rst | 2 ++ 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-13-23-18-39.gh-issue-149801.S_FfGr.rst diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 9259ab10d5850b..059ed2c03acfa3 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1155,7 +1155,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp857 | 857, IBM857 | Turkish | +-----------------+--------------------------------+--------------------------------+ -| cp858 | 858, IBM858 | Western Europe | +| cp858 | 858, IBM00858 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp860 | 860, IBM860 | Portuguese | +-----------------+--------------------------------+--------------------------------+ @@ -1192,7 +1192,7 @@ particular, the following variants typically exist: | | | | | | | .. versionadded:: 3.4 | +-----------------+--------------------------------+--------------------------------+ -| cp1140 | ibm1140 | Western Europe | +| cp1140 | IBM01140 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp1250 | windows-1250 | Central and Eastern Europe | +-----------------+--------------------------------+--------------------------------+ diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index f4b1b8dd43f920..e5e50630f33d14 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -71,6 +71,10 @@ # cp1140 codec '1140' : 'cp1140', + 'cp01140' : 'cp1140', + 'csibm01140' : 'cp1140', + 'ebcdic_us_37_euro' : 'cp1140', + 'ibm01140' : 'cp1140', 'ibm1140' : 'cp1140', # cp1250 codec @@ -159,8 +163,12 @@ # cp858 codec '858' : 'cp858', + 'cp00858' : 'cp858', + 'csibm00858' : 'cp858', 'csibm858' : 'cp858', + 'ibm00858' : 'cp858', 'ibm858' : 'cp858', + 'pc_multilingual_850_euro' : 'cp858', # cp860 codec '860' : 'cp860', diff --git a/Misc/NEWS.d/next/Library/2026-05-13-23-18-39.gh-issue-149801.S_FfGr.rst b/Misc/NEWS.d/next/Library/2026-05-13-23-18-39.gh-issue-149801.S_FfGr.rst new file mode 100644 index 00000000000000..f9e8538527d204 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-13-23-18-39.gh-issue-149801.S_FfGr.rst @@ -0,0 +1,2 @@ +Add IANA registered names and aliases with leading zeros before number (like +IBM00858, CP00858, IBM01140, CP01140) for corresponding codecs. From b3819aeff3dde744bab025b5b451182e3fc5a3a6 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 15 May 2026 15:07:18 +0200 Subject: [PATCH 065/110] [3.15] gh-149816: Fix race condition in `memoryview` with free-threading (GH-149858) (#149875) gh-149816: Fix race condition in `memoryview` with free-threading (GH-149858) (cherry picked from commit 1fdf0337742762cc47837042747cc607f024a202) Co-authored-by: sobolevn --- ...-05-15-11-31-57.gh-issue-149816.ugN2rx.rst | 1 + Objects/memoryobject.c | 20 ++++++------------- 2 files changed, 7 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-15-11-31-57.gh-issue-149816.ugN2rx.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-15-11-31-57.gh-issue-149816.ugN2rx.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-15-11-31-57.gh-issue-149816.ugN2rx.rst new file mode 100644 index 00000000000000..016c17dd66b19e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-15-11-31-57.gh-issue-149816.ugN2rx.rst @@ -0,0 +1 @@ +Fix a race condition in :class:`memoryview` with free-threading. diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 900db864621a84..9d1ca633780f92 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1629,11 +1629,7 @@ memory_getbuf(PyObject *_self, Py_buffer *view, int flags) view->obj = Py_NewRef(self); -#ifdef Py_GIL_DISABLED - _Py_atomic_add_ssize(&self->exports, 1); -#else - self->exports++; -#endif + FT_ATOMIC_ADD_SSIZE(self->exports, 1); return 0; } @@ -1642,11 +1638,7 @@ static void memory_releasebuf(PyObject *_self, Py_buffer *view) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; -#ifdef Py_GIL_DISABLED - _Py_atomic_add_ssize(&self->exports, -1); -#else - self->exports--; -#endif + FT_ATOMIC_ADD_SSIZE(self->exports, -1); return; /* PyBuffer_Release() decrements view->obj after this function returns. */ } @@ -2434,9 +2426,9 @@ memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, // Prevent 'self' from being freed if computing len(sep) mutates 'self' // in _Py_strhex_with_sep(). // See: https://github.com/python/cpython/issues/143195. - self->exports++; + FT_ATOMIC_ADD_SSIZE(self->exports, 1); PyObject *ret = _Py_strhex_with_sep(src->buf, src->len, sep, bytes_per_sep); - self->exports--; + FT_ATOMIC_ADD_SSIZE(self->exports, -1); return ret; } @@ -3363,9 +3355,9 @@ memory_hash(PyObject *_self) if (view->obj != NULL) { // Prevent 'self' from being freed when computing the item's hash. // See https://github.com/python/cpython/issues/142664. - self->exports++; + FT_ATOMIC_ADD_SSIZE(self->exports, 1); Py_hash_t h = PyObject_Hash(view->obj); - self->exports--; + FT_ATOMIC_ADD_SSIZE(self->exports, -1); if (h == -1) { /* Keep the original error message */ return -1; From a57855561e6c80be88e828183eb7a6ebc68395c2 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 15 May 2026 15:19:39 +0200 Subject: [PATCH 066/110] [3.15] gh-138489: Add build-details.json generation to PC/layout (GH-149153) (cherry picked from commit 4aa296f9c4a85a7badc09bf7ca6ede36cd8cd14c) Co-authored-by: Steve Dower --- ...-04-29-14-44-51.gh-issue-138489.234aj6.rst | 4 + PC/layout/main.py | 4 + PC/layout/support/builddetails.py | 119 ++++++++++++++++++ PC/layout/support/constants.py | 9 ++ PC/layout/support/options.py | 6 + 5 files changed, 142 insertions(+) create mode 100644 Misc/NEWS.d/next/Windows/2026-04-29-14-44-51.gh-issue-138489.234aj6.rst create mode 100644 PC/layout/support/builddetails.py diff --git a/Misc/NEWS.d/next/Windows/2026-04-29-14-44-51.gh-issue-138489.234aj6.rst b/Misc/NEWS.d/next/Windows/2026-04-29-14-44-51.gh-issue-138489.234aj6.rst new file mode 100644 index 00000000000000..4afb8f737b692e --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-04-29-14-44-51.gh-issue-138489.234aj6.rst @@ -0,0 +1,4 @@ +Windows distributions now include a :file:`build-details.json` file (see +:pep:`739`). The legacy installer does not install it, but all other +distributions from python.org and all preset configurations in the +``PC\layout`` script will include one. diff --git a/PC/layout/main.py b/PC/layout/main.py index 3a62ea91420c9e..f70a26b2b29659 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -22,6 +22,7 @@ __path__ = [str(Path(__file__).resolve().parent)] from .support.appxmanifest import * +from .support.builddetails import * from .support.catalog import * from .support.constants import * from .support.filesets import * @@ -317,6 +318,9 @@ def _c(d): for dest, src in get_appx_layout(ns): yield dest, src + for dest, src in get_builddetails(ns): + yield dest, src + if ns.include_cat: if ns.flat_dlls: yield ns.include_cat.name, ns.include_cat diff --git a/PC/layout/support/builddetails.py b/PC/layout/support/builddetails.py new file mode 100644 index 00000000000000..6ef860eeb04354 --- /dev/null +++ b/PC/layout/support/builddetails.py @@ -0,0 +1,119 @@ +import io +import json +from . import constants + +_LEVELS = { + 0xA0: "alpha", + 0xB0: "beta", + 0xC0: "candidate", + 0xF0: "final", +} + + +_TEMPLATE = { + "schema_version": "1.0", + "base_prefix": ".", + "base_interpreter": "python.exe", + "platform": None, # Set later + "language": { + "version": f"{constants.VER_MAJOR}.{constants.VER_MINOR}", + "version_info": { + "major": constants.VER_MAJOR, + "minor": constants.VER_MINOR, + "micro": constants.VER_MICRO, + "releaselevel": _LEVELS.get(constants.VER_FIELD4 & 0xF0, "final"), + "serial": constants.VER_FIELD4 & 0x0F, + }, + }, + "implementation": { + "name": "cpython", + "cache_tag": f"cpython-{constants.VER_MAJOR}{constants.VER_MINOR}", + "version": { + "major": constants.VER_MAJOR, + "minor": constants.VER_MINOR, + "micro": constants.VER_MICRO, + "releaselevel": _LEVELS.get(constants.VER_FIELD4 & 0xF0, "final"), + "serial": constants.VER_FIELD4 & 0x0F, + }, + "hexversion": constants.VER_HEXVERSION, + }, + "abi": { + "flags": [], + "extension_suffix": ".pyd", + "stable_abi_suffix": ".pyd", + }, + "suffixes": { + "source": [".py", ".pyw"], + "bytecode": [".pyc"], + "extensions": [".pyd"], + }, + "libpython": { + "dynamic": constants.PYTHON_DLL_NAME, + "dynamic_stableabi": constants.PYTHON_STABLE_DLL_NAME, + "link_extensions": True, + }, + "c_api": { + }, +} + + +def _with_d(path): + pre, sep, post = path.partition(".") + return pre + "_d" + sep + post + + +def _add_d(data, *args): + for a in args[:-1]: + data = data[a] + a = args[-1] + v = data[a] + if isinstance(v, list): + data[a] = [_with_d(i) for i in data[a]] + else: + data[a] = _with_d(data[a]) + + +def get_builddetails(ns): + if not ns.include_builddetails_json: + return + + details = dict(_TEMPLATE) + + plat = { + "win32": "win32", + "amd64": "win-amd64", + "arm64": "win-arm64", + }.get(ns.arch, ns.arch) + + pyd_abi_flags = "" + if ns.include_freethreaded: + details["abi"]["flags"].append("t") + pyd_abi_flags += "t" + if ns.debug: + details["abi"]["flags"].append("d") + + norm_plat = plat.replace("-", "_") + ext_suffix = f".cp{constants.VER_MAJOR}{constants.VER_MINOR}{pyd_abi_flags}-{norm_plat}.pyd" + details["abi"]["extension_suffix"] = ext_suffix + details["suffixes"]["extensions"].insert(0, ext_suffix) + + details["platform"] = plat + + if ns.include_dev: + details["c_api"]["headers"] = "Include" + + if ns.include_freethreaded: + details["libpython"]["dynamic"] = constants.FREETHREADED_PYTHON_DLL_NAME + details["libpython"]["dynamic_stableabi"] = constants.FREETHREADED_PYTHON_STABLE_DLL_NAME + + if ns.debug: + _add_d(details, "base_interpreter") + _add_d(details, "abi", "stable_abi_suffix") + _add_d(details, "abi", "extension_suffix") + _add_d(details, "suffixes", "extensions") + _add_d(details, "libpython", "dynamic") + _add_d(details, "libpython", "dynamic_stableabi") + + buffer = io.StringIO() + json.dump(details, buffer, indent=2) + yield "build-details.json", ("build-details.json", buffer.getvalue().encode()) diff --git a/PC/layout/support/constants.py b/PC/layout/support/constants.py index 6b8c915e519743..cb16f534685c8f 100644 --- a/PC/layout/support/constants.py +++ b/PC/layout/support/constants.py @@ -23,6 +23,14 @@ def _unpack_hexversion(): return _read_patchlevel_version(pathlib.Path(os.getenv("PYTHONINCLUDE"))) except OSError: pass + # Manual search for a '-s ` arument + try: + src = sys.argv[sys.argv.index("-s") + 1] + return _read_patchlevel_version(pathlib.Path(src) / "Include") + except (IndexError, ValueError): + pass + except OSError: + pass return struct.pack(">i", sys.hexversion) @@ -68,6 +76,7 @@ def check_patchlevel_version(sources): VER_MAJOR, VER_MINOR, VER_MICRO, VER_FIELD4 = _unpack_hexversion() +VER_HEXVERSION = (VER_MAJOR << 24) | (VER_MINOR << 16) | (VER_MICRO << 8) | (VER_FIELD4) VER_SUFFIX = _get_suffix(VER_FIELD4) VER_FIELD3 = VER_MICRO << 8 | VER_FIELD4 VER_DOT = "{}.{}".format(VER_MAJOR, VER_MINOR) diff --git a/PC/layout/support/options.py b/PC/layout/support/options.py index e8c393385425e7..3a6e00f720f01f 100644 --- a/PC/layout/support/options.py +++ b/PC/layout/support/options.py @@ -39,6 +39,7 @@ def public(f): "install-json": {"help": "a PyManager __install__.json file"}, "install-embed-json": {"help": "a PyManager __install__.json file for embeddable distro"}, "install-test-json": {"help": "a PyManager __install__.json for the test distro"}, + "builddetails-json": {"help": "a PEP 739 build-details.json"}, } @@ -69,6 +70,7 @@ def public(f): "props", "nuspec", "alias", + "builddetails-json", ], }, "iot": {"help": "Windows IoT Core", "options": ["alias", "stable", "pip"]}, @@ -85,6 +87,7 @@ def public(f): "symbols", "html-doc", "alias", + "builddetails-json", ], }, "embed": { @@ -96,6 +99,7 @@ def public(f): "flat-dlls", "underpth", "precompile", + "builddetails-json", ], }, "pymanager": { @@ -109,6 +113,7 @@ def public(f): "dev", "html-doc", "install-json", + "builddetails-json", ], }, "pymanager-test": { @@ -124,6 +129,7 @@ def public(f): "symbols", "tests", "install-test-json", + "builddetails-json", ], }, } From 0e2184aca0dabfe7009f1316c75d443085e2b3d6 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 15 May 2026 16:12:05 +0200 Subject: [PATCH 067/110] [3.15] gh-149786: Fixes venvlauncher builds on Windows free-threaded (GH-149847) (cherry picked from commit 1c5fe21eb2a65190c04bb3f4c0931d76f5ccf415) Co-authored-by: Steve Dower --- Lib/test/test_venv.py | 12 ++++++------ .../2026-05-14-22-09-46.gh-issue-149786.UI-HZM.rst | 1 + PC/layout/support/options.py | 2 ++ PC/layout/support/pymanager.py | 5 +++-- PCbuild/python.vcxproj | 8 ++++++++ PCbuild/pythonw.vcxproj | 8 ++++++++ PCbuild/venvlauncher.vcxproj | 7 +++++-- PCbuild/venvwlauncher.vcxproj | 7 +++++-- 8 files changed, 38 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2026-05-14-22-09-46.gh-issue-149786.UI-HZM.rst diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index a42787f261bfe8..9d2960664abfad 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -301,9 +301,9 @@ def test_sysconfig(self): self.assertEqual(out.strip(), expected, err) for attr, expected in ( ('executable', self.envpy()), - # Usually compare to sys.executable, but if we're running in our own - # venv then we really need to compare to our base executable - ('_base_executable', sys._base_executable), + # Usually compare to sys.prefix, but if we're running in our own + # venv then we really need to compare to our base prefix + ('base_prefix', sys.base_prefix), ): with self.subTest(attr): cmd[2] = f'import sys; print(sys.{attr})' @@ -916,10 +916,10 @@ def test_venvwlauncher(self): exename = exename.replace("python", "pythonw") envpyw = os.path.join(self.env_dir, self.bindir, exename) try: - subprocess.check_call([envpyw, "-c", "import sys; " - "assert sys._base_executable.endswith('%s')" % exename]) + subprocess.check_call([envpyw, "-c", "import fnmatch, sys; " + "assert fnmatch.fnmatch(sys._base_executable, '**/pythonw*.exe')"]) except subprocess.CalledProcessError: - self.fail("venvwlauncher.exe did not run %s" % exename) + self.fail("venvwlauncher.exe did not run pythonw.exe") @requireVenvCreate diff --git a/Misc/NEWS.d/next/Windows/2026-05-14-22-09-46.gh-issue-149786.UI-HZM.rst b/Misc/NEWS.d/next/Windows/2026-05-14-22-09-46.gh-issue-149786.UI-HZM.rst new file mode 100644 index 00000000000000..64ca91a01f41af --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-05-14-22-09-46.gh-issue-149786.UI-HZM.rst @@ -0,0 +1 @@ +Fixes virtual environment launchers on Windows free-threaded builds. diff --git a/PC/layout/support/options.py b/PC/layout/support/options.py index 3a6e00f720f01f..f67d8ba04d9070 100644 --- a/PC/layout/support/options.py +++ b/PC/layout/support/options.py @@ -112,6 +112,7 @@ def public(f): "venv", "dev", "html-doc", + "alias", "install-json", "builddetails-json", ], @@ -128,6 +129,7 @@ def public(f): "html-doc", "symbols", "tests", + "alias", "install-test-json", "builddetails-json", ], diff --git a/PC/layout/support/pymanager.py b/PC/layout/support/pymanager.py index 831d49ea3f9b46..f6316e0295c74a 100644 --- a/PC/layout/support/pymanager.py +++ b/PC/layout/support/pymanager.py @@ -66,8 +66,9 @@ def calculate_install_json(ns, *, for_embed=False, for_test=False): if ns.include_freethreaded: # Free-threaded distro comes with a tag suffix TAG_SUFFIX = "t" - TARGET = f"python{VER_MAJOR}.{VER_MINOR}t.exe" - TARGETW = f"pythonw{VER_MAJOR}.{VER_MINOR}t.exe" + if not ns.include_alias: + TARGET = f"python{VER_MAJOR}.{VER_MINOR}t.exe" + TARGETW = f"pythonw{VER_MAJOR}.{VER_MINOR}t.exe" DISPLAY_TAGS.append("free-threaded") FILE_SUFFIX = f"t-{ns.arch}" diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index 70dabaa3c8bc02..417ede34c54af3 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -135,6 +135,14 @@ set PYTHONPATH=$(PySourcePath)Lib "$(OutDir)$(PyExeName)$(PyDebugExt).exe" "$(PySourcePath)PC\validate_ucrtbase.py" $(UcrtName)' ContinueOnError="true" /> + + + + <_Content>@rem This script invokes the most recently built Python with all arguments diff --git a/PCbuild/pythonw.vcxproj b/PCbuild/pythonw.vcxproj index c6a5b8ce90a0d9..244cdf622ad915 100644 --- a/PCbuild/pythonw.vcxproj +++ b/PCbuild/pythonw.vcxproj @@ -115,4 +115,12 @@ + + + + \ No newline at end of file diff --git a/PCbuild/venvlauncher.vcxproj b/PCbuild/venvlauncher.vcxproj index abaf3a979af268..a2e8ffa82b10eb 100644 --- a/PCbuild/venvlauncher.vcxproj +++ b/PCbuild/venvlauncher.vcxproj @@ -89,10 +89,13 @@ - + + $(PyExeName)$(PyDebugExt).exe + $(PyExeName)$(MajorVersionNumber).$(MinorVersionNumber)t$(PyDebugExt).exe + - EXENAME=L"$(PyExeName)$(PyDebugExt).exe";_CONSOLE;%(PreprocessorDefinitions) + EXENAME=L"$(ExeName)";_CONSOLE;%(PreprocessorDefinitions) MultiThreaded diff --git a/PCbuild/venvwlauncher.vcxproj b/PCbuild/venvwlauncher.vcxproj index c58280deb8abeb..f2aaf83fe2b378 100644 --- a/PCbuild/venvwlauncher.vcxproj +++ b/PCbuild/venvwlauncher.vcxproj @@ -89,10 +89,13 @@ - + + $(PyWExeName)$(PyDebugExt).exe + $(PyWExeName)$(MajorVersionNumber).$(MinorVersionNumber)t$(PyDebugExt).exe + - EXENAME=L"$(PyWExeName)$(PyDebugExt).exe";_WINDOWS;%(PreprocessorDefinitions) + EXENAME=L"$(ExeName)";_WINDOWS;%(PreprocessorDefinitions) MultiThreaded From 6136ad68010756edfc4474ad24793e62e1b9907e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 15 May 2026 18:57:59 +0200 Subject: [PATCH 068/110] [3.15] gh-142349: Add `help("lazy")` support (GH-149886) (#149889) gh-142349: Add `help("lazy")` support (GH-149886) (cherry picked from commit 8be3fb1b50ce6b01bf0924f0a0362a5e04af83b4) Co-authored-by: sobolevn --- Doc/tools/extensions/pydoc_topics.py | 1 + Lib/pydoc.py | 1 + Lib/test/test_pydoc/test_pydoc.py | 2 +- .../next/Library/2026-05-15-18-44-20.gh-issue-142349.fHK3v1.rst | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-15-18-44-20.gh-issue-142349.fHK3v1.rst diff --git a/Doc/tools/extensions/pydoc_topics.py b/Doc/tools/extensions/pydoc_topics.py index a65d77433b255b..35878e2d1e43e9 100644 --- a/Doc/tools/extensions/pydoc_topics.py +++ b/Doc/tools/extensions/pydoc_topics.py @@ -68,6 +68,7 @@ "in", "integers", "lambda", + "lazy", "lists", "naming", "nonlocal", diff --git a/Lib/pydoc.py b/Lib/pydoc.py index a1a6aad434ddf4..497cc7d90a4245 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1845,6 +1845,7 @@ class Helper: 'in': ('in', 'SEQUENCEMETHODS'), 'is': 'COMPARISON', 'lambda': ('lambda', 'FUNCTIONS'), + 'lazy': ('lazy', 'MODULES'), 'nonlocal': ('nonlocal', 'global NAMESPACES'), 'not': 'BOOLEAN', 'or': 'BOOLEAN', diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 2e190d1b81be8e..5cd26923f75c31 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -2172,7 +2172,7 @@ def mock_getline(prompt): def test_keywords(self): self.assertEqual(sorted(pydoc.Helper.keywords), - sorted(keyword.kwlist)) + sorted(keyword.kwlist + ['lazy'])) def test_interact_empty_line_continues(self): # gh-138568: test pressing Enter without input should continue in help session diff --git a/Misc/NEWS.d/next/Library/2026-05-15-18-44-20.gh-issue-142349.fHK3v1.rst b/Misc/NEWS.d/next/Library/2026-05-15-18-44-20.gh-issue-142349.fHK3v1.rst new file mode 100644 index 00000000000000..fa667c4110941e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-15-18-44-20.gh-issue-142349.fHK3v1.rst @@ -0,0 +1 @@ +Add :keyword:`lazy` to the list of support topic by :func:`help`. From ec3aa6ab4847ebd6f8f3e9b2b7023c5ab3a6e39a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 16 May 2026 11:06:14 +0200 Subject: [PATCH 069/110] [3.15] gh-149816: Fix a race condition in `_PyBytes_FromList` with free-threading (GH-149909) (#149911) gh-149816: Fix a race condition in `_PyBytes_FromList` with free-threading (GH-149909) (cherry picked from commit 46afba7b9324bc9492c3527d0fe47dd74f1f598c) Co-authored-by: sobolevn --- .../2026-05-16-11-03-54.gh-issue-149816.X_gqMT.rst | 1 + Objects/bytesobject.c | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-16-11-03-54.gh-issue-149816.X_gqMT.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-16-11-03-54.gh-issue-149816.X_gqMT.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-16-11-03-54.gh-issue-149816.X_gqMT.rst new file mode 100644 index 00000000000000..d35f0857a1aefe --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-16-11-03-54.gh-issue-149816.X_gqMT.rst @@ -0,0 +1 @@ +Fix a race condition in ``_PyBytes_FromList`` in free-threading mode. diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 8a9d1b133affb3..2d694922557429 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -11,6 +11,7 @@ #include "pycore_global_objects.h"// _Py_GET_GLOBAL_OBJECT() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_long.h" // _PyLong_DigitValue +#include "pycore_list.h" // _PyList_GetItemRef #include "pycore_object.h" // _PyObject_GC_TRACK #include "pycore_pymem.h" // PYMEM_CLEANBYTE #include "pycore_strhex.h" // _Py_strhex_with_sep() @@ -2991,8 +2992,10 @@ _PyBytes_FromList(PyObject *x) size = _PyBytesWriter_GetAllocated(writer); for (Py_ssize_t i = 0; i < PyList_GET_SIZE(x); i++) { - PyObject *item = PyList_GET_ITEM(x, i); - Py_INCREF(item); + PyObject *item = _PyList_GetItemRef((PyListObject *)x, i); + if (item == NULL) { + goto error; + } Py_ssize_t value = PyNumber_AsSsize_t(item, NULL); Py_DECREF(item); if (value == -1 && PyErr_Occurred()) From bdc44c55a277b7c26efdb63885dec606e2abf1b7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 17 May 2026 10:33:11 +0200 Subject: [PATCH 070/110] [3.15] gh-149916: Restore the commented out part of test_body_encode in test_email (GH-149917) (GH-149937) (cherry picked from commit 1cbe035723698f15aa1b1af5deef615b28aae2e5) Co-authored-by: Serhiy Storchaka --- Lib/test/test_email/test_email.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 7778566492d8f4..d2c2261edbe04e 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -4995,15 +4995,8 @@ def test_body_encode(self): # Try the convert argument, where input codec != output codec c = Charset('euc-jp') # With apologies to Tokio Kikuchi ;) - # XXX FIXME -## try: -## eq('\x1b$B5FCO;~IW\x1b(B', -## c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7')) -## eq('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', -## c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', False)) -## except LookupError: -## # We probably don't have the Japanese codecs installed -## pass + eq('\x1b$B5FCO;~IW\x1b(B', + c.body_encode('\u83ca\u5730\u6642\u592b')) # Testing SF bug #625509, which we have to fake, since there are no # built-in encodings where the header encoding is QP but the body # encoding is not. From 42ff9b4959667cf31bde13a53fca01b1ec381168 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 17 May 2026 13:01:15 +0200 Subject: [PATCH 071/110] [3.15] gh-149921: Fix reference leaks in _interpchannels and _interpqueues modules (GH-149922) (#149943) gh-149921: Fix reference leaks in _interpchannels and _interpqueues modules (GH-149922) (cherry picked from commit acefff95eab3db6b7cf837f3ce2707bbf9199376) Co-authored-by: AN Long --- .../Library/2026-05-16-21-08-33.gh-issue-149921.I1yNML.rst | 2 ++ Modules/_interpchannelsmodule.c | 4 ++-- Modules/_interpqueuesmodule.c | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-16-21-08-33.gh-issue-149921.I1yNML.rst diff --git a/Misc/NEWS.d/next/Library/2026-05-16-21-08-33.gh-issue-149921.I1yNML.rst b/Misc/NEWS.d/next/Library/2026-05-16-21-08-33.gh-issue-149921.I1yNML.rst new file mode 100644 index 00000000000000..113bd1a802f799 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-16-21-08-33.gh-issue-149921.I1yNML.rst @@ -0,0 +1,2 @@ +Fix reference leaks in error paths of the :mod:`!_interpchannels` and +:mod:`!_interpqueues` extension modules. diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index 3c356cb40d2bca..c6d107d243dda0 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -2586,6 +2586,7 @@ static PyObject * _channelid_from_xid(_PyXIData_t *data) { struct _channelid_xid *xid = (struct _channelid_xid *)_PyXIData_DATA(data); + PyObject *cidobj = NULL; // It might not be imported yet, so we can't use _get_current_module(). PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR); @@ -2595,11 +2596,10 @@ _channelid_from_xid(_PyXIData_t *data) assert(mod != Py_None); module_state *state = get_module_state(mod); if (state == NULL) { - return NULL; + goto done; } // Note that we do not preserve the "resolve" flag. - PyObject *cidobj = NULL; int err = newchannelid(state->ChannelIDType, xid->cid, xid->end, _global_channels(), 0, 0, (channelid **)&cidobj); diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 777b6854749884..b23aa5f39489d9 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -1363,6 +1363,7 @@ _queueobj_from_xid(_PyXIData_t *data) if (mod == NULL) { mod = PyImport_ImportModule(MODULE_NAME_STR); if (mod == NULL) { + Py_DECREF(qidobj); return NULL; } } From 6e83c55e643ac1c189535604e70b0328c0ebec44 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 18 May 2026 11:00:19 +0200 Subject: [PATCH 072/110] [3.15] gh-149953: Fix null pointer dereference order in `code_objects.c` (GH-149956) (#149976) gh-149953: Fix null pointer dereference order in `code_objects.c` (GH-149956) Move check before (cherry picked from commit 0ed497a350d76dd20de1a1689c84426c7c1d6e22) Co-authored-by: Nezuko Agent --- Modules/_remote_debugging/code_objects.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_remote_debugging/code_objects.c b/Modules/_remote_debugging/code_objects.c index 7b95c0f2d4fa8d..97c6ba772e88f1 100644 --- a/Modules/_remote_debugging/code_objects.c +++ b/Modules/_remote_debugging/code_objects.c @@ -432,7 +432,7 @@ parse_code_object(RemoteUnwinderObject *unwinder, #ifdef Py_GIL_DISABLED // Handle thread-local bytecode (TLBC) in free threading builds - if (ctx->tlbc_index == 0 || unwinder->debug_offsets.code_object.co_tlbc == 0 || unwinder == NULL) { + if (ctx->tlbc_index == 0 || unwinder == NULL || unwinder->debug_offsets.code_object.co_tlbc == 0) { // No TLBC or no unwinder - use main bytecode directly addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; goto done_tlbc; From 34e4005c35f92ef3257e4ec2d1d4dc0eeebb321e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 18 May 2026 15:52:17 +0200 Subject: [PATCH 073/110] [3.15] gh-149887: Install python3t.lib for GIL-enabled Windows install (GH-149900) gh-149887: Install python3t.lib for GIL-enabled Windows install (GH-149900) (cherry picked from commit bd6bf91fcba8a8fba8b9aea6cc971333c9be3ad9) Co-authored-by: Nathan Goldbaum --- Tools/msi/dev/dev_files.wxs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Tools/msi/dev/dev_files.wxs b/Tools/msi/dev/dev_files.wxs index 21f9c848cc6be5..a9039d03f5f6fa 100644 --- a/Tools/msi/dev/dev_files.wxs +++ b/Tools/msi/dev/dev_files.wxs @@ -13,6 +13,9 @@ + + + @@ -24,6 +27,9 @@ + + + From 01b07df858a9594f4421dacfd10a1c1b26102e29 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 18 May 2026 18:16:55 +0200 Subject: [PATCH 074/110] [3.15] gh-149816: Fix a RC in `_random.Random.__init__` method (GH-149824) (#149997) gh-149816: Fix a RC in `_random.Random.__init__` method (GH-149824) (cherry picked from commit 14af19e6c0d9dd05b525596fdd53373f726253d8) Co-authored-by: sobolevn --- ...-05-14-15-55-28.gh-issue-149816.ZaXQ0q.rst | 2 ++ Modules/_randommodule.c | 35 ++++++++----------- Modules/clinic/_randommodule.c.h | 33 ++++++++++++++++- 3 files changed, 48 insertions(+), 22 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-14-15-55-28.gh-issue-149816.ZaXQ0q.rst diff --git a/Misc/NEWS.d/next/Library/2026-05-14-15-55-28.gh-issue-149816.ZaXQ0q.rst b/Misc/NEWS.d/next/Library/2026-05-14-15-55-28.gh-issue-149816.ZaXQ0q.rst new file mode 100644 index 00000000000000..3ea70071ec3c75 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-14-15-55-28.gh-issue-149816.ZaXQ0q.rst @@ -0,0 +1,2 @@ +Fix a race condition in ``_random.Random.__init__`` method in free-threading +mode. diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 0fb73481651748..a06966be23be1e 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -123,9 +123,9 @@ typedef struct { /*[clinic input] module _random -class _random.Random "RandomObject *" "_randomstate_type(type)->Random_Type" +class _random.Random "RandomObject *" "(PyTypeObject *)_randomstate_type(Py_TYPE(self))->Random_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=70a2c99619474983]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=f04bcbfba61a322e]*/ /* Random methods */ @@ -549,27 +549,20 @@ _random_Random_getrandbits_impl(RandomObject *self, uint64_t k) return result; } -static int -random_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - PyObject *arg = NULL; - _randomstate *state = _randomstate_type(Py_TYPE(self)); - - if ((Py_IS_TYPE(self, (PyTypeObject *)state->Random_Type) || - Py_TYPE(self)->tp_init == ((PyTypeObject*)state->Random_Type)->tp_init) && - !_PyArg_NoKeywords("Random", kwds)) { - return -1; - } - - if (PyTuple_GET_SIZE(args) > 1) { - PyErr_SetString(PyExc_TypeError, "Random() requires 0 or 1 argument"); - return -1; - } +/*[clinic input] +@critical_section +@text_signature "($self, [seed])" +_random.Random.__init__ as random_init - if (PyTuple_GET_SIZE(args) == 1) - arg = PyTuple_GET_ITEM(args, 0); + seed: object = NULL + / +[clinic start generated code]*/ - return random_seed(RandomObject_CAST(self), arg); +static int +random_init_impl(RandomObject *self, PyObject *seed) +/*[clinic end generated code: output=260734a3739c394f input=e516bf32e8a05e28]*/ +{ + return random_seed(self, seed); } diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h index 2563a16aea0b6f..ca9cad7a572dad 100644 --- a/Modules/clinic/_randommodule.c.h +++ b/Modules/clinic/_randommodule.c.h @@ -143,4 +143,35 @@ _random_Random_getrandbits(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=7ce97b2194eecaf7 input=a9049054013a1b77]*/ + +static int +random_init_impl(RandomObject *self, PyObject *seed); + +static int +random_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = (PyTypeObject *)_randomstate_type(Py_TYPE(self))->Random_Type; + PyObject *seed = NULL; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Random", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("Random", PyTuple_GET_SIZE(args), 0, 1)) { + goto exit; + } + if (PyTuple_GET_SIZE(args) < 1) { + goto skip_optional; + } + seed = PyTuple_GET_ITEM(args, 0); +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = random_init_impl((RandomObject *)self, seed); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} +/*[clinic end generated code: output=ec95f7df0c3f3c19 input=a9049054013a1b77]*/ From 84ea87cde0132ca59f0390031d17b1f867dba8e0 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 18 May 2026 21:24:06 +0200 Subject: [PATCH 075/110] [3.15] gh-149977: Fix extra output of `-m test test_lazy_import` (GH-149978) (#150016) gh-149977: Fix extra output of `-m test test_lazy_import` (GH-149978) (cherry picked from commit 6d5be4b1d6ca91a18e76ae8dad2c5e94837d6309) Co-authored-by: sobolevn --- Lib/test/test_lazy_import/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index 5d770eeae07a15..bcbf1a23233ba8 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -10,6 +10,7 @@ import unittest import tempfile import os +import contextlib from test import support from test.support.script_helper import assert_python_ok @@ -441,10 +442,14 @@ def tearDown(self): def test_lazy_import_pkg(self): """lazy import of package submodule should load the package.""" - import test.test_lazy_import.data.lazy_import_pkg + out = io.StringIO() + + with contextlib.redirect_stdout(out): + import test.test_lazy_import.data.lazy_import_pkg self.assertIn("test.test_lazy_import.data.pkg", sys.modules) self.assertIn("test.test_lazy_import.data.pkg.bar", sys.modules) + self.assertIn("BAR_MODULE_LOADED", out.getvalue()) def test_lazy_import_pkg_cross_import(self): """Cross-imports within package should preserve lazy imports.""" From 16f8ed5a82961cbfd3f74a529f940527957627f7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 18 May 2026 23:47:19 +0200 Subject: [PATCH 076/110] [3.15] gh-95816: Fix TLS version range example in docs (GH-148574) (#150008) gh-95816: Fix TLS version range example in docs (GH-148574) docs(ssl): Fix TLS version range example (cherry picked from commit dbd8985e8262055ed091de9a72660b7c112a4ce7) Co-authored-by: Jan Brasna <1784648+janbrasna@users.noreply.github.com> --- Doc/library/ssl.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index d9c736d27dcaec..b180673f22973e 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -2076,7 +2076,7 @@ to speed up repeated connections from the same clients. :attr:`~SSLContext.minimum_version` and :attr:`SSLContext.options` all affect the supported SSL and TLS versions of the context. The implementation does not prevent - invalid combination. For example a context with + invalid combinations. For example a context with :attr:`OP_NO_TLSv1_2` in :attr:`~SSLContext.options` and :attr:`~SSLContext.maximum_version` set to :attr:`TLSVersion.TLSv1_2` will not be able to establish a TLS 1.2 connection. @@ -2891,11 +2891,11 @@ disabled by default. :: >>> client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_2 >>> client_context.maximum_version = ssl.TLSVersion.TLSv1_3 -The SSL context created above will only allow TLSv1.3 and later (if +The SSL client context created above will only allow TLSv1.2 and TLSv1.3 (if supported by your system) connections to a server. :const:`PROTOCOL_TLS_CLIENT` implies certificate validation and hostname checks by default. You have to load certificates into the context. From c417fcabfdd56ed3b7e835ea66117a304d128200 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 19 May 2026 01:28:04 +0200 Subject: [PATCH 077/110] [3.15] gh-149590: Remove faulthandler_traverse (GH-150023) (#150037) gh-149590: Remove faulthandler_traverse (GH-150023) `faulthandler_traverse` visits Python objects owned by `_PyRuntime`, not by the module instance. With multi-phase init allowing multiple module instances, each instance's GC traversal decrements `gc_refs` on the same runtime-owned objects, driving it negative when two instances are collected simultaneously. (cherry picked from commit 56737483c2ffdaadfec648fd38d409c6b10941c0) Co-authored-by: Armaan Vakharia <43391096+armaan-v924@users.noreply.github.com> --- ...026-05-18-13-47-17.gh-issue-149590.IPBeQx.rst | 1 + Modules/faulthandler.c | 16 ---------------- 2 files changed, 1 insertion(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-13-47-17.gh-issue-149590.IPBeQx.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-13-47-17.gh-issue-149590.IPBeQx.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-13-47-17.gh-issue-149590.IPBeQx.rst new file mode 100644 index 00000000000000..8d3b29d69cc857 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-13-47-17.gh-issue-149590.IPBeQx.rst @@ -0,0 +1 @@ +Fix crash when faulthandler is imported more than once. diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 1b4f0c2302daae..fa7fb7085d7e8b 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1349,21 +1349,6 @@ faulthandler__stack_overflow_impl(PyObject *module) #endif /* defined(FAULTHANDLER_USE_ALT_STACK) && defined(HAVE_SIGACTION) */ -static int -faulthandler_traverse(PyObject *module, visitproc visit, void *arg) -{ - Py_VISIT(thread.file); -#ifdef FAULTHANDLER_USER - if (user_signals != NULL) { - for (size_t signum=0; signum < Py_NSIG; signum++) - Py_VISIT(user_signals[signum].file); - } -#endif - Py_VISIT(fatal_error.file); - return 0; -} - - #ifdef MS_WINDOWS /*[clinic input] faulthandler._raise_exception @@ -1459,7 +1444,6 @@ static struct PyModuleDef module_def = { .m_name = "faulthandler", .m_doc = module_doc, .m_methods = module_methods, - .m_traverse = faulthandler_traverse, .m_slots = faulthandler_slots }; From 66ade2861fec1d6c18998710938a1c71fde5f76b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 19 May 2026 01:53:57 +0200 Subject: [PATCH 078/110] [3.15] gh-146452: Improve locking granularity in pickle's batch_dict_exact and fix race condition (GH-150025) (#150039) gh-146452: Improve locking granularity in pickle's batch_dict_exact and fix race condition (GH-150025) Remove assertion that could fail in rare race condition. Replace the coarse critical section wrapping the entire function with fine-grained sections covering only PyDict_Next + Py_INCREF. Also handle PyDict_Next returning 0 in the single-item fast path. (cherry picked from commit 57a0e570d36f41b953a91bbaf4262a5d05d0391b) Co-authored-by: Saul Cooperman <58375603+scopreon@users.noreply.github.com> --- ...-05-18-15-30-34.gh-issue-146452.RM0EVJ.rst | 2 ++ Modules/_pickle.c | 35 +++++++++++++++---- 2 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-18-15-30-34.gh-issue-146452.RM0EVJ.rst diff --git a/Misc/NEWS.d/next/Library/2026-05-18-15-30-34.gh-issue-146452.RM0EVJ.rst b/Misc/NEWS.d/next/Library/2026-05-18-15-30-34.gh-issue-146452.RM0EVJ.rst new file mode 100644 index 00000000000000..66f9acf6c710a7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-18-15-30-34.gh-issue-146452.RM0EVJ.rst @@ -0,0 +1,2 @@ +Fix race condition when pickling dictionaries in free threaded builds. Also +reduce critical section cover. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 9874f9475ac029..15d95c658d6f90 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -3450,6 +3450,9 @@ batch_dict(PickleState *state, PicklerObject *self, PyObject *iter, PyObject *or * Returns 0 on success, -1 on error. * * Note that this currently doesn't work for protocol 0. + + * gh-146452: Wrap the dict iteration in a critical sections to prevent + * concurrent mutation from invalidating PyDict_Next() iteration state. */ static int batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) @@ -3466,15 +3469,24 @@ batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) assert(self->proto > 0); dict_size = PyDict_GET_SIZE(obj); - assert(dict_size); /* Write in batches of BATCHSIZE. */ Py_ssize_t total = 0; do { if (dict_size - total == 1) { - PyDict_Next(obj, &ppos, &key, &value); - Py_INCREF(key); - Py_INCREF(value); + int next; + Py_BEGIN_CRITICAL_SECTION(obj); + next = PyDict_Next(obj, &ppos, &key, &value); + if (next) { + Py_INCREF(key); + Py_INCREF(value); + } + Py_END_CRITICAL_SECTION(); + if (!next) { + PyErr_SetString(PyExc_RuntimeError, + "dictionary changed size during iteration"); + goto error; + } if (save(state, self, key, 0) < 0) { goto error; } @@ -3492,9 +3504,18 @@ batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) i = 0; if (_Pickler_Write(self, &mark_op, 1) < 0) return -1; - while (PyDict_Next(obj, &ppos, &key, &value)) { - Py_INCREF(key); - Py_INCREF(value); + int next; + while (1) { + Py_BEGIN_CRITICAL_SECTION(obj); + next = PyDict_Next(obj, &ppos, &key, &value); + if (next) { + Py_INCREF(key); + Py_INCREF(value); + } + Py_END_CRITICAL_SECTION(); + if (!next) { + break; + } if (save(state, self, key, 0) < 0) { goto error; } From 94c8bac2cd50bd05aa1811d35f4b00f1080a40a9 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Mon, 18 May 2026 17:53:54 -0700 Subject: [PATCH 079/110] =?UTF-8?q?[3.15]=20gh-148587:=20Make=20sys.lazy?= =?UTF-8?q?=5Fmodules=20match=20PEP=20and=20keep=20internal=20lazy=20submo?= =?UTF-8?q?dules=20tra=E2=80=A6=20(#150014)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make sys.lazy_modules match PEP and keep internal lazy submodules tracking internal --- Include/internal/pycore_interp_structs.h | 8 +++ Lib/test/test_lazy_import/__init__.py | 27 ++++------ Lib/test/test_lazy_import/__main__.py | 3 ++ ...-05-18-18-36-28.gh-issue-148587.-RD3z5.rst | 1 + Python/import.c | 54 ++++++++++++++----- 5 files changed, 62 insertions(+), 31 deletions(-) create mode 100644 Lib/test/test_lazy_import/__main__.py create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-18-36-28.gh-issue-148587.-RD3z5.rst diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index f13bc2178b1e7e..d8e83cf2ff5c9a 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -349,7 +349,15 @@ struct _import_state { int lazy_imports_mode; PyObject *lazy_imports_filter; PyObject *lazy_importing_modules; + // The set stored in sys.lazy_modules if values that have been + // lazily imported. This value is only for debugging/introspection + // purposes and is not used by the runtime. PyObject *lazy_modules; + // A dict mapping package names to a set of submodule names that + // have been imported lazily from packages which have been imported + // lazily. When the package is reified we need to add a + // LazyImportObject which refers to the submodule on the module. + PyObject *lazy_pending_submodules; #ifdef Py_GIL_DISABLED PyMutex lazy_mutex; #endif diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index bcbf1a23233ba8..366cb203f8f256 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -38,8 +38,7 @@ def test_basic_unused(self): """Lazy imported module should not be loaded if never accessed.""" import test.test_lazy_import.data.basic_unused self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) - self.assertIn("test.test_lazy_import.data", sys.lazy_modules) - self.assertEqual(sys.lazy_modules["test.test_lazy_import.data"], {"basic2"}) + self.assertIn("test.test_lazy_import.data.basic2", sys.lazy_modules) def test_sys_lazy_modules(self): try: @@ -49,7 +48,7 @@ def test_sys_lazy_modules(self): self.assertFalse("test.test_lazy_import.data.basic2" in sys.modules) self.assertIn("test.test_lazy_import.data", sys.lazy_modules) - self.assertEqual(sys.lazy_modules["test.test_lazy_import.data"], {"basic2"}) + self.assertIn("test.test_lazy_import.data.basic2", sys.lazy_modules) test.test_lazy_import.data.basic_from_unused.basic2 self.assertNotIn("test.test_import.data", sys.lazy_modules) @@ -574,8 +573,8 @@ def my_filter(name): self.assertIs(sys.get_lazy_imports_filter(), my_filter) def test_lazy_modules_attribute_is_dict(self): - """sys.lazy_modules should be a dict per PEP 810.""" - self.assertIsInstance(sys.lazy_modules, dict) + """sys.lazy_modules should be a set per PEP 810.""" + self.assertIsInstance(sys.lazy_modules, set) @support.requires_subprocess() def test_lazy_modules_tracks_lazy_imports(self): @@ -584,8 +583,7 @@ def test_lazy_modules_tracks_lazy_imports(self): import sys initial_count = len(sys.lazy_modules) import test.test_lazy_import.data.basic_unused - assert "test.test_lazy_import.data" in sys.lazy_modules - assert sys.lazy_modules["test.test_lazy_import.data"] == {"basic2"} + assert "test.test_lazy_import.data.basic2" in sys.lazy_modules assert len(sys.lazy_modules) > initial_count print("OK") """) @@ -1034,15 +1032,14 @@ def test_module_added_to_lazy_modules_on_lazy_import(self): lazy import test.test_lazy_import.data.basic2 # Should be in lazy_modules after lazy import - assert "test.test_lazy_import.data" in sys.lazy_modules - assert sys.lazy_modules["test.test_lazy_import.data"] == {"basic2"} + assert "test.test_lazy_import.data.basic2" in sys.lazy_modules assert len(sys.lazy_modules) > initial_count # Trigger reification _ = test.test_lazy_import.data.basic2.x # Module should still be tracked (for diagnostics per PEP 810) - assert "test.test_lazy_import.data" not in sys.lazy_modules + assert "test.test_lazy_import.data.basic2" not in sys.lazy_modules print("OK") """) result = subprocess.run( @@ -1055,8 +1052,8 @@ def test_module_added_to_lazy_modules_on_lazy_import(self): def test_lazy_modules_is_per_interpreter(self): """Each interpreter should have independent sys.lazy_modules.""" - # Basic test that sys.lazy_modules exists and is a dict - self.assertIsInstance(sys.lazy_modules, dict) + # Basic test that sys.lazy_modules exists and is a set + self.assertIsInstance(sys.lazy_modules, set) def test_lazy_module_without_children_is_tracked(self): code = textwrap.dedent(""" @@ -1065,10 +1062,6 @@ def test_lazy_module_without_children_is_tracked(self): assert "json" in sys.lazy_modules, ( f"expected 'json' in sys.lazy_modules, got {set(sys.lazy_modules)}" ) - assert sys.lazy_modules["json"] == set(), ( - f"expected empty set for sys.lazy_modules['json'], " - f"got {sys.lazy_modules['json']!r}" - ) print("OK") """) assert_python_ok("-c", code) @@ -1937,7 +1930,7 @@ def create_lazy_imports(idx): t.join() assert not errors, f"Errors: {errors}" - assert isinstance(sys.lazy_modules, dict), "sys.lazy_modules is not a dict" + assert isinstance(sys.lazy_modules, set), "sys.lazy_modules is not a dict" print("OK") """) diff --git a/Lib/test/test_lazy_import/__main__.py b/Lib/test/test_lazy_import/__main__.py new file mode 100644 index 00000000000000..d6c94efaf30833 --- /dev/null +++ b/Lib/test/test_lazy_import/__main__.py @@ -0,0 +1,3 @@ +import unittest + +unittest.main('test.test_lazy_import') diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-18-36-28.gh-issue-148587.-RD3z5.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-18-36-28.gh-issue-148587.-RD3z5.rst new file mode 100644 index 00000000000000..61bfdcdd37362c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-18-36-28.gh-issue-148587.-RD3z5.rst @@ -0,0 +1 @@ +``sys.lazy_modules`` is now a set instead of a dict as initially spelled out in PEP 810. diff --git a/Python/import.c b/Python/import.c index 60a5ee6e770f59..c5cc7b52922d5b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -94,6 +94,8 @@ static struct _inittab *inittab_copy = NULL; (interp)->imports.modules_by_index #define LAZY_MODULES(interp) \ (interp)->imports.lazy_modules +#define LAZY_PENDING_SUBMODULES(interp) \ + (interp)->imports.lazy_pending_submodules #define IMPORTLIB(interp) \ (interp)->imports.importlib #define OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp) \ @@ -271,8 +273,11 @@ import_get_module(PyThreadState *tstate, PyObject *name) PyObject * _PyImport_InitLazyModules(PyInterpreterState *interp) { - assert(LAZY_MODULES(interp) == NULL); - LAZY_MODULES(interp) = PyDict_New(); + assert(LAZY_MODULES(interp) == NULL && + LAZY_PENDING_SUBMODULES(interp) == NULL); + + LAZY_PENDING_SUBMODULES(interp) = PyDict_New(); + LAZY_MODULES(interp) = PySet_New(0); return LAZY_MODULES(interp); } @@ -280,6 +285,7 @@ void _PyImport_ClearLazyModules(PyInterpreterState *interp) { Py_CLEAR(LAZY_MODULES(interp)); + Py_CLEAR(LAZY_PENDING_SUBMODULES(interp)); } static int @@ -4339,7 +4345,7 @@ get_mod_dict(PyObject *module) // ensure we have the set for the parent module name in sys.lazy_modules. // Returns a new reference. static PyObject * -ensure_lazy_submodules(PyDictObject *lazy_modules, PyObject *parent) +ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) { PyObject *lazy_submodules; Py_BEGIN_CRITICAL_SECTION(lazy_modules); @@ -4358,6 +4364,9 @@ ensure_lazy_submodules(PyDictObject *lazy_modules, PyObject *parent) return lazy_submodules; } +// Ensures that we have a LazyImportObject on the parent module for +// all children modules which have been lazily imported. If the parent +// module overrides the child attribute then the value is not replaced. static int register_lazy_on_parent(PyThreadState *tstate, PyObject *name, PyObject *builtins) @@ -4369,16 +4378,16 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, PyObject *parent_dict = NULL; PyInterpreterState *interp = tstate->interp; - PyObject *lazy_modules = LAZY_MODULES(interp); - assert(lazy_modules != NULL); + PyObject *lazy_pending_submodules = LAZY_PENDING_SUBMODULES(interp); + assert(lazy_pending_submodules != NULL); Py_INCREF(name); while (true) { Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, PyUnicode_GET_LENGTH(name), -1); if (dot < 0) { - PyObject *lazy_submodules = ensure_lazy_submodules( - (PyDictObject *)lazy_modules, name); + PyObject *lazy_submodules = ensure_lazy_pending_submodules( + (PyDictObject *)lazy_pending_submodules, name); if (lazy_submodules == NULL) { goto done; } @@ -4400,8 +4409,8 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, } // Record the child as being lazily imported from the parent. - PyObject *lazy_submodules = ensure_lazy_submodules( - (PyDictObject *)lazy_modules, parent); + PyObject *lazy_submodules = ensure_lazy_pending_submodules( + (PyDictObject *)lazy_pending_submodules, parent); if (lazy_submodules == NULL) { goto done; } @@ -4464,6 +4473,14 @@ register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, if (fromname == NULL) { return -1; } + + // Add the module name to sys.lazy_modules set (PEP 810). + PyObject *lazy_modules = LAZY_MODULES(tstate->interp); + if (PySet_Add(lazy_modules, fromname) < 0) { + Py_DECREF(fromname); + return -1; + } + int res = register_lazy_on_parent(tstate, fromname, builtins); Py_DECREF(fromname); return res; @@ -4555,6 +4572,13 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, Py_DECREF(abs_name); return NULL; } + + // Add the module name to sys.lazy_modules set (PEP 810). + PyObject *lazy_modules = LAZY_MODULES(tstate->interp); + if (PySet_Add(lazy_modules, abs_name) < 0) { + goto error; + } + if (fromlist && PyUnicode_Check(fromlist)) { if (register_from_lazy_on_parent(tstate, abs_name, fromlist, builtins) < 0) { @@ -4791,6 +4815,7 @@ _PyImport_ClearCore(PyInterpreterState *interp) Py_CLEAR(IMPORTLIB(interp)); Py_CLEAR(IMPORT_FUNC(interp)); Py_CLEAR(LAZY_IMPORT_FUNC(interp)); + Py_CLEAR(interp->imports.lazy_pending_submodules); Py_CLEAR(interp->imports.lazy_modules); Py_CLEAR(interp->imports.lazy_importing_modules); Py_CLEAR(interp->imports.lazy_imports_filter); @@ -5636,11 +5661,13 @@ _imp__set_lazy_attributes_impl(PyObject *module, PyObject *modobj, PyThreadState *tstate = _PyThreadState_GET(); PyObject *module_dict = NULL; PyObject *ret = NULL; - PyObject *lazy_modules = LAZY_MODULES(tstate->interp); - assert(lazy_modules != NULL); + PyObject *lazy_pending_modules = LAZY_PENDING_SUBMODULES(tstate->interp); + assert(lazy_pending_modules != NULL); PyObject *lazy_submodules; - if (PyDict_GetItemRef(lazy_modules, name, &lazy_submodules) < 0) { + if (PySet_Discard(LAZY_MODULES(tstate->interp), name) < 0) { + return NULL; + } else if (PyDict_GetItemRef(lazy_pending_modules, name, &lazy_submodules) < 0) { return NULL; } else if (lazy_submodules == NULL) { @@ -5659,8 +5686,7 @@ _imp__set_lazy_attributes_impl(PyObject *module, PyObject *modobj, Py_END_CRITICAL_SECTION(); Py_DECREF(lazy_submodules); - // once a module is imported it is removed from sys.lazy_modules - if (PyDict_DelItem(lazy_modules, name) < 0) { + if (PyDict_DelItem(lazy_pending_modules, name) < 0) { goto error; } From 0b92f01c59c82a111bb6b9929fbfc0f9cb1241db Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 19 May 2026 06:41:43 +0200 Subject: [PATCH 080/110] [3.15] gh-149816: fix thread safety of deletion of list slice (GH-149936) (#150003) gh-149816: fix thread safety of deletion of list slice (GH-149936) (cherry picked from commit 00ea77613b942a9e08df6e3eb74b2ccd37641ba6) Co-authored-by: Kumar Aditya --- Objects/listobject.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Objects/listobject.c b/Objects/listobject.c index 10e25bbdcdcb6c..c76721c5d2ac9e 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3793,16 +3793,13 @@ list_ass_subscript_lock_held(PyObject *_self, PyObject *item, PyObject *value) lim = Py_SIZE(self) - cur - 1; } - memmove(self->ob_item + cur - i, - self->ob_item + cur + 1, - lim * sizeof(PyObject *)); + ptr_wise_atomic_memmove(self, self->ob_item + cur - i, + self->ob_item + cur + 1, lim); } cur = start + (size_t)slicelength * step; if (cur < (size_t)Py_SIZE(self)) { - memmove(self->ob_item + cur - slicelength, - self->ob_item + cur, - (Py_SIZE(self) - cur) * - sizeof(PyObject *)); + ptr_wise_atomic_memmove(self, self->ob_item + cur - slicelength, + self->ob_item + cur, Py_SIZE(self) - cur); } Py_SET_SIZE(self, Py_SIZE(self) - slicelength); From d36e08099d56a54028174429a946c9816f284374 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 19 May 2026 06:42:25 +0200 Subject: [PATCH 081/110] [3.15] gh-149816: fix `dict.clear()` race on split-table dict with non-embedded values (GH-149914) (#150000) gh-149816: fix `dict.clear()` race on split-table dict with non-embedded values (GH-149914) (cherry picked from commit 169285470630b697c5e6e0e4c8091c31f25ffb04) Co-authored-by: Kumar Aditya --- Lib/test/test_free_threading/test_dict.py | 28 +++++++++++++++++++++++ Objects/dictobject.c | 6 +++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py index 55272a00c3ad50..dfe0634211d4b0 100644 --- a/Lib/test/test_free_threading/test_dict.py +++ b/Lib/test/test_free_threading/test_dict.py @@ -268,6 +268,34 @@ def watcher(): finally: _testcapi.clear_dict_watcher(wid) + def test_racing_split_dict_clear_and_lookup(self): + class C: + pass + + keys = [f"a{i}" for i in range(16)] + + def make_split_nonembedded(): + inst = C() + for key in keys: + setattr(inst, key, keys.index(key)) + # dict.copy() of a split instance dict yields a split table + # with non-embedded values + return inst.__dict__.copy() + + d = make_split_nonembedded() + + def clearer(): + for _ in range(1000): + d.clear() + d.update(make_split_nonembedded()) + + def reader(): + for _ in range(1000): + for k in keys: + d.get(k) + + threading_helper.run_concurrently([clearer, reader, reader]) + def test_racing_dict_update_and_method_lookup(self): # gh-144295: test race between dict modifications and method lookups. # Uses BytesIO because the race requires a type without Py_TPFLAGS_INLINE_VALUES diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b33a273dac3b95..a7d67812bec925 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -3083,10 +3083,12 @@ clear_lock_held(PyObject *op) set_keys(mp, Py_EMPTY_KEYS); n = oldkeys->dk_nentries; for (i = 0; i < n; i++) { - Py_CLEAR(oldvalues->values[i]); + PyObject *tmp = oldvalues->values[i]; + FT_ATOMIC_STORE_PTR_RELEASE(oldvalues->values[i], NULL); + Py_XDECREF(tmp); } free_values(oldvalues, IS_DICT_SHARED(mp)); - dictkeys_decref(oldkeys, false); + dictkeys_decref(oldkeys, IS_DICT_SHARED(mp)); } ASSERT_CONSISTENT(mp); } From 28f275f713124a1425effe3c998394e829b20db7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 19 May 2026 07:10:01 +0200 Subject: [PATCH 082/110] [3.15] gh-86533: Restore os.makedirs() ability to apply *mode* recursively (GH-150011) (#150036) bpo-42367: Restore os.makedirs() and pathlib.mkdir() ability to apply *mode* recursively via a new parent_mode= keyword argument. (cherry picked from commit 9770e32ce07110f0c8c7a381604ec9a490028eed) + Make Path.mkdir parent_mode tests umask-independent test_mkdir_with_parent_mode, test_mkdir_parent_mode_deep_hierarchy and test_mkdir_parent_mode_same_as_mode assert exact directory mode bits but did not pin the process umask. On buildbots running with a restrictive umask (e.g. 0o077) the 0o755 leaf was masked down to 0o700, failing the assertions. Wrap them in os_helper.temp_umask(0o022), matching the other umask-aware mkdir tests in this file. --------- Co-authored-by: nessita <124304+nessita@users.noreply.github.com> Co-authored-by: Zackery Spytz Co-authored-by: Erlend E. Aasland Co-authored-by: Gregory P. Smith --- Doc/library/os.rst | 14 ++- Doc/library/pathlib.rst | 12 +- Doc/whatsnew/3.15.rst | 8 ++ Lib/os.py | 15 ++- Lib/pathlib/__init__.py | 8 +- Lib/test/test_os/test_os.py | 100 ++++++++++++++-- Lib/test/test_pathlib/test_pathlib.py | 110 ++++++++++++++++++ ...-08-30-07-44-30.gh-issue-86533.pathlib.rst | 4 + 8 files changed, 254 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-08-30-07-44-30.gh-issue-86533.pathlib.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index d2534b3e974f36..27a032a8a97c63 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2549,7 +2549,8 @@ features: Windows now handles a *mode* of ``0o700``. -.. function:: makedirs(name, mode=0o777, exist_ok=False) +.. function:: makedirs(name, mode=0o777, exist_ok=False, *, \ + parent_mode=None) .. index:: single: directory; creating @@ -2567,6 +2568,12 @@ features: If *exist_ok* is ``False`` (the default), a :exc:`FileExistsError` is raised if the target directory already exists. + If *parent_mode* is not ``None``, it is used as the mode for any + newly-created, intermediate-level directories. Like *mode*, it is + combined with the process's umask value; see :ref:`the mkdir() + description `. Otherwise, intermediate directories are + created with the default mode, which is also subject to the umask. + .. note:: :func:`makedirs` will become confused if the path elements to create @@ -2593,6 +2600,11 @@ features: The *mode* argument no longer affects the file permission bits of newly created intermediate-level directories. + .. versionadded:: 3.15 + The *parent_mode* parameter. To match the behavior from Python 3.6 and + earlier (where *mode* was applied to all created directories), pass + ``parent_mode=mode``. + .. function:: mkfifo(path, mode=0o666, *, dir_fd=None) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 2867015042ee16..45b5797058f623 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1514,7 +1514,8 @@ Creating files and directories :meth:`~Path.write_bytes` methods are often used to create files. -.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) +.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False, *, \ + parent_mode=None) Create a new directory at this given path. If *mode* is given, it is combined with the process's ``umask`` value to determine the file mode @@ -1525,6 +1526,12 @@ Creating files and directories as needed; they are created with the default permissions without taking *mode* into account (mimicking the POSIX ``mkdir -p`` command). + If *parent_mode* is not ``None``, it is used as the mode for any + newly-created, intermediate-level directories when *parents* is true. + Like *mode*, it is combined with the process's ``umask`` value. + Otherwise, intermediate directories are created with the default + permissions (also subject to the umask). + If *parents* is false (the default), a missing parent raises :exc:`FileNotFoundError`. @@ -1538,6 +1545,9 @@ Creating files and directories .. versionchanged:: 3.5 The *exist_ok* parameter was added. + .. versionadded:: 3.15 + The *parent_mode* parameter. + .. method:: Path.symlink_to(target, target_is_directory=False) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index fb0755e8ffec5b..9bef7aa61d23cd 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1285,6 +1285,10 @@ os glibc versions 2.28 and later. (Contributed by Jeffrey Bosboom and Victor Stinner in :gh:`83714`.) +* :func:`os.makedirs` function now has a *parent_mode* parameter that allows + specifying the mode for intermediate directories. This can be used to match + the behavior from Python 3.6 and earlier by passing ``parent_mode=mode``. + (Contributed by Zackery Spytz and Gregory P. Smith in :gh:`86533`.) os.path ------- @@ -2057,6 +2061,10 @@ importlib.resources pathlib ------- +* :meth:`pathlib.Path.mkdir` now has a *parent_mode* parameter that allows + specifying the mode for intermediate directories when ``parents=True``. + (Contributed by Gregory P. Smith in :gh:`86533`.) + * Removed deprecated :meth:`!pathlib.PurePath.is_reserved`. Use :func:`os.path.isreserved` to detect reserved paths on Windows. (Contributed by Nikita Sobolev in :gh:`133875`.) diff --git a/Lib/os.py b/Lib/os.py index 52cbc5bc85864e..1ca4648cc95c3e 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -219,14 +219,17 @@ def _add(str, fn): # Super directory utilities. # (Inspired by Eric Raymond; the doc strings are mostly his) -def makedirs(name, mode=0o777, exist_ok=False): - """makedirs(name [, mode=0o777][, exist_ok=False]) +def makedirs(name, mode=0o777, exist_ok=False, *, parent_mode=None): + """makedirs(name [, mode=0o777][, exist_ok=False][, parent_mode=None]) Super-mkdir; create a leaf directory and all intermediate ones. Works like mkdir, except that any intermediate path segment (not just the rightmost) will be created if it does not exist. If the target directory already exists, raise an OSError if exist_ok is False. Otherwise no exception is - raised. This is recursive. + raised. If parent_mode is not None, it will be used as the mode for any + newly-created, intermediate-level directories. Otherwise, intermediate + directories are created with the default permissions (respecting umask). + This is recursive. """ head, tail = path.split(name) @@ -234,7 +237,11 @@ def makedirs(name, mode=0o777, exist_ok=False): head, tail = path.split(head) if head and tail and not path.exists(head): try: - makedirs(head, exist_ok=exist_ok) + if parent_mode is not None: + makedirs(head, mode=parent_mode, exist_ok=exist_ok, + parent_mode=parent_mode) + else: + makedirs(head, exist_ok=exist_ok) except FileExistsError: # Defeats race condition when another thread created the path pass diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index a32e4b5320ff6d..8dd16c6225b927 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -1202,7 +1202,7 @@ def touch(self, mode=0o666, exist_ok=True): fd = os.open(self, flags, mode) os.close(fd) - def mkdir(self, mode=0o777, parents=False, exist_ok=False): + def mkdir(self, mode=0o777, parents=False, exist_ok=False, *, parent_mode=None): """ Create a new directory at this given path. """ @@ -1211,7 +1211,11 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): except FileNotFoundError: if not parents or self.parent == self: raise - self.parent.mkdir(parents=True, exist_ok=True) + if parent_mode is not None: + self.parent.mkdir(mode=parent_mode, parents=True, exist_ok=True, + parent_mode=parent_mode) + else: + self.parent.mkdir(parents=True, exist_ok=True) self.mkdir(mode, parents=False, exist_ok=exist_ok) except OSError: # Cannot rely on checking for EEXIST, since the operating system diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 7e670e5a139d99..6fcf94fc825385 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -2137,6 +2137,94 @@ def test_mode(self): self.assertEqual(os.stat(path).st_mode & 0o777, 0o555) self.assertEqual(os.stat(parent).st_mode & 0o777, 0o775) + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mode_with_parent_mode(self): + # Test the parent_mode parameter + parent = os.path.join(os_helper.TESTFN, 'dir1') + path = os.path.join(parent, 'dir2') + with os_helper.temp_umask(0o002): + # Specify mode for both leaf and parent directories + os.makedirs(path, 0o770, parent_mode=0o750) + self.assertTrue(os.path.exists(path)) + self.assertTrue(os.path.isdir(path)) + if os.name != 'nt': + # Leaf directory gets the mode parameter + self.assertEqual(os.stat(path).st_mode & 0o777, 0o770) + # Parent directory gets the parent_mode parameter + self.assertEqual(os.stat(parent).st_mode & 0o777, 0o750) + + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_parent_mode_deep_hierarchy(self): + # Test parent_mode with deep directory hierarchy + base = os.path.join(os_helper.TESTFN, 'dir1', 'dir2', 'dir3') + with os_helper.temp_umask(0o002): + os.makedirs(base, 0o755, parent_mode=0o700) + self.assertTrue(os.path.exists(base)) + if os.name != 'nt': + # Check that all parent directories have parent_mode + level1 = os.path.join(os_helper.TESTFN, 'dir1') + level2 = os.path.join(level1, 'dir2') + self.assertEqual(os.stat(level1).st_mode & 0o777, 0o700) + self.assertEqual(os.stat(level2).st_mode & 0o777, 0o700) + # Leaf directory has the regular mode + self.assertEqual(os.stat(base).st_mode & 0o777, 0o755) + + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_parent_mode_same_as_mode(self): + # Test emulating Python 3.6 behavior by setting parent_mode=mode + parent = os.path.join(os_helper.TESTFN, 'dir1') + path = os.path.join(parent, 'dir2') + with os_helper.temp_umask(0o002): + os.makedirs(path, 0o705, parent_mode=0o705) + self.assertTrue(os.path.exists(path)) + if os.name != 'nt': + # Both directories should have the same mode + self.assertEqual(os.stat(path).st_mode & 0o777, 0o705) + self.assertEqual(os.stat(parent).st_mode & 0o777, 0o705) + + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_parent_mode_combined_with_umask(self): + # parent_mode, like mode, is combined with the process umask; it does + # not bypass it. + parent = os.path.join(os_helper.TESTFN, 'dir1') + path = os.path.join(parent, 'dir2') + with os_helper.temp_umask(0o022): + os.makedirs(path, 0o777, parent_mode=0o777) + self.assertTrue(os.path.isdir(path)) + if os.name != 'nt': + # 0o777 is masked down to 0o755 by the 0o022 umask, for both + # the leaf (mode) and the parent (parent_mode). + self.assertEqual(os.stat(path).st_mode & 0o777, 0o755) + self.assertEqual(os.stat(parent).st_mode & 0o777, 0o755) + @unittest.skipIf( support.is_wasi, "WASI's umask is a stub." @@ -2210,15 +2298,9 @@ def test_win32_mkdir_700(self): ) def tearDown(self): - path = os.path.join(os_helper.TESTFN, 'dir1', 'dir2', 'dir3', - 'dir4', 'dir5', 'dir6') - # If the tests failed, the bottom-most directory ('../dir6') - # may not have been created, so we look for the outermost directory - # that exists. - while not os.path.exists(path) and path != os_helper.TESTFN: - path = os.path.dirname(path) - - os.removedirs(path) + # Remove the whole tree regardless of which sub-directories a test + # created and regardless of their permission bits. + os_helper.rmtree(os_helper.TESTFN) @unittest.skipUnless(hasattr(os, "chown"), "requires os.chown()") diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 09d1b5d725e5ba..2cb4876f5c6400 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -2492,6 +2492,116 @@ def my_mkdir(path, mode=0o777): self.assertNotIn(str(p12), concurrently_created) self.assertTrue(p.exists()) + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_parents_umask(self): + # Test that parent directories respect umask when parent_mode is not set + p = self.cls(self.base, 'umasktest', 'child') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o002): + p.mkdir(0o755, parents=True) + self.assertTrue(p.exists()) + # Leaf directory gets the specified mode + self.assertEqual(p.stat().st_mode & 0o777, 0o755) + # Parent directory respects umask (0o777 & ~0o002 = 0o775) + self.assertEqual(p.parent.stat().st_mode & 0o777, 0o775) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_with_parent_mode(self): + # Test the parent_mode parameter + p = self.cls(self.base, 'newdirPM', 'subdirPM') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o022): + # Specify different modes for parent and leaf directories + p.mkdir(0o755, parents=True, parent_mode=0o750) + self.assertTrue(p.exists()) + self.assertTrue(p.is_dir()) + # Leaf directory gets the mode parameter + self.assertEqual(p.stat().st_mode & 0o777, 0o755) + # Parent directory gets the parent_mode parameter + self.assertEqual(p.parent.stat().st_mode & 0o777, 0o750) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_parent_mode_deep_hierarchy(self): + # Test parent_mode with deep directory hierarchy + p = self.cls(self.base, 'level1PM', 'level2PM', 'level3PM') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o022): + p.mkdir(0o755, parents=True, parent_mode=0o700) + self.assertTrue(p.exists()) + # Check that all parent directories have parent_mode + level1 = self.cls(self.base, 'level1PM') + level2 = level1 / 'level2PM' + self.assertEqual(level1.stat().st_mode & 0o777, 0o700) + self.assertEqual(level2.stat().st_mode & 0o777, 0o700) + # Leaf directory has the regular mode + self.assertEqual(p.stat().st_mode & 0o777, 0o755) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_parent_mode_combined_with_umask(self): + # parent_mode, like mode, is combined with the process umask; it does + # not bypass it. + p = self.cls(self.base, 'umaskPM', 'child') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o022): + p.mkdir(0o777, parents=True, parent_mode=0o777) + self.assertTrue(p.exists()) + # 0o777 is masked down to 0o755 by the 0o022 umask, for both + # the leaf (mode) and the parent (parent_mode). + self.assertEqual(p.stat().st_mode & 0o777, 0o755) + self.assertEqual(p.parent.stat().st_mode & 0o777, 0o755) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_parent_mode_same_as_mode(self): + # Test setting parent_mode same as mode + p = self.cls(self.base, 'samedirPM', 'subdirPM') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o022): + p.mkdir(0o705, parents=True, parent_mode=0o705) + self.assertTrue(p.exists()) + # Both directories should have the same mode + self.assertEqual(p.stat().st_mode & 0o777, 0o705) + self.assertEqual(p.parent.stat().st_mode & 0o777, 0o705) + @needs_symlinks def test_symlink_to(self): P = self.cls(self.base) diff --git a/Misc/NEWS.d/next/Library/2025-08-30-07-44-30.gh-issue-86533.pathlib.rst b/Misc/NEWS.d/next/Library/2025-08-30-07-44-30.gh-issue-86533.pathlib.rst new file mode 100644 index 00000000000000..9c32671173e0ad --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-30-07-44-30.gh-issue-86533.pathlib.rst @@ -0,0 +1,4 @@ +The :func:`os.makedirs` function and :meth:`pathlib.Path.mkdir` method now have +a *parent_mode* parameter to specify the mode for intermediate directories when +creating parent directories. This allows one to match the behavior from Python +3.6 and earlier for :func:`os.makedirs`. From 65b255416ae217bf0e22085be3c1976cea18bd8c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 19 May 2026 10:44:35 +0200 Subject: [PATCH 083/110] [3.15] gh-146581: Update docs for dangerous filenames in ZIP files (GH-149994) (GH-150064) gh-146581: Update docs for dangerous filenames in ZIP files (GH-149994) (cherry picked from commit ba0aca3bffce431fe2fbd53ca4cd6a717a2e2c19) Co-authored-by: Serhiy Storchaka Co-authored-by: Sebastian Gassner --- Doc/library/shutil.rst | 4 ++-- Doc/library/zipfile.rst | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index d289ba58c24065..e0300a38e2f357 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -749,8 +749,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Never extract archives from untrusted sources without prior inspection. It is possible that files are created outside of the path specified in - the *extract_dir* argument, e.g. members that have absolute filenames - starting with "/" or filenames with two dots "..". + the *extract_dir* argument, for example, members that have absolute filenames + or filenames with ".." components. Since Python 3.14, the defaults for both built-in formats (zip and tar files) will prevent the most dangerous of such security issues, diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 9999ac26999910..ebafcb977803d4 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -411,9 +411,9 @@ ZipFile objects .. warning:: Never extract archives from untrusted sources without prior inspection. - It is possible that files are created outside of *path*, e.g. members - that have absolute filenames starting with ``"/"`` or filenames with two - dots ``".."``. This module attempts to prevent that. + It is possible that files are created outside of *path*, for example, members + that have absolute filenames or filenames with ".." components. + This module attempts to prevent that. See :meth:`extract` note. .. versionchanged:: 3.6 @@ -590,7 +590,7 @@ Path objects The :class:`Path` class does not sanitize filenames within the ZIP archive. Unlike the :meth:`ZipFile.extract` and :meth:`ZipFile.extractall` methods, it is the caller's responsibility to validate or sanitize filenames to prevent path traversal - vulnerabilities (e.g., filenames containing ".." or absolute paths). When handling + vulnerabilities (for example, absolute paths or paths with ".." components). When handling untrusted archives, consider resolving filenames using :func:`os.path.abspath` and checking against the target directory with :func:`os.path.commonpath`. From bec4336badaac36c8080ff3e55676362775f61d8 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 19 May 2026 19:48:48 +0200 Subject: [PATCH 084/110] [3.15] gh-69619: Clarify whitespace definition in str.strip docs (#150049) (cherry picked from commit 17eb17d43f66a0f7985fca05c7c9684bc01fabcd) Co-authored-by: Daniil --- Doc/library/stdtypes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index e3bd1a46891adc..b0388c4e1f0bd4 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2747,6 +2747,8 @@ expression support in the :mod:`re` module). The *chars* argument is not a prefix or suffix; rather, all combinations of its values are stripped. + Whitespace characters are defined by :meth:`str.isspace`. + For example: .. doctest:: From 3227857de8ad895fdef7c3d18a9e031f29980029 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 19 May 2026 22:38:47 +0200 Subject: [PATCH 085/110] [3.15] gh-149945: Fix potential OOM for gzip with large header (GH-149979) (GH-150093) Do not read the whole filename and comment to memory for calculating the CRC. (cherry picked from commit 51a5715df9c56f616944cf1b39323bd6ae009143) Co-authored-by: Serhiy Storchaka --- Lib/gzip.py | 50 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/Lib/gzip.py b/Lib/gzip.py index a89ebf806c8572..1e05f43c0c9e24 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -484,14 +484,22 @@ def _read_exact(fp, n): return data -def _read_until_null(fp, append_to): +def _read_until_null(fp, crc=None): '''Read until the first encountered null byte in fp. - Append to given byte array object''' - while True: - s = fp.read(1) - append_to += s - if not s or s == b'\000': - break + If crc is not None, update and return the CRC. + ''' + if crc is None: + while True: + s = fp.read(1) + if not s or s == b'\000': + break + else: + while True: + s = fp.read(1) + crc = zlib.crc32(s, crc) + if not s or s == b'\000': + break + return crc def _read_gzip_header(fp): @@ -517,30 +525,32 @@ def _read_gzip_header(fp): return last_mtime if flag == FNAME: # Read and discard a null-terminated string containing the filename - while True: - s = fp.read(1) - if not s or s==b'\000': - break + _read_until_null(fp) return last_mtime # Processing for more complex flags. Save header parts for FHCRC checking. - header = bytearray(magic + base_header) + if flag & FHCRC: + crc = zlib.crc32(magic + base_header) + else: + crc = None if flag & FEXTRA: extra_len_bytes = _read_exact(fp, 2) extra_len, = struct.unpack(" Date: Tue, 19 May 2026 22:57:21 +0200 Subject: [PATCH 086/110] [3.15] gh-124111: Keep tests passing for Tcl prior to 9.0 (GH-150102) Also disables the UWP build in CI, since it was breaking (and is no longer released). (cherry picked from commit ec9ce3ee98c68f235be6d075fa4bbd8f56d20256) Co-authored-by: Steve Dower --- .github/workflows/reusable-windows.yml | 2 -- Lib/test/test_tcl.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 4c8d0c8a2f984f..c6e8128884e90c 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -22,8 +22,6 @@ permissions: env: FORCE_COLOR: 1 - IncludeUwp: >- - true jobs: build: diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 81a5477b496b5c..70731d3222ced9 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -55,7 +55,7 @@ def test_eval_null_in_result(self): def test_eval_surrogates_in_result(self): tcl = self.interp result = tcl.eval(r'set a "<\ud83d\udcbb>"') - if sys.platform == 'win32': + if sys.platform == 'win32' and tcl_version >= (9, 0): self.assertEqual('<\ud83d\udcbb>', result) else: self.assertEqual('<\U0001f4bb>', result) @@ -294,7 +294,7 @@ def test_evalfile_surrogates_in_result(self): """) tcl.evalfile(filename) result = tcl.eval('set b') - if sys.platform == 'win32': + if sys.platform == 'win32' and tcl_version >= (9, 0): self.assertEqual('<\ud83d\udcbb>', result) else: self.assertEqual('<\U0001f4bb>', result) From f46b39797519d7c98f5561fbb06b99a76338642a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 19 May 2026 23:11:13 +0200 Subject: [PATCH 087/110] [3.15] gh-149983: Fix PyErr_NoMemory call without GIL in winconsoleio.c (GH-149984) (GH-150113) (cherry picked from commit 3d2aa899bad1c0e274640dc0c4323f1744e73435) Co-authored-by: AN Long --- Modules/_io/winconsoleio.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 677d7e85d4e626..4a3fc586fa3a14 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -673,12 +673,13 @@ read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) { maxlen += 1; Py_BLOCK_THREADS newbuf = (wchar_t*)PyMem_Realloc(buf, maxlen * sizeof(wchar_t)); - Py_UNBLOCK_THREADS if (!newbuf) { sig = -1; PyErr_NoMemory(); + Py_UNBLOCK_THREADS break; } + Py_UNBLOCK_THREADS buf = newbuf; /* Only advance by n and not BUFSIZ in this case */ off += n; From 6847f4bc60f984a8d8e619de4b824d1b1da9524a Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Tue, 19 May 2026 14:23:30 -0700 Subject: [PATCH 088/110] [3.15] gh-150052: Resolve un-loaded lazily loaded submodules via module.__getattr__ instead of publishing lazy values (#150055) --- Include/internal/pycore_import.h | 2 + Lib/test/test_lazy_import/__init__.py | 42 +++--- Objects/moduleobject.c | 34 +++++ Python/import.c | 198 ++++++++------------------ 4 files changed, 123 insertions(+), 153 deletions(-) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 32ed3a62b2b4a7..a1078828afa572 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -39,6 +39,8 @@ extern PyObject * _PyImport_GetAbsName( // Symbol is exported for the JIT on Windows builds. PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate( PyThreadState *tstate, PyObject *lazy_import); +extern PyObject * _PyImport_TryLoadLazySubmodule( + PyObject *mod_name, PyObject *attr_name); extern PyObject * _PyImport_LazyImportModuleLevelObject( PyThreadState *tstate, PyObject *name, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *fromlist, int level); diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index 366cb203f8f256..9f2cc92bcfcc78 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -450,6 +450,14 @@ def test_lazy_import_pkg(self): self.assertIn("test.test_lazy_import.data.pkg.bar", sys.modules) self.assertIn("BAR_MODULE_LOADED", out.getvalue()) + def test_lazy_submodule_stored_in_parent_dict(self): + """Accessing a lazy submodule should store it in the parent's __dict__.""" + import test.test_lazy_import.data.lazy_import_pkg + + pkg = sys.modules["test.test_lazy_import.data.pkg"] + self.assertIn("bar", pkg.__dict__) + self.assertIs(pkg.__dict__["bar"], sys.modules["test.test_lazy_import.data.pkg.bar"]) + def test_lazy_import_pkg_cross_import(self): """Cross-imports within package should preserve lazy imports.""" import test.test_lazy_import.data.pkg.c @@ -462,6 +470,18 @@ def test_lazy_import_pkg_cross_import(self): self.assertEqual(type(g["x"]), int) self.assertEqual(type(g["b"]), types.LazyImportType) + @support.requires_subprocess() + def test_lazy_from_import_does_not_pollute_parent(self): + """Lazy from import should not add the name to the parent module's dict.""" + code = textwrap.dedent(""" + lazy from json import nonexistent_attr + import json + assert "nonexistent_attr" not in json.__dict__, ( + "lazy from import should not publish attributes on the parent module" + ) + """) + assert_python_ok("-c", code) + @support.requires_subprocess() def test_package_from_import_with_module_getattr(self): """Lazy from import should respect a package's __getattr__.""" @@ -613,19 +633,14 @@ def tearDown(self): sys.set_lazy_imports("normal") def test_import_error_shows_chained_traceback(self): - """ImportError during reification should chain to show both definition and access.""" - # Errors at reification must show where the lazy import was defined - # AND where the access happened, per PEP 810 "Reification" section + """Accessing a nonexistent lazy submodule via parent attr raises AttributeError.""" code = textwrap.dedent(""" import sys lazy import test.test_lazy_import.data.nonexistent_module try: x = test.test_lazy_import.data.nonexistent_module - except ImportError as e: - # Should have __cause__ showing the original error - # The exception chain shows both where import was defined and where access happened - assert e.__cause__ is not None, "Expected chained exception" + except AttributeError as e: print("OK") """) result = subprocess.run( @@ -673,7 +688,7 @@ def test_reification_retries_on_failure(self): # First access - should fail try: x = test.test_lazy_import.data.broken_module - except ValueError: + except AttributeError: pass # The lazy object should still be a lazy proxy (not reified) @@ -683,7 +698,7 @@ def test_reification_retries_on_failure(self): # Second access - should also fail (retry the import) try: x = test.test_lazy_import.data.broken_module - except ValueError: + except AttributeError: print("OK - retry worked") """) result = subprocess.run( @@ -696,7 +711,6 @@ def test_reification_retries_on_failure(self): def test_error_during_module_execution_propagates(self): """Errors in module code during reification should propagate correctly.""" - # Module that raises during import should propagate with chaining code = textwrap.dedent(""" import sys lazy import test.test_lazy_import.data.broken_module @@ -704,12 +718,8 @@ def test_error_during_module_execution_propagates(self): try: _ = test.test_lazy_import.data.broken_module print("FAIL - should have raised") - except ValueError as e: - # The ValueError from the module should be the cause - if "always fails" in str(e) or (e.__cause__ and "always fails" in str(e.__cause__)): - print("OK") - else: - print(f"FAIL - wrong error: {e}") + except AttributeError: + print("OK") """) result = subprocess.run( [sys.executable, "-c", code], diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index f7b83c1d111cde..f447403ef31b43 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1299,6 +1299,33 @@ _PyModule_IsPossiblyShadowing(PyObject *origin) return result; } +// Check if `name` is a lazily pending submodule of module `m`. +// Returns a new reference on success, or NULL with no error set. +static PyObject * +try_load_lazy_submodule(PyModuleObject *m, PyObject *name) +{ + PyObject *mod_name; + int rc = PyDict_GetItemRef(m->md_dict, &_Py_ID(__name__), &mod_name); + if (rc <= 0) { + return NULL; + } + if (!PyUnicode_Check(mod_name)) { + Py_DECREF(mod_name); + return NULL; + } + PyObject *result = _PyImport_TryLoadLazySubmodule(mod_name, name); + Py_DECREF(mod_name); + if (result == NULL) { + PyErr_Clear(); + return NULL; + } + if (PyDict_SetItem(m->md_dict, name, result) < 0) { + Py_DECREF(result); + return NULL; + } + return result; +} + PyObject* _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) { @@ -1363,6 +1390,13 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) PyErr_Clear(); } assert(m->md_dict != NULL); + attr = try_load_lazy_submodule(m, name); + if (attr != NULL) { + return attr; + } + if (PyErr_Occurred()) { + return NULL; + } if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { return NULL; } diff --git a/Python/import.c b/Python/import.c index c5cc7b52922d5b..ef6f5274a23665 100644 --- a/Python/import.c +++ b/Python/import.c @@ -4332,16 +4332,6 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, return final_mod; } -static PyObject * -get_mod_dict(PyObject *module) -{ - if (PyModule_Check(module)) { - return Py_NewRef(_PyModule_GetDict(module)); - } - - return PyObject_GetAttr(module, &_Py_ID(__dict__)); -} - // ensure we have the set for the parent module name in sys.lazy_modules. // Returns a new reference. static PyObject * @@ -4364,18 +4354,16 @@ ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) return lazy_submodules; } -// Ensures that we have a LazyImportObject on the parent module for -// all children modules which have been lazily imported. If the parent -// module overrides the child attribute then the value is not replaced. +// Records all parent-child relationships in lazy_pending_submodules +// for a lazily imported module name. When a parent module's attribute +// is accessed, _Py_module_getattro_impl will check lazy_pending_submodules +// and trigger the import. static int -register_lazy_on_parent(PyThreadState *tstate, PyObject *name, - PyObject *builtins) +register_lazy_on_parent(PyThreadState *tstate, PyObject *name) { int ret = -1; PyObject *parent = NULL; PyObject *child = NULL; - PyObject *parent_module = NULL; - PyObject *parent_dict = NULL; PyInterpreterState *interp = tstate->interp; PyObject *lazy_pending_submodules = LAZY_PENDING_SUBMODULES(interp); @@ -4396,9 +4384,6 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, goto done; } parent = PyUnicode_Substring(name, 0, dot); - // If `parent` is NULL then this has hit the end of the import, no - // more "parent.child" in the import name. The entire import will be - // resolved lazily. if (parent == NULL) { goto done; } @@ -4408,7 +4393,6 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, goto done; } - // Record the child as being lazily imported from the parent. PyObject *lazy_submodules = ensure_lazy_pending_submodules( (PyDictObject *)lazy_pending_submodules, parent); if (lazy_submodules == NULL) { @@ -4421,44 +4405,11 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, } Py_DECREF(lazy_submodules); - // Add the lazy import for the child to the parent. - Py_XSETREF(parent_module, PyImport_GetModule(parent)); - if (parent_module != NULL) { - Py_XSETREF(parent_dict, get_mod_dict(parent_module)); - if (parent_dict == NULL) { - goto done; - } - if (PyDict_CheckExact(parent_dict)) { - int contains = PyDict_Contains(parent_dict, child); - if (contains < 0) { - goto done; - } - if (!contains) { - PyObject *lazy_module_attr = _PyLazyImport_New( - tstate->current_frame, builtins, parent, child - ); - if (lazy_module_attr == NULL) { - goto done; - } - if (PyDict_SetItem(parent_dict, child, - lazy_module_attr) < 0) { - Py_DECREF(lazy_module_attr); - goto done; - } - Py_DECREF(lazy_module_attr); - } - } - ret = 0; - goto done; - } - Py_SETREF(name, parent); parent = NULL; } done: - Py_XDECREF(parent_dict); - Py_XDECREF(parent_module); Py_XDECREF(child); Py_XDECREF(parent); Py_XDECREF(name); @@ -4467,7 +4418,7 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, static int register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, - PyObject *from, PyObject *builtins) + PyObject *from) { PyObject *fromname = PyUnicode_FromFormat("%U.%U", abs_name, from); if (fromname == NULL) { @@ -4481,11 +4432,59 @@ register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, return -1; } - int res = register_lazy_on_parent(tstate, fromname, builtins); + int res = register_lazy_on_parent(tstate, fromname); Py_DECREF(fromname); return res; } +PyObject * +_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(interp); + if (lazy_pending == NULL) { + return NULL; + } + + PyObject *pending_set; + int rc = PyDict_GetItemRef(lazy_pending, mod_name, &pending_set); + if (rc <= 0) { + return NULL; + } + + int contains = PySet_Contains(pending_set, attr_name); + if (contains <= 0) { + Py_DECREF(pending_set); + return NULL; + } + + PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); + if (full_name == NULL) { + Py_DECREF(pending_set); + return NULL; + } + + PyObject *mod = PyImport_ImportModuleLevelObject( + full_name, NULL, NULL, NULL, 0); + if (mod == NULL) { + Py_DECREF(pending_set); + Py_DECREF(full_name); + return NULL; + } + Py_DECREF(mod); + + if (PySet_Discard(pending_set, attr_name) < 0) { + Py_DECREF(pending_set); + Py_DECREF(full_name); + return NULL; + } + Py_DECREF(pending_set); + + PyObject *submod = PyImport_GetModule(full_name); + Py_DECREF(full_name); + return submod; +} + PyObject * _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, PyObject *name, PyObject *builtins, @@ -4580,8 +4579,7 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, } if (fromlist && PyUnicode_Check(fromlist)) { - if (register_from_lazy_on_parent(tstate, abs_name, fromlist, - builtins) < 0) { + if (register_from_lazy_on_parent(tstate, abs_name, fromlist) < 0) { goto error; } } @@ -4589,14 +4587,13 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, PyTuple_GET_SIZE(fromlist)) { for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(fromlist); i++) { if (register_from_lazy_on_parent(tstate, abs_name, - PyTuple_GET_ITEM(fromlist, i), - builtins) < 0) + PyTuple_GET_ITEM(fromlist, i)) < 0) { goto error; } } } - else if (register_lazy_on_parent(tstate, abs_name, builtins) < 0) { + else if (register_lazy_on_parent(tstate, abs_name) < 0) { goto error; } @@ -5605,46 +5602,6 @@ _imp_source_hash_impl(PyObject *module, long key, Py_buffer *source) return PyBytes_FromStringAndSize(hash.data, sizeof(hash.data)); } -static int -publish_lazy_imports_on_module(PyThreadState *tstate, - PyObject *lazy_submodules, - PyObject *name, - PyObject *module_dict) -{ - PyObject *builtins = _PyEval_GetBuiltins(tstate); - PyObject *attr_name; - Py_ssize_t pos = 0; - Py_hash_t hash; - - // Enumerate the set of lazy submodules which have been imported from the - // parent module. - while (_PySet_NextEntryRef(lazy_submodules, &pos, &attr_name, &hash)) { - if (_PyDict_Contains_KnownHash(module_dict, attr_name, hash)) { - Py_DECREF(attr_name); - continue; - } - // Create a new lazy module attr for the subpackage which was - // previously lazily imported. - PyObject *lazy_module_attr = _PyLazyImport_New(tstate->current_frame, builtins, - name, attr_name); - if (lazy_module_attr == NULL) { - Py_DECREF(attr_name); - return -1; - } - - // Publish on the module that was just imported. - if (PyDict_SetItem(module_dict, attr_name, - lazy_module_attr) < 0) { - Py_DECREF(lazy_module_attr); - Py_DECREF(attr_name); - return -1; - } - Py_DECREF(lazy_module_attr); - Py_DECREF(attr_name); - } - return 0; -} - /*[clinic input] _imp._set_lazy_attributes modobj: object @@ -5658,44 +5615,11 @@ _imp__set_lazy_attributes_impl(PyObject *module, PyObject *modobj, PyObject *name) /*[clinic end generated code: output=3369bb3242b1f043 input=38ea6f30956dd7d6]*/ { - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *module_dict = NULL; - PyObject *ret = NULL; - PyObject *lazy_pending_modules = LAZY_PENDING_SUBMODULES(tstate->interp); - assert(lazy_pending_modules != NULL); - - PyObject *lazy_submodules; - if (PySet_Discard(LAZY_MODULES(tstate->interp), name) < 0) { - return NULL; - } else if (PyDict_GetItemRef(lazy_pending_modules, name, &lazy_submodules) < 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (PySet_Discard(LAZY_MODULES(interp), name) < 0) { return NULL; } - else if (lazy_submodules == NULL) { - Py_RETURN_NONE; - } - - module_dict = get_mod_dict(modobj); - if (module_dict == NULL || !PyDict_CheckExact(module_dict)) { - Py_DECREF(lazy_submodules); - goto done; - } - - assert(PyAnySet_CheckExact(lazy_submodules)); - Py_BEGIN_CRITICAL_SECTION(lazy_submodules); - publish_lazy_imports_on_module(tstate, lazy_submodules, name, module_dict); - Py_END_CRITICAL_SECTION(); - Py_DECREF(lazy_submodules); - - if (PyDict_DelItem(lazy_pending_modules, name) < 0) { - goto error; - } - -done: - ret = Py_NewRef(Py_None); - -error: - Py_XDECREF(module_dict); - return ret; + Py_RETURN_NONE; } PyDoc_STRVAR(doc_imp, From de401ef6a5fc9d064ec89d6ee15844af95d4fbe8 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 May 2026 00:05:19 +0200 Subject: [PATCH 089/110] [3.15] gh-150114: Reduce memory usage of test_free_threading.test_iteration (gh-150115) (#150124) Reduce NUMITEMS from 100000 to 5000. Peak RSS for the full test_free_threading suite drops from ~850 MB to ~175 MB. (cherry picked from commit 61f12211fc40aef4a2dcccb9c94aae8108042edb) Co-authored-by: Sam Gross --- Lib/test/test_free_threading/test_iteration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_free_threading/test_iteration.py b/Lib/test/test_free_threading/test_iteration.py index a51ad0cf83a006..44d3e9ccfdd14e 100644 --- a/Lib/test/test_free_threading/test_iteration.py +++ b/Lib/test/test_free_threading/test_iteration.py @@ -12,7 +12,7 @@ NUMITEMS = 1000 NUMTHREADS = 2 else: - NUMITEMS = 100000 + NUMITEMS = 5000 NUMTHREADS = 5 NUMMUTATORS = 2 From d6dda0d23c4a2c7db360235b2198c8f18e0f5596 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 May 2026 02:46:42 +0200 Subject: [PATCH 090/110] [3.15] gh-150042: queue.SimpleQueue.put: fix minor refleak. (GH-150043) (GH-150127) If queue.SimpleQueue.put can't handoff the item to a waiting thread, and fails to allocate memory when adding the item to a ringbuf, it would leak a reference. Fixed. (cherry picked from commit 79088e0d82931c21fa72eadc416a18b7b0fdf9c1) Co-authored-by: larryhastings --- ...-05-18-16-54-54.gh-issue-150042.LSr5W8.rst | 1 + Modules/_queuemodule.c | 37 ++++++++----------- 2 files changed, 17 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-16-54-54.gh-issue-150042.LSr5W8.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-16-54-54.gh-issue-150042.LSr5W8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-16-54-54.gh-issue-150042.LSr5W8.rst new file mode 100644 index 00000000000000..18a4fbd9dadd60 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-16-54-54.gh-issue-150042.LSr5W8.rst @@ -0,0 +1 @@ +Fix refleak in queue.SimpleQueue.put if memory allocation fails. diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index ed925f3525a9a7..d5ba36273c8262 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -154,8 +154,6 @@ RingBuf_Get(RingBuf *buf) } // Returns 0 on success or -1 if the buffer failed to grow. -// -// Steals a reference to item. static int RingBuf_Put(RingBuf *buf, PyObject *item) { @@ -165,11 +163,10 @@ RingBuf_Put(RingBuf *buf, PyObject *item) // Buffer is full, grow it. if (resize_ringbuf(buf, buf->items_cap * 2) < 0) { PyErr_NoMemory(); - Py_DECREF(item); return -1; } } - buf->items[buf->put_idx] = item; + buf->items[buf->put_idx] = Py_NewRef(item); buf->put_idx = (buf->put_idx + 1) % buf->items_cap; buf->num_items++; return 0; @@ -276,16 +273,13 @@ maybe_handoff_item(void *arg, void *park_arg, int has_more_waiters) { HandoffData *data = (HandoffData*)arg; PyObject **item = (PyObject**)park_arg; - if (item == NULL) { - // No threads were waiting - data->handed_off = false; - } - else { + data->queue->has_threads_waiting = has_more_waiters; + + data->handed_off = item != NULL; + if (data->handed_off) { // There was at least one waiting thread, hand off the item - *item = data->item; - data->handed_off = true; + *item = Py_NewRef(data->item); } - data->queue->has_threads_waiting = has_more_waiters; } /*[clinic input] @@ -307,21 +301,22 @@ _queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item, int block, PyObject *timeout) /*[clinic end generated code: output=4333136e88f90d8b input=a16dbb33363c0fa8]*/ { - HandoffData data = { - .handed_off = 0, - .item = Py_NewRef(item), - .queue = self, - }; if (self->has_threads_waiting) { + HandoffData data = { + .handed_off = 0, + .item = item, + .queue = self, + }; // Try to hand the item off directly if there are threads waiting _PyParkingLot_Unpark(&self->has_threads_waiting, maybe_handoff_item, &data); - } - if (!data.handed_off) { - if (RingBuf_Put(&self->buf, item) < 0) { - return NULL; + if (data.handed_off) { + Py_RETURN_NONE; } } + if (RingBuf_Put(&self->buf, item) < 0) { + return NULL; + } Py_RETURN_NONE; } From 4baf3e5b0dede9c8d954ad0ac2ad01047a573c3d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 May 2026 02:54:59 +0200 Subject: [PATCH 091/110] [3.15] gh-134887: Add references to `locale` module for locale-aware number formatting references in `string` module docs (GH-134888) (GH-150120) (cherry picked from commit 47723af4e74ae1a65108837fe15795e2f70f9d02) Co-authored-by: Stefanie Molin <24376333+stefmolin@users.noreply.github.com> --- Doc/library/string.rst | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 08ccdfa3f454f8..be968a3c53d843 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -472,7 +472,9 @@ of a number respectively. It can be one of the following: | | this option is not supported. | +---------+----------------------------------------------------------+ -For a locale aware separator, use the ``'n'`` presentation type instead. +For a locale-aware separator, use the ``'n'`` +:ref:`float presentation type ` or +:ref:`integer presentation type ` instead. .. versionchanged:: 3.1 Added the ``','`` option (see also :pep:`378`). @@ -518,9 +520,14 @@ The available integer presentation types are: | | In case ``'#'`` is specified, the prefix ``'0x'`` will | | | be upper-cased to ``'0X'`` as well. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'d'``, except that it uses | + | ``'n'`` | .. _n-format-integer: | + | | | + | | Number. This is the same as ``'d'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators. | + | | digit group separators. Note that the default locale is | + | | not the system locale. Depending on your use case, you | + | | may wish to set :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | None | The same as ``'d'``. | +---------+----------------------------------------------------------+ @@ -603,10 +610,15 @@ The available presentation types for :class:`float` and | | ``'E'`` if the number gets too large. The | | | representations of infinity and NaN are uppercased, too. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'g'``, except that it uses | + | ``'n'`` | .. _n-format-float: | + | | | + | | Number. This is the same as ``'g'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators | - | | for the integral part of a number. | + | | digit group separators for the integral part of a | + | | number. Note that the default locale is not the system | + | | locale. Depending on your use case, you may wish to set | + | | :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | From eb7be9ab52593f7a27d2046cd44b1fdfd6cac430 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 May 2026 03:00:54 +0200 Subject: [PATCH 092/110] [3.15] gh-72088: clarify `inspect.ismethod` and `inspect.isfunction` (and related) usage with class-level access (GH-150013) (GH-150119) (cherry picked from commit 0aa59ce2d4f007a9d19740eb2f6230ed302096f7) Co-authored-by: Stefanie Molin <24376333+stefmolin@users.noreply.github.com> Co-authored-by: CHINMAY <89741289+Das-Chinmay@users.noreply.github.com> --- Doc/library/inspect.rst | 45 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 48ae9147587c64..92840e702fbbfe 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -424,12 +424,39 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a bound method written in Python. + .. note:: -.. function:: ispackage(object) + For example, given this class:: - Return ``True`` if the object is a :term:`package`. + >>> class Greeter: + ... def say_hello(self): + ... print('hello!') - .. versionadded:: 3.14 + A bound method (also known as an *instance method*) is created when + accessing ``say_hello`` (a :term:`function` defined in the + ``Greeter`` namespace) through an instance of the ``Greeter`` class:: + + >>> instance = Greeter() + + >>> instance.say_hello + > + >>> ismethod(instance.say_hello) + True + >>> isfunction(instance.say_hello) + False + + Accessing ``say_hello`` through the ``Greeter`` class will return the + function itself. For this function, :func:`ismethod` will return + ``False``, but :func:`isfunction` will return ``True``:: + + >>> Greeter.say_hello + + >>> ismethod(Greeter.say_hello) + False + >>> isfunction(Greeter.say_hello) + True + + See :ref:`typesmethods` for details. .. function:: isfunction(object) @@ -437,11 +464,23 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a Python function, which includes functions created by a :term:`lambda` expression. + See the note for :func:`~inspect.ismethod` for an example. + + +.. function:: ispackage(object) + + Return ``True`` if the object is a :term:`package`. + + .. versionadded:: 3.14 + .. function:: isgeneratorfunction(object) Return ``True`` if the object is a Python generator function. + It also returns ``True`` for bound methods created from Python generator functions + (see :ref:`typesmethods` for more information). + .. versionchanged:: 3.8 Functions wrapped in :func:`functools.partial` now return ``True`` if the wrapped function is a Python generator function. From 653f12b2bfdbbe104f9463816d5f0c02e60e7c1d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 May 2026 03:03:47 +0200 Subject: [PATCH 093/110] [3.15] gh-143387: Update docs to reflect the behavior and note the changed version. (GH-150095) (#150106) gh-143387: Update docs to reflect the behavior and note the changed version. (GH-150095) (cherry picked from commit 192796cfd4793cd7c9e88261795394ab016d5984) Co-authored-by: Jason R. Coombs --- Doc/library/importlib.metadata.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 63de4f91f4ba5f..e11db37b9fad50 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -105,6 +105,13 @@ You can also get a :ref:`distribution's version number `, list its current Python environment. +.. exception:: MetadataNotFound + + Subclass of :class:`FileNotFoundError` raised when attempting to load metadata + from a distribution folder that is empty or otherwise does not contain a + metadata file. + + Functional API ============== @@ -224,6 +231,9 @@ Distribution metadata Raises :exc:`PackageNotFoundError` if the named distribution package is not installed in the current Python environment. + Raises :exc:`MetadataNotFound` if a distribution package is + present but no METADATA file is present. + .. class:: PackageMetadata A concrete implementation of the @@ -252,6 +262,12 @@ all the metadata in a JSON-compatible form per :PEP:`566`:: The full set of available metadata is not described here. See the PyPA `Core metadata specification `_ for additional details. +.. versionchanged:: 3.15 + Previously and incidentally, if a METADATA file was missing from a distribution, an + empty ``PackageMetadata`` would be returned, indistinguishable from + an empty METADATA file. Now, a missing METADATA file triggers a + ``MetadataNotFound`` exception. + .. versionchanged:: 3.10 The ``Description`` is now included in the metadata when presented through the payload. Line continuation characters have been removed. @@ -465,6 +481,9 @@ The same applies for :func:`entry_points` and :func:`files`. .. attribute:: metadata :type: PackageMetadata + Raises :exc:`MetadataNotFound` if the METADATA file is not present in + the distribution. + There are all kinds of additional metadata available on :class:`!Distribution` instances as a :class:`PackageMetadata` instance:: From c555e260615d8d820f6a98e60a1d0ad0a1f13141 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 May 2026 13:48:24 +0200 Subject: [PATCH 094/110] [3.15] gh-148260: Use at least 1 MiB stack size on musl (GH-149993) (#150150) gh-148260: Use at least 1 MiB stack size on musl (GH-149993) On Linux when Python is linked to the musl C library, use a thread stack size of at least 1 MiB instead of musl default which is 128 kiB. (cherry picked from commit df6c157e51430e8e7458012417c534ad8c33119f) Co-authored-by: Victor Stinner --- ...-05-18-16-00-41.gh-issue-148260.UwFiIX.rst | 3 + configure | 55 +++++++++++++++++++ configure.ac | 48 ++++++++++++++++ 3 files changed, 106 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2026-05-18-16-00-41.gh-issue-148260.UwFiIX.rst diff --git a/Misc/NEWS.d/next/Build/2026-05-18-16-00-41.gh-issue-148260.UwFiIX.rst b/Misc/NEWS.d/next/Build/2026-05-18-16-00-41.gh-issue-148260.UwFiIX.rst new file mode 100644 index 00000000000000..8248c24cbd511a --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-05-18-16-00-41.gh-issue-148260.UwFiIX.rst @@ -0,0 +1,3 @@ +On Linux when Python is linked to the musl C library, use a thread stack +size of at least 1 MiB instead of musl default which is 128 kiB. Patch by +Victor Stinner. diff --git a/configure b/configure index 8979ec294bcdf0..acba294d55de8f 100755 --- a/configure +++ b/configure @@ -9866,6 +9866,61 @@ fi ;; esac +if test "$ac_sys_system" = "Linux" -a "$cross_compiling" = no; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for thread stack size" >&5 +printf %s "checking for thread stack size... " >&6; } +if test ${ac_cv_thread_stack_size+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + cat > conftest.c < + +int main() +{ + pthread_attr_t attrs; + size_t size; + + int rc = pthread_attr_init(&attrs); + if (rc != 0) { + return 2; + } + + rc = pthread_attr_getstacksize(&attrs, &size); + if (rc != 0) { + return 2; + } + + if (size < 1024 * 1024) { + return 1; + } + return 0; +} +EOF + + ac_cv_thread_stack_size=unknown + if $CC -pthread $CFLAGS conftest.c -o conftest &>/dev/null; then + ./conftest &>/dev/null + exitcode=$? + if test $exitcode -eq 1; then + ac_cv_thread_stack_size=1048576 + elif test $exitcode -eq 0; then + ac_cv_thread_stack_size="default" + fi + fi + rm -f conftest.c conftest + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_thread_stack_size" >&5 +printf "%s\n" "$ac_cv_thread_stack_size" >&6; } + + if test "$ac_cv_thread_stack_size" != "default" -a "$ac_cv_thread_stack_size" != "unknown"; then + LDFLAGS="$LDFLAGS -Wl,-z,stack-size=$ac_cv_thread_stack_size" + fi +fi + case $enable_wasm_dynamic_linking in #( yes) : ac_cv_func_dlopen=yes ;; #( diff --git a/configure.ac b/configure.ac index 842e010f20cb74..d909c2fc92894e 100644 --- a/configure.ac +++ b/configure.ac @@ -2462,6 +2462,54 @@ AS_CASE([$ac_sys_system], ] ) +dnl On Linux, check the thread stack size. musl (ex: Alpine Linux) uses +dnl a default thread stack size of 128 kB, whereas the glibc uses 8 MiB. +dnl Python needs at least 1 MiB. +if test "$ac_sys_system" = "Linux" -a "$cross_compiling" = no; then + AC_CACHE_CHECK([for thread stack size], [ac_cv_thread_stack_size], [ + cat > conftest.c < + +int main() +{ + pthread_attr_t attrs; + size_t size; + + int rc = pthread_attr_init(&attrs); + if (rc != 0) { + return 2; + } + + rc = pthread_attr_getstacksize(&attrs, &size); + if (rc != 0) { + return 2; + } + + if (size < 1024 * 1024) { + return 1; + } + return 0; +} +EOF + + ac_cv_thread_stack_size=unknown + if $CC -pthread $CFLAGS conftest.c -o conftest &>/dev/null; then + ./conftest &>/dev/null + exitcode=$? + if test $exitcode -eq 1; then + ac_cv_thread_stack_size=1048576 + elif test $exitcode -eq 0; then + ac_cv_thread_stack_size="default" + fi + fi + rm -f conftest.c conftest + ]) + + if test "$ac_cv_thread_stack_size" != "default" -a "$ac_cv_thread_stack_size" != "unknown"; then + LDFLAGS="$LDFLAGS -Wl,-z,stack-size=$ac_cv_thread_stack_size" + fi +fi + AS_CASE([$enable_wasm_dynamic_linking], [yes], [ac_cv_func_dlopen=yes], [no], [ac_cv_func_dlopen=no], From 7f29fa5032cc5a9d02cccbb39a3f7cd116e0f145 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 May 2026 13:51:54 +0200 Subject: [PATCH 095/110] [3.15] gh-149807: Fix hash(frozendict): compute (key, value) pair hash (GH-149841) (#150149) gh-149807: Fix hash(frozendict): compute (key, value) pair hash (GH-149841) (cherry picked from commit 244300162d2e863a0588d1754e224d68931ada37) Co-authored-by: Victor Stinner --- Lib/test/test_dict.py | 25 ++++++++++ ...-05-14-19-41-03.gh-issue-149807.IwGaCo.rst | 2 + Objects/dictobject.c | 50 +++++++++++++++---- Objects/tupleobject.c | 3 ++ 4 files changed, 69 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-14-19-41-03.gh-issue-149807.IwGaCo.rst diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 4efb066d4fd01c..f26586809238f0 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1903,10 +1903,35 @@ def test_hash(self): self.assertEqual(hash(frozendict(x=1, y=2)), hash(frozendict(y=2, x=1))) + # Check that hash() computes the hash of (key, value) pairs + cases = [ + frozendict(a=False, b=True, c=True), + frozendict(a=True, b=False, c=True), + frozendict(a=True, b=True, c=False), + frozendict({False: "a", "b": True, "c": True}), + frozendict({"a": "b", False: True, True: "c"}), + ] + hashes = {hash(fd) for fd in cases} + self.assertEqual(len(hashes), len(cases)) + fd = frozendict(x=[1], y=[2]) with self.assertRaisesRegex(TypeError, "unhashable type: 'list'"): hash(fd) + @support.cpython_only + def test_hash_cpython(self): + # Check that hash(frozendict) implementation is: + # hash(frozenset(fd.items())) + for fd in ( + frozendict(), + frozendict(x=1, y=2), + frozendict(y=2, x=1), + frozendict(a=False, b=True, c=True), + frozendict.fromkeys('abc'), + ): + with self.subTest(fd=fd): + self.assertEqual(hash(fd), hash(frozenset(fd.items()))) + def test_fromkeys(self): self.assertEqual(frozendict.fromkeys('abc'), frozendict(a=None, b=None, c=None)) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-14-19-41-03.gh-issue-149807.IwGaCo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-14-19-41-03.gh-issue-149807.IwGaCo.rst new file mode 100644 index 00000000000000..a94c737e73619d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-14-19-41-03.gh-issue-149807.IwGaCo.rst @@ -0,0 +1,2 @@ +Fix ``hash(frozendict)``: compute the hash of each ``(key, value)`` pair +correctly. Patch by Victor Stinner. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a7d67812bec925..3830fedd42bd27 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -8230,6 +8230,39 @@ _shuffle_bits(Py_uhash_t h) return ((h ^ 89869747UL) ^ (h << 16)) * 3644798167UL; } +// Compute hash((key, value)). +// Code copied from tuple_hash(). +static Py_hash_t +frozendict_pair_hash(Py_hash_t key_hash, PyObject *value) +{ + assert(key_hash != -1); + + const Py_ssize_t len = 2; + Py_uhash_t acc = _PyTuple_HASH_XXPRIME_5; + + Py_uhash_t lane = key_hash; + acc += lane * _PyTuple_HASH_XXPRIME_2; + acc = _PyTuple_HASH_XXROTATE(acc); + acc *= _PyTuple_HASH_XXPRIME_1; + + lane = PyObject_Hash(value); + if (lane == (Py_uhash_t)-1) { + return -1; + } + acc += lane * _PyTuple_HASH_XXPRIME_2; + acc = _PyTuple_HASH_XXROTATE(acc); + acc *= _PyTuple_HASH_XXPRIME_1; + + /* Add input length, mangled to keep the historical value of hash(()). */ + acc += len ^ (_PyTuple_HASH_XXPRIME_5 ^ 3527539UL); + + if (acc == (Py_uhash_t)-1) { + acc = 1546275796; + } + return acc; +} + + // Code copied from frozenset_hash() static Py_hash_t frozendict_hash(PyObject *op) @@ -8243,20 +8276,15 @@ frozendict_hash(PyObject *op) PyDictObject *mp = _PyAnyDict_CAST(op); Py_uhash_t hash = 0; - PyObject *key, *value; // borrowed refs + PyObject *value; // borrowed ref Py_ssize_t pos = 0; - while (PyDict_Next(op, &pos, &key, &value)) { - Py_hash_t key_hash = PyObject_Hash(key); - if (key_hash == -1) { - return -1; - } - hash ^= _shuffle_bits(key_hash); - - Py_hash_t value_hash = PyObject_Hash(value); - if (value_hash == -1) { + Py_hash_t key_hash; + while (_PyDict_Next(op, &pos, NULL, &value, &key_hash)) { + Py_hash_t pair_hash = frozendict_pair_hash(key_hash, value); + if (pair_hash == -1) { return -1; } - hash ^= _shuffle_bits(value_hash); + hash ^= _shuffle_bits(pair_hash); } /* Factor in the number of active entries */ diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 753c270f525976..6de9487432a398 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -363,6 +363,9 @@ tuple_repr(PyObject *self) https://github.com/Cyan4973/xxHash/blob/master/doc/xxhash_spec.md The constants for the hash function are defined in pycore_tuple.h. + + If you update this code, update also frozendict_pair_hash() which copied + this code. */ static Py_hash_t From 034c536d56aa89350dcdd29bf14bc54042abca04 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 May 2026 13:59:10 +0200 Subject: [PATCH 096/110] [3.15] gh-149584: Fix excessive overhead in the Tachyon profiler regarding the cache behavior (GH-149649) (#150152) --- Lib/profiling/sampling/sample.py | 27 ++ Lib/test/test_external_inspection.py | 7 + ...-05-10-19-26-50.gh-issue-149584.x7Qm9A.rst | 4 + Modules/_remote_debugging/_remote_debugging.h | 83 ++++- Modules/_remote_debugging/clinic/module.c.h | 9 +- Modules/_remote_debugging/code_objects.c | 13 + Modules/_remote_debugging/frame_cache.c | 26 ++ Modules/_remote_debugging/frames.c | 101 ++++--- Modules/_remote_debugging/module.c | 284 +++++++++++++++--- Modules/_remote_debugging/threads.c | 112 ++++++- Python/remote_debug.h | 98 ++++-- .../benchmark_external_inspection.py | 100 +++++- 12 files changed, 738 insertions(+), 126 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-10-19-26-50.gh-issue-149584.x7Qm9A.rst diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index b9e7e2625d09e4..2d379e1e16a35e 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -327,6 +327,33 @@ def _print_unwinder_stats(self): print(f" Hits: {code_hits:n} ({ANSIColors.GREEN}{fmt(code_hits_pct)}%{ANSIColors.RESET})") print(f" Misses: {code_misses:n} ({ANSIColors.RED}{fmt(code_misses_pct)}%{ANSIColors.RESET})") + batched_attempts = stats.get('batched_read_attempts', 0) + batched_successes = stats.get('batched_read_successes', 0) + batched_misses = stats.get('batched_read_misses', 0) + segments_requested = stats.get('batched_read_segments_requested', 0) + segments_completed = stats.get('batched_read_segments_completed', 0) + if batched_attempts > 0: + batched_success_rate = stats.get('batched_read_success_rate', 0.0) + batched_miss_rate = 100.0 - batched_success_rate + segment_completion_rate = stats.get( + 'batched_read_segment_completion_rate', 0.0 + ) + + print(f" {ANSIColors.CYAN}Batched Reads:{ANSIColors.RESET}") + print(f" Attempts: {batched_attempts:n}") + print( + f" Successes: {batched_successes:n} " + f"({ANSIColors.GREEN}{fmt(batched_success_rate)}%{ANSIColors.RESET})" + ) + print( + f" Misses: {batched_misses:n} " + f"({ANSIColors.RED}{fmt(batched_miss_rate)}%{ANSIColors.RESET})" + ) + print( + f" Segments read: {segments_completed:n}/{segments_requested:n} " + f"({ANSIColors.GREEN}{fmt(segment_completion_rate)}%{ANSIColors.RESET})" + ) + # Memory operations memory_reads = stats.get('memory_reads', 0) memory_bytes = stats.get('memory_bytes_read', 0) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index a29e6cdbbf6c78..6b1529aa173f01 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -3767,6 +3767,13 @@ def test_get_stats(self): "frames_read_from_cache", "frames_read_from_memory", "frame_cache_hit_rate", + "batched_read_attempts", + "batched_read_successes", + "batched_read_misses", + "batched_read_segments_requested", + "batched_read_segments_completed", + "batched_read_success_rate", + "batched_read_segment_completion_rate", ] for key in expected_keys: self.assertIn(key, stats) diff --git a/Misc/NEWS.d/next/Library/2026-05-10-19-26-50.gh-issue-149584.x7Qm9A.rst b/Misc/NEWS.d/next/Library/2026-05-10-19-26-50.gh-issue-149584.x7Qm9A.rst new file mode 100644 index 00000000000000..6734250fdd6af3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-10-19-26-50.gh-issue-149584.x7Qm9A.rst @@ -0,0 +1,4 @@ +Fix excessive overhead in the Tachyon profiler when inspecting a remote +process by avoiding repeated remote page-cache scans, batching predicted +remote reads, and reusing cached profiler result objects. Patch by Pablo +Galindo and Maurycy Pawłowski-Wieroński. diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 7369cd1514c296..d91ce54a18c813 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -30,6 +30,7 @@ extern "C" { #include "internal/pycore_llist.h" // struct llist_node #include "internal/pycore_long.h" // _PyLong_GetZero #include "internal/pycore_pyerrors.h" // _PyErr_FormatFromCause +#include "internal/pycore_pyhash.h" // _Py_HashPointerRaw #include "internal/pycore_stackref.h" // Py_TAG_BITS #include "../../Python/remote_debug.h" @@ -215,6 +216,8 @@ typedef struct { PyObject *file_name; int first_lineno; PyObject *linetable; // bytes + PyObject *last_frame_info; + ptrdiff_t last_addrq; uintptr_t addr_code_adaptive; } CachedCodeMetadata; @@ -224,11 +227,41 @@ typedef struct { typedef struct { uint64_t thread_id; // 0 = empty slot + uintptr_t thread_state_addr; uintptr_t addrs[FRAME_CACHE_MAX_FRAMES]; Py_ssize_t num_addrs; + PyObject *thread_id_obj; // owned reference, NULL if empty PyObject *frame_list; // owned reference, NULL if empty } FrameCacheEntry; +#define INTERPRETER_THREAD_CACHE_SIZE 32 +#if (INTERPRETER_THREAD_CACHE_SIZE & (INTERPRETER_THREAD_CACHE_SIZE - 1)) != 0 +# error "INTERPRETER_THREAD_CACHE_SIZE must be a power of two" +#endif + +// The two per-interpreter L2 caches below are split into per-field tables so +// that a writer rebinding one slot cannot leave stale data in a field owned by +// the other when the slot is reused across interpreters. +typedef struct { + uintptr_t interpreter_addr; + uintptr_t thread_state_addr; +} InterpreterTstateCacheEntry; +typedef struct { + uintptr_t interpreter_addr; + uint64_t code_object_generation; +} InterpreterGenerationCacheEntry; + +// Carries already-read thread state and/or frame buffers across helpers so the +// downstream callee can skip a remote read. Address fields are caller-supplied +// inputs; buffer pointers (tstate, frame) are NULL unless a prior batched read +// successfully populated them. +typedef struct { + const char *tstate; + uintptr_t tstate_addr; + const char *frame; + uintptr_t frame_addr; +} RemoteReadPrefetch; + /* Statistics for profiling performance analysis */ typedef struct { uint64_t total_samples; // Total number of get_stack_trace calls @@ -242,14 +275,44 @@ typedef struct { uint64_t code_object_cache_hits; // Code object cache hits uint64_t code_object_cache_misses; // Code object cache misses uint64_t stale_cache_invalidations; // Times stale entries were cleared + uint64_t batched_read_attempts; // Batched remote-read attempts + uint64_t batched_read_successes; // Attempts that read all requested segments + uint64_t batched_read_misses; // Attempts that fell back or partially read + uint64_t batched_read_segments_requested; // Segments requested by batched reads + uint64_t batched_read_segments_completed; // Segments completed by batched reads } UnwinderStats; +#if defined(__GNUC__) || defined(__clang__) +# define REMOTE_DEBUG_UNLIKELY(value) __builtin_expect(!!(value), 0) +#else +# define REMOTE_DEBUG_UNLIKELY(value) (value) +#endif + /* Stats tracking macros - no-op when stats collection is disabled */ #define STATS_INC(unwinder, field) \ - do { if ((unwinder)->collect_stats) (unwinder)->stats.field++; } while(0) + do { if (REMOTE_DEBUG_UNLIKELY((unwinder)->collect_stats)) (unwinder)->stats.field++; } while(0) #define STATS_ADD(unwinder, field, val) \ - do { if ((unwinder)->collect_stats) (unwinder)->stats.field += (val); } while(0) + do { if (REMOTE_DEBUG_UNLIKELY((unwinder)->collect_stats)) (unwinder)->stats.field += (val); } while(0) + +#if HAVE_PROCESS_VM_READV +# define STATS_BATCHED_READ(unwinder, requested, completed) \ + do { \ + if (REMOTE_DEBUG_UNLIKELY((unwinder)->collect_stats)) { \ + (unwinder)->stats.batched_read_attempts++; \ + (unwinder)->stats.batched_read_segments_requested += (uint64_t)(requested); \ + (unwinder)->stats.batched_read_segments_completed += (uint64_t)(completed); \ + if ((completed) == (requested)) { \ + (unwinder)->stats.batched_read_successes++; \ + } \ + else { \ + (unwinder)->stats.batched_read_misses++; \ + } \ + } \ + } while(0) +#else +# define STATS_BATCHED_READ(unwinder, requested, completed) ((void)0) +#endif typedef struct { PyTypeObject *RemoteDebugging_Type; @@ -290,7 +353,6 @@ typedef struct { struct _Py_AsyncioModuleDebugOffsets async_debug_offsets; uintptr_t interpreter_addr; uintptr_t tstate_addr; - uint64_t code_object_generation; _Py_hashtable_t *code_object_cache; int debug; int only_active_thread; @@ -302,9 +364,17 @@ typedef struct { int cache_frames; int collect_stats; // whether to collect statistics uint32_t stale_invalidation_counter; // counter for throttling frame_cache_invalidate_stale + // L1 single-entry shortcut over cached_tstates[]: most workloads sample one + // interpreter, so check these pairs before hashing into the table below. + uintptr_t cached_tstate_interpreter_addr; + uintptr_t cached_tstate_addr; + uintptr_t cached_generation_interpreter_addr; + uint64_t cached_code_object_generation; RemoteDebuggingState *cached_state; FrameCacheEntry *frame_cache; // preallocated array of FRAME_CACHE_MAX_THREADS entries UnwinderStats stats; // statistics for performance analysis + InterpreterTstateCacheEntry cached_tstates[INTERPRETER_THREAD_CACHE_SIZE]; + InterpreterGenerationCacheEntry cached_generations[INTERPRETER_THREAD_CACHE_SIZE]; #ifdef Py_GIL_DISABLED uint32_t tlbc_generation; _Py_hashtable_t *tlbc_cache; @@ -361,11 +431,13 @@ typedef struct { typedef struct { /* Inputs */ uintptr_t frame_addr; // Starting frame address + uintptr_t thread_state_addr; // Owning thread state address uintptr_t base_frame_addr; // Sentinel at bottom (for validation) uintptr_t gc_frame; // GC frame address (0 if not tracking) uintptr_t last_profiled_frame; // Last cached frame (0 if no cache) StackChunkList *chunks; // Pre-copied stack chunks int skip_first_frame; // Skip frame_addr itself (continue from its caller) + RemoteReadPrefetch prefetch; // Optional already-read thread/frame buffers /* Outputs */ PyObject *frame_info; // List to append FrameInfo objects @@ -548,6 +620,7 @@ extern int process_frame_chain( extern int frame_cache_init(RemoteUnwinderObject *unwinder); extern void frame_cache_cleanup(RemoteUnwinderObject *unwinder); extern FrameCacheEntry *frame_cache_find(RemoteUnwinderObject *unwinder, uint64_t thread_id); +extern FrameCacheEntry *frame_cache_find_by_tstate(RemoteUnwinderObject *unwinder, uintptr_t tstate_addr); extern int clear_last_profiled_frames(RemoteUnwinderObject *unwinder); extern void frame_cache_invalidate_stale(RemoteUnwinderObject *unwinder, PyObject *result); extern int frame_cache_lookup_and_extend( @@ -566,6 +639,7 @@ extern int frame_cache_store( PyObject *frame_list, const uintptr_t *addrs, Py_ssize_t num_addrs, + uintptr_t thread_state_addr, uintptr_t base_frame_addr, uintptr_t last_frame_visited); @@ -605,7 +679,8 @@ extern PyObject* unwind_stack_for_thread( uintptr_t *current_tstate, uintptr_t gil_holder_tstate, uintptr_t gc_frame, - uintptr_t main_thread_tstate + uintptr_t main_thread_tstate, + const RemoteReadPrefetch *prefetch ); /* Thread stopping functions (for blocking mode) */ diff --git a/Modules/_remote_debugging/clinic/module.c.h b/Modules/_remote_debugging/clinic/module.c.h index d56622fb82ab56..78b1d3e8d80962 100644 --- a/Modules/_remote_debugging/clinic/module.c.h +++ b/Modules/_remote_debugging/clinic/module.c.h @@ -411,8 +411,15 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_stats__doc__, " - code_object_cache_hits: Code object cache hits\n" " - code_object_cache_misses: Code object cache misses\n" " - stale_cache_invalidations: Times stale cache entries were cleared\n" +" - batched_read_attempts: Batched remote-read attempts\n" +" - batched_read_successes: Attempts that read all requested segments\n" +" - batched_read_misses: Attempts that fell back or partially read\n" +" - batched_read_segments_requested: Segments requested by batched reads\n" +" - batched_read_segments_completed: Segments completed by batched reads\n" " - frame_cache_hit_rate: Percentage of samples that hit the cache\n" " - code_object_cache_hit_rate: Percentage of code object lookups that hit cache\n" +" - batched_read_success_rate: Percentage of batched reads that completed all segments\n" +" - batched_read_segment_completion_rate: Percentage of requested segments read by batched reads\n" "\n" "Raises:\n" " RuntimeError: If stats collection was not enabled (stats=False)"); @@ -1540,4 +1547,4 @@ _remote_debugging_get_gc_stats(PyObject *module, PyObject *const *args, Py_ssize exit: return return_value; } -/*[clinic end generated code: output=5e2a29746a0c5d65 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=884914b100e9c90c input=a9049054013a1b77]*/ diff --git a/Modules/_remote_debugging/code_objects.c b/Modules/_remote_debugging/code_objects.c index 97c6ba772e88f1..3af58f2b3c379e 100644 --- a/Modules/_remote_debugging/code_objects.c +++ b/Modules/_remote_debugging/code_objects.c @@ -405,6 +405,8 @@ parse_code_object(RemoteUnwinderObject *unwinder, meta->func_name = func; meta->file_name = file; meta->linetable = linetable; + meta->last_frame_info = NULL; + meta->last_addrq = -1; meta->first_lineno = GET_MEMBER(int, code_object, unwinder->debug_offsets.code_object.firstlineno); meta->addr_code_adaptive = real_address + (uintptr_t)unwinder->debug_offsets.code_object.co_code_adaptive; @@ -482,6 +484,12 @@ parse_code_object(RemoteUnwinderObject *unwinder, addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; #endif ; // Empty statement to avoid C23 extension warning + + if (!unwinder->opcodes && meta->last_frame_info != NULL && meta->last_addrq == addrq) { + *result = Py_NewRef(meta->last_frame_info); + return 0; + } + LocationInfo info = {0}; bool ok = parse_linetable(addrq, PyBytes_AS_STRING(meta->linetable), PyBytes_GET_SIZE(meta->linetable), @@ -529,6 +537,11 @@ parse_code_object(RemoteUnwinderObject *unwinder, goto error; } + if (!unwinder->opcodes) { + Py_XSETREF(meta->last_frame_info, Py_NewRef(tuple)); + meta->last_addrq = addrq; + } + *result = tuple; return 0; diff --git a/Modules/_remote_debugging/frame_cache.c b/Modules/_remote_debugging/frame_cache.c index b6566d7cff7b54..19fc406bca9ac9 100644 --- a/Modules/_remote_debugging/frame_cache.c +++ b/Modules/_remote_debugging/frame_cache.c @@ -30,6 +30,7 @@ frame_cache_cleanup(RemoteUnwinderObject *unwinder) return; } for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + Py_CLEAR(unwinder->frame_cache[i].thread_id_obj); Py_CLEAR(unwinder->frame_cache[i].frame_list); } PyMem_Free(unwinder->frame_cache); @@ -53,6 +54,21 @@ frame_cache_find(RemoteUnwinderObject *unwinder, uint64_t thread_id) return NULL; } +FrameCacheEntry * +frame_cache_find_by_tstate(RemoteUnwinderObject *unwinder, uintptr_t tstate_addr) +{ + if (!unwinder->frame_cache || tstate_addr == 0) { + return NULL; + } + for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + if (unwinder->frame_cache[i].thread_state_addr == tstate_addr) { + assert(unwinder->frame_cache[i].num_addrs <= FRAME_CACHE_MAX_FRAMES); + return &unwinder->frame_cache[i]; + } + } + return NULL; +} + // Allocate a cache slot for a thread // Returns NULL if cache is full (graceful degradation) static FrameCacheEntry * @@ -127,8 +143,10 @@ frame_cache_invalidate_stale(RemoteUnwinderObject *unwinder, PyObject *result) } if (!found) { // Clear this entry + Py_CLEAR(unwinder->frame_cache[i].thread_id_obj); Py_CLEAR(unwinder->frame_cache[i].frame_list); unwinder->frame_cache[i].thread_id = 0; + unwinder->frame_cache[i].thread_state_addr = 0; unwinder->frame_cache[i].num_addrs = 0; STATS_INC(unwinder, stale_cache_invalidations); } @@ -216,6 +234,7 @@ frame_cache_store( PyObject *frame_list, const uintptr_t *addrs, Py_ssize_t num_addrs, + uintptr_t thread_state_addr, uintptr_t base_frame_addr, uintptr_t last_frame_visited) { @@ -257,6 +276,13 @@ frame_cache_store( return -1; } entry->thread_id = thread_id; + entry->thread_state_addr = thread_state_addr; + if (entry->thread_id_obj == NULL) { + entry->thread_id_obj = PyLong_FromUnsignedLongLong(thread_id); + if (entry->thread_id_obj == NULL) { + return -1; + } + } memcpy(entry->addrs, addrs, num_addrs * sizeof(uintptr_t)); entry->num_addrs = num_addrs; assert(entry->num_addrs == num_addrs); diff --git a/Modules/_remote_debugging/frames.c b/Modules/_remote_debugging/frames.c index bbdfce3f7201d9..8d8019396b3e31 100644 --- a/Modules/_remote_debugging/frames.c +++ b/Modules/_remote_debugging/frames.c @@ -186,30 +186,16 @@ is_frame_valid( return 1; } -int -parse_frame_object( +static int +parse_frame_buffer( RemoteUnwinderObject *unwinder, PyObject** result, - uintptr_t address, + const char *frame, uintptr_t* address_of_code_object, uintptr_t* previous_frame ) { - char frame[SIZEOF_INTERP_FRAME]; *address_of_code_object = 0; - Py_ssize_t bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - address, - SIZEOF_INTERP_FRAME, - frame - ); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter frame"); - return -1; - } - STATS_INC(unwinder, memory_reads); - STATS_ADD(unwinder, memory_bytes_read, SIZEOF_INTERP_FRAME); - *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); uintptr_t code_object = GET_MEMBER_NO_TAG(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.executable); int frame_valid = is_frame_valid(unwinder, (uintptr_t)frame, code_object); @@ -237,6 +223,31 @@ parse_frame_object( return parse_code_object(unwinder, result, &code_ctx); } +int +parse_frame_object( + RemoteUnwinderObject *unwinder, + PyObject** result, + uintptr_t address, + uintptr_t* address_of_code_object, + uintptr_t* previous_frame +) { + char frame[SIZEOF_INTERP_FRAME]; + Py_ssize_t bytes_read = _Py_RemoteDebug_ReadRemoteMemory( + &unwinder->handle, + address, + SIZEOF_INTERP_FRAME, + frame + ); + if (bytes_read < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter frame"); + return -1; + } + STATS_INC(unwinder, memory_reads); + STATS_ADD(unwinder, memory_bytes_read, SIZEOF_INTERP_FRAME); + + return parse_frame_buffer(unwinder, result, frame, address_of_code_object, previous_frame); +} + int parse_frame_from_chunks( RemoteUnwinderObject *unwinder, @@ -312,15 +323,32 @@ process_frame_chain( } assert(frame_count <= MAX_FRAMES); - if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, &stackpointer, ctx->chunks) < 0) { + if (ctx->chunks && ctx->chunks->count > 0) { + if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, &stackpointer, ctx->chunks) == 0) { + goto parsed_frame; + } PyErr_Clear(); + } + { uintptr_t address_of_code_object = 0; - if (parse_frame_object(unwinder, &frame, frame_addr, &address_of_code_object, &next_frame_addr) < 0) { + int parse_result; + if (ctx->prefetch.frame && ctx->prefetch.frame_addr == frame_addr) { + parse_result = parse_frame_buffer( + unwinder, &frame, ctx->prefetch.frame, + &address_of_code_object, &next_frame_addr); + } + else { + parse_result = parse_frame_object( + unwinder, &frame, frame_addr, + &address_of_code_object, &next_frame_addr); + } + if (parse_result < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in chain"); return -1; } } +parsed_frame: // Skip first frame if requested (used for cache miss continuation) if (ctx->skip_first_frame && frame_count == 1) { Py_XDECREF(frame); @@ -501,41 +529,37 @@ try_full_cache_hit( PyObject *current_frame = NULL; uintptr_t code_object_addr = 0; uintptr_t previous_frame = 0; - int parse_result = parse_frame_object(unwinder, ¤t_frame, ctx->frame_addr, + int parse_result; + if (ctx->prefetch.frame && ctx->prefetch.frame_addr == ctx->frame_addr) { + parse_result = parse_frame_buffer(unwinder, ¤t_frame, + ctx->prefetch.frame, &code_object_addr, &previous_frame); + } + else { + parse_result = parse_frame_object(unwinder, ¤t_frame, ctx->frame_addr, + &code_object_addr, &previous_frame); + } if (parse_result < 0) { return -1; } - Py_ssize_t cached_size = PyList_GET_SIZE(entry->frame_list); - PyObject *parent_slice = NULL; - if (cached_size > 1) { - parent_slice = PyList_GetSlice(entry->frame_list, 1, cached_size); - if (!parent_slice) { - Py_XDECREF(current_frame); - return -1; - } - } - if (current_frame != NULL) { if (PyList_Append(ctx->frame_info, current_frame) < 0) { Py_DECREF(current_frame); - Py_XDECREF(parent_slice); return -1; } Py_DECREF(current_frame); STATS_ADD(unwinder, frames_read_from_memory, 1); } - if (parent_slice) { - Py_ssize_t cur_size = PyList_GET_SIZE(ctx->frame_info); - int result = PyList_SetSlice(ctx->frame_info, cur_size, cur_size, parent_slice); - Py_DECREF(parent_slice); - if (result < 0) { + Py_ssize_t cached_size = PyList_GET_SIZE(entry->frame_list); + for (Py_ssize_t i = 1; i < cached_size; i++) { + PyObject *cached_frame = PyList_GET_ITEM(entry->frame_list, i); + if (PyList_Append(ctx->frame_info, cached_frame) < 0) { return -1; } - STATS_ADD(unwinder, frames_read_from_cache, cached_size - 1); } + STATS_ADD(unwinder, frames_read_from_cache, cached_size > 1 ? cached_size - 1 : 0); STATS_INC(unwinder, frame_cache_hits); return 1; @@ -606,7 +630,8 @@ collect_frames_with_cache( } if (frame_cache_store(unwinder, thread_id, ctx->frame_info, ctx->frame_addrs, ctx->num_addrs, - ctx->base_frame_addr, ctx->last_frame_visited) < 0) { + ctx->thread_state_addr, ctx->base_frame_addr, + ctx->last_frame_visited) < 0) { return -1; } diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index efdd2e1a2d7b7a..ae2f7e7f31ba77 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -166,6 +166,7 @@ cached_code_metadata_destroy(void *ptr) Py_DECREF(meta->func_name); Py_DECREF(meta->file_name); Py_DECREF(meta->linetable); + Py_XDECREF(meta->last_frame_info); PyMem_RawFree(meta); } @@ -360,6 +361,10 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, self->cache_frames = cache_frames; self->collect_stats = stats; self->stale_invalidation_counter = 0; + self->cached_tstate_interpreter_addr = 0; + self->cached_tstate_addr = 0; + memset(self->cached_tstates, 0, sizeof(self->cached_tstates)); + memset(self->cached_generations, 0, sizeof(self->cached_generations)); self->debug = debug; self->only_active_thread = only_active_thread; self->mode = mode; @@ -473,6 +478,172 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, return 0; } +static inline size_t +interpreter_thread_cache_index(uintptr_t interpreter_addr) +{ + // Direct-mapped table indexed by the remote interpreter address. Each entry + // stores the full address and verifies it on lookup, so hash collisions + // degrade to misses and cannot return a value from the wrong interpreter. + return (size_t)_Py_HashPointerRaw((const void *)interpreter_addr) + & (INTERPRETER_THREAD_CACHE_SIZE - 1); +} + +static inline uintptr_t +get_cached_tstate_for_interpreter( + RemoteUnwinderObject *self, + uintptr_t interpreter_addr) +{ + if (interpreter_addr == 0) { + return 0; + } + + if (self->cached_tstate_interpreter_addr == interpreter_addr) { + return self->cached_tstate_addr; + } + + InterpreterTstateCacheEntry *entry = + &self->cached_tstates[interpreter_thread_cache_index(interpreter_addr)]; + if (entry->interpreter_addr == interpreter_addr) { + self->cached_tstate_interpreter_addr = interpreter_addr; + self->cached_tstate_addr = entry->thread_state_addr; + return entry->thread_state_addr; + } + return 0; +} + +static inline void +set_cached_tstate_for_interpreter( + RemoteUnwinderObject *self, + uintptr_t interpreter_addr, + uintptr_t thread_state_addr) +{ + if (interpreter_addr == 0 || thread_state_addr == 0) { + return; + } + + self->cached_tstate_interpreter_addr = interpreter_addr; + self->cached_tstate_addr = thread_state_addr; + + InterpreterTstateCacheEntry *entry = + &self->cached_tstates[interpreter_thread_cache_index(interpreter_addr)]; + entry->interpreter_addr = interpreter_addr; + entry->thread_state_addr = thread_state_addr; +} + +static void +refresh_generation_caches_from_interp_state( + RemoteUnwinderObject *self, + uintptr_t interpreter_addr, + const char *interp_state_buffer) +{ + uint64_t code_object_generation = GET_MEMBER(uint64_t, interp_state_buffer, + self->debug_offsets.interpreter_state.code_object_generation); + + if (self->cached_generation_interpreter_addr == interpreter_addr) { + if (code_object_generation != self->cached_code_object_generation) { + self->cached_code_object_generation = code_object_generation; + _Py_hashtable_clear(self->code_object_cache); + } + } + else { + InterpreterGenerationCacheEntry *entry = + &self->cached_generations[interpreter_thread_cache_index(interpreter_addr)]; + // A slot rebound from another interpreter must be treated as changed: + // the code_object_cache is global, so even if the new generation + // numerically matches what the previous occupant had, stale entries + // from that occupant could still be served. + int changed = entry->interpreter_addr != interpreter_addr + || entry->code_object_generation != code_object_generation; + entry->interpreter_addr = interpreter_addr; + entry->code_object_generation = code_object_generation; + if (changed) { + _Py_hashtable_clear(self->code_object_cache); + } + self->cached_generation_interpreter_addr = interpreter_addr; + self->cached_code_object_generation = code_object_generation; + } + +#ifdef Py_GIL_DISABLED + uint32_t current_tlbc_generation = GET_MEMBER(uint32_t, interp_state_buffer, + self->debug_offsets.interpreter_state.tlbc_generation); + if (current_tlbc_generation != self->tlbc_generation) { + self->tlbc_generation = current_tlbc_generation; + _Py_hashtable_clear(self->tlbc_cache); + } +#endif +} + +static int +refresh_generation_caches_for_interpreter( + RemoteUnwinderObject *self, + uintptr_t interpreter_addr) +{ + char interp_state_buffer[INTERP_STATE_BUFFER_SIZE]; + if (_Py_RemoteDebug_ReadRemoteMemory( + &self->handle, + interpreter_addr, + INTERP_STATE_BUFFER_SIZE, + interp_state_buffer) < 0) { + set_exception_cause(self, PyExc_RuntimeError, + "Failed to read interpreter state buffer"); + return -1; + } + refresh_generation_caches_from_interp_state(self, interpreter_addr, interp_state_buffer); + return 0; +} + +static int +read_interp_state_and_maybe_thread_frame( + RemoteUnwinderObject *unwinder, + uintptr_t interpreter_addr, + char *interp_state_buffer, + char *tstate_buffer, + char *frame_buffer, + RemoteReadPrefetch *prefetch) +{ + prefetch->tstate = NULL; + prefetch->frame = NULL; + if (prefetch->tstate_addr != 0) { + size_t tstate_size = (size_t)unwinder->debug_offsets.thread_state.size; + _Py_RemoteReadSegment segments[3] = { + {interpreter_addr, interp_state_buffer, INTERP_STATE_BUFFER_SIZE}, + {prefetch->tstate_addr, tstate_buffer, tstate_size}, + {prefetch->frame_addr, frame_buffer, SIZEOF_INTERP_FRAME}, + }; + int nsegs = prefetch->frame_addr != 0 ? 3 : 2; + Py_ssize_t nread = _Py_RemoteDebug_BatchedReadRemoteMemory( + &unwinder->handle, segments, nsegs); + int completed = 0; + if (nread >= (Py_ssize_t)INTERP_STATE_BUFFER_SIZE) { + completed = 1; + Py_ssize_t with_tstate = (Py_ssize_t)INTERP_STATE_BUFFER_SIZE + + (Py_ssize_t)tstate_size; + if (nread >= with_tstate) { + completed = 2; + } + if (nsegs == 3 + && nread == with_tstate + (Py_ssize_t)SIZEOF_INTERP_FRAME) { + completed = 3; + } + } + STATS_BATCHED_READ(unwinder, nsegs, completed); + if (completed >= 1) { + if (completed >= 2) { + prefetch->tstate = tstate_buffer; + } + if (completed >= 3) { + prefetch->frame = frame_buffer; + } + return 0; + } + } + return _Py_RemoteDebug_ReadRemoteMemory( + &unwinder->handle, + interpreter_addr, + INTERP_STATE_BUFFER_SIZE, + interp_state_buffer); +} + /*[clinic input] @permit_long_docstring_body @critical_section @@ -537,15 +708,32 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self while (current_interpreter != 0) { // Read interpreter state to get the interpreter ID char interp_state_buffer[INTERP_STATE_BUFFER_SIZE]; - if (_Py_RemoteDebug_PagedReadRemoteMemory( - &self->handle, + char prefetched_tstate[SIZEOF_THREAD_STATE]; + char prefetched_frame[SIZEOF_INTERP_FRAME]; + RemoteReadPrefetch prefetch = {0}; + if (self->cache_frames) { + prefetch.tstate_addr = get_cached_tstate_for_interpreter( + self, current_interpreter); + } + if (prefetch.tstate_addr != 0) { + FrameCacheEntry *entry = frame_cache_find_by_tstate(self, prefetch.tstate_addr); + if (entry && entry->num_addrs > 0) { + prefetch.frame_addr = entry->addrs[0]; + } + } + + if (read_interp_state_and_maybe_thread_frame( + self, current_interpreter, - INTERP_STATE_BUFFER_SIZE, - interp_state_buffer) < 0) { + interp_state_buffer, + prefetched_tstate, + prefetched_frame, + &prefetch) < 0) { set_exception_cause(self, PyExc_RuntimeError, "Failed to read interpreter state buffer"); Py_CLEAR(result); goto exit; } + refresh_generation_caches_from_interp_state(self, current_interpreter, interp_state_buffer); uintptr_t gc_frame = 0; if (self->gc) { @@ -557,25 +745,6 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self int64_t interpreter_id = GET_MEMBER(int64_t, interp_state_buffer, self->debug_offsets.interpreter_state.id); - // Get code object generation from buffer - uint64_t code_object_generation = GET_MEMBER(uint64_t, interp_state_buffer, - self->debug_offsets.interpreter_state.code_object_generation); - - if (code_object_generation != self->code_object_generation) { - self->code_object_generation = code_object_generation; - _Py_hashtable_clear(self->code_object_cache); - } - -#ifdef Py_GIL_DISABLED - // Check TLBC generation and invalidate cache if needed - uint32_t current_tlbc_generation = GET_MEMBER(uint32_t, interp_state_buffer, - self->debug_offsets.interpreter_state.tlbc_generation); - if (current_tlbc_generation != self->tlbc_generation) { - self->tlbc_generation = current_tlbc_generation; - _Py_hashtable_clear(self->tlbc_cache); - } -#endif - // Create a list to hold threads for this interpreter PyObject *interpreter_threads = PyList_New(0); if (!interpreter_threads) { @@ -611,6 +780,9 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self // Target specific thread (only process first interpreter) current_tstate = self->tstate_addr; } + if (current_tstate != 0 && self->cache_frames) { + set_cached_tstate_for_interpreter(self, current_interpreter, current_tstate); + } // Acquire main thread state information uintptr_t main_thread_tstate = GET_MEMBER(uintptr_t, interp_state_buffer, @@ -621,7 +793,8 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self PyObject* frame_info = unwind_stack_for_thread(self, ¤t_tstate, gil_holder_tstate, gc_frame, - main_thread_tstate); + main_thread_tstate, + &prefetch); if (!frame_info) { // Check if this was an intentional skip due to mode-based filtering if ((self->mode == PROFILING_MODE_CPU || self->mode == PROFILING_MODE_GIL || @@ -771,6 +944,9 @@ _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *s if (ensure_async_debug_offsets(self) < 0) { return NULL; } + if (refresh_generation_caches_for_interpreter(self, self->interpreter_addr) < 0) { + return NULL; + } PyObject *result = PyList_New(0); if (result == NULL) { @@ -860,6 +1036,9 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject if (ensure_async_debug_offsets(self) < 0) { return NULL; } + if (refresh_generation_caches_for_interpreter(self, self->interpreter_addr) < 0) { + return NULL; + } PyObject *result = PyList_New(0); if (result == NULL) { @@ -904,8 +1083,15 @@ RemoteUnwinder was created with stats=True. - code_object_cache_hits: Code object cache hits - code_object_cache_misses: Code object cache misses - stale_cache_invalidations: Times stale cache entries were cleared + - batched_read_attempts: Batched remote-read attempts + - batched_read_successes: Attempts that read all requested segments + - batched_read_misses: Attempts that fell back or partially read + - batched_read_segments_requested: Segments requested by batched reads + - batched_read_segments_completed: Segments completed by batched reads - frame_cache_hit_rate: Percentage of samples that hit the cache - code_object_cache_hit_rate: Percentage of code object lookups that hit cache + - batched_read_success_rate: Percentage of batched reads that completed all segments + - batched_read_segment_completion_rate: Percentage of requested segments read by batched reads Raises: RuntimeError: If stats collection was not enabled (stats=False) @@ -913,7 +1099,7 @@ RemoteUnwinder was created with stats=True. static PyObject * _remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=21e36477122be2a0 input=75fef4134c12a8c9]*/ +/*[clinic end generated code: output=21e36477122be2a0 input=0392d62b278e9c35]*/ { if (!self->collect_stats) { PyErr_SetString(PyExc_RuntimeError, @@ -948,9 +1134,24 @@ _remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self) ADD_STAT(code_object_cache_hits); ADD_STAT(code_object_cache_misses); ADD_STAT(stale_cache_invalidations); + ADD_STAT(batched_read_attempts); + ADD_STAT(batched_read_successes); + ADD_STAT(batched_read_misses); + ADD_STAT(batched_read_segments_requested); + ADD_STAT(batched_read_segments_completed); #undef ADD_STAT +#define ADD_DERIVED_STAT(name, value) do { \ + PyObject *val = PyFloat_FromDouble(value); \ + if (!val || PyDict_SetItemString(result, name, val) < 0) { \ + Py_XDECREF(val); \ + Py_DECREF(result); \ + return NULL; \ + } \ + Py_DECREF(val); \ +} while(0) + // Calculate and add derived statistics // Hit rate is calculated as (hits + partial_hits) / total_cache_lookups double frame_cache_hit_rate = 0.0; @@ -959,26 +1160,33 @@ _remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self) frame_cache_hit_rate = 100.0 * (double)(self->stats.frame_cache_hits + self->stats.frame_cache_partial_hits) / (double)total_cache_lookups; } - PyObject *hit_rate = PyFloat_FromDouble(frame_cache_hit_rate); - if (!hit_rate || PyDict_SetItemString(result, "frame_cache_hit_rate", hit_rate) < 0) { - Py_XDECREF(hit_rate); - Py_DECREF(result); - return NULL; - } - Py_DECREF(hit_rate); + ADD_DERIVED_STAT("frame_cache_hit_rate", frame_cache_hit_rate); double code_object_hit_rate = 0.0; uint64_t total_code_lookups = self->stats.code_object_cache_hits + self->stats.code_object_cache_misses; if (total_code_lookups > 0) { code_object_hit_rate = 100.0 * (double)self->stats.code_object_cache_hits / (double)total_code_lookups; } - PyObject *code_hit_rate = PyFloat_FromDouble(code_object_hit_rate); - if (!code_hit_rate || PyDict_SetItemString(result, "code_object_cache_hit_rate", code_hit_rate) < 0) { - Py_XDECREF(code_hit_rate); - Py_DECREF(result); - return NULL; + ADD_DERIVED_STAT("code_object_cache_hit_rate", code_object_hit_rate); + + double batched_read_success_rate = 0.0; + if (self->stats.batched_read_attempts > 0) { + batched_read_success_rate = + 100.0 * (double)self->stats.batched_read_successes + / (double)self->stats.batched_read_attempts; } - Py_DECREF(code_hit_rate); + ADD_DERIVED_STAT("batched_read_success_rate", batched_read_success_rate); + + double batched_read_segment_completion_rate = 0.0; + if (self->stats.batched_read_segments_requested > 0) { + batched_read_segment_completion_rate = + 100.0 * (double)self->stats.batched_read_segments_completed + / (double)self->stats.batched_read_segments_requested; + } + ADD_DERIVED_STAT("batched_read_segment_completion_rate", + batched_read_segment_completion_rate); + +#undef ADD_DERIVED_STAT return result; } diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index d775234b8d78d7..ae120a26d5f4ec 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -289,28 +289,110 @@ typedef struct { unsigned int :24; } _thread_status; +static int +read_thread_state_and_maybe_frame( + RemoteUnwinderObject *unwinder, + uintptr_t tstate_addr, + size_t tstate_size, + char *tstate_buffer, + uintptr_t predicted_frame_addr, + char *frame_buffer, + int *frame_read) +{ + *frame_read = 0; + if (predicted_frame_addr != 0) { + _Py_RemoteReadSegment segments[2] = { + {tstate_addr, tstate_buffer, tstate_size}, + {predicted_frame_addr, frame_buffer, SIZEOF_INTERP_FRAME}, + }; + Py_ssize_t nread = _Py_RemoteDebug_BatchedReadRemoteMemory( + &unwinder->handle, segments, 2); + int completed = 0; + if (nread >= (Py_ssize_t)tstate_size) { + completed = 1; + if (nread == (Py_ssize_t)(tstate_size + SIZEOF_INTERP_FRAME)) { + completed = 2; + } + } + STATS_BATCHED_READ(unwinder, 2, completed); + if (completed >= 1) { + *frame_read = completed == 2; + return 0; + } + } + return _Py_RemoteDebug_ReadRemoteMemory( + &unwinder->handle, tstate_addr, tstate_size, tstate_buffer); +} + PyObject* unwind_stack_for_thread( RemoteUnwinderObject *unwinder, uintptr_t *current_tstate, uintptr_t gil_holder_tstate, uintptr_t gc_frame, - uintptr_t main_thread_tstate + uintptr_t main_thread_tstate, + const RemoteReadPrefetch *prefetch ) { PyObject *frame_info = NULL; PyObject *thread_id = NULL; PyObject *result = NULL; StackChunkList chunks = {0}; - char ts[SIZEOF_THREAD_STATE]; - int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, *current_tstate, (size_t)unwinder->debug_offsets.thread_state.size, ts); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); - goto error; + char local_ts[SIZEOF_THREAD_STATE]; + char local_prefetched_frame[SIZEOF_INTERP_FRAME]; + const char *ts; + RemoteReadPrefetch ctx_prefetch = {0}; + if (prefetch->tstate && prefetch->tstate_addr == *current_tstate) { + ts = prefetch->tstate; + if (prefetch->frame) { + ctx_prefetch.frame = prefetch->frame; + ctx_prefetch.frame_addr = prefetch->frame_addr; + } + } + else if (unwinder->cache_frames) { + uintptr_t predicted_frame_addr = 0; + int have_prefetched_frame = 0; + FrameCacheEntry *entry = frame_cache_find_by_tstate(unwinder, *current_tstate); + if (entry && entry->num_addrs > 0) { + predicted_frame_addr = entry->addrs[0]; + } + + int rc = read_thread_state_and_maybe_frame( + unwinder, + *current_tstate, + (size_t)unwinder->debug_offsets.thread_state.size, + local_ts, + predicted_frame_addr, + local_prefetched_frame, + &have_prefetched_frame); + if (rc < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); + goto error; + } + ts = local_ts; + if (have_prefetched_frame) { + ctx_prefetch.frame = local_prefetched_frame; + ctx_prefetch.frame_addr = predicted_frame_addr; + } + } + else { + int rc = _Py_RemoteDebug_ReadRemoteMemory( + &unwinder->handle, + *current_tstate, + (size_t)unwinder->debug_offsets.thread_state.size, + local_ts); + if (rc < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); + goto error; + } + ts = local_ts; } STATS_INC(unwinder, memory_reads); STATS_ADD(unwinder, memory_bytes_read, unwinder->debug_offsets.thread_state.size); + if (ctx_prefetch.frame) { + STATS_INC(unwinder, memory_reads); + STATS_ADD(unwinder, memory_bytes_read, SIZEOF_INTERP_FRAME); + } long tid = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.native_thread_id); @@ -432,9 +514,11 @@ unwind_stack_for_thread( uintptr_t addrs[FRAME_CACHE_MAX_FRAMES]; FrameWalkContext ctx = { .frame_addr = frame_addr, + .thread_state_addr = *current_tstate, .base_frame_addr = base_frame_addr, .gc_frame = gc_frame, .chunks = &chunks, + .prefetch = ctx_prefetch, .frame_info = frame_info, .frame_addrs = addrs, .num_addrs = 0, @@ -467,10 +551,18 @@ unwind_stack_for_thread( *current_tstate = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.next); - thread_id = PyLong_FromLongLong(tid); + if (unwinder->cache_frames) { + FrameCacheEntry *entry = frame_cache_find(unwinder, (uint64_t)tid); + if (entry && entry->thread_id_obj) { + thread_id = Py_NewRef(entry->thread_id_obj); + } + } if (thread_id == NULL) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID"); - goto error; + thread_id = PyLong_FromLongLong(tid); + if (thread_id == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID"); + goto error; + } } RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); diff --git a/Python/remote_debug.h b/Python/remote_debug.h index 6c089a834dcd40..7b2c4f3bcb8077 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -147,6 +147,7 @@ typedef struct { int memfd; #endif page_cache_entry_t pages[MAX_PAGES]; + int page_cache_count; Py_ssize_t page_size; } proc_handle_t; @@ -185,14 +186,16 @@ _Py_RemoteDebug_FreePageCache(proc_handle_t *handle) handle->pages[i].data = NULL; handle->pages[i].valid = 0; } + handle->page_cache_count = 0; } UNUSED static void _Py_RemoteDebug_ClearCache(proc_handle_t *handle) { - for (int i = 0; i < MAX_PAGES; i++) { + for (int i = 0; i < handle->page_cache_count; i++) { handle->pages[i].valid = 0; } + handle->page_cache_count = 0; } #if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX @@ -222,6 +225,7 @@ _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { handle->memfd = -1; #endif handle->page_size = get_page_size(); + handle->page_cache_count = 0; for (int i = 0; i < MAX_PAGES; i++) { handle->pages[i].data = NULL; handle->pages[i].valid = 0; @@ -1287,8 +1291,9 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); } - // Search for valid cached page - for (int i = 0; i < MAX_PAGES; i++) { + // Search only the pages used since the last clear. The cache is cleared + // between profiler samples, so entries are packed at the front. + for (int i = 0; i < handle->page_cache_count; i++) { page_cache_entry_t *entry = &handle->pages[i]; if (entry->valid && entry->page_addr == page_base) { memcpy(out, entry->data + offset_in_page, size); @@ -1296,33 +1301,31 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, } } - // Find reusable slot - for (int i = 0; i < MAX_PAGES; i++) { - page_cache_entry_t *entry = &handle->pages[i]; - if (!entry->valid) { + if (handle->page_cache_count < MAX_PAGES) { + page_cache_entry_t *entry = &handle->pages[handle->page_cache_count]; + if (entry->data == NULL) { + entry->data = PyMem_RawMalloc(page_size); if (entry->data == NULL) { - entry->data = PyMem_RawMalloc(page_size); - if (entry->data == NULL) { - PyErr_NoMemory(); - _set_debug_exception_cause(PyExc_MemoryError, - "Cannot allocate %zu bytes for page cache entry " - "during read from PID %d at address 0x%lx", - page_size, handle->pid, addr); - return -1; - } - } - - if (_Py_RemoteDebug_ReadRemoteMemory(handle, page_base, page_size, entry->data) < 0) { - // Try to just copy the exact amount as a fallback - PyErr_Clear(); - goto fallback; + PyErr_NoMemory(); + _set_debug_exception_cause(PyExc_MemoryError, + "Cannot allocate %zu bytes for page cache entry " + "during read from PID %d at address 0x%lx", + page_size, handle->pid, addr); + return -1; } + } - entry->page_addr = page_base; - entry->valid = 1; - memcpy(out, entry->data + offset_in_page, size); - return 0; + if (_Py_RemoteDebug_ReadRemoteMemory(handle, page_base, page_size, entry->data) < 0) { + // Try to just copy the exact amount as a fallback + PyErr_Clear(); + goto fallback; } + + entry->page_addr = page_base; + entry->valid = 1; + handle->page_cache_count++; + memcpy(out, entry->data + offset_in_page, size); + return 0; } fallback: @@ -1330,6 +1333,49 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); } +typedef struct { + uintptr_t remote_addr; + void *local_buf; + size_t size; +} _Py_RemoteReadSegment; + +#define _PY_REMOTE_DEBUG_MAX_BATCHED_SEGMENTS 4 + +// Batched read of multiple remote regions in a single syscall when supported. +// Returns total bytes read (>= 0) on success, -1 if batched reads are +// unavailable or the syscall failed. Callers compare the return value against +// cumulative segment sizes to determine which segments were fully populated. +UNUSED static Py_ssize_t +_Py_RemoteDebug_BatchedReadRemoteMemory( + proc_handle_t *handle, + const _Py_RemoteReadSegment *segments, + int nsegs) +{ +#if defined(__linux__) && HAVE_PROCESS_VM_READV + if (handle->memfd == -1 + && nsegs > 0 + && nsegs <= _PY_REMOTE_DEBUG_MAX_BATCHED_SEGMENTS) { + struct iovec local[_PY_REMOTE_DEBUG_MAX_BATCHED_SEGMENTS]; + struct iovec remote[_PY_REMOTE_DEBUG_MAX_BATCHED_SEGMENTS]; + for (int i = 0; i < nsegs; i++) { + local[i].iov_base = segments[i].local_buf; + local[i].iov_len = segments[i].size; + remote[i].iov_base = (void *)segments[i].remote_addr; + remote[i].iov_len = segments[i].size; + } + ssize_t nread = process_vm_readv(handle->pid, local, nsegs, remote, nsegs, 0); + if (nread >= 0) { + return (Py_ssize_t)nread; + } + } +#else + (void)handle; + (void)segments; + (void)nsegs; +#endif + return -1; +} + UNUSED static int _Py_RemoteDebug_ReadDebugOffsets( proc_handle_t *handle, diff --git a/Tools/inspection/benchmark_external_inspection.py b/Tools/inspection/benchmark_external_inspection.py index fee3435496da0b..8e367422a961da 100644 --- a/Tools/inspection/benchmark_external_inspection.py +++ b/Tools/inspection/benchmark_external_inspection.py @@ -151,6 +151,45 @@ def create_threads(n): time.sleep(0.05) ''' +ASYNC_CODE = '''\ +import asyncio +import contextlib +import math + +def compute_slice(seed): + result = 0.0 + for i in range(2000): + result += math.sin(seed + i) * math.sqrt(i + 1) + return result + +async def leaf_task(seed): + total = 0.0 + while True: + total += compute_slice(seed) + await asyncio.sleep(0) + +async def parent_task(seed): + child = asyncio.create_task(leaf_task(seed + 1000), name=f"leaf-{seed}") + try: + while True: + compute_slice(seed) + await asyncio.sleep(0.001) + finally: + child.cancel() + with contextlib.suppress(asyncio.CancelledError): + await child + +async def main(): + tasks = [ + asyncio.create_task(parent_task(i), name=f"parent-{i}") + for i in range(8) + ] + await asyncio.gather(*tasks) + +if __name__ == "__main__": + asyncio.run(main()) +''' + CODE_EXAMPLES = { "basic": { "code": CODE, @@ -164,10 +203,29 @@ def create_threads(n): "code": CODE_WITH_TONS_OF_THREADS, "description": "Tons of threads doing mixed CPU/IO work", }, + "asyncio": { + "code": ASYNC_CODE, + "description": "Asyncio tasks with active and awaited coroutine chains", + }, +} + +OPERATIONS = { + "stack_trace": { + "method": "get_stack_trace", + "label": "get_stack_trace()", + }, + "async_stack_trace": { + "method": "get_async_stack_trace", + "label": "get_async_stack_trace()", + }, + "all_awaited_by": { + "method": "get_all_awaited_by", + "label": "get_all_awaited_by()", + }, } -def benchmark(unwinder, duration_seconds=10, blocking=False): +def benchmark(unwinder, duration_seconds=10, blocking=False, operation="stack_trace"): """Benchmark mode - measure raw sampling speed for specified duration""" sample_count = 0 fail_count = 0 @@ -175,11 +233,14 @@ def benchmark(unwinder, duration_seconds=10, blocking=False): start_time = time.perf_counter() end_time = start_time + duration_seconds total_attempts = 0 + operation_info = OPERATIONS[operation] + operation_method = getattr(unwinder, operation_info["method"]) colors = get_colors(can_colorize()) print( - f"{colors.BOLD_BLUE}Benchmarking sampling speed for {duration_seconds} seconds...{colors.RESET}" + f"{colors.BOLD_BLUE}Benchmarking {operation_info['label']} speed " + f"for {duration_seconds} seconds...{colors.RESET}" ) try: @@ -190,8 +251,8 @@ def benchmark(unwinder, duration_seconds=10, blocking=False): if blocking: unwinder.pause_threads() try: - stack_trace = unwinder.get_stack_trace() - if stack_trace: + sample = operation_method() + if sample: sample_count += 1 finally: if blocking: @@ -239,6 +300,7 @@ def benchmark(unwinder, duration_seconds=10, blocking=False): (sample_count / total_attempts) * 100 if total_attempts > 0 else 0 ), "total_work_time": total_work_time, + "operation": operation_info["label"], "avg_work_time_us": ( (total_work_time / total_attempts) * 1e6 if total_attempts > 0 else 0 ), @@ -252,7 +314,7 @@ def print_benchmark_results(results): colors = get_colors(can_colorize()) print(f"\n{colors.BOLD_GREEN}{'='*60}{colors.RESET}") - print(f"{colors.BOLD_GREEN}get_stack_trace() Benchmark Results{colors.RESET}") + print(f"{colors.BOLD_GREEN}{results['operation']} Benchmark Results{colors.RESET}") print(f"{colors.BOLD_GREEN}{'='*60}{colors.RESET}") # Basic statistics @@ -329,6 +391,8 @@ def parse_arguments(): %(prog)s -d 60 # Run basic benchmark for 60 seconds %(prog)s --code deep_static # Run deep static call stack benchmark %(prog)s --code deep_static -d 30 # Run deep static benchmark for 30 seconds + %(prog)s --operation async_stack_trace + %(prog)s --operation all_awaited_by Available code examples: {examples_desc} @@ -348,8 +412,15 @@ def parse_arguments(): "--code", "-c", choices=list(CODE_EXAMPLES.keys()), - default="basic", - help="Code example to benchmark (default: basic)", + default=None, + help="Code example to benchmark (default: basic, or asyncio for async operations)", + ) + + parser.add_argument( + "--operation", + choices=list(OPERATIONS.keys()), + default="stack_trace", + help="Remote unwinder operation to benchmark (default: stack_trace)", ) parser.add_argument( @@ -365,7 +436,10 @@ def parse_arguments(): help="Stop all threads before sampling for consistent snapshots", ) - return parser.parse_args() + args = parser.parse_args() + if args.code is None: + args.code = "asyncio" if args.operation != "stack_trace" else "basic" + return args def create_target_process(temp_file, code_example="basic"): @@ -420,6 +494,9 @@ def main(): print( f"{colors.CYAN}Benchmark Duration:{colors.RESET} {colors.YELLOW}{args.duration}{colors.RESET} seconds" ) + print( + f"{colors.CYAN}Operation:{colors.RESET} {colors.GREEN}{OPERATIONS[args.operation]['label']}{colors.RESET}" + ) print( f"{colors.CYAN}Blocking Mode:{colors.RESET} {colors.GREEN if args.blocking else colors.YELLOW}{'enabled' if args.blocking else 'disabled'}{colors.RESET}" ) @@ -451,7 +528,12 @@ def main(): unwinder = _remote_debugging.RemoteUnwinder( process.pid, cache_frames=True, **kwargs ) - results = benchmark(unwinder, duration_seconds=args.duration, blocking=args.blocking) + results = benchmark( + unwinder, + duration_seconds=args.duration, + blocking=args.blocking, + operation=args.operation, + ) finally: cleanup_process(process, temp_file_path) From 9c2620964e46cc9d3e31f3cdea329e44ebbb86fb Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 May 2026 14:00:49 +0200 Subject: [PATCH 097/110] [3.15] gh-150034: Print JSONL filename when profiling finishes (GH-150035) (#150151) --- Lib/profiling/sampling/jsonl_collector.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/profiling/sampling/jsonl_collector.py b/Lib/profiling/sampling/jsonl_collector.py index 7d26129b80de86..5aa42ef09024dc 100644 --- a/Lib/profiling/sampling/jsonl_collector.py +++ b/Lib/profiling/sampling/jsonl_collector.py @@ -164,6 +164,7 @@ def export(self, filename): self._iter_final_agg_entries(), ) self._write_message(output, self._build_end_record()) + print(f"JSONL profile written to {filename}") def _build_meta_record(self): record = { From 61444f60a1d82f8458d078c600405dff7256f196 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 May 2026 15:55:30 +0200 Subject: [PATCH 098/110] [3.15] gh-150146: Fix NULL dereference in `_Py_subs_parameters` (GH-150147) (#150153) gh-150146: Fix NULL dereference in `_Py_subs_parameters` (GH-150147) (cherry picked from commit f621ba16b72510e1abc9646a844a632df4ac275c) Co-authored-by: sobolevn --- Lib/test/test_genericalias.py | 13 ++++++++++--- .../2026-05-20-13-06-17.gh-issue-150146.i5m_SL.rst | 5 +++++ Objects/genericaliasobject.c | 3 +++ 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-05-20-13-06-17.gh-issue-150146.i5m_SL.rst diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index a5969b7a47d948..7816775620bc01 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -55,15 +55,14 @@ from unittest.case import _AssertRaisesContext from queue import Queue, SimpleQueue from weakref import WeakSet, ReferenceType, ref -import typing -from typing import Unpack try: from tkinter import Event except ImportError: Event = None from string.templatelib import Template, Interpolation -from typing import TypeVar +import typing +from typing import TypeVar, Unpack T = TypeVar('T') K = TypeVar('K') V = TypeVar('V') @@ -621,6 +620,14 @@ def test_nested_paramspec_specialization(self): self.assertEqual(deeply_nested_specialized.__args__, ([str, [float], int], float)) self.assertEqual(deeply_nested_specialized.__parameters__, ()) + def test_gh150146(self): + # It used to crash: + for container in [memoryview, list, tuple]: + with self.subTest(container=container): + x = container[TypeVar("")] + with self.assertRaises(TypeError): + x[*typing.Mapping[..., ...]] + class TypeIterationTests(unittest.TestCase): _UNITERABLE_TYPES = (list, tuple) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-20-13-06-17.gh-issue-150146.i5m_SL.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-20-13-06-17.gh-issue-150146.i5m_SL.rst new file mode 100644 index 00000000000000..f373f0bee7023e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-20-13-06-17.gh-issue-150146.i5m_SL.rst @@ -0,0 +1,5 @@ +Fix a crash on a complex type variable substitution. + +``from typing import TypeVar; memoryview[TypeVar("")][*typing.Mapping[..., +...]]`` used to fail due to missing ``NULL`` check on ``_unpack_args`` C +function call. diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index e3bc8eb2739e3f..9c797e8dd6fd2c 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -412,6 +412,9 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje self); } item = _unpack_args(item); + if (item == NULL) { + return NULL; + } for (Py_ssize_t i = 0; i < nparams; i++) { PyObject *param = PyTuple_GET_ITEM(parameters, i); PyObject *prepare, *tmp; From ad2f0cb997a3711ae3a729cfc453d8004ba1a34e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 May 2026 17:17:28 +0200 Subject: [PATCH 099/110] [3.15] PEP 810 - Update some error strings (GH-150126) (#150135) --- Objects/lazyimportobject.c | 2 +- Python/import.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index 451f335e033f16..fa1eb25047d961 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -135,7 +135,7 @@ PyDoc_STRVAR(lazy_import_doc, "lazy_import(builtins, name, fromlist=None, /)\n" "--\n" "\n" -"Represents a deferred import that will be resolved on first use.\n" +"Represents a lazy import that will be resolved on first use.\n" "\n" "Instances of this object accessed from the global scope will be\n" "automatically imported based upon their name and then replaced with\n" diff --git a/Python/import.c b/Python/import.c index ef6f5274a23665..352941a836ef21 100644 --- a/Python/import.c +++ b/Python/import.c @@ -4055,7 +4055,7 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) // Create a cause exception showing where the lazy import was declared. PyObject *msg = PyUnicode_FromFormat( - "deferred import of '%U' raised an exception during resolution", + "lazy import of '%U' raised an exception during resolution", import_name ); Py_DECREF(import_name); // Done with import_name. From fb6984f305db00aa2dad623cf7b1236eddbd0387 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 May 2026 21:48:04 +0200 Subject: [PATCH 100/110] [3.15] gh-143387: Add news blurb for importlib.metadata highlighting the change. (GH-150169) (#150171) gh-143387: Add news blurb for importlib.metadata highlighting the change. (GH-150169) (cherry picked from commit cb3b4b98d8d141c9de0462a0fa7e227a2104c1c7) Co-authored-by: Jason R. Coombs --- Doc/whatsnew/3.15.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 9bef7aa61d23cd..1670f033401f2b 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1208,6 +1208,19 @@ http.server (Contributed by Anton I. Sipos in :gh:`135057`.) +importlib.metadata +------------------ + +* Previously, when accessing a distribution metadata directory not + containing a metadata file, ``metadata()`` and ``Distribution.metadata()`` + would return an empty ``PackageMetadata`` object as if the file + was present but empty. Now, a ``MetadataNotFound`` exception is raised. + See `importlib_metadata#493 `_ + for background and rationale and and :gh:`143387` for rationale on the + compatibility concerns. + (Contributed by Jason R. Coombs.) + + inspect ------- From ca0da94f09d22794da6c5c8f8cdc622cf5536965 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 May 2026 21:49:13 +0200 Subject: [PATCH 101/110] [3.15] gh-134261: ZipFile - Don't rely on local time for reproducible builds & tests (GH-134264) (#150137) gh-134261: ZipFile - Don't rely on local time for reproducible builds & tests (GH-134264) --------- (cherry picked from commit 9dcf94e906906ff39c7955227c2b044b515ee162) Co-authored-by: Caleb <23644849+ctrlaltf2@users.noreply.github.com> Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Emma Smith Co-authored-by: Jason R. Coombs --- Lib/test/test_zipfile/test_core.py | 17 +++++++++-------- Lib/zipfile/__init__.py | 9 ++++++--- ...25-05-19-21-08-25.gh-issue-134261.ravGYm.rst | 1 + 3 files changed, 16 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-19-21-08-25.gh-issue-134261.ravGYm.rst diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 0d407371f40a0f..30550263ad50aa 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -1886,11 +1886,8 @@ def test_write_with_source_date_epoch(self): with zipfile.ZipFile(TESTFN, "r") as zf: zip_info = zf.getinfo("test_source_date_epoch.txt") - get_time = time.localtime(int(os.environ['SOURCE_DATE_EPOCH']))[:6] - # Compare each element of the date_time tuple - # Allow for a 1-second difference - for z_time, g_time in zip(zip_info.date_time, get_time): - self.assertAlmostEqual(z_time, g_time, delta=1) + expected_utc = (2025, 1, 1, 7, 19, 58) + self.assertEqual(zip_info.date_time, expected_utc) def test_write_without_source_date_epoch(self): with os_helper.EnvironmentVarGuard() as env: @@ -1901,9 +1898,13 @@ def test_write_without_source_date_epoch(self): with zipfile.ZipFile(TESTFN, "r") as zf: zip_info = zf.getinfo("test_no_source_date_epoch.txt") - current_time = time.localtime()[:6] - for z_time, c_time in zip(zip_info.date_time, current_time): - self.assertAlmostEqual(z_time, c_time, delta=2) + self.assertTimestampAlmostEqual(time.localtime(), zip_info.date_time, tolerance=2) + + def assertTimestampAlmostEqual(self, time1, time2, tolerance): + import datetime + dt1 = datetime.datetime(*time1[:6]) + dt2 = datetime.datetime(*time2[:6]) + self.assertLessEqual((dt1 - dt2).total_seconds(), tolerance) def test_close(self): """Check that the zipfile is closed after the 'with' block.""" diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 86c3bc36b695c7..c5c6ac03fb7b8c 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -663,9 +663,12 @@ def _for_archive(self, archive): Return self. """ # gh-91279: Set the SOURCE_DATE_EPOCH to a specific timestamp - epoch = os.environ.get('SOURCE_DATE_EPOCH') - get_time = int(epoch) if epoch else time.time() - self.date_time = time.localtime(get_time)[:6] + source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH') + + if source_date_epoch: + self.date_time = time.gmtime(int(source_date_epoch))[:6] + else: + self.date_time = time.localtime(time.time())[:6] self.compress_type = archive.compression self.compress_level = archive.compresslevel diff --git a/Misc/NEWS.d/next/Library/2025-05-19-21-08-25.gh-issue-134261.ravGYm.rst b/Misc/NEWS.d/next/Library/2025-05-19-21-08-25.gh-issue-134261.ravGYm.rst new file mode 100644 index 00000000000000..bf552fee814acb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-19-21-08-25.gh-issue-134261.ravGYm.rst @@ -0,0 +1 @@ +zip: On reproducible builds, ZipFile uses UTC instead of the local time when writing file datetimes to avoid underflows. From dea552c1b67192020d8be7f47d6e8ef974100ed2 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 20 May 2026 22:10:21 +0200 Subject: [PATCH 102/110] [3.15] Add summary table to the `unicodedata` doc (GH-149957) (#150161) (cherry picked from commit 87a879f4d0ec2e545e84c898c5ce452a6c87b09e) Co-authored-by: Stan Ulbrych --- Doc/library/unicodedata.rst | 43 ++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index f5c11fd849f58b..25bf872e0ab55a 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -18,8 +18,7 @@ this database is compiled from the `UCD version 17.0.0 The module uses the same names and symbols as defined by Unicode Standard Annex #44, `"Unicode Character Database" -`_. It defines the -following functions: +`_. .. seealso:: @@ -27,6 +26,44 @@ following functions: this module. +============================================================ =========================================================== +**Lookup** +------------------------------------------------------------------------------------------------------------------------- +:func:`lookup(name) ` Look up character by name +:func:`name(chr) ` Return the name assigned to a character + +**Numeric values** +------------------------------------------------------------------------------------------------------------------------- +:func:`decimal(chr) ` Decimal value of a character +:func:`digit(chr) ` Digit value of a character +:func:`numeric(chr) ` Numeric value of a character + +**Properties** +------------------------------------------------------------------------------------------------------------------------- +:func:`bidirectional(chr) ` Bidirectional class of a character +:func:`block(chr) ` Unicode block of a character +:func:`category(chr) ` General category of a character +:func:`combining(chr) ` Canonical combining class of a character +:func:`decomposition(chr) ` Character decomposition mapping +:func:`east_asian_width(chr) ` East Asian width of a character +:func:`extended_pictographic(chr) ` Check if a character has the Extended_Pictographic property +:func:`grapheme_cluster_break(chr) ` Grapheme_Cluster_Break property of a character +:func:`indic_conjunct_break(chr) ` Indic_Conjunct_Break property of a character +:func:`isxidcontinue(chr) ` Check if a character is a valid identifier continuation +:func:`isxidstart(chr) ` Check if a character is a valid identifier start +:func:`mirrored(chr) ` Mirrored property of a character + +**Normalization** +------------------------------------------------------------------------------------------------------------------------- +:func:`normalize(form, unistr) ` Return the normalized form of a string +:func:`is_normalized(form, unistr) ` Check if a Unicode string is normalized + +**Text segmentation** +------------------------------------------------------------------------------------------------------------------------- +:func:`iter_graphemes(unistr) ` Iterate over grapheme clusters in a string +============================================================ =========================================================== + + .. function:: lookup(name, /) Look up character by name. If a character with the given name is found, return @@ -273,7 +310,7 @@ following functions: .. versionadded:: 3.15 -In addition, the module exposes the following constant: +In addition, the module exposes the following constants: .. data:: unidata_version From a56a27100084c3011667ded3e93d552b4020ca68 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Thu, 21 May 2026 12:53:00 +0300 Subject: [PATCH 103/110] [3.15] gh-149981: Test lazy import corner cases with module-level `__getattr__` (GH-149982) (#150185) (cherry picked from commit 6dbf4ba403cd38d0219d3c7514f61c2ac8f6a74f) --- Lib/test/test_lazy_import/__init__.py | 106 ++++++++++++++++++ .../data/module_with_getattr.py | 8 ++ .../test_lazy_import/data/pkg/__init__.py | 8 ++ 3 files changed, 122 insertions(+) diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index 9f2cc92bcfcc78..4340efd31095ea 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -97,6 +97,59 @@ def test_from_import_with_module_getattr(self): """) assert_python_ok("-c", code) + @support.requires_subprocess() + def test_from_import_with_module_getattr_raising(self): + """Lazy from import should respect module-level __getattr__.""" + code = textwrap.dedent(""" + lazy from test.test_lazy_import.data.module_with_getattr import raising_attr + + try: + raising_attr + except ValueError as exc: + assert str(exc) == 'from_getattr', exc + else: + assert False, f'ValueError is not raised: {raising_attr}' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_module_getattr_missing(self): + """Lazy from import should respect module-level __getattr__.""" + for attr in ("missing_attr", "import_error_attr"): + with self.subTest(attr=attr): + code = textwrap.dedent(f""" + lazy from test.test_lazy_import.data.module_with_getattr import {attr} + + try: + {attr} + except ImportError as exc: + assert '{attr}' in str(exc), exc + assert exc.__cause__ is not None + else: + assert False, ('ImportError is not raised', {attr}) + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_module_getattr_warning(self): + """Lazy from import should respect module-level __getattr__.""" + code = textwrap.dedent(""" + import warnings + + with warnings.catch_warnings(record=True) as log: + lazy from test.test_lazy_import.data.module_with_getattr import warning_attr + + assert log == [] + + with warnings.catch_warnings(record=True) as log: + warning_attr + assert warning_attr == 'from_warning_attr', warning_attr + assert len(log) == 1, log + assert isinstance(log[0].message, UserWarning), log + assert str(log[0].message) == 'from_getattr', log + """) + assert_python_ok("-c", code) + @support.requires_subprocess() def test_from_import_with_imported_module_getattr(self): """Lazy from import should not shadow an imported module's __getattr__.""" @@ -482,6 +535,59 @@ def test_lazy_from_import_does_not_pollute_parent(self): """) assert_python_ok("-c", code) + @support.requires_subprocess() + def test_package_from_import_with_module_getattr_raising(self): + """Lazy from import should respect a package's __getattr__.""" + code = textwrap.dedent(""" + lazy from test.test_lazy_import.data.pkg import raising_attr + + try: + raising_attr + except ValueError as exc: + assert str(exc) == 'from_getattr', exc + else: + assert False, f'ValueError is not raised: {raising_attr}' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_package_from_import_with_module_getattr_missing(self): + """Lazy from import should respect package's __getattr__.""" + for attr in ("missing_attr", "import_error_attr"): + with self.subTest(attr=attr): + code = textwrap.dedent(f""" + lazy from test.test_lazy_import.data.pkg import {attr} + + try: + {attr} + except ImportError as exc: + assert '{attr}' in str(exc), exc + assert exc.__cause__ is not None + else: + assert False, ('ImportError is not raised', {attr}) + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_module_getattr_warning(self): + """Lazy from import should respect package's __getattr__.""" + code = textwrap.dedent(""" + import warnings + + with warnings.catch_warnings(record=True) as log: + lazy from test.test_lazy_import.data.pkg import warning_attr + + assert log == [] + + with warnings.catch_warnings(record=True) as log: + warning_attr + assert warning_attr == 'from_warning_attr', warning_attr + assert len(log) == 1, log + assert isinstance(log[0].message, UserWarning), log + assert str(log[0].message) == 'from_getattr', log + """) + assert_python_ok("-c", code) + @support.requires_subprocess() def test_package_from_import_with_module_getattr(self): """Lazy from import should respect a package's __getattr__.""" diff --git a/Lib/test/test_lazy_import/data/module_with_getattr.py b/Lib/test/test_lazy_import/data/module_with_getattr.py index 2ac01a90d76e62..db3a2301075c2e 100644 --- a/Lib/test/test_lazy_import/data/module_with_getattr.py +++ b/Lib/test/test_lazy_import/data/module_with_getattr.py @@ -1,4 +1,12 @@ def __getattr__(name): if name == "dynamic_attr": return "from_getattr" + elif name == "raising_attr": + raise ValueError("from_getattr") + elif name == "import_error_attr": + raise ImportError(name) + elif name == "warning_attr": + import warnings + warnings.warn("from_getattr", category=UserWarning) + return "from_warning_attr" raise AttributeError(name) diff --git a/Lib/test/test_lazy_import/data/pkg/__init__.py b/Lib/test/test_lazy_import/data/pkg/__init__.py index e526aab94131b8..5f7b8662596cac 100644 --- a/Lib/test/test_lazy_import/data/pkg/__init__.py +++ b/Lib/test/test_lazy_import/data/pkg/__init__.py @@ -3,4 +3,12 @@ def __getattr__(name): if name == "dynamic_attr": return "from_getattr" + elif name == "raising_attr": + raise ValueError("from_getattr") + elif name == "import_error_attr": + raise ImportError(name) + elif name == "warning_attr": + import warnings + warnings.warn("from_getattr", category=UserWarning) + return "from_warning_attr" raise AttributeError(name) From 9d042ad9aec4fce19f87f7a7729c0a4a85ee4967 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 21 May 2026 17:22:24 +0200 Subject: [PATCH 104/110] [3.15] gh-149219: Test `frozendict` in `Lib/test/test_crossinterp.py` (GH-149220) (#150186) gh-149219: Test `frozendict` in `Lib/test/test_crossinterp.py` (GH-149220) (cherry picked from commit c35b0f2b624ecc4d649a808acdb07a7fbcea60ac) Co-authored-by: sobolevn --- Lib/test/test_crossinterp.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_crossinterp.py b/Lib/test/test_crossinterp.py index 4e536211168747..f4bf5a66ad2155 100644 --- a/Lib/test/test_crossinterp.py +++ b/Lib/test/test_crossinterp.py @@ -157,6 +157,10 @@ def ignore_byteswarning(): {}, {1: 7, 2: 8, 3: 9}, {1: [1], 2: (2,), 3: {3: 4}}, + # frozendict + frozendict(), + frozendict({1: 7, 2: 8, 3: 9}), + frozendict({1: [1], 2: (2,), 3: {3: 4}, 4: frozendict({5: 6})}), # set set(), {1, 2, 3}, From e6e3b9f490b6a24dcbb461418f214e54431191f0 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 21 May 2026 22:02:10 +0200 Subject: [PATCH 105/110] [3.15] gh-107398: Fix tarfile stream mode exception when process the file with the gzip extra field (GH-126304) (GH-150199) (cherry picked from commit 65f99329edf5d0df3ee14d9a242e1a4c8b842211) Co-authored-by: Nadeshiko Manju Co-authored-by: Serhiy Storchaka --- Lib/tarfile.py | 2 +- Lib/test/test_tarfile.py | 33 +++++++++++++++++-- ...-11-02-02-02-31.gh-issue-107398.uUtA6Q.rst | 1 + 3 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-02-02-02-31.gh-issue-107398.uUtA6Q.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 1394a26f2096ff..5e43b4c19c0a8a 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -498,7 +498,7 @@ def _init_read_gz(self): if flag & 4: xlen = ord(self.__read(1)) + 256 * ord(self.__read(1)) - self.read(xlen) + self.__read(xlen) if flag & 8: while True: s = self.__read(1) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 192c948edc6056..4be207e8cbf4e6 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -893,10 +893,39 @@ def test_extractall_hardlink_on_symlink(self): self._assert_on_file_content(hardlink_filepath, sha256_regtype) +class GzipReadTestBase: + + def test_read_with_extra_field(self): + with open(self.tarname, 'rb') as f: + data = bytearray(f.read()) + flags = data[3] + self.assertEqual(flags, 8) + data[3] = flags | 4 + data[10:10] = b'\x05\x00extra' + with open(tmpname, 'wb') as f: + f.write(data) + print(self.mode) + with tarfile.open(tmpname, mode=self.mode): + pass + + def test_read_with_file_comment(self): + with open(self.tarname, 'rb') as f: + data = bytearray(f.read()) + flags = data[3] + self.assertEqual(flags, 8) + data[3] = flags | 16 + i = data.index(0, 10) + 1 + data[i:i] = b'comment\x00' + with open(tmpname, 'wb') as f: + f.write(data) + with tarfile.open(tmpname, mode=self.mode): + pass + + class MiscReadTest(MiscReadTestBase, unittest.TestCase): test_fail_comp = None -class GzipMiscReadTest(GzipTest, MiscReadTestBase, unittest.TestCase): +class GzipMiscReadTest(GzipTest, GzipReadTestBase, MiscReadTestBase, unittest.TestCase): pass class Bz2MiscReadTest(Bz2Test, MiscReadTestBase, unittest.TestCase): @@ -970,7 +999,7 @@ def test_compare_members(self): finally: tar1.close() -class GzipStreamReadTest(GzipTest, StreamReadTest): +class GzipStreamReadTest(GzipTest, GzipReadTestBase, StreamReadTest): pass class Bz2StreamReadTest(Bz2Test, StreamReadTest): diff --git a/Misc/NEWS.d/next/Library/2024-11-02-02-02-31.gh-issue-107398.uUtA6Q.rst b/Misc/NEWS.d/next/Library/2024-11-02-02-02-31.gh-issue-107398.uUtA6Q.rst new file mode 100644 index 00000000000000..d5af322d68d309 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-02-02-02-31.gh-issue-107398.uUtA6Q.rst @@ -0,0 +1 @@ +Fix :mod:`tarfile` stream mode exception when process the file with the gzip extra field. From 233cf75d6db0d8a915114c4cc4f8182afe510ed1 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 22 May 2026 00:31:41 +0200 Subject: [PATCH 106/110] [3.15] gh-149819: fix .pth and .start file processing in subprocess when inheriting PYTHONPATH (GH-150177) (#150202) gh-149819: fix .pth and .start file processing in subprocess when inheriting PYTHONPATH (GH-150177) * gh-149819: Fix .pth files not loaded in Python subprocesses After PR gh-149583 (Fix double evaluation of .pth and .site files in venvs), .pth files are no longer loaded in subprocesses started with subprocess.run([sys.executable, ...]). The root cause: main() seeds known_paths from removeduppaths() with all sys.path entries inherited from the parent process. addsitedir() then skips .pth processing for every directory already in known_paths. Fix: - main(): call removeduppaths() for dedup but start known_paths as a fresh empty set, so that addsitedir() processes .pth files in every site-packages directory regardless of inherited sys.path. - addsitedir(): move known_paths.add() before the sys.path.append and guard the append with 'sitedir not in sys.path' to avoid creating duplicate entries when called with a fresh known_paths. This preserves the gh-75723 dedup guarantee while allowing subprocesses to load .pth files. * Fill out the tests for GH#149888 * Extend _make_start() and _make_pth() to take an optional `basedir` which is used instead of `site.tmpdir` if given. * Add test_pth_processed_when_sitedir_already_on_path() to test the core GH#149819 bug: .pth files in subprocesses aren't handled if PYTHONPATH pointing to the .pth directory is inherited. * Similarly add test_start_processed_when_sitedir_already_on_path() to verify that .start files in the same circumstances are also now processed. * Update Lib/site.py * Oops! Remove redundant code --------- (cherry picked from commit 3c298e2e385fc6f462abaada2fd680deb1a2b58e) Co-authored-by: Barry Warsaw Co-authored-by: BugBounty Mind Co-authored-by: scoder --- Lib/site.py | 17 ++-- Lib/test/test_site.py | 98 +++++++++++++++++-- ...-05-15-16-28-00.gh-issue-149819.fixpth.rst | 4 + 3 files changed, 105 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-15-16-28-00.gh-issue-149819.fixpth.rst diff --git a/Lib/site.py b/Lib/site.py index 64e8192a9ac81a..239ee0d6f57bce 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -490,13 +490,16 @@ def addsitedir(sitedir, known_paths=None, *, defer_processing_start_files=False) reset = False sitedir, sitedircase = makepath(sitedir) - # If the normcase'd new sitedir isn't already known, append it to - # sys.path, keep a record of it, and process all .pth and .start files - # found in that directory. If the new sitedir is known, be sure not - # to process all of those more than once! gh-75723 + # If the normcase'd new sitedir isn't already known, record it to + # prevent re-processing, append it to sys.path (only if not already + # present), and process all .pth and .start files found in that + # directory. Use a direct sys.path membership check for the append + # guard so that callers (like main()) can pass a fresh known_paths + # set while avoiding duplicate sys.path entries (gh-149819). if sitedircase not in known_paths: - sys.path.append(sitedir) known_paths.add(sitedircase) + if sitedir not in sys.path: + sys.path.append(sitedir) try: names = os.listdir(sitedir) @@ -1000,13 +1003,13 @@ def main(): global ENABLE_USER_SITE orig_path = sys.path[:] - known_paths = removeduppaths() + removeduppaths() if orig_path != sys.path: # removeduppaths() might make sys.path absolute. # Fix __file__ of already imported modules too. abs_paths() - known_paths = venv(known_paths) + known_paths = venv(known_paths=set()) if ENABLE_USER_SITE is None: ENABLE_USER_SITE = check_enableusersite() known_paths = addusersitepackages(known_paths, defer_processing_start_files=True) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 0e6f352f49cd38..e2a81b82321ede 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -456,6 +456,7 @@ def cleanup(self, prep=False): if os.path.exists(self.bad_dir_path): os.rmdir(self.bad_dir_path) + class ImportSideEffectTests(unittest.TestCase): """Test side-effects from importing 'site'.""" @@ -545,7 +546,6 @@ def test_customization_modules_on_startup(self): output = subprocess.check_output([sys.executable, '-s', '-c', '""']) self.assertNotIn(eyecatcher, output.decode('utf-8')) - @unittest.skipUnless(hasattr(urllib.request, "HTTPSHandler"), 'need SSL support to download license') @test.support.requires_resource('network') @@ -926,18 +926,28 @@ def setUp(self): def _reset_startup_state(self): site._startup_state = None - def _make_start(self, content, name='testpkg'): - """Write a .start file and return its basename.""" + def _make_start(self, content, name='testpkg', basedir=None): + """Write a .start file and return its basename. + + ``basedir`` defaults to ``self.tmpdir``. Pass an explicit directory + when the .start file needs to live somewhere other than the test's + primary tmpdir (e.g. a nested user-site). + """ basename = f"{name}.start" - filepath = os.path.join(self.tmpdir, basename) + filepath = os.path.join(self.tmpdir if basedir is None else basedir, basename) with open(filepath, 'w', encoding='utf-8') as f: f.write(content) return basename - def _make_pth(self, content, name='testpkg'): - """Write a .pth file and return its basename.""" + def _make_pth(self, content, name='testpkg', basedir=None): + """Write a .pth file and return its basename. + + ``basedir`` defaults to ``self.tmpdir``. Pass an explicit directory + when the .pth file needs to live somewhere other than the test's + primary tmpdir (e.g. a nested user-site). + """ basename = f"{name}.pth" - filepath = os.path.join(self.tmpdir, basename) + filepath = os.path.join(self.tmpdir if basedir is None else basedir, basename) with open(filepath, 'w', encoding='utf-8') as f: f.write(content) return basename @@ -1640,6 +1650,80 @@ def bootstrap(): self.assertIn(overlay, sys.path) self.assertIn(pkgdir, sys.path) + # gh-149819 + @unittest.skipUnless(site.ENABLE_USER_SITE, "requires user-site") + @support.requires_subprocess() + def test_pth_processed_when_sitedir_already_on_path(self): + # A .pth file in a site-packages directory must still be processed by + # site.main() when that directory is already on sys.path at + # interpreter start up, for example in a subprocess that inherits + # PYTHONPATH from its parent. Before the fix, main() seeded + # known_paths with all entries derived from removeduppaths(), and + # addsitedir() then skipped .pth processing for any directory already + # in known_paths. + user_base = self.tmpdir + user_site = site._get_path(user_base) + os.makedirs(user_site) + sentinel = "GH149819_PTH_RAN" + # Writing some text to stderr is the simplest observable side effect. + self._make_pth(f"""\ +import sys; sys.stderr.write({sentinel!r}); sys.stderr.flush() +""", + name='gh149819', + basedir=user_site) + with EnvironmentVarGuard() as env: + # PYTHONUSERBASE points USER_SITE at our temp directory so + # site.main() will call addsitedir() on it, rather than on the + # host interpreter's real user-site. + env['PYTHONUSERBASE'] = user_base + # PYTHONPATH puts that same directory on sys.path before + # site.main() runs in the subprocess. This is what triggers the + # bug: removeduppaths() records it in known_paths, and the unfixed + # addsitedir() then skips .pth processing. + env['PYTHONPATH'] = user_site + result = subprocess.run( + [sys.executable, '-c', ''], + capture_output=True, + check=True, + ) + self.assertIn(sentinel.encode(), result.stderr) + + @unittest.skipUnless(site.ENABLE_USER_SITE, "requires user-site") + @support.requires_subprocess() + def test_start_processed_when_sitedir_already_on_path(self): + # Companion to test_pth_processed_when_sitedir_already_on_path: + # the same dedup-guard skip in addsitedir() suppressed both .pth + # and .start file processing, so verify .start entry points also + # run for a site-packages directory inherited via PYTHONPATH. + user_base = self.tmpdir + user_site = site._get_path(user_base) + os.makedirs(user_site) + sentinel = "GH149819_START_RAN" + # The .start entry point resolves to a callable, so we write a + # tiny importable module that outputs the sentinel text. It lands in + # /extdir. That path is added to PYTHONPATH below so + # the subprocess can import it. + extdir = self._make_mod(f"""\ +import sys +def run(): + sys.stderr.write({sentinel!r}) + sys.stderr.flush() +""", name='gh149819mod') + self._make_start( + 'gh149819mod:run\n', name='gh149819', basedir=user_site + ) + with EnvironmentVarGuard() as env: + # See above for details. + env['PYTHONUSERBASE'] = user_base + env['PYTHONPATH'] = os.pathsep.join([user_site, extdir]) + result = subprocess.run( + [sys.executable, '-c', ''], + capture_output=True, + check=True, + ) + self.assertIn(sentinel.encode(), result.stderr) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2026-05-15-16-28-00.gh-issue-149819.fixpth.rst b/Misc/NEWS.d/next/Library/2026-05-15-16-28-00.gh-issue-149819.fixpth.rst new file mode 100644 index 00000000000000..66e6da0ecf0d87 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-15-16-28-00.gh-issue-149819.fixpth.rst @@ -0,0 +1,4 @@ +Fix regression in :func:`site.addsitedir` where ``.pth`` files were no +longer processed in Python subprocesses. This happened because +:func:`site.main` seeded ``known_paths`` with entries inherited from +the parent process, causing ``addsitedir`` to skip ``.pth`` processing. From 072246a7803246f562bea6d099d2c3640d59db96 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 22 May 2026 04:18:28 +0200 Subject: [PATCH 107/110] [3.15] gh-148294: Make configure find g++ correctly (GH-150212) The `AC_PATH_TOOL` calls had included a duplicated argument, causing a "`PATH`" consisting of `notfound` to be searched instead of `$PATH`. (cherry picked from commit c613f72eeef83340cb369287f7c1a195e086d1d5) Co-authored-by: sendaoYan --- ...-05-21-15-14-59.gh-issue-148294.VtFaW4.rst | 2 ++ configure | 24 +++++++++---------- configure.ac | 8 +++---- 3 files changed, 18 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2026-05-21-15-14-59.gh-issue-148294.VtFaW4.rst diff --git a/Misc/NEWS.d/next/Build/2026-05-21-15-14-59.gh-issue-148294.VtFaW4.rst b/Misc/NEWS.d/next/Build/2026-05-21-15-14-59.gh-issue-148294.VtFaW4.rst new file mode 100644 index 00000000000000..861261dd97269f --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-05-21-15-14-59.gh-issue-148294.VtFaW4.rst @@ -0,0 +1,2 @@ +Corrected the use of ``AC_PATH_TOOL`` in ``configure.ac`` to allow a C++ +compiler to be found on :envvar:`!PATH`. diff --git a/configure b/configure index acba294d55de8f..9ad2171460f7ac 100755 --- a/configure +++ b/configure @@ -6645,7 +6645,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6694,7 +6694,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6726,7 +6726,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="g++" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -6755,7 +6755,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6804,7 +6804,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6836,7 +6836,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="c++" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -6865,7 +6865,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6914,7 +6914,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6946,7 +6946,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="clang++" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -6975,7 +6975,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -7024,7 +7024,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -7056,7 +7056,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="icpc" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) diff --git a/configure.ac b/configure.ac index d909c2fc92894e..a51e173e5293f2 100644 --- a/configure.ac +++ b/configure.ac @@ -1137,10 +1137,10 @@ preset_cxx="$CXX" if test -z "$CXX" then case "$ac_cv_cc_name" in - gcc) AC_PATH_TOOL([CXX], [g++], [g++], [notfound]) ;; - cc) AC_PATH_TOOL([CXX], [c++], [c++], [notfound]) ;; - clang) AC_PATH_TOOL([CXX], [clang++], [clang++], [notfound]) ;; - icc) AC_PATH_TOOL([CXX], [icpc], [icpc], [notfound]) ;; + gcc) AC_PATH_TOOL([CXX], [g++], [notfound]) ;; + cc) AC_PATH_TOOL([CXX], [c++], [notfound]) ;; + clang) AC_PATH_TOOL([CXX], [clang++], [notfound]) ;; + icc) AC_PATH_TOOL([CXX], [icpc], [notfound]) ;; esac if test "$CXX" = "notfound" then From 591c4ffdd93ff4e89c61f4b9f84fd68a1f36d8db Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 22 May 2026 08:41:43 +0200 Subject: [PATCH 108/110] [3.15] gh-149995: Update typing.py docstrings and documentation (GH-149996) (#150215) gh-149995: Update typing.py docstrings and documentation (GH-149996) Some of these docstrings read as if they were written when typing.py was first written, and things have evolved since then. A few motivations: - Call protocols protocols instead of ABCs. They are also ABCs, but the fact they are protocols is more relevant to typing. - Avoid recommending direct use of .__annotations__ and steer users to annotationlib instead. - For TypedDict, mention NotRequired before total=False since it is more general and probably more frequently useful. - For overloads, mention runtime use first instead of stub use. I think early on there was talk of allowing overload only in stubs, but it is now heavily used at runtime too and that's more likely to be relevant to users. (cherry picked from commit f159419ae2ef1aebbd90ce9427b55e27738c960c) Co-authored-by: Jelle Zijlstra --- Doc/library/typing.rst | 46 +++---- Lib/typing.py | 115 +++++++++--------- ...-05-18-07-44-46.gh-issue-149995.vvtFHn.rst | 1 + 3 files changed, 81 insertions(+), 81 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-18-07-44-46.gh-issue-149995.vvtFHn.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 71b395c80166cc..b2167cbc63a1ff 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -719,8 +719,8 @@ The :data:`Any` type ==================== A special kind of type is :data:`Any`. A static type checker will treat -every type as being compatible with :data:`Any` and :data:`Any` as being -compatible with every type. +every type as assignable to :data:`Any` and :data:`Any` as assignable to +every type. This means that it is possible to perform any operation or method call on a value of type :data:`Any` and assign it to any variable:: @@ -785,7 +785,7 @@ it as a return value) of a more specialized type is a type error. For example:: hash_a(42) hash_a("foo") - # Passes type checking, since Any is compatible with all types + # Passes type checking, since Any is assignable to all types hash_b(42) hash_b("foo") @@ -851,8 +851,8 @@ using ``[]``. Special type indicating an unconstrained type. - * Every type is compatible with :data:`Any`. - * :data:`Any` is compatible with every type. + * Every type is assignable to :data:`Any`. + * :data:`Any` is assignable to every type. .. versionchanged:: 3.11 :data:`Any` can now be used as a base class. This can be useful for @@ -1292,10 +1292,10 @@ These can be used as types in annotations. They all support subscription using :data:`ClassVar` accepts only types and cannot be further subscribed. - :data:`ClassVar` is not a class itself, and should not + :data:`ClassVar` is not a class itself, and cannot be used with :func:`isinstance` or :func:`issubclass`. :data:`ClassVar` does not change Python runtime behavior, but - it can be used by third-party type checkers. For example, a type checker + it can be used by static type checkers. For example, a type checker might flag the following code as an error:: enterprise_d = Starship(3000) @@ -1365,7 +1365,7 @@ These can be used as types in annotations. They all support subscription using def mutate_movie(m: Movie) -> None: m["year"] = 1999 # allowed - m["title"] = "The Matrix" # typechecker error + m["title"] = "The Matrix" # type checker error There is no runtime checking for this property. @@ -2472,9 +2472,9 @@ types. Fields with a default value must come after any fields without a default. - The resulting class has an extra attribute ``__annotations__`` giving a - dict that maps the field names to the field types. (The field names are in - the ``_fields`` attribute and the default values are in the + The types for each field name can be retrieved by calling + :func:`annotationlib.get_annotations` on the resulting class. (The field + names are in the ``_fields`` attribute and the default values are in the ``_field_defaults`` attribute, both of which are part of the :func:`~collections.namedtuple` API.) @@ -2535,7 +2535,7 @@ types. Helper class to create low-overhead :ref:`distinct types `. - A ``NewType`` is considered a distinct type by a typechecker. At runtime, + A ``NewType`` is considered a distinct type by a type checker. At runtime, however, calling a ``NewType`` returns its argument unchanged. Usage:: @@ -2616,7 +2616,7 @@ types. Mark a protocol class as a runtime protocol. Such a protocol can be used with :func:`isinstance` and :func:`issubclass`. - This allows a simple-minded structural check, very similar to "one trick ponies" + This allows a simple-minded structural check, very similar to "one-trick ponies" in :mod:`collections.abc` such as :class:`~collections.abc.Iterable`. For example:: @runtime_checkable @@ -2855,7 +2855,7 @@ types. key: T group: list[T] - A ``TypedDict`` can be introspected via annotations dicts + A ``TypedDict`` can be introspected via :func:`annotationlib.get_annotations` (see :ref:`annotations-howto` for more information on annotations best practices) and the following attributes: @@ -2898,7 +2898,7 @@ types. For backwards compatibility with Python 3.10 and below, it is also possible to use inheritance to declare both required and - non-required keys in the same ``TypedDict`` . This is done by declaring a + non-required keys in the same ``TypedDict``. This is done by declaring a ``TypedDict`` with one value for the ``total`` argument and then inheriting from it in another ``TypedDict`` with a different value for ``total``: @@ -2982,34 +2982,34 @@ with :deco:`runtime_checkable`. .. class:: SupportsAbs - An ABC with one abstract method ``__abs__`` that is covariant + A protocol with one abstract method ``__abs__`` that is covariant in its return type. .. class:: SupportsBytes - An ABC with one abstract method ``__bytes__``. + A protocol with one abstract method ``__bytes__``. .. class:: SupportsComplex - An ABC with one abstract method ``__complex__``. + A protocol with one abstract method ``__complex__``. .. class:: SupportsFloat - An ABC with one abstract method ``__float__``. + A protocol with one abstract method ``__float__``. .. class:: SupportsIndex - An ABC with one abstract method ``__index__``. + A protocol with one abstract method ``__index__``. .. versionadded:: 3.8 .. class:: SupportsInt - An ABC with one abstract method ``__int__``. + A protocol with one abstract method ``__int__``. .. class:: SupportsRound - An ABC with one abstract method ``__round__`` + A protocol with one abstract method ``__round__`` that is covariant in its return type. .. _typing-io: @@ -3763,7 +3763,7 @@ Constant .. data:: TYPE_CHECKING - A special constant that is assumed to be ``True`` by 3rd party static + A special constant that is assumed to be ``True`` by static type checkers. It's ``False`` at runtime. A module which is expensive to import, and which only contain types diff --git a/Lib/typing.py b/Lib/typing.py index 5b1e223d59641e..bd1f6448894e8f 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -5,7 +5,7 @@ * Generic, Protocol, and internal machinery to support generic aliases. All subscripted types like X[int], Union[int, str] are generic aliases. * Various "special forms" that have unique meanings in type annotations: - NoReturn, Never, ClassVar, Self, Concatenate, Unpack, and others. + Any, Never, ClassVar, Self, Concatenate, Unpack, and others. * Classes whose instances can be type arguments to generic classes and functions: TypeVar, ParamSpec, TypeVarTuple. * Public helper functions: get_type_hints, overload, cast, final, and others. @@ -604,12 +604,12 @@ def __repr__(self): class Any(metaclass=_AnyMeta): """Special type indicating an unconstrained type. - - Any is compatible with every type. - - Any assumed to have all methods. - - All values assumed to be instances of Any. + - Any is assignable to every type. + - Any assumed to have all methods and attributes. + - All values are assignable to Any. Note that all the above statements are true from the point of view of - static type checkers. At runtime, Any should not be used with instance + static type checkers. At runtime, Any cannot be used with instance checks. """ @@ -728,7 +728,7 @@ class Starship: ClassVar accepts only types and cannot be further subscribed. - Note that ClassVar is not a class itself, and should not + Note that ClassVar is not a class itself, and cannot be used with isinstance() or issubclass(). """ item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) @@ -758,7 +758,7 @@ class FastConnector(Connection): @_SpecialForm def Optional(self, parameters): - """Optional[X] is equivalent to Union[X, None].""" + """Optional[X] is equivalent to X | None.""" arg = _type_check(parameters, f"{self} requires a single type.") return Union[arg, type(None)] @@ -801,7 +801,7 @@ def open_helper(file: str, mode: MODE) -> str: def TypeAlias(self, parameters): """Special form for marking type aliases. - Use TypeAlias to indicate that an assignment should + TypeAlias can be used to indicate that an assignment should be recognized as a proper type alias definition by type checkers. @@ -1809,7 +1809,7 @@ class Movie(TypedDict): def foo(**kwargs: Unpack[Movie]): ... Note that there is only some runtime checking of this operator. Not - everything the runtime allows may be accepted by static type checkers. + everything the runtime allows is accepted by static type checkers. For more information, see PEPs 646 and 692. """ @@ -2320,7 +2320,7 @@ def runtime_checkable(cls): Such protocol can be used with isinstance() and issubclass(). Raise TypeError if applied to a non-protocol class. This allows a simple-minded structural check very similar to - one trick ponies in collections.abc such as Iterable. + one-trick ponies in collections.abc such as Iterable. For example:: @@ -2390,8 +2390,8 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, *, format=None): """Return type hints for an object. - This is often the same as obj.__annotations__, but it handles - forward references encoded as string literals and recursively replaces all + This is often the same as annotationlib.get_annotations(obj) or obj.__annotations__, + but it handles forward references encoded as string literals and recursively replaces all 'Annotated[T, ...]' with 'T' (unless 'include_extras=True'). The argument may be a module, class, method, or function. The annotations @@ -2603,7 +2603,7 @@ def get_args(tp): def is_typeddict(tp): - """Check if an annotation is a TypedDict class. + """Check if an object is a TypedDict class. For example:: @@ -2700,10 +2700,10 @@ def _overload_dummy(*args, **kwds): def overload(func): """Decorator for overloaded functions/methods. - In a stub file, place two or more stub definitions for the same - function in a row, each decorated with @overload. - - For example:: + In a non-stub file, place two or more stub definitions for the same + function in a row, each decorated with @overload, followed + by an implementation. The implementation should *not* + be decorated with @overload:: @overload def utf8(value: None) -> None: ... @@ -2711,10 +2711,11 @@ def utf8(value: None) -> None: ... def utf8(value: bytes) -> bytes: ... @overload def utf8(value: str) -> bytes: ... + def utf8(value): + ... # implementation goes here - In a non-stub file (i.e. a regular .py file), do the same but - follow it with an implementation. The implementation should *not* - be decorated with @overload:: + In a stub file or in an abstract method (for example, in a Protocol definition), + the implementation may be omitted:: @overload def utf8(value: None) -> None: ... @@ -2722,8 +2723,6 @@ def utf8(value: None) -> None: ... def utf8(value: bytes) -> bytes: ... @overload def utf8(value: str) -> bytes: ... - def utf8(value): - ... # implementation goes here The overloads for a function can be retrieved at runtime using the get_overloads() function. @@ -2759,7 +2758,7 @@ def final(f): """Decorator to indicate final methods and final classes. Use this decorator to indicate to type checkers that the decorated - method cannot be overridden, and decorated class cannot be subclassed. + method cannot be overridden, and the decorated class cannot be subclassed. For example:: @@ -2824,7 +2823,7 @@ class Disjoint3(Disjoint1, Disjoint2): pass # Type checker error V_co = TypeVar('V_co', covariant=True) # Any type covariant containers. VT_co = TypeVar('VT_co', covariant=True) # Value type covariant containers. T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. -# Internal type variable used for Type[]. +# Internal type bound to class object types. CT_co = TypeVar('CT_co', covariant=True, bound=type) @@ -2912,7 +2911,7 @@ class TeamUser(User): ... And a function that takes a class argument that's a subclass of User and returns an instance of the corresponding class:: - def new_user[U](user_class: Type[U]) -> U: + def new_user[U](user_class: type[U]) -> U: user = user_class() # (Here we could write the user object to a database) return user @@ -2925,7 +2924,7 @@ def new_user[U](user_class: Type[U]) -> U: @runtime_checkable class SupportsInt(Protocol): - """An ABC with one abstract method __int__.""" + """A protocol with one abstract method __int__.""" __slots__ = () @@ -2936,7 +2935,7 @@ def __int__(self) -> int: @runtime_checkable class SupportsFloat(Protocol): - """An ABC with one abstract method __float__.""" + """A protocol with one abstract method __float__.""" __slots__ = () @@ -2947,7 +2946,7 @@ def __float__(self) -> float: @runtime_checkable class SupportsComplex(Protocol): - """An ABC with one abstract method __complex__.""" + """A protocol with one abstract method __complex__.""" __slots__ = () @@ -2958,7 +2957,7 @@ def __complex__(self) -> complex: @runtime_checkable class SupportsBytes(Protocol): - """An ABC with one abstract method __bytes__.""" + """A protocol with one abstract method __bytes__.""" __slots__ = () @@ -2969,7 +2968,7 @@ def __bytes__(self) -> bytes: @runtime_checkable class SupportsIndex(Protocol): - """An ABC with one abstract method __index__.""" + """A protocol with one abstract method __index__.""" __slots__ = () @@ -2980,7 +2979,7 @@ def __index__(self) -> int: @runtime_checkable class SupportsAbs[T](Protocol): - """An ABC with one abstract method __abs__ that is covariant in its return type.""" + """A protocol with one abstract method __abs__ that is covariant in its return type.""" __slots__ = () @@ -2991,7 +2990,7 @@ def __abs__(self) -> T: @runtime_checkable class SupportsRound[T](Protocol): - """An ABC with one abstract method __round__ that is covariant in its return type.""" + """A protocol with one abstract method __round__ that is covariant in its return type.""" __slots__ = () @@ -3108,7 +3107,7 @@ def annotate(format): def NamedTuple(typename, fields, /): - """Typed version of namedtuple. + """Typed version of collections.namedtuple. Usage:: @@ -3120,8 +3119,8 @@ class Employee(NamedTuple): Employee = collections.namedtuple('Employee', ['name', 'id']) - The resulting class has an extra __annotations__ attribute, giving a - dict that maps field names to types. (The field names are also in + The types for each field name can be retrieved by calling + annotationlib.get_annotations(Employee). (The field names are also in the _fields attribute, which is part of the namedtuple API.) An alternative equivalent functional syntax is also accepted:: @@ -3174,7 +3173,7 @@ def __new__(cls, name, bases, ns, total=True, closed=None, This method is called when TypedDict is subclassed, or when TypedDict is instantiated. This way - TypedDict supports all three syntax forms described in its docstring. + TypedDict classes can be created through both class-based and functional syntax. Subclasses and instances of TypedDict return actual dictionaries. """ for base in bases: @@ -3328,14 +3327,22 @@ def TypedDict(typename, fields, /, *, total=True, closed=None, >>> Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') True - The type info can be accessed via the Point2D.__annotations__ dict, and - the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. + The type info can be accessed by calling annotationlib.get_annotations(Point2D), and + via the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. TypedDict supports an additional equivalent form:: Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) By default, all keys must be present in a TypedDict. It is possible - to override this by specifying totality:: + to override this by using the NotRequired and Required special forms:: + + class Point2D(TypedDict): + x: int # the "x" key must always be present (Required is the default) + y: NotRequired[int] # the "y" key can be omitted + + This means that a Point2D TypedDict can have the "y" key omitted, but the "x" key must be present. + Items are required by default, so the Required special form is not necessary in this example. + In addition, the total argument to the TypedDict function can be used to make all items not required:: class Point2D(TypedDict, total=False): x: int @@ -3344,16 +3351,8 @@ class Point2D(TypedDict, total=False): This means that a Point2D TypedDict can have any of the keys omitted. A type checker is only expected to support a literal False or True as the value of the total argument. True is the default, and makes all items defined in the - class body be required. - - The Required and NotRequired special forms can also be used to mark - individual keys as being required or not required:: - - class Point2D(TypedDict): - x: int # the "x" key must always be present (Required is the default) - y: NotRequired[int] # the "y" key can be omitted - - See PEP 655 for more details on Required and NotRequired. + class body be required. The Required special form can be used to mark individual + keys as required in a total=False TypedDict. The ReadOnly special form can be used to mark individual keys as immutable for type checkers:: @@ -3387,7 +3386,7 @@ class Point3D(Point2D): by default, and it may not be used with the closed argument at the same time. - See PEP 728 for more information about closed and extra_items. + See PEPs 589, 655, 705, and 728 for more information. """ ns = {'__annotations__': dict(fields)} module = _caller() @@ -3417,7 +3416,7 @@ class Movie(TypedDict, total=False): year: int m = Movie( - title='The Matrix', # typechecker error if key is omitted + title='The Matrix', # type checker error if key is omitted year=1999, ) @@ -3439,7 +3438,7 @@ class Movie(TypedDict): year: NotRequired[int] m = Movie( - title='The Matrix', # typechecker error if key is omitted + title='The Matrix', # type checker error if key is omitted year=1999, ) """ @@ -3459,7 +3458,7 @@ class Movie(TypedDict): def mutate_movie(m: Movie) -> None: m["year"] = 1992 # allowed - m["title"] = "The Matrix" # typechecker error + m["title"] = "The Matrix" # type checker error There is no runtime checking for this property. """ @@ -3546,8 +3545,8 @@ class IO(Generic[AnyStr]): classes (text vs. binary, read vs. write vs. read/write, append-only, unbuffered). The TextIO and BinaryIO subclasses below capture the distinctions between text vs. binary, which is - pervasive in the interface; however we currently do not offer a - way to track the other distinctions in the type system. + pervasive in the interface. For more precise types, define a custom + Protocol. """ __slots__ = () @@ -3637,7 +3636,7 @@ def __exit__(self, type, value, traceback, /) -> None: class BinaryIO(IO[bytes]): - """Typed version of the return of open() in binary mode.""" + """Typed approximation of the return of open() in binary mode.""" __slots__ = () @@ -3651,7 +3650,7 @@ def __enter__(self) -> BinaryIO: class TextIO(IO[str]): - """Typed version of the return of open() in text mode.""" + """Typed approximation of the return of open() in text mode.""" __slots__ = () @@ -3718,7 +3717,7 @@ def dataclass_transform( field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (), **kwargs: Any, ) -> _IdentityCallable: - """Decorator to mark an object as providing dataclass-like behaviour. + """Decorator to mark an object as providing dataclass-like behavior. The decorator can be applied to a function, class, or metaclass. diff --git a/Misc/NEWS.d/next/Library/2026-05-18-07-44-46.gh-issue-149995.vvtFHn.rst b/Misc/NEWS.d/next/Library/2026-05-18-07-44-46.gh-issue-149995.vvtFHn.rst new file mode 100644 index 00000000000000..a8e412b578da37 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-18-07-44-46.gh-issue-149995.vvtFHn.rst @@ -0,0 +1 @@ +Update various docstrings in :mod:`typing`. From b039d1bd976819320af15494a11c973e0eca02f1 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 22 May 2026 09:48:42 +0200 Subject: [PATCH 109/110] [3.15] gh-133998: Fix gzip file creation when time is out of range (GH-134278) (GH-150221) (cherry picked from commit 1daad8a1630c9ee011f6ff3796c4e7aef243463b) Co-authored-by: adang1345 Co-authored-by: Serhiy Storchaka --- Doc/library/gzip.rst | 10 ++++++--- Lib/gzip.py | 10 +++++++-- Lib/test/test_gzip.py | 21 +++++++++++++++++++ ...-05-19-20-29-35.gh-issue-133998.KmElUw.rst | 5 +++++ 4 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-19-20-29-35.gh-issue-133998.KmElUw.rst diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index ed9fdaf1d727b0..2c667ddc522399 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -108,9 +108,13 @@ The module defines the following items: is no compression. The default is ``9``. The optional *mtime* argument is the timestamp requested by gzip. The time - is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. - If *mtime* is omitted or ``None``, the current time is used. Use *mtime* = 0 - to generate a compressed stream that does not depend on creation time. + is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. Set + *mtime* to ``0`` to generate a compressed stream that does not depend on + creation time. If *mtime* is omitted or ``None``, the current time is used; + however, if the current time is outside the range 00:00:00 UTC, January 1, + 1970 through 06:28:15 UTC, February 7, 2106, or explicitly passed *mtime* + argument is outside the range ``0`` to ``2**32-1``, then the value ``0`` + is used instead. See below for the :attr:`mtime` attribute that is set when decompressing. diff --git a/Lib/gzip.py b/Lib/gzip.py index 1e05f43c0c9e24..8720acc4db9976 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -188,8 +188,10 @@ def __init__(self, filename=None, mode=None, The optional mtime argument is the timestamp requested by gzip. The time is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. - If mtime is omitted or None, the current time is used. Use mtime = 0 - to generate a compressed stream that does not depend on creation time. + Set mtime to 0 to generate a compressed stream that does not depend on + creation time. If mtime is omitted or None, the current time is used. + If the resulting mtime is outside the range 0 to 2**32-1, then the + value 0 is used instead. """ @@ -295,6 +297,8 @@ def _write_gzip_header(self, compresslevel): mtime = self._write_mtime if mtime is None: mtime = time.time() + if not 0 <= mtime < 2**32: + mtime = 0 write32u(self.fileobj, int(mtime)) if compresslevel == _COMPRESS_LEVEL_BEST: xfl = b'\002' @@ -663,6 +667,8 @@ def compress(data, compresslevel=_COMPRESS_LEVEL_TRADEOFF, *, mtime=0): gzip_data = zlib.compress(data, level=compresslevel, wbits=31) if mtime is None: mtime = time.time() + if not 0 <= mtime < 2**32: + mtime = 0 # Reuse gzip header created by zlib, replace mtime and OS byte for # consistency. header = struct.pack("<4sLBB", gzip_data, int(mtime), gzip_data[8], 255) diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index b3b7c8f87e4f9f..cafac9d3c8be6e 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -10,6 +10,7 @@ import sys import unittest from subprocess import PIPE, Popen +from unittest import mock from test.support import catch_unraisable_exception from test.support import force_not_colorized_test_class, import_helper from test.support import os_helper @@ -350,6 +351,26 @@ def test_mtime(self): self.assertEqual(dataRead, data1) self.assertEqual(fRead.mtime, mtime) + def test_mtime_out_of_range(self): + for mtime in (-1, 2**32): + with gzip.GzipFile(self.filename, 'w', mtime=mtime) as fWrite: + fWrite.write(data1) + with gzip.GzipFile(self.filename) as fRead: + fRead.read(1) + self.assertEqual(fRead.mtime, 0) + datac = gzip.compress(data1, mtime=mtime) + with gzip.GzipFile(fileobj=io.BytesIO(datac)) as fRead: + fRead.read(1) + self.assertEqual(fRead.mtime, 0) + + for mtime in (-1, 2**32): + with mock.patch('time.time', return_value=float(mtime)): + with gzip.GzipFile(self.filename, 'w') as fWrite: + fWrite.write(data1) + with gzip.GzipFile(self.filename) as fRead: + fRead.read(1) + self.assertEqual(fRead.mtime, 0) + def test_metadata(self): mtime = 123456789 diff --git a/Misc/NEWS.d/next/Library/2025-05-19-20-29-35.gh-issue-133998.KmElUw.rst b/Misc/NEWS.d/next/Library/2025-05-19-20-29-35.gh-issue-133998.KmElUw.rst new file mode 100644 index 00000000000000..77d92628beefac --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-19-20-29-35.gh-issue-133998.KmElUw.rst @@ -0,0 +1,5 @@ +Fix :exc:`struct.error` exception when creating a file with +:class:`gzip.GzipFile` or compressing data with :func:`gzip.compress` +if the system time is outside the range 00:00:00 UTC, January 1, 1970 +through 06:28:15 UTC, February 7, 2106, or explicitly passed *mtime* +argument is outside the range ``0`` to ``2**32-1``. From e192a0ea521cdeac1531ddefb08df0db76c0207f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 22 May 2026 11:46:14 +0200 Subject: [PATCH 110/110] [3.15] gh-137571: Protect against possible UnboundLocalError in gzip._GzipReader.read() (GH-150222) (GH-150229) This has not been observed in practice, but we cannot be 100% sure that it will not happen with some weird gzip data. (cherry picked from commit 28eac9a7263ad8dcfa9b536aa238549131857e0f) Co-authored-by: Serhiy Storchaka --- Lib/gzip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/gzip.py b/Lib/gzip.py index 8720acc4db9976..0713b922522ee1 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -610,10 +610,10 @@ def read(self, size=-1): # Read a chunk of data from the file if self._decompressor.needs_input: buf = self._fp.read(READ_BUFFER_SIZE) - uncompress = self._decompressor.decompress(buf, size) else: - uncompress = self._decompressor.decompress(b"", size) + buf = b"" + uncompress = self._decompressor.decompress(buf, size) if self._decompressor.unused_data != b"": # Prepend the already read bytes to the fileobj so they can # be seen by _read_eof() and _read_gzip_header()