diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index af904a567cfb7e2..bdf134254121e58 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -156,7 +156,7 @@ Misc/libabigail.abignore @encukou # ---------------------------------------------------------------------------- # Android -Android/ @mhsmith @freakboy3742 +Platforms/Android/ @mhsmith @freakboy3742 Doc/using/android.rst @mhsmith @freakboy3742 Lib/_android_support.py @mhsmith @freakboy3742 Lib/test/test_android.py @mhsmith @freakboy3742 @@ -164,8 +164,7 @@ Lib/test/test_android.py @mhsmith @freakboy3742 # iOS Doc/using/ios.rst @freakboy3742 Lib/_ios_support.py @freakboy3742 -Apple/ @freakboy3742 -iOS/ @freakboy3742 +Platforms/Apple/ @freakboy3742 # macOS Mac/ @python/macos-team @@ -176,8 +175,8 @@ Lib/test/test__osx_support.py @python/macos-team Tools/wasm/README.md @brettcannon @freakboy3742 @emmatyping # WebAssembly (Emscripten) -Tools/wasm/config.site-wasm32-emscripten @freakboy3742 @emmatyping -Tools/wasm/emscripten @freakboy3742 @emmatyping +Platforms/emscripten @freakboy3742 @emmatyping +Tools/wasm/emscripten @freakboy3742 @emmatyping # WebAssembly (WASI) Platforms/WASI @brettcannon @emmatyping @savannahostrowski @@ -574,9 +573,9 @@ Lib/shutil.py @giampaolo Lib/test/test_shutil.py @giampaolo # Site -Lib/site.py @FFY00 -Lib/test/test_site.py @FFY00 -Doc/library/site.rst @FFY00 +Lib/site.py @FFY00 @warsaw +Lib/test/test_site.py @FFY00 @warsaw +Doc/library/site.rst @FFY00 @warsaw # string.templatelib Doc/library/string.templatelib.rst @lysnikolaou @AA-Turner @@ -587,10 +586,10 @@ Lib/test/test_string/test_templatelib.py @lysnikolaou @AA-Turner **/*sysconfig* @FFY00 # SQLite 3 -Doc/library/sqlite3.rst @berkerpeksag @erlend-aasland -Lib/sqlite3/ @berkerpeksag @erlend-aasland -Lib/test/test_sqlite3/ @berkerpeksag @erlend-aasland -Modules/_sqlite/ @berkerpeksag @erlend-aasland +Doc/library/sqlite3.rst @erlend-aasland +Lib/sqlite3/ @erlend-aasland +Lib/test/test_sqlite3/ @erlend-aasland +Modules/_sqlite/ @erlend-aasland # Subprocess Lib/subprocess.py @gpshead @@ -623,9 +622,6 @@ Modules/_typesmodule.c @AA-Turner Lib/unittest/mock.py @cjw296 Lib/test/test_unittest/testmock/ @cjw296 -# Urllib -**/*robotparser* @berkerpeksag - # Venv **/*venv* @vsajip @FFY00 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1af3a0607f9ad2a..12bf160178e3c72 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,6 +49,53 @@ jobs: if: fromJSON(needs.build-context.outputs.run-docs) uses: ./.github/workflows/reusable-docs.yml + check-abi: + name: 'Check if the ABI has changed' + runs-on: ubuntu-22.04 # 24.04 causes spurious errors + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.x' + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + sudo apt-get install -yq --no-install-recommends abigail-tools + - name: Build CPython + env: + CFLAGS: -g3 -O0 + run: | + # Build Python with the libpython dynamic library + ./configure --enable-shared + make -j4 + - name: Check for changes in the ABI + id: check + run: | + if ! make check-abidump; then + echo "Generated ABI file is not up to date." + echo "Please add the release manager of this branch as a reviewer of this PR." + echo "" + echo "The up to date ABI file should be attached to this build as an artifact." + echo "" + echo "To learn more about this check: https://devguide.python.org/getting-started/setup-building/index.html#regenerate-the-abi-dump" + echo "" + exit 1 + fi + - name: Generate updated ABI files + if: ${{ failure() && steps.check.conclusion == 'failure' }} + run: | + make regen-abidump + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + name: Publish updated ABI files + if: ${{ failure() && steps.check.conclusion == 'failure' }} + with: + name: abi-data + path: ./Doc/data/*.abi + check-autoconf-regen: name: 'Check if Autoconf files are up to date' # Don't use ubuntu-latest but a specific version to make the job diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 490c32ecfc9a629..d748b6ff63e68a1 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -69,12 +69,11 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: python-version: "3.15" - allow-prereleases: true - cache: pip - cache-dependency-path: Tools/requirements-dev.txt - - run: pip install -r Tools/requirements-dev.txt + activate-environment: true + cache-dependency-glob: Tools/requirements-dev.txt + - run: uv pip install -r Tools/requirements-dev.txt - run: python3 Misc/mypy/make_symlinks.py --symlink - - run: mypy --config-file ${{ matrix.target }}/mypy.ini + - run: mypy --num-workers 4 --config-file ${{ matrix.target }}/mypy.ini diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 4c8d0c8a2f984fc..c6e8128884e90c2 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -22,8 +22,6 @@ permissions: env: FORCE_COLOR: 1 - IncludeUwp: >- - true jobs: build: diff --git a/.gitignore b/.gitignore index 118eb5ee76e8051..78b6d4efb0e1097 100644 --- a/.gitignore +++ b/.gitignore @@ -177,7 +177,3 @@ Python/frozen_modules/MANIFEST # People's custom https://docs.anthropic.com/en/docs/claude-code/memory configs. /.claude/ CLAUDE.local.md - -#### main branch only stuff below this line, things to backport go above. #### -# main branch only: ABI files are not checked/maintained. -Doc/data/python*.abi diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index a2a0d0d80657ebf..556113a97bf772f 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -151,7 +151,7 @@ Dictionary objects * If the key is present, set *\*result* to a new :term:`strong reference` to the value and return ``1``. * If the key is missing, set *\*result* to ``NULL`` and return ``0``. - * On error, raise an exception and return ``-1``. + * On error, raise an exception, set *\*result* to ``NULL`` and return ``-1``. The first argument can be a :class:`dict` or a :class:`frozendict`. diff --git a/Doc/c-api/sentinel.rst b/Doc/c-api/sentinel.rst index 89e0a28bf3b835b..937cae18e86f507 100644 --- a/Doc/c-api/sentinel.rst +++ b/Doc/c-api/sentinel.rst @@ -14,8 +14,20 @@ Sentinel objects .. c:function:: int PySentinel_Check(PyObject *o) - Return true if *o* is a :class:`sentinel` object. The :class:`sentinel` type - does not allow subclasses, so this check is exact. + Return true if *o* is a :class:`sentinel` object or a subtype. + The :class:`sentinel` type does not currently allow subclasses, + so this check is exact. + Future Python versions may choose to allow subtyping. + This function always succeeds. + + .. versionadded:: 3.15 + +.. c:function:: int PySentinel_CheckExact(PyObject *o) + + Return true if *o* is a :class:`sentinel` object, but not a subtype. + The :class:`sentinel` type does not currently allow subclasses. + Future Python versions may choose to allow subtyping. + This function always succeeds. .. versionadded:: 3.15 diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 0ff066680b8c733..13e5d5c96135c0e 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -114,7 +114,7 @@ versions of Python. All functions in Stable ABI are present as functions in Python's shared library, not solely as macros. -This makes them usable are usable from languages that don't use the C +This makes them usable in languages that don't use the C preprocessor, including Python's :py:mod:`ctypes`. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 059a7ef399ae0f5..401c99ebeb0fec6 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -762,7 +762,7 @@ APIs: The string must not have been “used” yet. See :c:func:`PyUnicode_New` for details. - Return the number of written character, or return ``-1`` and raise an + Return the number of written characters, or return ``-1`` and raise an exception on error. .. versionadded:: 3.3 @@ -1174,7 +1174,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. .. versionchanged:: 3.10 This function is a part of the :ref:`limited API `. @@ -1196,7 +1196,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. UTF-32 Codecs diff --git a/Doc/data/python3.15.abi b/Doc/data/python3.15.abi new file mode 100644 index 000000000000000..04211b6e4e274ae --- /dev/null +++ b/Doc/data/python3.15.abi @@ -0,0 +1,33491 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index a722607b2c1f198..8af40a2f8a65e3f 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -16,8 +16,10 @@ This module provides functions for encoding binary data to printable ASCII characters and decoding such encodings back to binary data. This includes the :ref:`encodings specified in ` -:rfc:`4648` (Base64, Base32 and Base16) -and the non-standard :ref:`Base85 encodings `. +:rfc:`4648` (Base64, Base32 and Base16), the :ref:`Base85 encoding +` specified in `PDF 2.0 +`_, and non-standard variants +of Base85 used elsewhere. There are two interfaces provided by this module. The modern interface supports encoding :term:`bytes-like objects ` to ASCII @@ -284,19 +286,28 @@ POST request. Base85 Encodings ----------------- -Base85 encoding is not formally specified but rather a de facto standard, -thus different systems perform the encoding differently. +Base85 encoding is a family of algorithms which represent four bytes +using five ASCII characters. Originally implemented in the Unix +``btoa(1)`` utility, a version of it was later adopted by Adobe in the +PostScript language and is standardized in PDF 2.0 (ISO 32000-2). +This version, in both its ``btoa`` and PDF variants, is implemented by +:func:`a85encode`. -The :func:`a85encode` and :func:`b85encode` functions in this module are two implementations of -the de facto standard. You should call the function with the Base85 -implementation used by the software you intend to work with. +A separate version, using a different output character set, was +defined as an April Fool's joke in :rfc:`1924` but is now used by Git +and other software. This version is implemented by :func:`b85encode`. -The two functions present in this module differ in how they handle the following: +Finally, a third version, using yet another output character set +designed for safe inclusion in programming language strings, is +defined by ZeroMQ and implemented here by :func:`z85encode`. -* Whether to include enclosing ``<~`` and ``~>`` markers -* Whether to include newline characters -* The set of ASCII characters used for encoding -* Handling of null bytes +The functions present in this module differ in how they handle the following: + +* Whether to include and expect enclosing ``<~`` and ``~>`` markers. +* Whether to fold the input into multiple lines. +* The set of ASCII characters used for encoding. +* Compact encodings of sequences of spaces and null bytes. +* The encoding of zero-padding bytes applied to the input. Refer to the documentation of the individual functions for more information. @@ -307,18 +318,22 @@ Refer to the documentation of the individual functions for more information. *foldspaces* is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Ascii85 encoding. + feature is not supported by the standard encoding used in PDF. If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. - Note that the ``btoa`` implementation always pads. + *pad* controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by ``btoa``, + producing an exact multiple of 5 bytes of output. This is not part + of the standard encoding used in PDF, as it does not preserve the + length of the data. - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. Note + that while ASCII85Decode streams in PDF documents *must* be + terminated with ``~>``, they *must not* use a leading ``<~``. .. versionadded:: 3.4 @@ -330,10 +345,12 @@ Refer to the documentation of the individual functions for more information. *foldspaces* is a flag that specifies whether the 'y' short sequence should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). - This feature is not supported by the "standard" Ascii85 encoding. + This feature is not supported by the standard Ascii85 encoding used in + PDF and PostScript. - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). + *adobe* controls whether the ``<~`` and ``~>`` markers are + present. While the leading ``<~`` is not required, the input must + end with ``~>``, or a :exc:`ValueError` is raised. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. @@ -356,8 +373,11 @@ Refer to the documentation of the individual functions for more information. Encode the :term:`bytes-like object` *b* using base85 (as used in e.g. git-style binary diffs) and return the encoded :class:`bytes`. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + The input is padded with ``b'\0'`` so its length is a multiple of 4 + bytes before encoding. If *pad* is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, and thus the length of the data may not be + preserved on decoding. If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. @@ -372,8 +392,7 @@ Refer to the documentation of the individual functions for more information. .. function:: b85decode(b, *, ignorechars=b'', canonical=False) Decode the base85-encoded :term:`bytes-like object` or ASCII string *b* and - return the decoded :class:`bytes`. Padding is implicitly removed, if - necessary. + return the decoded :class:`bytes`. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. @@ -392,11 +411,12 @@ Refer to the documentation of the individual functions for more information. .. function:: z85encode(s, pad=False, *, wrapcol=0) Encode the :term:`bytes-like object` *s* using Z85 (as used in ZeroMQ) - and return the encoded :class:`bytes`. See `Z85 specification - `_ for more information. + and return the encoded :class:`bytes`. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + The input is padded with ``b'\0'`` so its length is a multiple of 4 + bytes before encoding. If *pad* is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, as required by the ZeroMQ standard. If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. @@ -414,8 +434,7 @@ Refer to the documentation of the individual functions for more information. .. function:: z85decode(s, *, ignorechars=b'', canonical=False) Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and - return the decoded :class:`bytes`. See `Z85 specification - `_ for more information. + return the decoded :class:`bytes`. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. @@ -499,3 +518,11 @@ recommended to review the security section for any code deployed to production. Section 5.2, "Base64 Content-Transfer-Encoding," provides the definition of the base64 encoding. + `ISO 32000-2 Portable document format - Part 2: PDF 2.0 `_ + Section 7.4.3, "ASCII85Decode Filter," provides the definition + of the Ascii85 encoding used in PDF and PostScript, including + the output character set and the details of data length preservation + using zero-padding and partial output groups. + + `ZeroMQ RFC 32/Z85 `_ + The "Formal Specification" section provides the character set used in Z85. diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 8b4ba6ae9fb2549..60afe9261d51fac 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -133,8 +133,11 @@ The :mod:`!binascii` module defines the following functions: should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is not supported by the "standard" Ascii85 encoding. - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. If + *adobe* is true, a leading ``<~`` is optionally accepted, while a + trailing ``~>`` is *required*, and :exc:`binascii.Error` is raised + if it is not found. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. @@ -164,12 +167,16 @@ The :mod:`!binascii` module defines the following functions: after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. - Note that the ``btoa`` implementation always pads. + If *pad* is true, the zero-padding applied to the end of the input + is fully retained in the output encoding, as done by ``btoa``, + producing an exact multiple of 5 bytes of output. This is not part + of the standard encoding used in PDF, as it does not preserve the + length of the data. - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. Note + that while ASCII85Decode streams in PDF documents *must* be + terminated with ``~>``, they *must not* use a leading ``<~``. .. versionadded:: 3.15 @@ -213,8 +220,10 @@ The :mod:`!binascii` module defines the following functions: after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + If *pad* is true, the zero-padding applied to the end of the input + is retained in the output, which will always be a multiple of 5 + bytes, and thus the length of the data may not be preserved on + decoding. .. versionadded:: 3.15 diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 9259ab10d5850b5..059ed2c03acfa38 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1155,7 +1155,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp857 | 857, IBM857 | Turkish | +-----------------+--------------------------------+--------------------------------+ -| cp858 | 858, IBM858 | Western Europe | +| cp858 | 858, IBM00858 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp860 | 860, IBM860 | Portuguese | +-----------------+--------------------------------+--------------------------------+ @@ -1192,7 +1192,7 @@ particular, the following variants typically exist: | | | | | | | .. versionadded:: 3.4 | +-----------------+--------------------------------+--------------------------------+ -| cp1140 | ibm1140 | Western Europe | +| cp1140 | IBM01140 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp1250 | windows-1250 | Central and Eastern Europe | +-----------------+--------------------------------+--------------------------------+ diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 0bce3e5b762b8be..a09c28ad9791584 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -498,7 +498,8 @@ Module contents .. function:: is_dataclass(obj) Return ``True`` if its parameter is a dataclass (including subclasses of a - dataclass) or an instance of one, otherwise return ``False``. + dataclass, but not including :ref:`generic aliases `) + or an instance of one, otherwise return ``False``. If you need to know if a class is an instance of a dataclass (and not a dataclass itself), then add a further check for ``not diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index ed9fdaf1d727b08..2c667ddc522399c 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -108,9 +108,13 @@ The module defines the following items: is no compression. The default is ``9``. The optional *mtime* argument is the timestamp requested by gzip. The time - is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. - If *mtime* is omitted or ``None``, the current time is used. Use *mtime* = 0 - to generate a compressed stream that does not depend on creation time. + is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. Set + *mtime* to ``0`` to generate a compressed stream that does not depend on + creation time. If *mtime* is omitted or ``None``, the current time is used; + however, if the current time is outside the range 00:00:00 UTC, January 1, + 1970 through 06:28:15 UTC, February 7, 2106, or explicitly passed *mtime* + argument is outside the range ``0`` to ``2**32-1``, then the value ``0`` + is used instead. See below for the :attr:`mtime` attribute that is set when decompressing. diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 63de4f91f4ba5f5..e11db37b9fad501 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -105,6 +105,13 @@ You can also get a :ref:`distribution's version number `, list its current Python environment. +.. exception:: MetadataNotFound + + Subclass of :class:`FileNotFoundError` raised when attempting to load metadata + from a distribution folder that is empty or otherwise does not contain a + metadata file. + + Functional API ============== @@ -224,6 +231,9 @@ Distribution metadata Raises :exc:`PackageNotFoundError` if the named distribution package is not installed in the current Python environment. + Raises :exc:`MetadataNotFound` if a distribution package is + present but no METADATA file is present. + .. class:: PackageMetadata A concrete implementation of the @@ -252,6 +262,12 @@ all the metadata in a JSON-compatible form per :PEP:`566`:: The full set of available metadata is not described here. See the PyPA `Core metadata specification `_ for additional details. +.. versionchanged:: 3.15 + Previously and incidentally, if a METADATA file was missing from a distribution, an + empty ``PackageMetadata`` would be returned, indistinguishable from + an empty METADATA file. Now, a missing METADATA file triggers a + ``MetadataNotFound`` exception. + .. versionchanged:: 3.10 The ``Description`` is now included in the metadata when presented through the payload. Line continuation characters have been removed. @@ -465,6 +481,9 @@ The same applies for :func:`entry_points` and :func:`files`. .. attribute:: metadata :type: PackageMetadata + Raises :exc:`MetadataNotFound` if the METADATA file is not present in + the distribution. + There are all kinds of additional metadata available on :class:`!Distribution` instances as a :class:`PackageMetadata` instance:: diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 8713765b8aebfbd..92840e702fbbfe6 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -416,17 +416,47 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a class, whether built-in or created in Python code. + This function returns ``False`` for :ref:`generic aliases ` of classes, + such as ``list[int]``. + .. function:: ismethod(object) Return ``True`` if the object is a bound method written in Python. + .. note:: -.. function:: ispackage(object) + For example, given this class:: - Return ``True`` if the object is a :term:`package`. + >>> class Greeter: + ... def say_hello(self): + ... print('hello!') - .. versionadded:: 3.14 + A bound method (also known as an *instance method*) is created when + accessing ``say_hello`` (a :term:`function` defined in the + ``Greeter`` namespace) through an instance of the ``Greeter`` class:: + + >>> instance = Greeter() + + >>> instance.say_hello + > + >>> ismethod(instance.say_hello) + True + >>> isfunction(instance.say_hello) + False + + Accessing ``say_hello`` through the ``Greeter`` class will return the + function itself. For this function, :func:`ismethod` will return + ``False``, but :func:`isfunction` will return ``True``:: + + >>> Greeter.say_hello + + >>> ismethod(Greeter.say_hello) + False + >>> isfunction(Greeter.say_hello) + True + + See :ref:`typesmethods` for details. .. function:: isfunction(object) @@ -434,11 +464,23 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a Python function, which includes functions created by a :term:`lambda` expression. + See the note for :func:`~inspect.ismethod` for an example. + + +.. function:: ispackage(object) + + Return ``True`` if the object is a :term:`package`. + + .. versionadded:: 3.14 + .. function:: isgeneratorfunction(object) Return ``True`` if the object is a Python generator function. + It also returns ``True`` for bound methods created from Python generator functions + (see :ref:`typesmethods` for more information). + .. versionchanged:: 3.8 Functions wrapped in :func:`functools.partial` now return ``True`` if the wrapped function is a Python generator function. diff --git a/Doc/library/json.rst b/Doc/library/json.rst index b354e7ba534835f..383ccad9df041b5 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -211,7 +211,7 @@ Basic Usage a string (such as ``"\t"``) is used to indent each level. If zero, negative, or ``""`` (the empty string), only newlines are inserted. - If ``None`` (the default), the most compact representation is used. + If ``None`` (the default), no newlines are inserted. :type indent: int | str | None :param separators: diff --git a/Doc/library/os.rst b/Doc/library/os.rst index d2534b3e974f368..27a032a8a97c637 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2549,7 +2549,8 @@ features: Windows now handles a *mode* of ``0o700``. -.. function:: makedirs(name, mode=0o777, exist_ok=False) +.. function:: makedirs(name, mode=0o777, exist_ok=False, *, \ + parent_mode=None) .. index:: single: directory; creating @@ -2567,6 +2568,12 @@ features: If *exist_ok* is ``False`` (the default), a :exc:`FileExistsError` is raised if the target directory already exists. + If *parent_mode* is not ``None``, it is used as the mode for any + newly-created, intermediate-level directories. Like *mode*, it is + combined with the process's umask value; see :ref:`the mkdir() + description `. Otherwise, intermediate directories are + created with the default mode, which is also subject to the umask. + .. note:: :func:`makedirs` will become confused if the path elements to create @@ -2593,6 +2600,11 @@ features: The *mode* argument no longer affects the file permission bits of newly created intermediate-level directories. + .. versionadded:: 3.15 + The *parent_mode* parameter. To match the behavior from Python 3.6 and + earlier (where *mode* was applied to all created directories), pass + ``parent_mode=mode``. + .. function:: mkfifo(path, mode=0o666, *, dir_fd=None) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 2867015042ee162..45b5797058f6239 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1514,7 +1514,8 @@ Creating files and directories :meth:`~Path.write_bytes` methods are often used to create files. -.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) +.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False, *, \ + parent_mode=None) Create a new directory at this given path. If *mode* is given, it is combined with the process's ``umask`` value to determine the file mode @@ -1525,6 +1526,12 @@ Creating files and directories as needed; they are created with the default permissions without taking *mode* into account (mimicking the POSIX ``mkdir -p`` command). + If *parent_mode* is not ``None``, it is used as the mode for any + newly-created, intermediate-level directories when *parents* is true. + Like *mode*, it is combined with the process's ``umask`` value. + Otherwise, intermediate directories are created with the default + permissions (also subject to the umask). + If *parents* is false (the default), a missing parent raises :exc:`FileNotFoundError`. @@ -1538,6 +1545,9 @@ Creating files and directories .. versionchanged:: 3.5 The *exist_ok* parameter was added. + .. versionadded:: 3.15 + The *parent_mode* parameter. + .. method:: Path.symlink_to(target, target_is_directory=False) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index f8975c2f4281d45..8eadc2cf2b1ef0d 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -56,7 +56,7 @@ The :mod:`!pickle` module differs from :mod:`marshal` in several significant way * :mod:`marshal` cannot be used to serialize user-defined classes and their instances. :mod:`!pickle` can save and restore class instances transparently, however the class definition must be importable and live in the same module as - when the object was stored. + when the object was pickled. * The :mod:`marshal` serialization format is not guaranteed to be portable across Python versions. Because its primary job in life is to support @@ -693,7 +693,10 @@ or both. If a string is returned, the string should be interpreted as the name of a global variable. It should be the object's local name relative to its module; the pickle module searches the module namespace to determine the - object's module. This behaviour is typically useful for singletons. + object's module: for a given ``obj`` to be pickled, the ``__module__`` + attribute is looked up on ``obj`` directly, which falls back to a lookup + on the type of ``obj`` if no ``__module__`` instance attribute is set. + This behaviour is typically useful for singletons. When a tuple is returned, it must be between two and six items long. Optional items can either be omitted, or ``None`` can be provided as their diff --git a/Doc/library/select.rst b/Doc/library/select.rst index 09563af14d018a4..6400005871746a5 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -37,7 +37,7 @@ The module defines the following: .. function:: devpoll() - (Only supported on Solaris and derivatives.) Returns a ``/dev/poll`` + Returns a ``/dev/poll`` polling object; see section :ref:`devpoll-objects` below for the methods supported by devpoll objects. @@ -54,9 +54,11 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: Solaris. + .. function:: epoll(sizehint=-1, flags=0) - (Only supported on Linux 2.5.44 and newer.) Return an edge polling object, + Return an edge polling object, which can be used as Edge or Level Triggered interface for I/O events. @@ -94,18 +96,22 @@ The module defines the following: When CPython is built, this function may be disabled using :option:`--disable-epoll`. + .. availability:: Linux >= 2.5.44. + .. function:: poll() - (Not supported by all operating systems.) Returns a polling object, which + Returns a polling object, which supports registering and unregistering file descriptors, and then polling them for I/O events; see section :ref:`poll-objects` below for the methods supported by polling objects. + .. availability:: Unix. + .. function:: kqueue() - (Only supported on BSD.) Returns a kernel queue object; see section + Returns a kernel queue object; see section :ref:`kqueue-objects` below for the methods supported by kqueue objects. The new file descriptor is :ref:`non-inheritable `. @@ -113,12 +119,16 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: BSD, macOS. + .. function:: kevent(ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0) - (Only supported on BSD.) Returns a kernel event object; see section + Returns a kernel event object; see section :ref:`kevent-objects` below for the methods supported by kevent objects. + .. availability:: BSD, macOS. + .. function:: select(rlist, wlist, xlist, timeout=None) @@ -190,7 +200,7 @@ The module defines the following: .. _devpoll-objects: -``/dev/poll`` Polling Objects +``/dev/poll`` polling objects ----------------------------- Solaris and derivatives have ``/dev/poll``. While :c:func:`!select` is @@ -285,52 +295,52 @@ object. .. _epoll-objects: -Edge and Level Trigger Polling (epoll) Objects +Edge and level trigger polling (epoll) objects ---------------------------------------------- https://linux.die.net/man/4/epoll - *eventmask* - - +-------------------------+-----------------------------------------------+ - | Constant | Meaning | - +=========================+===============================================+ - | :const:`EPOLLIN` | Available for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLOUT` | Available for write | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLPRI` | Urgent data for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLERR` | Error condition happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLHUP` | Hang up happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | - | | Level Trigger behavior | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | - | | pulled out, the fd is internally disabled | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | - | | associated fd has an event. The default (if | - | | this flag is not set) is to wake all epoll | - | | objects polling on a fd. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | - | | down writing half of connection. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDBAND` | Priority data band can be read. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRBAND` | Priority data may be written. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLMSG` | Ignored. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | - +-------------------------+-----------------------------------------------+ + The *eventmask* is a bit mask using the following constants: + + +-------------------------+------------------------------------------------+ + | Constant | Meaning | + +=========================+================================================+ + | :const:`EPOLLIN` | Available for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLOUT` | Available for write. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLPRI` | Urgent data for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLERR` | Error condition happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLHUP` | Hang up happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | + | | Level Trigger behavior. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | + | | pulled out, the fd is internally disabled. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | + | | associated fd has an event. The default (if | + | | this flag is not set) is to wake all epoll | + | | objects polling on an fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | + | | down writing half of connection. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDBAND` | Priority data band can be read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT`. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRBAND` | Priority data may be written. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLMSG` | Ignored. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | + +-------------------------+------------------------------------------------+ .. versionadded:: 3.6 :const:`EPOLLEXCLUSIVE` was added. It's only supported by Linux Kernel 4.5 @@ -362,12 +372,12 @@ Edge and Level Trigger Polling (epoll) Objects .. method:: epoll.register(fd[, eventmask]) - Register a fd descriptor with the epoll object. + Register a file descriptor *fd* with the epoll object. .. method:: epoll.modify(fd, eventmask) - Modify a registered file descriptor. + Modify a registered file descriptor *fd*. .. method:: epoll.unregister(fd) @@ -396,7 +406,7 @@ Edge and Level Trigger Polling (epoll) Objects .. _poll-objects: -Polling Objects +Polling objects --------------- The :c:func:`!poll` system call, supported on most Unix systems, provides better @@ -421,24 +431,24 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w :const:`POLLPRI`, and :const:`POLLOUT`, described in the table below. If not specified, the default value used will check for all 3 types of events. - +-------------------+------------------------------------------+ - | Constant | Meaning | - +===================+==========================================+ - | :const:`POLLIN` | There is data to read | - +-------------------+------------------------------------------+ - | :const:`POLLPRI` | There is urgent data to read | - +-------------------+------------------------------------------+ - | :const:`POLLOUT` | Ready for output: writing will not block | - +-------------------+------------------------------------------+ - | :const:`POLLERR` | Error condition of some sort | - +-------------------+------------------------------------------+ - | :const:`POLLHUP` | Hung up | - +-------------------+------------------------------------------+ - | :const:`POLLRDHUP`| Stream socket peer closed connection, or | - | | shut down writing half of connection | - +-------------------+------------------------------------------+ - | :const:`POLLNVAL` | Invalid request: descriptor not open | - +-------------------+------------------------------------------+ + +-------------------+-------------------------------------------+ + | Constant | Meaning | + +===================+===========================================+ + | :const:`POLLIN` | There is data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLPRI` | There is urgent data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLOUT` | Ready for output: writing will not block. | + +-------------------+-------------------------------------------+ + | :const:`POLLERR` | Error condition of some sort. | + +-------------------+-------------------------------------------+ + | :const:`POLLHUP` | Hung up. | + +-------------------+-------------------------------------------+ + | :const:`POLLRDHUP`| Stream socket peer closed connection, or | + | | shut down writing half of connection. | + +-------------------+-------------------------------------------+ + | :const:`POLLNVAL` | Invalid request: descriptor not open. | + +-------------------+-------------------------------------------+ Registering a file descriptor that's already registered is not an error, and has the same effect as registering the descriptor exactly once. @@ -489,7 +499,7 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w .. _kqueue-objects: -Kqueue Objects +Kqueue objects -------------- .. method:: kqueue.close() @@ -533,7 +543,7 @@ Kqueue Objects .. _kevent-objects: -Kevent Objects +Kevent objects -------------- https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 @@ -553,66 +563,66 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 | Constant | Meaning | +===========================+=============================================+ | :const:`KQ_FILTER_READ` | Takes a descriptor and returns whenever | - | | there is data available to read | + | | there is data available to read. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_WRITE` | Takes a descriptor and returns whenever | - | | there is data available to write | + | | there is data available to write. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_AIO` | AIO requests | + | :const:`KQ_FILTER_AIO` | AIO requests. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_VNODE` | Returns when one or more of the requested | - | | events watched in *fflag* occurs | + | | events watched in *fflag* occurs. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_PROC` | Watch for events on a process id | + | :const:`KQ_FILTER_PROC` | Watch for events on a process ID. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_NETDEV` | Watch for events on a network device | - | | [not available on macOS] | + | | (not available on macOS). | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_SIGNAL` | Returns whenever the watched signal is | - | | delivered to the process | + | | delivered to the process. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer | + | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer. | +---------------------------+---------------------------------------------+ .. attribute:: kevent.flags Filter action. - +---------------------------+---------------------------------------------+ - | Constant | Meaning | - +===========================+=============================================+ - | :const:`KQ_EV_ADD` | Adds or modifies an event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DELETE` | Removes an event from the queue | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ENABLE` | Permits control() to return the event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DISABLE` | Disables event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_SYSFLAGS` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_FLAG1` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_EOF` | Filter specific EOF condition | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ERROR` | See return values | - +---------------------------+---------------------------------------------+ + +---------------------------+----------------------------------------------+ + | Constant | Meaning | + +===========================+==============================================+ + | :const:`KQ_EV_ADD` | Adds or modifies an event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DELETE` | Removes an event from the queue. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ENABLE` | Permits control() to return the event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DISABLE` | Disables event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_SYSFLAGS` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_FLAG1` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_EOF` | Filter-specific EOF condition. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ERROR` | See return values. | + +---------------------------+----------------------------------------------+ .. attribute:: kevent.fflags - Filter specific flags. + Filter-specific flags. :const:`KQ_FILTER_READ` and :const:`KQ_FILTER_WRITE` filter flags: +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LOWAT` | low water mark of a socket buffer | + | :const:`KQ_NOTE_LOWAT` | Low water mark of a socket buffer. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_VNODE` filter flags: @@ -620,19 +630,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_DELETE` | *unlink()* was called | + | :const:`KQ_NOTE_DELETE` | *unlink()* was called. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_WRITE` | a write occurred | + | :const:`KQ_NOTE_WRITE` | A write occurred. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXTEND` | the file was extended | + | :const:`KQ_NOTE_EXTEND` | The file was extended. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_ATTRIB` | an attribute was changed | + | :const:`KQ_NOTE_ATTRIB` | An attribute was changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINK` | the link count has changed | + | :const:`KQ_NOTE_LINK` | The link count has changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_RENAME` | the file was renamed | + | :const:`KQ_NOTE_RENAME` | The file was renamed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_REVOKE` | access to the file was revoked | + | :const:`KQ_NOTE_REVOKE` | Access to the file was revoked. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_PROC` filter flags: @@ -640,22 +650,22 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_EXIT` | the process has exited | + | :const:`KQ_NOTE_EXIT` | The process has exited. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_FORK` | the process has called *fork()* | + | :const:`KQ_NOTE_FORK` | The process has called *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXEC` | the process has executed a new process | + | :const:`KQ_NOTE_EXEC` | The process has executed a new process. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PCTRLMASK` | internal filter flag | + | :const:`KQ_NOTE_PCTRLMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PDATAMASK` | internal filter flag | + | :const:`KQ_NOTE_PDATAMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACK` | follow a process across *fork()* | + | :const:`KQ_NOTE_TRACK` | Follow a process across *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_CHILD` | returned on the child process for | - | | *NOTE_TRACK* | + | :const:`KQ_NOTE_CHILD` | Returned on the child process for | + | | *NOTE_TRACK*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACKERR` | unable to attach to a child | + | :const:`KQ_NOTE_TRACKERR` | Unable to attach to a child. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_NETDEV` filter flags (not available on macOS): @@ -663,19 +673,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LINKUP` | link is up | + | :const:`KQ_NOTE_LINKUP` | Link is up. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKDOWN` | link is down | + | :const:`KQ_NOTE_LINKDOWN` | Link is down. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKINV` | link state is invalid | + | :const:`KQ_NOTE_LINKINV` | Link state is invalid. | +----------------------------+--------------------------------------------+ .. attribute:: kevent.data - Filter specific data. + Filter-specific data. .. attribute:: kevent.udata - User defined value. + User-defined value. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index d289ba58c240658..e0300a38e2f357d 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -749,8 +749,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Never extract archives from untrusted sources without prior inspection. It is possible that files are created outside of the path specified in - the *extract_dir* argument, e.g. members that have absolute filenames - starting with "/" or filenames with two dots "..". + the *extract_dir* argument, for example, members that have absolute filenames + or filenames with ".." components. Since Python 3.14, the defaults for both built-in formats (zip and tar files) will prevent the most dangerous of such security issues, diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index d9c736d27dcaecc..b180673f22973e2 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -2076,7 +2076,7 @@ to speed up repeated connections from the same clients. :attr:`~SSLContext.minimum_version` and :attr:`SSLContext.options` all affect the supported SSL and TLS versions of the context. The implementation does not prevent - invalid combination. For example a context with + invalid combinations. For example a context with :attr:`OP_NO_TLSv1_2` in :attr:`~SSLContext.options` and :attr:`~SSLContext.maximum_version` set to :attr:`TLSVersion.TLSv1_2` will not be able to establish a TLS 1.2 connection. @@ -2891,11 +2891,11 @@ disabled by default. :: >>> client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_2 >>> client_context.maximum_version = ssl.TLSVersion.TLSv1_3 -The SSL context created above will only allow TLSv1.3 and later (if +The SSL client context created above will only allow TLSv1.2 and TLSv1.3 (if supported by your system) connections to a server. :const:`PROTOCOL_TLS_CLIENT` implies certificate validation and hostname checks by default. You have to load certificates into the context. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 3d943566be34ff1..b0388c4e1f0bd45 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2747,6 +2747,8 @@ expression support in the :mod:`re` module). The *chars* argument is not a prefix or suffix; rather, all combinations of its values are stripped. + Whitespace characters are defined by :meth:`str.isspace`. + For example: .. doctest:: @@ -5858,7 +5860,8 @@ type and the :class:`bytes` data type: ``GenericAlias`` objects are instances of the class :class:`types.GenericAlias`, which can also be used to create ``GenericAlias`` -objects directly. +objects directly. Specializations of user-defined :ref:`generic classes ` +may not be instances of :class:`types.GenericAlias`, but they provide similar functionality. .. describe:: T[X, Y, ...] diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 08ccdfa3f454f8d..be968a3c53d8430 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -472,7 +472,9 @@ of a number respectively. It can be one of the following: | | this option is not supported. | +---------+----------------------------------------------------------+ -For a locale aware separator, use the ``'n'`` presentation type instead. +For a locale-aware separator, use the ``'n'`` +:ref:`float presentation type ` or +:ref:`integer presentation type ` instead. .. versionchanged:: 3.1 Added the ``','`` option (see also :pep:`378`). @@ -518,9 +520,14 @@ The available integer presentation types are: | | In case ``'#'`` is specified, the prefix ``'0x'`` will | | | be upper-cased to ``'0X'`` as well. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'d'``, except that it uses | + | ``'n'`` | .. _n-format-integer: | + | | | + | | Number. This is the same as ``'d'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators. | + | | digit group separators. Note that the default locale is | + | | not the system locale. Depending on your use case, you | + | | may wish to set :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | None | The same as ``'d'``. | +---------+----------------------------------------------------------+ @@ -603,10 +610,15 @@ The available presentation types for :class:`float` and | | ``'E'`` if the number gets too large. The | | | representations of infinity and NaN are uppercased, too. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'g'``, except that it uses | + | ``'n'`` | .. _n-format-float: | + | | | + | | Number. This is the same as ``'g'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators | - | | for the integral part of a number. | + | | digit group separators for the integral part of a | + | | number. Note that the default locale is not the system | + | | locale. Depending on your use case, you may wish to set | + | | :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index dca51b8014da5a4..b2167cbc63a1ffd 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -719,8 +719,8 @@ The :data:`Any` type ==================== A special kind of type is :data:`Any`. A static type checker will treat -every type as being compatible with :data:`Any` and :data:`Any` as being -compatible with every type. +every type as assignable to :data:`Any` and :data:`Any` as assignable to +every type. This means that it is possible to perform any operation or method call on a value of type :data:`Any` and assign it to any variable:: @@ -785,7 +785,7 @@ it as a return value) of a more specialized type is a type error. For example:: hash_a(42) hash_a("foo") - # Passes type checking, since Any is compatible with all types + # Passes type checking, since Any is assignable to all types hash_b(42) hash_b("foo") @@ -851,8 +851,8 @@ using ``[]``. Special type indicating an unconstrained type. - * Every type is compatible with :data:`Any`. - * :data:`Any` is compatible with every type. + * Every type is assignable to :data:`Any`. + * :data:`Any` is assignable to every type. .. versionchanged:: 3.11 :data:`Any` can now be used as a base class. This can be useful for @@ -1292,10 +1292,10 @@ These can be used as types in annotations. They all support subscription using :data:`ClassVar` accepts only types and cannot be further subscribed. - :data:`ClassVar` is not a class itself, and should not + :data:`ClassVar` is not a class itself, and cannot be used with :func:`isinstance` or :func:`issubclass`. :data:`ClassVar` does not change Python runtime behavior, but - it can be used by third-party type checkers. For example, a type checker + it can be used by static type checkers. For example, a type checker might flag the following code as an error:: enterprise_d = Starship(3000) @@ -1365,7 +1365,7 @@ These can be used as types in annotations. They all support subscription using def mutate_movie(m: Movie) -> None: m["year"] = 1999 # allowed - m["title"] = "The Matrix" # typechecker error + m["title"] = "The Matrix" # type checker error There is no runtime checking for this property. @@ -2472,9 +2472,9 @@ types. Fields with a default value must come after any fields without a default. - The resulting class has an extra attribute ``__annotations__`` giving a - dict that maps the field names to the field types. (The field names are in - the ``_fields`` attribute and the default values are in the + The types for each field name can be retrieved by calling + :func:`annotationlib.get_annotations` on the resulting class. (The field + names are in the ``_fields`` attribute and the default values are in the ``_field_defaults`` attribute, both of which are part of the :func:`~collections.namedtuple` API.) @@ -2535,7 +2535,7 @@ types. Helper class to create low-overhead :ref:`distinct types `. - A ``NewType`` is considered a distinct type by a typechecker. At runtime, + A ``NewType`` is considered a distinct type by a type checker. At runtime, however, calling a ``NewType`` returns its argument unchanged. Usage:: @@ -2616,7 +2616,7 @@ types. Mark a protocol class as a runtime protocol. Such a protocol can be used with :func:`isinstance` and :func:`issubclass`. - This allows a simple-minded structural check, very similar to "one trick ponies" + This allows a simple-minded structural check, very similar to "one-trick ponies" in :mod:`collections.abc` such as :class:`~collections.abc.Iterable`. For example:: @runtime_checkable @@ -2855,7 +2855,7 @@ types. key: T group: list[T] - A ``TypedDict`` can be introspected via annotations dicts + A ``TypedDict`` can be introspected via :func:`annotationlib.get_annotations` (see :ref:`annotations-howto` for more information on annotations best practices) and the following attributes: @@ -2898,7 +2898,7 @@ types. For backwards compatibility with Python 3.10 and below, it is also possible to use inheritance to declare both required and - non-required keys in the same ``TypedDict`` . This is done by declaring a + non-required keys in the same ``TypedDict``. This is done by declaring a ``TypedDict`` with one value for the ``total`` argument and then inheriting from it in another ``TypedDict`` with a different value for ``total``: @@ -2982,34 +2982,34 @@ with :deco:`runtime_checkable`. .. class:: SupportsAbs - An ABC with one abstract method ``__abs__`` that is covariant + A protocol with one abstract method ``__abs__`` that is covariant in its return type. .. class:: SupportsBytes - An ABC with one abstract method ``__bytes__``. + A protocol with one abstract method ``__bytes__``. .. class:: SupportsComplex - An ABC with one abstract method ``__complex__``. + A protocol with one abstract method ``__complex__``. .. class:: SupportsFloat - An ABC with one abstract method ``__float__``. + A protocol with one abstract method ``__float__``. .. class:: SupportsIndex - An ABC with one abstract method ``__index__``. + A protocol with one abstract method ``__index__``. .. versionadded:: 3.8 .. class:: SupportsInt - An ABC with one abstract method ``__int__``. + A protocol with one abstract method ``__int__``. .. class:: SupportsRound - An ABC with one abstract method ``__round__`` + A protocol with one abstract method ``__round__`` that is covariant in its return type. .. _typing-io: @@ -3633,14 +3633,27 @@ Introspection helpers Determine if a type is a :class:`Protocol`. - For example:: + For example: + + .. testcode:: class P(Protocol): def a(self) -> str: ... b: int - is_protocol(P) # => True - is_protocol(int) # => False + assert is_protocol(P) + assert not is_protocol(int) + + This function only returns true for ``Protocol`` classes, not for + :ref:`generic aliases ` of them: + + .. testcode:: + + class GenericP[T](Protocol): + def a(self) -> T: ... + b: int + + assert not is_protocol(GenericP[int]) .. versionadded:: 3.13 @@ -3663,6 +3676,17 @@ Introspection helpers # not a typed dict itself assert not is_typeddict(TypedDict) + This function only returns true for ``TypedDict`` classes, not for + :ref:`generic aliases ` of them: + + .. testcode:: + + class GenericFilm[T](TypedDict): + title: str + year: T + + assert not is_typeddict(GenericFilm[int]) + .. versionadded:: 3.10 .. class:: ForwardRef @@ -3739,7 +3763,7 @@ Constant .. data:: TYPE_CHECKING - A special constant that is assumed to be ``True`` by 3rd party static + A special constant that is assumed to be ``True`` by static type checkers. It's ``False`` at runtime. A module which is expensive to import, and which only contain types diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index f5c11fd849f58b3..25bf872e0ab55a8 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -18,8 +18,7 @@ this database is compiled from the `UCD version 17.0.0 The module uses the same names and symbols as defined by Unicode Standard Annex #44, `"Unicode Character Database" -`_. It defines the -following functions: +`_. .. seealso:: @@ -27,6 +26,44 @@ following functions: this module. +============================================================ =========================================================== +**Lookup** +------------------------------------------------------------------------------------------------------------------------- +:func:`lookup(name) ` Look up character by name +:func:`name(chr) ` Return the name assigned to a character + +**Numeric values** +------------------------------------------------------------------------------------------------------------------------- +:func:`decimal(chr) ` Decimal value of a character +:func:`digit(chr) ` Digit value of a character +:func:`numeric(chr) ` Numeric value of a character + +**Properties** +------------------------------------------------------------------------------------------------------------------------- +:func:`bidirectional(chr) ` Bidirectional class of a character +:func:`block(chr) ` Unicode block of a character +:func:`category(chr) ` General category of a character +:func:`combining(chr) ` Canonical combining class of a character +:func:`decomposition(chr) ` Character decomposition mapping +:func:`east_asian_width(chr) ` East Asian width of a character +:func:`extended_pictographic(chr) ` Check if a character has the Extended_Pictographic property +:func:`grapheme_cluster_break(chr) ` Grapheme_Cluster_Break property of a character +:func:`indic_conjunct_break(chr) ` Indic_Conjunct_Break property of a character +:func:`isxidcontinue(chr) ` Check if a character is a valid identifier continuation +:func:`isxidstart(chr) ` Check if a character is a valid identifier start +:func:`mirrored(chr) ` Mirrored property of a character + +**Normalization** +------------------------------------------------------------------------------------------------------------------------- +:func:`normalize(form, unistr) ` Return the normalized form of a string +:func:`is_normalized(form, unistr) ` Check if a Unicode string is normalized + +**Text segmentation** +------------------------------------------------------------------------------------------------------------------------- +:func:`iter_graphemes(unistr) ` Iterate over grapheme clusters in a string +============================================================ =========================================================== + + .. function:: lookup(name, /) Look up character by name. If a character with the given name is found, return @@ -273,7 +310,7 @@ following functions: .. versionadded:: 3.15 -In addition, the module exposes the following constant: +In addition, the module exposes the following constants: .. data:: unidata_version diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index c54f3e2792c3888..ff619f979233251 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1262,10 +1262,10 @@ Test cases | :meth:`assertNotEndsWith(a, b) | ``not a.endswith(b)`` | 3.14 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertHasAttr(a, b) | ``hastattr(a, b)`` | 3.14 | + | :meth:`assertHasAttr(a, b) | ``hasattr(a, b)`` | 3.14 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertNotHasAttr(a, b) | ``not hastattr(a, b)`` | 3.14 | + | :meth:`assertNotHasAttr(a, b) | ``not hasattr(a, b)`` | 3.14 | | ` | | | +---------------------------------------+--------------------------------+--------------+ diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 9999ac26999910b..ebafcb977803d41 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -411,9 +411,9 @@ ZipFile objects .. warning:: Never extract archives from untrusted sources without prior inspection. - It is possible that files are created outside of *path*, e.g. members - that have absolute filenames starting with ``"/"`` or filenames with two - dots ``".."``. This module attempts to prevent that. + It is possible that files are created outside of *path*, for example, members + that have absolute filenames or filenames with ".." components. + This module attempts to prevent that. See :meth:`extract` note. .. versionchanged:: 3.6 @@ -590,7 +590,7 @@ Path objects The :class:`Path` class does not sanitize filenames within the ZIP archive. Unlike the :meth:`ZipFile.extract` and :meth:`ZipFile.extractall` methods, it is the caller's responsibility to validate or sanitize filenames to prevent path traversal - vulnerabilities (e.g., filenames containing ".." or absolute paths). When handling + vulnerabilities (for example, absolute paths or paths with ".." components). When handling untrusted archives, consider resolving filenames using :func:`os.path.abspath` and checking against the target directory with :func:`os.path.commonpath`. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 72e1cad3bbd8924..a819c41d834aa70 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -618,8 +618,8 @@ The match statement is used for pattern matching. Syntax: .. productionlist:: python-grammar match_stmt: 'match' `subject_expr` ":" NEWLINE INDENT `case_block`+ DEDENT - subject_expr: `!star_named_expression` "," `!star_named_expressions`? - : | `!named_expression` + subject_expr: `flexible_expression` "," [`flexible_expression_list` [',']] + : | `assignment_expression` case_block: 'case' `patterns` [`guard`] ":" `!block` .. note:: @@ -709,7 +709,7 @@ Guards .. index:: ! guard .. productionlist:: python-grammar - guard: "if" `!named_expression` + guard: "if" `assignment_expression` A ``guard`` (which is part of the ``case``) must succeed for code inside the ``case`` block to execute. It takes the form: :keyword:`if` followed by an diff --git a/Doc/tools/extensions/pydoc_topics.py b/Doc/tools/extensions/pydoc_topics.py index a65d77433b255bc..35878e2d1e43e9b 100644 --- a/Doc/tools/extensions/pydoc_topics.py +++ b/Doc/tools/extensions/pydoc_topics.py @@ -68,6 +68,7 @@ "in", "integers", "lambda", + "lazy", "lists", "naming", "nonlocal", diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 9e2f789334ff02b..1670f033401f2bf 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -411,8 +411,8 @@ embedding applications, and native libraries. unwinding for the whole Python process. (Contributed by Pablo Galindo Salgado and Savannah Ostrowski in -:gh:`149201`; PEP 831 written by Pablo Galindo Salgado, Ken Jin, and -Savannah Ostrowski.) +:gh:`149201`; PEP 831 written by Pablo Galindo Salgado, Ken Jin, +Savannah Ostrowski, and Diego Russo.) .. seealso:: :pep:`831` for further details. @@ -798,7 +798,7 @@ Other language changes n = buffer.find(b'\n') data = bytes(buffer[:n + 1]) del buffer[:n + 1] - assert data == b'abc' + assert data == b'abc\n' assert buffer == bytearray(b'def') - .. code:: python @@ -1208,6 +1208,19 @@ http.server (Contributed by Anton I. Sipos in :gh:`135057`.) +importlib.metadata +------------------ + +* Previously, when accessing a distribution metadata directory not + containing a metadata file, ``metadata()`` and ``Distribution.metadata()`` + would return an empty ``PackageMetadata`` object as if the file + was present but empty. Now, a ``MetadataNotFound`` exception is raised. + See `importlib_metadata#493 `_ + for background and rationale and and :gh:`143387` for rationale on the + compatibility concerns. + (Contributed by Jason R. Coombs.) + + inspect ------- @@ -1285,6 +1298,10 @@ os glibc versions 2.28 and later. (Contributed by Jeffrey Bosboom and Victor Stinner in :gh:`83714`.) +* :func:`os.makedirs` function now has a *parent_mode* parameter that allows + specifying the mode for intermediate directories. This can be used to match + the behavior from Python 3.6 and earlier by passing ``parent_mode=mode``. + (Contributed by Zackery Spytz and Gregory P. Smith in :gh:`86533`.) os.path ------- @@ -2057,6 +2074,10 @@ importlib.resources pathlib ------- +* :meth:`pathlib.Path.mkdir` now has a *parent_mode* parameter that allows + specifying the mode for intermediate directories when ``parents=True``. + (Contributed by Gregory P. Smith in :gh:`86533`.) + * Removed deprecated :meth:`!pathlib.PurePath.is_reserved`. Use :func:`os.path.isreserved` to detect reserved paths on Windows. (Contributed by Nikita Sobolev in :gh:`133875`.) diff --git a/Include/cpython/sentinelobject.h b/Include/cpython/sentinelobject.h index 0b6ff0f17e6f8c1..15643ef966af86e 100644 --- a/Include/cpython/sentinelobject.h +++ b/Include/cpython/sentinelobject.h @@ -1,15 +1,18 @@ /* Sentinel object interface */ #ifndef Py_LIMITED_API -#ifndef Py_SENTINELOBJECT_H -#define Py_SENTINELOBJECT_H +#ifndef _Py_SENTINELOBJECT_H +#define _Py_SENTINELOBJECT_H #ifdef __cplusplus extern "C" { #endif PyAPI_DATA(PyTypeObject) PySentinel_Type; -#define PySentinel_Check(op) Py_IS_TYPE((op), &PySentinel_Type) +#define PySentinel_CheckExact(op) Py_IS_TYPE((op), &PySentinel_Type) + +/* Alias as long as subclasses are not allowed. */ +#define PySentinel_Check(op) PySentinel_CheckExact(op) PyAPI_FUNC(PyObject *) PySentinel_New( const char *name, @@ -18,5 +21,5 @@ PyAPI_FUNC(PyObject *) PySentinel_New( #ifdef __cplusplus } #endif -#endif /* !Py_SENTINELOBJECT_H */ +#endif /* !_Py_SENTINELOBJECT_H */ #endif /* !Py_LIMITED_API */ diff --git a/Include/cpython/sliceobject.h b/Include/cpython/sliceobject.h index 4c3ea1facebc4ed..137206eff15b33f 100644 --- a/Include/cpython/sliceobject.h +++ b/Include/cpython/sliceobject.h @@ -1,4 +1,4 @@ -#ifndef Py_CPYTHON_SLICEOBJECT_H +#ifndef _Py_CPYTHON_SLICEOBJECT_H # error "this header file must not be included directly" #endif diff --git a/Include/cpython/structseq.h b/Include/cpython/structseq.h index 328fbe86143b024..83a1abcd6f3b347 100644 --- a/Include/cpython/structseq.h +++ b/Include/cpython/structseq.h @@ -1,4 +1,4 @@ -#ifndef Py_CPYTHON_STRUCTSEQ_H +#ifndef _Py_CPYTHON_STRUCTSEQ_H # error "this header file must not be included directly" #endif diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index 11932b8d1e1ab60..bb6fe2625975596 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -13,6 +13,8 @@ extern "C" { struct _Py_dict_state { uint32_t next_keys_version; + PyMutex watcher_mutex; // Protects the watchers array (free-threaded builds) + _PyOnceFlag watcher_setup_once; // One-time optimizer watcher setup PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 32ed3a62b2b4a7c..a1078828afa572e 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -39,6 +39,8 @@ extern PyObject * _PyImport_GetAbsName( // Symbol is exported for the JIT on Windows builds. PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate( PyThreadState *tstate, PyObject *lazy_import); +extern PyObject * _PyImport_TryLoadLazySubmodule( + PyObject *mod_name, PyObject *attr_name); extern PyObject * _PyImport_LazyImportModuleLevelObject( PyThreadState *tstate, PyObject *name, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *fromlist, int level); diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index f13bc2178b1e7eb..d8e83cf2ff5c9a9 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -349,7 +349,15 @@ struct _import_state { int lazy_imports_mode; PyObject *lazy_imports_filter; PyObject *lazy_importing_modules; + // The set stored in sys.lazy_modules if values that have been + // lazily imported. This value is only for debugging/introspection + // purposes and is not used by the runtime. PyObject *lazy_modules; + // A dict mapping package names to a set of submodule names that + // have been imported lazily from packages which have been imported + // lazily. When the package is reified we need to add a + // LazyImportObject which refers to the submodule on the module. + PyObject *lazy_pending_submodules; #ifdef Py_GIL_DISABLED PyMutex lazy_mutex; #endif diff --git a/Include/internal/pycore_jit_unwind.h b/Include/internal/pycore_jit_unwind.h index 508caee97c43ab5..7099b88812ce7be 100644 --- a/Include/internal/pycore_jit_unwind.h +++ b/Include/internal/pycore_jit_unwind.h @@ -11,7 +11,7 @@ #if defined(_Py_JIT) && defined(__linux__) && defined(__ELF__) # define PY_HAVE_JIT_GDB_UNWIND # if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ - defined(HAVE_LIBGCC_EH_FRAME_REGISTRATION) + defined(_Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION) # define PY_HAVE_JIT_GNU_BACKTRACE_UNWIND # endif #endif diff --git a/Include/internal/pycore_mmap.h b/Include/internal/pycore_mmap.h index 897816db01077f6..c117cbd16283da9 100644 --- a/Include/internal/pycore_mmap.h +++ b/Include/internal/pycore_mmap.h @@ -11,12 +11,12 @@ extern "C" { #include "pycore_pystate.h" -#if defined(HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) +#if defined(_Py_HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) # include # include #endif -#if defined(HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) +#if defined(_Py_HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) static inline int _PyAnnotateMemoryMap(void *addr, size_t size, const char *name) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index fafdd728a8229a9..d8ec306a0dae3fc 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -138,6 +138,7 @@ extern "C" { #define FT_ATOMIC_ADD_SSIZE(value, new_value) \ (void)_Py_atomic_add_ssize(&value, new_value) #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) +#define FT_MUTEX_LOCK_FLAGS(lock, flags) PyMutex_LockFlags(lock, flags) #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) #else @@ -201,6 +202,7 @@ extern "C" { #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_ADD_SSIZE(value, new_value) (void)(value += new_value) #define FT_MUTEX_LOCK(lock) do {} while (0) +#define FT_MUTEX_LOCK_FLAGS(lock, flags) do {} while (0) #define FT_MUTEX_UNLOCK(lock) do {} while (0) #endif diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 8d48cf6605ca7e3..785b77d3e3be81e 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -122,6 +122,8 @@ extern PyObject* _Py_BaseObject_RichCompare(PyObject* self, PyObject* other, int extern PyObject* _Py_slot_tp_getattro(PyObject *self, PyObject *name); extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); +extern int _PyType_HasSlotTpIternext(PyTypeObject *type); + extern PyTypeObject _PyBufferWrapper_Type; PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 32c85792d550c3f..cdca931566577fc 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -27,7 +27,7 @@ #define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.15.0b1" +#define PY_VERSION "3.15.0b1+" /*--end constants--*/ diff --git a/Include/sliceobject.h b/Include/sliceobject.h index 00c70a6e911b417..9d6a16da95fe2f5 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -45,9 +45,9 @@ PyAPI_FUNC(Py_ssize_t) PySlice_AdjustIndices(Py_ssize_t length, #endif #ifndef Py_LIMITED_API -# define Py_CPYTHON_SLICEOBJECT_H +# define _Py_CPYTHON_SLICEOBJECT_H # include "cpython/sliceobject.h" -# undef Py_CPYTHON_SLICEOBJECT_H +# undef _Py_CPYTHON_SLICEOBJECT_H #endif #ifdef __cplusplus diff --git a/Include/structseq.h b/Include/structseq.h index e52d6188030af9d..e5da785f13d46b8 100644 --- a/Include/structseq.h +++ b/Include/structseq.h @@ -29,9 +29,9 @@ PyAPI_FUNC(void) PyStructSequence_SetItem(PyObject*, Py_ssize_t, PyObject*); PyAPI_FUNC(PyObject*) PyStructSequence_GetItem(PyObject*, Py_ssize_t); #ifndef Py_LIMITED_API -# define Py_CPYTHON_STRUCTSEQ_H +# define _Py_CPYTHON_STRUCTSEQ_H # include "cpython/structseq.h" -# undef Py_CPYTHON_STRUCTSEQ_H +# undef _Py_CPYTHON_STRUCTSEQ_H #endif #ifdef __cplusplus diff --git a/Lib/argparse.py b/Lib/argparse.py index 6d21823e6524293..29e6ebb9634261a 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -163,6 +163,8 @@ class _ColorlessTheme: def __getattr__(self, name): # _colorize's no_color themes are just all empty strings # by directly using empty strings the import is avoided + if name.startswith("_"): + raise AttributeError(name) return "" _colorless_theme = _ColorlessTheme() diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index a51319cb72a6a9b..6727065bbe323ff 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -18,8 +18,8 @@ def _is_debug_mode(): def iscoroutinefunction(func): - import warnings """Return True if func is a decorated coroutine function.""" + import warnings warnings._deprecated("asyncio.iscoroutinefunction", f"{warnings._DEPRECATED_MSG}; " "use inspect.iscoroutinefunction() instead", diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py index acd49441131b042..d6393f0b1ffee5d 100644 --- a/Lib/asyncio/windows_utils.py +++ b/Lib/asyncio/windows_utils.py @@ -111,8 +111,9 @@ def fileno(self): def close(self, *, CloseHandle=_winapi.CloseHandle): if self._handle is not None: - CloseHandle(self._handle) + handle = self._handle self._handle = None + CloseHandle(handle) def __del__(self, _warn=warnings.warn): if self._handle is not None: diff --git a/Lib/base64.py b/Lib/base64.py index 4b810e08569e5ba..4a0e9d446edb0bc 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -315,16 +315,20 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): foldspaces is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Adobe encoding. + feature is not supported by the standard encoding used in PDF. If wrapcol is non-zero, insert a newline (b'\\n') character after at most every wrapcol characters. - pad controls whether the input is padded to a multiple of 4 before - encoding. Note that the btoa implementation always pads. + pad controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by btoa, + producing an exact multiple of 5 bytes of output. + + adobe controls whether the encoded byte sequence is framed with <~ + and ~>, as in a PostScript base-85 string literal. Note that + while ASCII85Decode streams in PDF documents must be terminated + with ~>, they must not use a leading <~. - adobe controls whether the encoded byte sequence is framed with <~ and ~>, - which is used by the Adobe implementation. """ return binascii.b2a_ascii85(b, foldspaces=foldspaces, adobe=adobe, wrapcol=wrapcol, pad=pad) @@ -333,12 +337,14 @@ def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v', canonical=False): """Decode the Ascii85 encoded bytes-like object or ASCII string b. - foldspaces is a flag that specifies whether the 'y' short sequence should be - accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is - not supported by the "standard" Adobe encoding. + foldspaces is a flag that specifies whether the 'y' short sequence + should be accepted as shorthand for 4 consecutive spaces (ASCII + 0x20). This feature is not supported by the standard Ascii85 + encoding used in PDF and PostScript. - adobe controls whether the input sequence is in Adobe Ascii85 format (i.e. - is framed with <~ and ~>). + adobe controls whether the <~ and ~> markers are present. While + the leading <~ is not required, the input must end with ~>, or a + ValueError is raised. ignorechars should be a byte string containing characters to ignore from the input. This should only contain whitespace characters, and by default @@ -358,8 +364,10 @@ def b85encode(b, pad=False, *, wrapcol=0): If wrapcol is non-zero, insert a newline (b'\\n') character after at most every wrapcol characters. - If pad is true, the input is padded with b'\\0' so its length is a multiple of - 4 bytes before encoding. + The input is padded with b'\0' so its length is a multiple of 4 + bytes before encoding. If pad is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes. """ return binascii.b2a_base85(b, wrapcol=wrapcol, pad=pad) @@ -379,8 +387,10 @@ def z85encode(s, pad=False, *, wrapcol=0): If wrapcol is non-zero, insert a newline (b'\\n') character after at most every wrapcol characters. - If pad is true, the input is padded with b'\\0' so its length is a multiple of - 4 bytes before encoding. + The input is padded with b'\0' so its length is a multiple of + bytes before encoding. If pad is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, as required by the ZeroMQ standard. """ return binascii.b2a_base85(s, wrapcol=wrapcol, pad=pad, alphabet=binascii.Z85_ALPHABET) diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 9873958f5c2790c..792072ab9f6128a 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -1461,6 +1461,16 @@ def get_phrase(value): else: try: token, value = get_word(value) + if (token[0].token_type == 'encoded-word' + and phrase + and phrase[-1].token_type == 'atom' + and len(phrase[-1]) > 1 + and phrase[-1][-2].token_type == 'encoded-word' + and phrase[-1][-1].token_type == 'cfws' + and not phrase[-1][-1].comments + ): + # linear ws between ews needs special handing... + phrase[-1][-1] = EWWhiteSpaceTerminal(phrase[-1], 'fws') except errors.HeaderParseError: if value[0] in CFWS_LEADER: token, value = get_cfws(value) diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index f4b1b8dd43f9205..e5e50630f33d14d 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -71,6 +71,10 @@ # cp1140 codec '1140' : 'cp1140', + 'cp01140' : 'cp1140', + 'csibm01140' : 'cp1140', + 'ebcdic_us_37_euro' : 'cp1140', + 'ibm01140' : 'cp1140', 'ibm1140' : 'cp1140', # cp1250 codec @@ -159,8 +163,12 @@ # cp858 codec '858' : 'cp858', + 'cp00858' : 'cp858', + 'csibm00858' : 'cp858', 'csibm858' : 'cp858', + 'ibm00858' : 'cp858', 'ibm858' : 'cp858', + 'pc_multilingual_850_euro' : 'cp858', # cp860 codec '860' : 'cp860', diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 640acc64f620cc9..2f092d50f31782b 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -883,7 +883,16 @@ def ftpcp(source, sourcename, target, targetname = '', type = 'I'): type = 'TYPE ' + type source.voidcmd(type) target.voidcmd(type) - sourcehost, sourceport = parse227(source.sendcmd('PASV')) + # Don't trust the IPv4 address the source server advertises in its PASV + # reply: a malicious source could otherwise point the target's data + # connection at an arbitrary host (SSRF). A caller that needs the old + # behavior can set trust_server_pasv_ipv4_address on the source FTP + # object. See FTP.makepasv(), which applies the same rule. + untrusted_host, sourceport = parse227(source.sendcmd('PASV')) + if source.trust_server_pasv_ipv4_address: + sourcehost = untrusted_host + else: + sourcehost = source.sock.getpeername()[0] target.sendport(sourcehost, sourceport) # RFC 959: the user must "listen" [...] BEFORE sending the # transfer request. diff --git a/Lib/functools.py b/Lib/functools.py index cd374631f167925..e03a77f204b5443 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -232,7 +232,7 @@ def __ge__(self, other): ### reduce() sequence to a single item ################################################################################ -_initial_missing = object() +_initial_missing = sentinel('_initial_missing') def reduce(function, sequence, initial=_initial_missing): """ diff --git a/Lib/gzip.py b/Lib/gzip.py index 971063aa24f8712..0713b922522ee18 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -188,8 +188,10 @@ def __init__(self, filename=None, mode=None, The optional mtime argument is the timestamp requested by gzip. The time is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. - If mtime is omitted or None, the current time is used. Use mtime = 0 - to generate a compressed stream that does not depend on creation time. + Set mtime to 0 to generate a compressed stream that does not depend on + creation time. If mtime is omitted or None, the current time is used. + If the resulting mtime is outside the range 0 to 2**32-1, then the + value 0 is used instead. """ @@ -295,6 +297,8 @@ def _write_gzip_header(self, compresslevel): mtime = self._write_mtime if mtime is None: mtime = time.time() + if not 0 <= mtime < 2**32: + mtime = 0 write32u(self.fileobj, int(mtime)) if compresslevel == _COMPRESS_LEVEL_BEST: xfl = b'\002' @@ -484,40 +488,73 @@ def _read_exact(fp, n): return data +def _read_until_null(fp, crc=None): + '''Read until the first encountered null byte in fp. + If crc is not None, update and return the CRC. + ''' + if crc is None: + while True: + s = fp.read(1) + if not s or s == b'\000': + break + else: + while True: + s = fp.read(1) + crc = zlib.crc32(s, crc) + if not s or s == b'\000': + break + return crc + + def _read_gzip_header(fp): '''Read a gzip header from `fp` and progress to the end of the header. Returns last mtime if header was present or None otherwise. ''' magic = fp.read(2) - if magic == b'': + if not magic: return None if magic != b'\037\213': raise BadGzipFile('Not a gzipped file (%r)' % magic) - - (method, flag, last_mtime) = struct.unpack("' % (self.__class__.__name__, self.OutputString()) - def _js_output(self, attrs=None): """Internal implementation without deprecation warning.""" - import base64 + import urllib.parse # Print javascript output_string = self.OutputString(attrs) if _has_control_character(output_string): raise CookieError("Control characters are not allowed in cookies") # Base64-encode value to avoid template # injection in cookie values. - output_encoded = base64.b64encode(output_string.encode('utf-8')).decode("ascii") + output_encoded = urllib.parse.quote(output_string, safe='', encoding='utf-8') return """ """ % (output_encoded,) diff --git a/Lib/inspect.py b/Lib/inspect.py index a96b3dc954ef0ca..dc5a6e3be883bb0 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2200,7 +2200,8 @@ def wrap_value(s): except NameError: raise ValueError - if isinstance(value, (str, int, float, bytes, bool, type(None))): + if isinstance(value, (str, int, float, bytes, bool, type(None), + sentinel)): return ast.Constant(value) raise ValueError diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index 251025efac14b87..94c177cafa0294f 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -142,8 +142,8 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent - level of 0 will only insert newlines. ``None`` is the most compact - representation. + level of 0 will only insert newlines. ``None`` is the default and gives + a representation with no newlines inserted. If specified, ``separators`` should be an ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is @@ -206,8 +206,8 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent - level of 0 will only insert newlines. ``None`` is the most compact - representation. + level of 0 will only insert newlines. ``None`` is the default and gives + a representation with no newlines inserted. If specified, ``separators`` should be an ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is diff --git a/Lib/os.py b/Lib/os.py index 52cbc5bc85864e7..1ca4648cc95c3ee 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -219,14 +219,17 @@ def _add(str, fn): # Super directory utilities. # (Inspired by Eric Raymond; the doc strings are mostly his) -def makedirs(name, mode=0o777, exist_ok=False): - """makedirs(name [, mode=0o777][, exist_ok=False]) +def makedirs(name, mode=0o777, exist_ok=False, *, parent_mode=None): + """makedirs(name [, mode=0o777][, exist_ok=False][, parent_mode=None]) Super-mkdir; create a leaf directory and all intermediate ones. Works like mkdir, except that any intermediate path segment (not just the rightmost) will be created if it does not exist. If the target directory already exists, raise an OSError if exist_ok is False. Otherwise no exception is - raised. This is recursive. + raised. If parent_mode is not None, it will be used as the mode for any + newly-created, intermediate-level directories. Otherwise, intermediate + directories are created with the default permissions (respecting umask). + This is recursive. """ head, tail = path.split(name) @@ -234,7 +237,11 @@ def makedirs(name, mode=0o777, exist_ok=False): head, tail = path.split(head) if head and tail and not path.exists(head): try: - makedirs(head, exist_ok=exist_ok) + if parent_mode is not None: + makedirs(head, mode=parent_mode, exist_ok=exist_ok, + parent_mode=parent_mode) + else: + makedirs(head, exist_ok=exist_ok) except FileExistsError: # Defeats race condition when another thread created the path pass diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index a32e4b5320ff6dd..8dd16c6225b927b 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -1202,7 +1202,7 @@ def touch(self, mode=0o666, exist_ok=True): fd = os.open(self, flags, mode) os.close(fd) - def mkdir(self, mode=0o777, parents=False, exist_ok=False): + def mkdir(self, mode=0o777, parents=False, exist_ok=False, *, parent_mode=None): """ Create a new directory at this given path. """ @@ -1211,7 +1211,11 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): except FileNotFoundError: if not parents or self.parent == self: raise - self.parent.mkdir(parents=True, exist_ok=True) + if parent_mode is not None: + self.parent.mkdir(mode=parent_mode, parents=True, exist_ok=True, + parent_mode=parent_mode) + else: + self.parent.mkdir(parents=True, exist_ok=True) self.mkdir(mode, parents=False, exist_ok=exist_ok) except OSError: # Cannot rely on checking for EEXIST, since the operating system diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap.js b/Lib/profiling/sampling/_heatmap_assets/heatmap.js index 2da1103b82a52a3..1f698779f3a46e3 100644 --- a/Lib/profiling/sampling/_heatmap_assets/heatmap.js +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap.js @@ -84,7 +84,7 @@ function showNavigationMenu(button, items, title) { item.appendChild(funcDiv); item.appendChild(createElement('div', 'callee-menu-file', linkData.file)); - item.addEventListener('click', () => window.location.href = linkData.link); + item.addEventListener('click', () => navigateToLine(linkData.link)); menu.appendChild(item); }); @@ -105,7 +105,7 @@ function handleNavigationClick(button, e) { const navData = button.getAttribute('data-nav'); if (navData) { - window.location.href = JSON.parse(navData).link; + navigateToLine(JSON.parse(navData).link); return; } @@ -117,11 +117,29 @@ function handleNavigationClick(button, e) { } } +function restartLineHighlight(target) { + target.style.animation = 'none'; + // Force style recalculation so restoring the animation restarts it. + void target.offsetWidth; + target.style.animation = ''; +} + +function navigateToLine(link) { + const url = new URL(link, window.location.href); + + if (url.href === window.location.href) { + scrollToTargetLine(); + } else { + window.location.href = link; + } +} + function scrollToTargetLine() { if (!window.location.hash) return; const target = document.querySelector(window.location.hash); if (target) { target.scrollIntoView({ behavior: 'smooth', block: 'start' }); + restartLineHighlight(target); } } diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index 0648713edc52af3..a5d9573ae6b6ddd 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -167,7 +167,9 @@ def _build_child_profiler_args(args): child_args.extend(["--mode", mode]) # Format options (skip pstats as it's the default) - if args.format != "pstats": + if args.format == "diff_flamegraph": + child_args.extend(["--diff-flamegraph", args.diff_baseline]) + elif args.format != "pstats": child_args.append(f"--{args.format}") return child_args diff --git a/Lib/profiling/sampling/collector.py b/Lib/profiling/sampling/collector.py index 81ec6344ebdea4a..8e0f0c44c4f8f36 100644 --- a/Lib/profiling/sampling/collector.py +++ b/Lib/profiling/sampling/collector.py @@ -143,6 +143,8 @@ def iter_async_frames(awaited_info_list): class Collector(ABC): + aggregating = False + @abstractmethod def collect(self, stack_frames, timestamps_us=None): """Collect profiling data from stack frames. diff --git a/Lib/profiling/sampling/gecko_collector.py b/Lib/profiling/sampling/gecko_collector.py index 8986194268b3ce4..54392af95000082 100644 --- a/Lib/profiling/sampling/gecko_collector.py +++ b/Lib/profiling/sampling/gecko_collector.py @@ -63,6 +63,8 @@ class GeckoCollector(Collector): + aggregating = True + def __init__(self, sample_interval_usec, *, skip_idle=False, opcodes=False): self.sample_interval_usec = sample_interval_usec self.skip_idle = skip_idle diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index 5c36d78f5535e71..6e650ec08f410bc 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -452,7 +452,8 @@ def process_frames(self, frames, thread_id, weight=1): next_lineno = extract_lineno(next_frame[1]) self._record_call_relationship( (filename, lineno, funcname), - (next_frame[0], next_lineno, next_frame[2]) + (next_frame[0], next_lineno, next_frame[2]), + weight=weight, ) def _is_valid_frame(self, filename, lineno): @@ -561,7 +562,7 @@ def _get_bytecode_data_for_line(self, filename, lineno): result.sort(key=lambda x: (-x['samples'], x['opcode'])) return result - def _record_call_relationship(self, callee_frame, caller_frame): + def _record_call_relationship(self, callee_frame, caller_frame, weight=1): """Record caller/callee relationship between adjacent frames.""" callee_filename, callee_lineno, callee_funcname = callee_frame caller_filename, caller_lineno, caller_funcname = caller_frame @@ -587,7 +588,7 @@ def _record_call_relationship(self, callee_frame, caller_frame): # Count this call edge for path analysis edge_key = (caller_key, callee_key) - self.edge_samples[edge_key] += 1 + self.edge_samples[edge_key] += weight def export(self, output_path): """Export heatmap data as HTML files in a directory. diff --git a/Lib/profiling/sampling/jsonl_collector.py b/Lib/profiling/sampling/jsonl_collector.py index 7d26129b80de868..5aa42ef09024dc3 100644 --- a/Lib/profiling/sampling/jsonl_collector.py +++ b/Lib/profiling/sampling/jsonl_collector.py @@ -164,6 +164,7 @@ def export(self, filename): self._iter_final_agg_entries(), ) self._write_message(output, self._build_end_record()) + print(f"JSONL profile written to {filename}") def _build_meta_record(self): record = { diff --git a/Lib/profiling/sampling/pstats_collector.py b/Lib/profiling/sampling/pstats_collector.py index 50500296c15acc9..43b1daf2a119d4e 100644 --- a/Lib/profiling/sampling/pstats_collector.py +++ b/Lib/profiling/sampling/pstats_collector.py @@ -8,6 +8,8 @@ class PstatsCollector(Collector): + aggregating = True + def __init__(self, sample_interval_usec, *, skip_idle=False): self.result = collections.defaultdict( lambda: dict(total_rec_calls=0, direct_calls=0, cumulative_calls=0) diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 5bbe24835813332..2d379e1e16a35e3 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -47,6 +47,9 @@ def _pause_threads(unwinder, blocking): # If fewer samples are collected, we skip the TUI and just print a message MIN_SAMPLES_FOR_TUI = 200 +# Maximum number of consecutive identical samples to keep before flushing. +MAX_PENDING_SAMPLES = 8192 + class SampleProfiler: def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, native=False, gc=True, opcodes=False, skip_non_matching_threads=True, collect_stats=False, blocking=False): self.pid = pid @@ -109,6 +112,20 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): last_sample_time = start_time realtime_update_interval = 1.0 # Update every second last_realtime_update = start_time + aggregating = getattr(collector, 'aggregating', False) is True + prev_stack = None + pending_count = 0 + pending_timestamps = [] if aggregating else None + + def flush_pending(): + nonlocal pending_count, pending_timestamps + if pending_count == 0: + return + pending_count = 0 + ts = pending_timestamps + pending_timestamps = [] + collector.collect(prev_stack, timestamps_us=ts) + try: while duration_sec is None or running_time_sec < duration_sec: # Check if live collector wants to stop @@ -116,6 +133,7 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): break current_time = time.perf_counter() + current_time_us = int(current_time * 1_000_000) if next_time > current_time: sleep_time = (next_time - current_time) * 0.9 if sleep_time > 0.0001: @@ -125,13 +143,24 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): stack_frames = self._get_stack_trace( async_aware=async_aware ) - collector.collect(stack_frames) + if aggregating: + if stack_frames != prev_stack: + flush_pending() + prev_stack = stack_frames + pending_count += 1 + pending_timestamps.append(current_time_us) + if pending_count >= MAX_PENDING_SAMPLES: + flush_pending() + else: + collector.collect(stack_frames) except ProcessLookupError as e: running_time_sec = current_time - start_time break except (RuntimeError, UnicodeDecodeError, MemoryError, OSError): + flush_pending() collector.collect_failed_sample() errors += 1 + prev_stack = None except Exception as e: if not _is_process_running(self.pid): break @@ -163,6 +192,8 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): interrupted = True running_time_sec = time.perf_counter() - start_time print("Interrupted by user.") + finally: + flush_pending() # Clear real-time stats line if it was being displayed if self.realtime_stats and len(self.sample_intervals) > 0: @@ -296,6 +327,33 @@ def _print_unwinder_stats(self): print(f" Hits: {code_hits:n} ({ANSIColors.GREEN}{fmt(code_hits_pct)}%{ANSIColors.RESET})") print(f" Misses: {code_misses:n} ({ANSIColors.RED}{fmt(code_misses_pct)}%{ANSIColors.RESET})") + batched_attempts = stats.get('batched_read_attempts', 0) + batched_successes = stats.get('batched_read_successes', 0) + batched_misses = stats.get('batched_read_misses', 0) + segments_requested = stats.get('batched_read_segments_requested', 0) + segments_completed = stats.get('batched_read_segments_completed', 0) + if batched_attempts > 0: + batched_success_rate = stats.get('batched_read_success_rate', 0.0) + batched_miss_rate = 100.0 - batched_success_rate + segment_completion_rate = stats.get( + 'batched_read_segment_completion_rate', 0.0 + ) + + print(f" {ANSIColors.CYAN}Batched Reads:{ANSIColors.RESET}") + print(f" Attempts: {batched_attempts:n}") + print( + f" Successes: {batched_successes:n} " + f"({ANSIColors.GREEN}{fmt(batched_success_rate)}%{ANSIColors.RESET})" + ) + print( + f" Misses: {batched_misses:n} " + f"({ANSIColors.RED}{fmt(batched_miss_rate)}%{ANSIColors.RESET})" + ) + print( + f" Segments read: {segments_completed:n}/{segments_requested:n} " + f"({ANSIColors.GREEN}{fmt(segment_completion_rate)}%{ANSIColors.RESET})" + ) + # Memory operations memory_reads = stats.get('memory_reads', 0) memory_bytes = stats.get('memory_bytes_read', 0) diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 04622a8c1e89ef6..42281dc6454c83c 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -16,6 +16,8 @@ class StackTraceCollector(Collector): + aggregating = True + def __init__(self, sample_interval_usec, *, skip_idle=False): self.sample_interval_usec = sample_interval_usec self.skip_idle = skip_idle @@ -698,6 +700,8 @@ def _add_elided_metadata(self, node, baseline_stats, scale, path): func_key = self._extract_func_key(node, self._baseline_collector._string_table) current_path = path + (func_key,) if func_key else path + baseline_self = 0 + baseline_total = 0 if func_key and current_path in baseline_stats: baseline_data = baseline_stats[current_path] baseline_self = baseline_data["self"] * scale diff --git a/Lib/pydoc.py b/Lib/pydoc.py index a1a6aad434ddf4d..497cc7d90a42456 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1845,6 +1845,7 @@ class Helper: 'in': ('in', 'SEQUENCEMETHODS'), 'is': 'COMPARISON', 'lambda': ('lambda', 'FUNCTIONS'), + 'lazy': ('lazy', 'MODULES'), 'nonlocal': ('nonlocal', 'global NAMESPACES'), 'not': 'BOOLEAN', 'or': 'BOOLEAN', diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index e8cef29d00467f7..6c6d9bb6b34244e 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -179,14 +179,14 @@ def attr_matches(self, text): if (word[:n] == attr and not (noprefix and word[:n+1] == noprefix)): match = "%s.%s" % (expr, word) - if isinstance(getattr(type(thisobject), word, None), - property): - # bpo-44752: thisobject.word is a method decorated by - # `@property`. What follows applies a postfix if - # thisobject.word is callable, but know we know that - # this is not callable (because it is a property). - # Also, getattr(thisobject, word) will evaluate the - # property method, which is not desirable. + + class_attr = getattr(type(thisobject), word, None) + if isinstance( + class_attr, + (property, types.GetSetDescriptorType, types.MemberDescriptorType) + ) or (hasattr(class_attr, '__get__') and not callable(class_attr)): + # Avoid evaluating descriptors, which could run + # arbitrary code or raise exceptions. matches.append(match) continue diff --git a/Lib/site.py b/Lib/site.py index 52dd9648734c3ec..239ee0d6f57bce4 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -154,13 +154,37 @@ def _init_pathinfo(): return d -# Accumulated entry points from .start files across all site-packages -# directories. Execution is deferred until all paths in .pth files have been -# appended to sys.path. Map the .pth/.start file the data is found in to the -# data. -_pending_entrypoints = {} -_pending_syspaths = {} -_pending_importexecs = {} +# PEP 829 implementation notes. +# +# Startup information (.pth and .start file information) can be processed in +# implicit or explicit batches. Implicit batches are handled by the site.py +# machinery automatically, while explicit batches are driven by user code and +# processed on boundaries defined by that code. +# +# addsitedir() calls which use the default defer_processing_start_files=False +# are self-contained: they create a per-call _StartupState, populate it from +# the site directory's .pth/.start files, run process() on it, and then throw +# the state away. This is implicit batching and in that case the +# _startup_state global variable stays None. +# +# main() needs different semantics: it accumulates state across multiple +# addsitedir() calls (user-site plus all global site-packages) so that +# every sys.path extension is visible *before* any startup code (.pth +# import lines and .start entry points) runs. Callers opt into this by +# passing defer_processing_start_files=True, which preserves the _StartupState +# into the global _startup_state. Subsequent addsitedir() calls (with +# or without defer_processing_start_files=True) then write into that +# same shared state, and a later process_startup_files() call flushes +# all the state and resets the global to None. +# +# Here's the CRITICAL reentrancy invariant: process_startup_files() must clear +# the global _startup_state *before* calling state.process(), so that any +# reentrant site.addsitedir() calls reached from an exec'd .pth import line or +# a .start entry point falls into the per-call branch and gets its own fresh +# state. Otherwise the recursive addsitedir() would mutate the very dicts +# that the outer state.process() is iterating. This is the bug reported in +# gh-149504. +_startup_state = None def _read_pthstart_file(sitedir, name, suffix): @@ -194,13 +218,13 @@ def _read_pthstart_file(sitedir, name, suffix): return None, filename try: - # Accept BOM markers in .start and .pth files as we do in source files (Windows PowerShell - # 5.1 makes it hard to emit UTF-8 files without a BOM). + # Accept BOM markers in .start and .pth files as we do in source files + # (Windows PowerShell 5.1 makes it hard to emit UTF-8 files without a BOM). content = raw_content.decode("utf-8-sig") except UnicodeDecodeError: _trace(f"Cannot read {filename!r} as UTF-8.") - # For .pth files only, and then only until Python 3.20, fallback to locale encoding for - # backward compatibility. + # For .pth files only, and then only until Python 3.20, fall back to + # locale encoding for backward compatibility. _warn_future_us( ".pth files decoded to locale encoding as a fallback", remove=(3, 20) @@ -214,153 +238,221 @@ def _read_pthstart_file(sitedir, name, suffix): return content.splitlines(), filename -def _read_pth_file(sitedir, name, known_paths): - """Parse a .pth file, accumulating sys.path extensions and import lines. - - Errors on individual lines do not abort processing of the rest of the - file (PEP 829). - """ - lines, filename = _read_pthstart_file(sitedir, name, ".pth") - if lines is None: - return - - for n, line in enumerate(lines, 1): - line = line.strip() - if not line or line.startswith("#"): - continue - - # In Python 3.18 and 3.19, `import` lines are silently ignored. In - # Python 3.20 and beyond, issue a warning when `import` lines in .pth - # files are detected. - if line.startswith(("import ", "import\t")): - _warn_future_us( - "import lines in .pth files are silently ignored", - remove=(3, 18) - ) - _warn_future_us( - "import lines in .pth files are noisily ignored", - remove=(3, 20) - ) - _pending_importexecs.setdefault(filename, []).append(line) - continue - - try: - dir_, dircase = makepath(sitedir, line) - except Exception as exc: - _trace(f"Error in {filename!r}, line {n:d}: {line!r}", exc) - continue - - if dircase in known_paths: - _trace(f"In {filename!r}, line {n:d}: " - f"skipping duplicate sys.path entry: {dir_}") - else: - _pending_syspaths.setdefault(filename, []).append(dir_) - known_paths.add(dircase) - +class _StartupState: + """Per-batch accumulator for .pth and .start file processing. -def _read_start_file(sitedir, name): - """Parse a .start file for a list of entry point strings.""" - lines, filename = _read_pthstart_file(sitedir, name, ".start") - if lines is None: - return + A _StartupState collects sys.path extensions, deprecated .pth import + lines, and .start entry points read from one or more site-packages + directories. Calling process() applies them in PEP 829 order: paths + are added to sys.path first, then import lines from .pth files (skipping + any with a matching .start), then entry points from .start files. - # PEP 829: the *presence* of a matching .start file disables `import` - # line processing in the matched .pth file, regardless of whether the - # .start file produced any entry points. Register the filename as a - # key now so an empty (or comment-only) .start file still suppresses. - entrypoints = _pending_entrypoints.setdefault(filename, []) + State lives entirely on the instance; there is no module-level pending + state. This is what makes the module reentrancy-safe: a site.addsitedir() + call reached recursively from an exec'd import line or a .start entry + point operates on a different _StartupState than the one being processed + by the outer call. - for n, line in enumerate(lines, 1): - line = line.strip() - if not line or line.startswith("#"): - continue - # Syntax validation is deferred to entry-point execution time, - # where pkgutil.resolve_name(strict=True) enforces the - # pkg.mod:callable form. - entrypoints.append(line) - - -def _extend_syspath(): - # We've already filtered out duplicates, either in the existing sys.path - # or in all the .pth files we've seen. We've also abspath/normpath'd all - # the entries, so all that's left to do is to ensure that the path exists. - for filename, dirs in _pending_syspaths.items(): - for dir_ in dirs: - if os.path.exists(dir_): - _trace(f"Extending sys.path with {dir_} from {filename}") - sys.path.append(dir_) - else: - _print_error( - f"In {filename}: {dir_} does not exist; " - f"skipping sys.path append") - - -def _exec_imports(): - # For all the `import` lines we've seen in .pth files, exec() them in - # order. However, if they come from a file with a matching .start, then - # we ignore these import lines. For the ones we do process, print a - # warning but only when -v was given. - for filename, imports in _pending_importexecs.items(): - name, dot, pth = filename.rpartition(".") - assert dot == "." and pth == "pth", f"Bad startup filename: {filename}" - - if f"{name}.start" in _pending_entrypoints: - # Skip import lines in favor of entry points. - continue + The internal data is intentionally private; the public methods + (read_pth_file, read_start_file, process) are the only supported write + APIs. + """ + __slots__ = ('_syspaths', '_importexecs', '_entrypoints') + + def __init__(self): + # All three dicts map "" -> list + # of items collected from that file. Mapping by filename lets us + # cross-reference a .pth and its matching .start (PEP 829 import + # suppression rule) and lets _print_error report the source file + # when an entry fails. + self._syspaths = {} + self._importexecs = {} + self._entrypoints = {} + + def read_pth_file(self, sitedir, name, known_paths): + """Parse a .pth file, accumulating sys.path extensions and import lines. + + Errors on individual lines do not abort processing of the rest of + the file (PEP 829). ``known_paths`` is the per-batch dedup + ledger: any path already in it is skipped, and newly accepted + paths are added to it so that subsequent .pth files in the same + batch don't add them more than once. + """ + lines, filename = _read_pthstart_file(sitedir, name, ".pth") + if lines is None: + return + + for n, line in enumerate(lines, 1): + line = line.strip() + if not line or line.startswith("#"): + continue - _trace( - f"import lines in {filename} are deprecated, " - f"use entry points in a {name}.start file instead." - ) + # In Python 3.18 and 3.19, `import` lines are silently + # ignored. In Python 3.20 and beyond, issue a warning when + # `import` lines in .pth files are detected. + if line.startswith(("import ", "import\t")): + _warn_future_us( + "import lines in .pth files are silently ignored", + remove=(3, 18), + ) + _warn_future_us( + "import lines in .pth files are noisily ignored", + remove=(3, 20), + ) + self._importexecs.setdefault(filename, []).append(line) + continue - for line in imports: try: - _trace(f"Exec'ing from {filename}: {line}") - exec(line) + dir_, dircase = makepath(sitedir, line) except Exception as exc: - _print_error( - f"Error in import line from {filename}: {line}", exc) - - -def _execute_start_entrypoints(): - """Execute all accumulated .start file entry points. + _trace(f"Error in {filename!r}, line {n:d}: {line!r}", exc) + continue - Called after all site-packages directories have been processed so that - sys.path is fully populated before any entry point code runs. Uses - pkgutil.resolve_name(strict=True) which both validates the strict - pkg.mod:callable form and resolves the entry point in one step. - """ - for filename, entrypoints in _pending_entrypoints.items(): - for entrypoint in entrypoints: - try: - _trace(f"Executing entry point: {entrypoint} from {filename}") - callable_ = pkgutil.resolve_name(entrypoint, strict=True) - except ValueError as exc: - _print_error( - f"Invalid entry point syntax in {filename}: " - f"{entrypoint!r}", exc) + # PEP 829 dedup: skip paths already seen in this batch. See + # _startup_state docstring above for batch lifetimes. + if dircase in known_paths: + _trace( + f"In {filename!r}, line {n:d}: " + f"skipping duplicate sys.path entry: {dir_}" + ) + else: + self._syspaths.setdefault(filename, []).append(dir_) + known_paths.add(dircase) + + def read_start_file(self, sitedir, name): + """Parse a .start file for a list of entry point strings.""" + lines, filename = _read_pthstart_file(sitedir, name, ".start") + if lines is None: + return + + # PEP 829: the *presence* of a matching .start file disables `import` + # line processing in the matched .pth file, regardless of whether this + # .start file contains any entry points. Register the filename as a + # key now so an empty (or comment-only) .start file still suppresses. + entrypoints = self._entrypoints.setdefault(filename, []) + + for n, line in enumerate(lines, 1): + line = line.strip() + if not line or line.startswith("#"): continue - except Exception as exc: - _print_error( - f"Error resolving entry point {entrypoint} " - f"from {filename}", exc) + # Syntax validation is deferred to entry point execution + # time, where pkgutil.resolve_name(strict=True) enforces the + # pkg.mod:callable form. + entrypoints.append(line) + + def process(self): + """Apply accumulated state in PEP 829 order. + + Phase order matters: all .pth path extensions are applied to + sys.path *before* any import line or .start entry point runs, so + that an entry point may live in a module reachable only via a + .pth-extended path. + """ + self._extend_syspath() + self._exec_imports() + self._execute_start_entrypoints() + + def _extend_syspath(self): + # Duplicates have already been filtered (in existing sys.path or + # across .pth files via known_paths), and entries are already + # abspath/normpath'd, so all that remains is to confirm the path + # exists. + for filename, dirs in self._syspaths.items(): + for dir_ in dirs: + if os.path.exists(dir_): + _trace(f"Extending sys.path with {dir_} from {filename}") + sys.path.append(dir_) + else: + _print_error( + f"In {filename}: {dir_} does not exist; " + f"skipping sys.path append" + ) + + def _exec_imports(self): + # For each `import` line we've seen in a .pth file, exec() it in + # order, unless the .pth has a matching .start file in this same + # batch. In that case, PEP 829 says the import lines are + # suppressed in favor of the .start's entry points. + for filename, imports in self._importexecs.items(): + # Given "/path/to/foo.pth", check whether "/path/to/foo.start" was + # registered in this same batch. + name, dot, pth = filename.rpartition(".") + assert dot == "." and pth == "pth", ( + f"Bad startup filename: {filename}" + ) + if f"{name}.start" in self._entrypoints: + _trace( + f"import lines in {filename} are suppressed " + f"due to matching {name}.start file." + ) continue - try: - callable_() - except Exception as exc: - _print_error( - f"Error in entry point {entrypoint} from {filename}", - exc) + + _trace( + f"import lines in {filename} are deprecated, " + f"use entry points in a {name}.start file instead." + ) + for line in imports: + try: + _trace(f"Exec'ing from {filename}: {line}") + exec(line) + except Exception as exc: + _print_error( + f"Error in import line from {filename}: {line}", + exc, + ) + + def _execute_start_entrypoints(self): + # Resolve each entry point string to a callable via + # pkgutil.resolve_name(strict=True), which both validates the + # required pkg.mod:callable form and performs the import in one + # step, then call it with no arguments. + for filename, entrypoints in self._entrypoints.items(): + for entrypoint in entrypoints: + try: + _trace( + f"Executing entry point: {entrypoint} from {filename}" + ) + callable_ = pkgutil.resolve_name(entrypoint, strict=True) + except ValueError as exc: + _print_error( + f"Invalid entry point syntax in {filename}: " + f"{entrypoint!r}", + exc, + ) + except Exception as exc: + _print_error( + f"Error resolving entry point {entrypoint} " + f"from {filename}", + exc, + ) + else: + try: + callable_() + except Exception as exc: + _print_error( + f"Error in entry point {entrypoint} from {filename}", + exc, + ) def process_startup_files(): - """Flush all pending sys.path and entry points.""" - _extend_syspath() - _exec_imports() - _execute_start_entrypoints() - _pending_syspaths.clear() - _pending_importexecs.clear() - _pending_entrypoints.clear() + """Flush any pending startup-file state accumulated during a batch. + + Used by main() (and any external caller that drove addsitedir() with + defer_processing_start_files=True) to apply the accumulated paths + and run the deferred import lines / entry points. + + Reentrancy: the active batch state is detached from _startup_state + *before* state.process() runs. This way, if an exec'd import line + or .start entry point itself calls site.addsitedir(), that call + creates its own per-call _StartupState rather than mutating the dicts + being iterated here. See gh-149504. + """ + global _startup_state + if _startup_state is None: + return + state, _startup_state = _startup_state, None + state.process() def addpackage(sitedir, name, known_paths): @@ -370,16 +462,26 @@ def addpackage(sitedir, name, known_paths): reset = True else: reset = False - _read_pth_file(sitedir, name, known_paths) - process_startup_files() - if reset: - known_paths = None - return known_paths + + # If a batch is already in progress (for example, main() is still + # accumulating sitedirs), participate in the batch by writing into the + # shared _startup_state and letting the eventual process_startup_files() + # flush it. Otherwise this is a standalone call, so create a unique + # per-call state, populate it, and process it before returning. + if _startup_state is None: + state = _StartupState() + state.read_pth_file(sitedir, name, known_paths) + state.process() + else: + _startup_state.read_pth_file(sitedir, name, known_paths) + + return None if reset else known_paths def addsitedir(sitedir, known_paths=None, *, defer_processing_start_files=False): """Add 'sitedir' argument to sys.path if missing and handle startup files.""" + global _startup_state _trace(f"Adding directory: {sitedir!r}") if known_paths is None: known_paths = _init_pathinfo() @@ -387,44 +489,77 @@ def addsitedir(sitedir, known_paths=None, *, defer_processing_start_files=False) else: reset = False sitedir, sitedircase = makepath(sitedir) - if not sitedircase in known_paths: - sys.path.append(sitedir) # Add path component - known_paths.add(sitedircase) - try: - names = os.listdir(sitedir) - except OSError: - return - # The following phases are defined by PEP 829. - # Phases 1-3: Read .pth files, accumulating paths and import lines. - pth_names = sorted( - name for name in names - if name.endswith(".pth") and not name.startswith(".") - ) - for name in pth_names: - _read_pth_file(sitedir, name, known_paths) - - # Phases 6-7: Discover .start files and accumulate their entry points. - # Import lines from .pth files with a matching .start file are discarded - # at flush time by _exec_imports(). - start_names = sorted( - name for name in names - if name.endswith(".start") and not name.startswith(".") - ) - for name in start_names: - _read_start_file(sitedir, name) + # If the normcase'd new sitedir isn't already known, record it to + # prevent re-processing, append it to sys.path (only if not already + # present), and process all .pth and .start files found in that + # directory. Use a direct sys.path membership check for the append + # guard so that callers (like main()) can pass a fresh known_paths + # set while avoiding duplicate sys.path entries (gh-149819). + if sitedircase not in known_paths: + known_paths.add(sitedircase) + if sitedir not in sys.path: + sys.path.append(sitedir) - # Generally, when addsitedir() is called explicitly, we'll want to process - # all the startup file data immediately. However, when called through - # main(), we'll want to batch up all the startup file processing. main() - # will set this flag to True to defer processing. - if not defer_processing_start_files: - process_startup_files() + try: + names = os.listdir(sitedir) + except OSError: + return None if reset else known_paths + + # Pick the _StartupState we'll write into. There are three cases: + # + # 1. A batch is already active (_startup_state is set, e.g. because + # main() previously called us with + # defer_processing_start_files=True). Participate in this batch by + # sharing the same state. Don't flush the state since the batch's + # eventual process_startup_files() will do that. + # + # 2. There is no active batch but the caller passed + # defer_processing_start_files=True. Preserve a fresh + # _StartupState into the global _startup_state so that subsequent + # addsitedir() calls participate in this batch, and so that the + # caller's later process_startup_files() finds it. + # + # 3. This is a standalone call (there is no active batch and + # defer_processing_start_files=False). Create a unique per-call + # state, populate it, process it, and then clear it. Per-call + # state is what makes reentrant addsitedir() safe; a recursive call + # from inside process() lands here too and gets its own independent + # state. + + if _startup_state is not None: + state = _startup_state + flush_now = False + elif defer_processing_start_files: + state = _startup_state = _StartupState() + flush_now = False + else: + state = _StartupState() + flush_now = True + + # The following phases are defined by PEP 829. + # Phases 1-3: Read .pth files, accumulating paths and import lines. + pth_names = sorted( + name for name in names + if name.endswith(".pth") and not name.startswith(".") + ) + for name in pth_names: + state.read_pth_file(sitedir, name, known_paths) + + # Phases 6-7: Discover .start files and accumulate their entry points. + # Import lines from .pth files with a matching .start file are + # discarded at flush time by _StartupState._exec_imports(). + start_names = sorted( + name for name in names + if name.endswith(".start") and not name.startswith(".") + ) + for name in start_names: + state.read_start_file(sitedir, name) - if reset: - known_paths = None + if flush_now: + state.process() - return known_paths + return None if reset else known_paths def check_enableusersite(): @@ -868,13 +1003,13 @@ def main(): global ENABLE_USER_SITE orig_path = sys.path[:] - known_paths = removeduppaths() + removeduppaths() if orig_path != sys.path: # removeduppaths() might make sys.path absolute. # Fix __file__ of already imported modules too. abs_paths() - known_paths = venv(known_paths) + known_paths = venv(known_paths=set()) if ENABLE_USER_SITE is None: ENABLE_USER_SITE = check_enableusersite() known_paths = addusersitepackages(known_paths, defer_processing_start_files=True) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index d0e7dec5575047a..5e43b4c19c0a8a7 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -498,7 +498,7 @@ def _init_read_gz(self): if flag & 4: xlen = ord(self.__read(1)) + 256 * ord(self.__read(1)) - self.read(xlen) + self.__read(xlen) if flag & 8: while True: s = self.__read(1) @@ -830,16 +830,22 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) + # A link member that resolves to the destination directory itself + # would replace it with a (sym)link, redirecting the destination + # for all subsequent members. + if target_path == dest_path: + raise OutsideDestinationError(member, target_path) normalized = os.path.normpath(member.linkname) if normalized != member.linkname: new_attrs['linkname'] = normalized if member.issym(): - target_path = os.path.join(dest_path, - os.path.dirname(name), - member.linkname) + # The symlink is created at `name` with trailing separators + # stripped, so its target is relative to the directory + # containing that path. + link_dir = os.path.dirname(name.rstrip('/' + os.sep)) + target_path = os.path.join(dest_path, link_dir, normalized) else: - target_path = os.path.join(dest_path, - member.linkname) + target_path = os.path.join(dest_path, normalized) target_path = os.path.realpath(target_path, strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index a893932169a089b..8be5bf8aa4f5469 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -208,6 +208,16 @@ def rl(name): else: return None + try: + import _remote_debugging + except ImportError: + _remote_debugging = None + + def rd(name): + if _remote_debugging: + return getattr(_remote_debugging, name, None) + return None + # Try a range of "open" functions. # All of them should fail with TestHook(raise_on_events={"open"}) as hook: @@ -225,6 +235,8 @@ def rl(name): (rl("append_history_file"), 0, None), (rl("read_init_file"), testfn), (rl("read_init_file"), None), + (rd("BinaryWriter"), testfn, 1000, 0), + (rd("BinaryReader"), testfn), ]: if not fn: continue @@ -258,6 +270,8 @@ def rl(name): ("~/.history", "a") if rl("append_history_file") else None, (testfn, "r") if readline else None, ("", "r") if readline else None, + (testfn, "wb") if rd("BinaryWriter") else None, + (testfn, "rb") if rd("BinaryReader") else None, ] if i is not None ], diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 958a915626ad241..d0759d2626989d6 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -145,7 +145,7 @@ def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: # Storage of uncollectable GC objects (gc.garbage) -GC_GARBAGE = [] +GC_GARBAGE: list[object] = [] def _load_run_test(result: TestResult, runtests: RunTests) -> None: diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 4ea5b6f53a04265..1dc3f538f4ad8ba 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -140,6 +140,48 @@ def test_parse_args(self): ) +class TestArgumentParserCopiable(unittest.TestCase): + def _get_parser(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo', type=int, default=42) + parser.add_argument('bar', nargs='?', default='baz') + return parser + + @force_not_colorized + def test_copiable(self): + import copy + parser = self._get_parser() + parser2 = copy.copy(parser) + ns = parser2.parse_args(['--foo', '123', 'quux']) + self.assertEqual(ns.foo, 123) + self.assertEqual(ns.bar, 'quux') + ns2 = parser2.parse_args([]) + self.assertEqual(ns2.foo, 42) + self.assertEqual(ns2.bar, 'baz') + + # Test shallow copy also gets new arguments + parser.add_argument("--extra") + ns3 = parser2.parse_args(["--extra", "bar"]) + self.assertEqual(ns3.extra, "bar") + + @force_not_colorized + def test_deepcopiable(self): + import copy + parser = self._get_parser() + parser2 = copy.deepcopy(parser) + ns = parser2.parse_args(['--foo', '123', 'quux']) + self.assertEqual(ns.foo, 123) + self.assertEqual(ns.bar, 'quux') + ns2 = parser2.parse_args([]) + self.assertEqual(ns2.foo, 42) + self.assertEqual(ns2.bar, 'baz') + + # Test deep copy does not get new arguments + parser.add_argument("--extra") + with self.assertRaises(argparse.ArgumentError): + parser2.parse_args(["--extra", "bar"]) + + class TestArgumentParserPickleable(unittest.TestCase): @force_not_colorized @@ -7863,12 +7905,25 @@ def fake_can_colorize(*, file=None): def test_fake_color_theme_matches_real(self): from argparse import _colorless_theme + + # Check the attributes match those of the 'real' theme _colorize_nocolor = _colorize.get_theme(force_no_color=True).argparse for k in _colorize_nocolor: self.assertEqual( getattr(_colorless_theme, k), getattr(_colorize_nocolor, k) ) + def test_fake_color_theme_raises(self): + from argparse import _colorless_theme + + # Make sure the _colorless_theme doesn't return empty strings + # for magic methods or private attributes + with self.assertRaises(AttributeError): + _colorless_theme.__unknown_dunder__ + + with self.assertRaises(AttributeError): + _colorless_theme._private_attribute + class TestModule(unittest.TestCase): def test_deprecated__version__(self): diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py index f9ee2f4f68150a1..509697613475953 100644 --- a/Lib/test/test_asyncio/test_windows_utils.py +++ b/Lib/test/test_asyncio/test_windows_utils.py @@ -77,6 +77,30 @@ def test_pipe_handle(self): else: raise RuntimeError('expected ERROR_INVALID_HANDLE') + def test_pipe_handle_close_after_external_close(self): + # gh-149388: PipeHandle.close() must clear ``_handle`` before calling + # CloseHandle so that if CloseHandle raises on a stale handle the + # PipeHandle is still marked closed and __del__ / subsequent close() + # calls are silent no-ops. + h1, h2 = windows_utils.pipe(overlapped=(False, False)) + try: + p = windows_utils.PipeHandle(h1) + # Simulate an external close of the underlying handle (e.g. + # a finalizer race or a concurrent close on the same object). + _winapi.CloseHandle(p.handle) + # First close() still propagates the OSError from CloseHandle, + # but must clear ``_handle`` first. + with self.assertRaises(OSError): + p.close() + self.assertIsNone(p.handle) + # Second close() is a no-op. + p.close() + # __del__ through GC is also a silent no-op — no unraisable. + del p + support.gc_collect() + finally: + _winapi.CloseHandle(h2) + class PopenTests(unittest.TestCase): diff --git a/Lib/test/test_frame_pointer_unwind.py b/Lib/test/test_c_stack_unwind.py similarity index 89% rename from Lib/test/test_frame_pointer_unwind.py rename to Lib/test/test_c_stack_unwind.py index faa012c9c00d8f9..91bf44e463473de 100644 --- a/Lib/test/test_frame_pointer_unwind.py +++ b/Lib/test/test_c_stack_unwind.py @@ -1,3 +1,12 @@ +"""Test in-process C stack unwinders against Python and JIT frames. + +The tests build a recursive Python call stack, ask each _testinternalcapi +unwinder for return addresses, and classify those addresses as Python, JIT, or +other frames. The backends include CPython's manual stack-chain unwinder and +GNU backtrace(), so this module is about in-process C stack unwinding rather +than a single unwind mechanism. GDB integration tests live in test_gdb. +""" + import json import os import platform @@ -20,7 +29,7 @@ STACK_DEPTH = 10 -def _frame_pointers_expected(machine): +def _manual_unwind_expected(machine): _Py_WITH_FRAME_POINTERS = getattr( _testinternalcapi, "_Py_WITH_FRAME_POINTERS", @@ -89,6 +98,21 @@ def _frame_pointers_expected(machine): return None +def _is_arm32_build(): + if sys.maxsize >= 2**32: + return False + + abi = " ".join( + value for value in ( + sysconfig.get_config_var("MULTIARCH"), + sysconfig.get_config_var("HOST_GNU_TYPE"), + sysconfig.get_config_var("SOABI"), + ) + if value + ).lower() + return "arm" in abi + + def _build_stack_and_unwind(unwinder): import operator @@ -180,7 +204,7 @@ def _annotate_unwind_after_executor_free(unwinder_name="gnu_backtrace_unwind"): def _run_unwind_helper(helper_name, unwinder_name, **env): code = ( - f"from test.test_frame_pointer_unwind import {helper_name}; " + f"from test.test_c_stack_unwind import {helper_name}; " f"print({helper_name}({unwinder_name!r}));" ) run_env = os.environ.copy() @@ -220,15 +244,17 @@ def _unwind_after_executor_free_result(unwinder_name, **env): @support.requires_gil_enabled("test requires the GIL enabled") @unittest.skipIf(support.is_wasi, "test not supported on WASI") -class FramePointerUnwindTests(unittest.TestCase): +class ManualStackUnwindTests(unittest.TestCase): def setUp(self): super().setUp() machine = platform.machine().lower() - expected = _frame_pointers_expected(machine) + expected = _manual_unwind_expected(machine) if expected is None: - self.skipTest(f"unsupported architecture for frame pointer check: {machine}") + self.skipTest( + f"unsupported architecture for manual stack unwind check: {machine}" + ) if expected == "crash": self.skipTest(f"test does crash on {machine}") @@ -236,12 +262,14 @@ def setUp(self): _testinternalcapi.manual_frame_pointer_unwind() except RuntimeError as exc: if "not supported" in str(exc): - self.skipTest("manual frame pointer unwinding not supported on this platform") + self.skipTest( + "manual stack unwinding not supported on this platform" + ) raise self.machine = machine - self.frame_pointers_expected = expected + self.manual_unwind_expected = expected - def test_manual_unwind_respects_frame_pointers(self): + def test_manual_unwind_finds_expected_frames(self): jit_available = hasattr(sys, "_jit") and sys._jit.is_available() envs = [({"PYTHON_JIT": "0"}, False)] if jit_available: @@ -253,7 +281,7 @@ def test_manual_unwind_respects_frame_pointers(self): jit_frames = result["jit_frames"] python_frames = result.get("python_frames", 0) jit_backend = result.get("jit_backend") - if self.frame_pointers_expected: + if self.manual_unwind_expected: self.assertGreaterEqual( python_frames, STACK_DEPTH, @@ -295,6 +323,10 @@ def test_manual_unwind_respects_frame_pointers(self): @support.requires_gil_enabled("test requires the GIL enabled") @unittest.skipIf(support.is_wasi, "test not supported on WASI") @unittest.skipUnless(sys.platform == "linux", "GNU backtrace unwinding test requires Linux") +@unittest.skipIf( + _is_arm32_build(), + "GNU backtrace unwinding skipped on Arm 32-bit", +) class GnuBacktraceUnwindTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py index 635deaa73f7efab..e6fd068dc20d8d4 100644 --- a/Lib/test/test_capi/test_object.py +++ b/Lib/test/test_capi/test_object.py @@ -71,6 +71,8 @@ def test_pysentinel_new(self): self.assertIs(type(marker), sentinel) self.assertTrue(_testcapi.pysentinel_check(marker)) self.assertFalse(_testcapi.pysentinel_check(object())) + self.assertTrue(_testcapi.pysentinel_checkexact(marker)) + self.assertFalse(_testcapi.pysentinel_checkexact(object())) self.assertEqual(marker.__name__, "CAPI_SENTINEL") self.assertEqual(marker.__module__, __name__) self.assertEqual(repr(marker), "CAPI_SENTINEL") diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index d80fec9a8a0d2b0..2f606c2c6eba2d6 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -598,7 +598,8 @@ def testfunc(n, m): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_ITER_NEXT_INLINE", uops) + self.assertIn("_FOR_ITER_TIER_TWO", uops) + self.assertNotIn("_ITER_NEXT_INLINE", uops) @requires_specialization @@ -6137,6 +6138,20 @@ def __init__(self, x): C(0) if i else str(0) """)) + def test_load_special_type_guard_deopt(self): + script_helper.assert_python_ok("-s", "-c", textwrap.dedent(f""" + def f1(): + class Context: + def __enter__(self): ... + def __exit__(self, e, v, t): ... + + with Context(): + pass + + for _ in range({TIER2_THRESHOLD + 5}): + f1() + """), PYTHON_JIT="1") + def global_identity(x): return x diff --git a/Lib/test/test_crossinterp.py b/Lib/test/test_crossinterp.py index 4e5362111687477..f4bf5a66ad21550 100644 --- a/Lib/test/test_crossinterp.py +++ b/Lib/test/test_crossinterp.py @@ -157,6 +157,10 @@ def ignore_byteswarning(): {}, {1: 7, 2: 8, 3: 9}, {1: [1], 2: (2,), 3: {3: 4}}, + # frozendict + frozendict(), + frozendict({1: 7, 2: 8, 3: 9}), + frozendict({1: [1], 2: (2,), 3: {3: 4}, 4: frozendict({5: 6})}), # set set(), {1, 2, 3}, diff --git a/Lib/test/test_ctypes/test_parameters.py b/Lib/test/test_ctypes/test_parameters.py index 46f8ff93efa9152..6dadb7b410d7034 100644 --- a/Lib/test/test_ctypes/test_parameters.py +++ b/Lib/test/test_ctypes/test_parameters.py @@ -1,6 +1,7 @@ import sys import unittest import test.support +import ctypes from ctypes import (CDLL, PyDLL, ArgumentError, Structure, Array, Union, _Pointer, _SimpleCData, _CFuncPtr, @@ -247,6 +248,13 @@ def test_parameter_repr(self): self.assertRegex(repr(c_char_p.from_param(b'hihi')), r"^$") self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^$") self.assertRegex(repr(c_void_p.from_param(0x12)), r"^$") + if hasattr(ctypes, 'c_double_complex'): + self.assertRegex(repr(ctypes.c_double_complex.from_param(0)), + r"^$") + self.assertRegex(repr(ctypes.c_float_complex.from_param(0)), + r"^$") + self.assertRegex(repr(ctypes.c_longdouble_complex.from_param(0)), + r"^$") @test.support.cpython_only def test_from_param_result_refcount(self): diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index b2f4363b23e7480..f26586809238f0e 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1868,6 +1868,11 @@ def test_merge(self): self.assertEqual(fd | {}, fd) self.assertEqual(frozendict() | fd, fd) + # gh-149676: Test hash(frozendict | frozendict) + a = frozendict({"a": 1}) + b = frozendict({"b": 2}) + self.assertEqual(hash(a | b), hash(frozendict({"a": 1, "b": 2}))) + def test_update(self): # test "a |= b" operator d = frozendict(x=1) @@ -1898,10 +1903,35 @@ def test_hash(self): self.assertEqual(hash(frozendict(x=1, y=2)), hash(frozendict(y=2, x=1))) + # Check that hash() computes the hash of (key, value) pairs + cases = [ + frozendict(a=False, b=True, c=True), + frozendict(a=True, b=False, c=True), + frozendict(a=True, b=True, c=False), + frozendict({False: "a", "b": True, "c": True}), + frozendict({"a": "b", False: True, True: "c"}), + ] + hashes = {hash(fd) for fd in cases} + self.assertEqual(len(hashes), len(cases)) + fd = frozendict(x=[1], y=[2]) with self.assertRaisesRegex(TypeError, "unhashable type: 'list'"): hash(fd) + @support.cpython_only + def test_hash_cpython(self): + # Check that hash(frozendict) implementation is: + # hash(frozenset(fd.items())) + for fd in ( + frozendict(), + frozendict(x=1, y=2), + frozendict(y=2, x=1), + frozendict(a=False, b=True, c=True), + frozendict.fromkeys('abc'), + ): + with self.subTest(fd=fd): + self.assertEqual(hash(fd), hash(frozenset(fd.items()))) + def test_fromkeys(self): self.assertEqual(frozendict.fromkeys('abc'), frozendict(a=None, b=None, c=None)) diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index aded44e85ee3368..9d9fe418ee4d067 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -1060,6 +1060,78 @@ def get_phrase_cfws_only_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_phrase(' (foo) ') + def test_get_phrase_adjacent_ew(self): + # "'linear-white-space' that separates a pair of adjacent + # 'encoded-word's is ignored" (rfc2047 section 6.2) + self._test_get_x(parser.get_phrase, '=?ascii?q?Joi?= \t =?ascii?q?ned?=', 'Joined', 'Joined', [], '') + + def test_get_phrase_adjacent_ew_different_encodings(self): + self._test_get_x( + parser.get_phrase, + '=?utf-8?q?B=C3=A9r?= =?iso-8859-1?q?=E9nice?=', 'Bérénice', 'Bérénice', [], '' + ) + + def test_get_phrase_adjacent_ew_encoded_spaces(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Encoded?= =?ascii?q?_spaces_?= =?ascii?q?preserved?=', + 'Encoded spaces preserved', + 'Encoded spaces preserved', + [], + '' + ) + + def test_get_phrase_adjacent_ew_comment_is_not_linear_white_space(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Comment?= (is not) =?ascii?q?linear-white-space?=', + 'Comment (is not) linear-white-space', + 'Comment linear-white-space', + [], + '', + comments=['is not'], + ) + + def test_get_phrase_adjacent_ew_no_error_on_defects(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Def?= =?ascii?q?ect still joins?=', + 'Defect still joins', + 'Defect still joins', + [errors.InvalidHeaderDefect], # whitespace inside encoded word + '' + ) + + def test_get_phrase_adjacent_ew_ignore_non_ew(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?No?= =?join?= for non-ew', + 'No =?join?= for non-ew', + 'No =?join?= for non-ew', + [], + '' + ) + + def test_get_phrase_adjacent_ew_ignore_invalid_ew(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?No?= =?ascii?rot13?wbva= for invalid ew', + 'No =?ascii?rot13?wbva= for invalid ew', + 'No =?ascii?rot13?wbva= for invalid ew', + [], + '' + ) + + def test_get_phrase_adjacent_ew_missing_space(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Joi?==?ascii?q?ned?=', + 'Joined', + 'Joined', + [errors.InvalidHeaderDefect], # missing trailing whitespace + '' + ) + # get_local_part def test_get_local_part_simple(self): @@ -2387,6 +2459,22 @@ def test_get_address_rfc2047_display_name(self): self.assertEqual(address[0].token_type, 'mailbox') + def test_get_address_rfc2047_display_name_adjacent_ews(self): + address = self._test_get_x(parser.get_address, + '=?utf-8?q?B=C3=A9r?= =?utf-8?q?=C3=A9nice?= ', + 'Bérénice ', + 'Bérénice ', + [], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 1) + self.assertEqual(address.mailboxes, + address.all_mailboxes) + self.assertEqual(address.mailboxes[0].display_name, + 'Bérénice') + self.assertEqual(address[0].token_type, + 'mailbox') + def test_get_address_empty_group(self): address = self._test_get_x(parser.get_address, 'Monty Python:;', diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 7778566492d8f44..d2c2261edbe04e1 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -4995,15 +4995,8 @@ def test_body_encode(self): # Try the convert argument, where input codec != output codec c = Charset('euc-jp') # With apologies to Tokio Kikuchi ;) - # XXX FIXME -## try: -## eq('\x1b$B5FCO;~IW\x1b(B', -## c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7')) -## eq('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', -## c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', False)) -## except LookupError: -## # We probably don't have the Japanese codecs installed -## pass + eq('\x1b$B5FCO;~IW\x1b(B', + c.body_encode('\u83ca\u5730\u6642\u592b')) # Testing SF bug #625509, which we have to fake, since there are no # built-in encodings where the header encoding is QP but the body # encoding is not. diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index a29e6cdbbf6c785..6b1529aa173f01c 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -3767,6 +3767,13 @@ def test_get_stats(self): "frames_read_from_cache", "frames_read_from_memory", "frame_cache_hit_rate", + "batched_read_attempts", + "batched_read_successes", + "batched_read_misses", + "batched_read_segments_requested", + "batched_read_segments_completed", + "batched_read_success_rate", + "batched_read_segment_completion_rate", ] for key in expected_keys: self.assertIn(key, stats) diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py index 55272a00c3ad501..dfe0634211d4b02 100644 --- a/Lib/test/test_free_threading/test_dict.py +++ b/Lib/test/test_free_threading/test_dict.py @@ -268,6 +268,34 @@ def watcher(): finally: _testcapi.clear_dict_watcher(wid) + def test_racing_split_dict_clear_and_lookup(self): + class C: + pass + + keys = [f"a{i}" for i in range(16)] + + def make_split_nonembedded(): + inst = C() + for key in keys: + setattr(inst, key, keys.index(key)) + # dict.copy() of a split instance dict yields a split table + # with non-embedded values + return inst.__dict__.copy() + + d = make_split_nonembedded() + + def clearer(): + for _ in range(1000): + d.clear() + d.update(make_split_nonembedded()) + + def reader(): + for _ in range(1000): + for k in keys: + d.get(k) + + threading_helper.run_concurrently([clearer, reader, reader]) + def test_racing_dict_update_and_method_lookup(self): # gh-144295: test race between dict modifications and method lookups. # Uses BytesIO because the race requires a type without Py_TPFLAGS_INLINE_VALUES diff --git a/Lib/test/test_free_threading/test_dict_watcher.py b/Lib/test/test_free_threading/test_dict_watcher.py new file mode 100644 index 000000000000000..6a6843f9344f640 --- /dev/null +++ b/Lib/test/test_free_threading/test_dict_watcher.py @@ -0,0 +1,89 @@ +import unittest + +from test.support import import_helper, threading_helper + +_testcapi = import_helper.import_module("_testcapi") + +ITERS = 100 +NTHREADS = 20 + + +@threading_helper.requires_working_threading() +class TestDictWatcherThreadSafety(unittest.TestCase): + # Watcher kinds from _testcapi + EVENTS = 0 # appends dict events as strings to global event list + + def test_concurrent_add_clear_watchers(self): + """Race AddWatcher and ClearWatcher from multiple threads. + + Uses more threads than available watcher slots (5 user slots out + of DICT_MAX_WATCHERS=8). + """ + results = [] + + def worker(): + for _ in range(ITERS): + try: + wid = _testcapi.add_dict_watcher(self.EVENTS) + except RuntimeError: + continue # All slots taken + self.assertGreaterEqual(wid, 0) + results.append(wid) + _testcapi.clear_dict_watcher(wid) + + threading_helper.run_concurrently(worker, NTHREADS) + + # Verify at least some watchers were successfully added + self.assertGreater(len(results), 0) + + def test_concurrent_watch_unwatch(self): + """Race Watch and Unwatch on the same dict from multiple threads.""" + wid = _testcapi.add_dict_watcher(self.EVENTS) + dicts = [{} for _ in range(10)] + + def worker(): + for _ in range(ITERS): + for d in dicts: + _testcapi.watch_dict(wid, d) + for d in dicts: + _testcapi.unwatch_dict(wid, d) + + try: + threading_helper.run_concurrently(worker, NTHREADS) + + # Verify watching still works after concurrent watch/unwatch + _testcapi.watch_dict(wid, dicts[0]) + dicts[0]["key"] = "value" + events = _testcapi.get_dict_watcher_events() + self.assertIn("new:key:value", events) + finally: + _testcapi.clear_dict_watcher(wid) + + def test_concurrent_modify_watched_dict(self): + """Race dict mutations (triggering callbacks) with watch/unwatch.""" + wid = _testcapi.add_dict_watcher(self.EVENTS) + d = {} + _testcapi.watch_dict(wid, d) + + def mutator(): + for i in range(ITERS): + d[f"key_{i}"] = i + d.pop(f"key_{i}", None) + + def toggler(): + for i in range(ITERS): + _testcapi.watch_dict(wid, d) + d[f"toggler_{i}"] = i + _testcapi.unwatch_dict(wid, d) + + workers = [mutator, toggler] * (NTHREADS // 2) + try: + threading_helper.run_concurrently(workers) + events = _testcapi.get_dict_watcher_events() + self.assertGreater(len(events), 0) + finally: + _testcapi.clear_dict_watcher(wid) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_iteration.py b/Lib/test/test_free_threading/test_iteration.py index a51ad0cf83a0065..44d3e9ccfdd14e0 100644 --- a/Lib/test/test_free_threading/test_iteration.py +++ b/Lib/test/test_free_threading/test_iteration.py @@ -12,7 +12,7 @@ NUMITEMS = 1000 NUMTHREADS = 2 else: - NUMITEMS = 100000 + NUMITEMS = 5000 NUMTHREADS = 5 NUMMUTATORS = 2 diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index c864d401f9ed67e..f1eff9430f7351c 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -16,7 +16,7 @@ except ImportError: ssl = None -from unittest import TestCase, skipUnless +from unittest import mock, TestCase, skipUnless from test import support from test.support import requires_subprocess from test.support import threading_helper @@ -1145,6 +1145,40 @@ def testTimeoutDirectAccess(self): ftp.close() +class TestFtpcpSecurity(TestCase): + """ftpcp() must not trust the host a source server advertises in PASV. + + A malicious source server can otherwise redirect the target server's + data connection to an arbitrary host:port (SSRF), so ftpcp() uses the + source server's actual peer address instead, the same as FTP.makepasv(). + """ + + def _make_pair(self, *, advertised_host, real_host, trust=False): + source = mock.Mock(spec=ftplib.FTP) + source.trust_server_pasv_ipv4_address = trust + source.sock.getpeername.return_value = (real_host, 21) + # PASV replies give the host as comma-separated octets, not dotted. + advertised = advertised_host.replace('.', ',') + source.sendcmd.side_effect = lambda cmd: ( + f'227 Entering Passive Mode ({advertised},1,2).' + if cmd == 'PASV' else '150 ok') + target = mock.Mock(spec=ftplib.FTP) + target.sendcmd.return_value = '150 ok' + return source, target + + def test_ftpcp_ignores_untrusted_pasv_host(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7') + ftplib.ftpcp(source, 'a', target, 'b') + target.sendport.assert_called_once_with('198.51.100.7', 258) + + def test_ftpcp_trust_server_pasv_ipv4_address(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7', trust=True) + ftplib.ftpcp(source, 'a', target, 'b') + target.sendport.assert_called_once_with('10.0.0.5', 258) + + class MiscTestCase(TestCase): def test__all__(self): not_exported = { diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index a5969b7a47d948b..7816775620bc013 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -55,15 +55,14 @@ from unittest.case import _AssertRaisesContext from queue import Queue, SimpleQueue from weakref import WeakSet, ReferenceType, ref -import typing -from typing import Unpack try: from tkinter import Event except ImportError: Event = None from string.templatelib import Template, Interpolation -from typing import TypeVar +import typing +from typing import TypeVar, Unpack T = TypeVar('T') K = TypeVar('K') V = TypeVar('V') @@ -621,6 +620,14 @@ def test_nested_paramspec_specialization(self): self.assertEqual(deeply_nested_specialized.__args__, ([str, [float], int], float)) self.assertEqual(deeply_nested_specialized.__parameters__, ()) + def test_gh150146(self): + # It used to crash: + for container in [memoryview, list, tuple]: + with self.subTest(container=container): + x = container[TypeVar("")] + with self.assertRaises(TypeError): + x[*typing.Mapping[..., ...]] + class TypeIterationTests(unittest.TestCase): _UNITERABLE_TYPES = (list, tuple) diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index 442d30fc970fa94..cafac9d3c8be6e7 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -10,6 +10,7 @@ import sys import unittest from subprocess import PIPE, Popen +from unittest import mock from test.support import catch_unraisable_exception from test.support import force_not_colorized_test_class, import_helper from test.support import os_helper @@ -350,6 +351,26 @@ def test_mtime(self): self.assertEqual(dataRead, data1) self.assertEqual(fRead.mtime, mtime) + def test_mtime_out_of_range(self): + for mtime in (-1, 2**32): + with gzip.GzipFile(self.filename, 'w', mtime=mtime) as fWrite: + fWrite.write(data1) + with gzip.GzipFile(self.filename) as fRead: + fRead.read(1) + self.assertEqual(fRead.mtime, 0) + datac = gzip.compress(data1, mtime=mtime) + with gzip.GzipFile(fileobj=io.BytesIO(datac)) as fRead: + fRead.read(1) + self.assertEqual(fRead.mtime, 0) + + for mtime in (-1, 2**32): + with mock.patch('time.time', return_value=float(mtime)): + with gzip.GzipFile(self.filename, 'w') as fWrite: + fWrite.write(data1) + with gzip.GzipFile(self.filename) as fRead: + fRead.read(1) + self.assertEqual(fRead.mtime, 0) + def test_metadata(self): mtime = 123456789 @@ -795,6 +816,35 @@ def test_decompress_missing_trailer(self): compressed_data = gzip.compress(data1) self.assertRaises(EOFError, gzip.decompress, compressed_data[:-8]) + def test_truncated_header(self): + truncated_headers = [ + b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00", # Missing OS byte + b"\x1f\x8b\x08\x02\x00\x00\x00\x00\x00\xff", # FHRC, but no checksum + b"\x1f\x8b\x08\x04\x00\x00\x00\x00\x00\xff", # FEXTRA, but no xlen + b"\x1f\x8b\x08\x04\x00\x00\x00\x00\x00\xff\xaa\x00", # FEXTRA, xlen, but no data + b"\x1f\x8b\x08\x08\x00\x00\x00\x00\x00\xff", # FNAME but no fname + b"\x1f\x8b\x08\x10\x00\x00\x00\x00\x00\xff", # FCOMMENT, but no fcomment + ] + for header in truncated_headers: + with self.subTest(header=header): + with self.assertRaises(EOFError): + gzip.decompress(header) + + def test_corrupted_gzip_header(self): + header = (b"\x1f\x8b\x08\x1f\x00\x00\x00\x00\x00\xff" # All flags set + b"\x05\x00" # Xlen = 5 + b"extra" + b"name\x00" + b"comment\x00") + true_crc = zlib.crc32(header) & 0xFFFF + corrupted_crc = true_crc ^ 0xFFFF + corrupted_header = header + corrupted_crc.to_bytes(2, "little") + with self.assertRaises(gzip.BadGzipFile) as err: + gzip.decompress(corrupted_header) + self.assertEqual(str(err.exception), + f"Corrupted gzip header. Checksums do not " + f"match: {true_crc:04x} != {corrupted_crc:04x}") + def test_read_truncated(self): data = data1*50 # Drop the CRC (4 bytes) and file size (4 bytes). diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index cde268e32418509..d1df2ec42f0d146 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -1,11 +1,11 @@ # Simple test suite for http/cookies.py -import base64 import copy import unittest import doctest from http import cookies import pickle from test import support +import urllib.parse class CookieTests(unittest.TestCase): @@ -152,21 +152,21 @@ def test_load(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - cookie_encoded = base64.b64encode(b'Customer="WILE_E_COYOTE"; Path=/acme; Version=1').decode('ascii') + cookie_encoded = urllib.parse.quote('Customer="WILE_E_COYOTE"; Path=/acme; Version=1', safe='', encoding='utf-8') with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): self.assertEqual(C.js_output(), fr""" """) - cookie_encoded = base64.b64encode(b'Customer="WILE_E_COYOTE"; Path=/acme').decode('ascii') + cookie_encoded = urllib.parse.quote('Customer="WILE_E_COYOTE"; Path=/acme', safe='', encoding='utf-8') with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): self.assertEqual(C.js_output(['path']), fr""" """) @@ -271,21 +271,21 @@ def test_quoted_meta(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - expected_encoded_cookie = base64.b64encode(b'Customer=\"WILE_E_COYOTE\"; Path=/acme; Version=1').decode('ascii') + expected_encoded_cookie = urllib.parse.quote('Customer=\"WILE_E_COYOTE\"; Path=/acme; Version=1', safe='', encoding='utf-8') with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): self.assertEqual(C.js_output(), fr""" """) - expected_encoded_cookie = base64.b64encode(b'Customer=\"WILE_E_COYOTE\"; Path=/acme').decode('ascii') + expected_encoded_cookie = urllib.parse.quote('Customer=\"WILE_E_COYOTE\"; Path=/acme', safe='', encoding='utf-8') with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): self.assertEqual(C.js_output(['path']), fr""" """) @@ -376,13 +376,14 @@ def test_setter(self): self.assertEqual( M.output(), "Set-Cookie: %s=%s; Path=/foo" % (i, "%s_coded_val" % i)) - expected_encoded_cookie = base64.b64encode( - ("%s=%s; Path=/foo" % (i, "%s_coded_val" % i)).encode("ascii") - ).decode('ascii') + expected_encoded_cookie = urllib.parse.quote( + "%s=%s; Path=/foo" % (i, "%s_coded_val" % i), + safe='', encoding='utf-8', + ) expected_js_output = """ """ % (expected_encoded_cookie,) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 9028d42c617fb4b..7351f97fd9a4b5c 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -6255,8 +6255,7 @@ def test_faulthandler_module_has_signatures(self): self._test_module_has_signatures(faulthandler, unsupported_signature=unsupported_signature) def test_functools_module_has_signatures(self): - unsupported_signature = {"reduce"} - self._test_module_has_signatures(functools, unsupported_signature=unsupported_signature) + self._test_module_has_signatures(functools) def test_gc_module_has_signatures(self): import gc diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index 1d1d2e00bd733f4..4340efd31095ea6 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -10,6 +10,7 @@ import unittest import tempfile import os +import contextlib from test import support from test.support.script_helper import assert_python_ok @@ -37,8 +38,7 @@ def test_basic_unused(self): """Lazy imported module should not be loaded if never accessed.""" import test.test_lazy_import.data.basic_unused self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) - self.assertIn("test.test_lazy_import.data", sys.lazy_modules) - self.assertEqual(sys.lazy_modules["test.test_lazy_import.data"], {"basic2"}) + self.assertIn("test.test_lazy_import.data.basic2", sys.lazy_modules) def test_sys_lazy_modules(self): try: @@ -48,7 +48,7 @@ def test_sys_lazy_modules(self): self.assertFalse("test.test_lazy_import.data.basic2" in sys.modules) self.assertIn("test.test_lazy_import.data", sys.lazy_modules) - self.assertEqual(sys.lazy_modules["test.test_lazy_import.data"], {"basic2"}) + self.assertIn("test.test_lazy_import.data.basic2", sys.lazy_modules) test.test_lazy_import.data.basic_from_unused.basic2 self.assertNotIn("test.test_import.data", sys.lazy_modules) @@ -88,6 +88,79 @@ def test_basic_used(self): import test.test_lazy_import.data.basic_used self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + @support.requires_subprocess() + def test_from_import_with_module_getattr(self): + """Lazy from import should respect module-level __getattr__.""" + code = textwrap.dedent(""" + lazy from test.test_lazy_import.data.module_with_getattr import dynamic_attr + assert dynamic_attr == "from_getattr" + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_module_getattr_raising(self): + """Lazy from import should respect module-level __getattr__.""" + code = textwrap.dedent(""" + lazy from test.test_lazy_import.data.module_with_getattr import raising_attr + + try: + raising_attr + except ValueError as exc: + assert str(exc) == 'from_getattr', exc + else: + assert False, f'ValueError is not raised: {raising_attr}' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_module_getattr_missing(self): + """Lazy from import should respect module-level __getattr__.""" + for attr in ("missing_attr", "import_error_attr"): + with self.subTest(attr=attr): + code = textwrap.dedent(f""" + lazy from test.test_lazy_import.data.module_with_getattr import {attr} + + try: + {attr} + except ImportError as exc: + assert '{attr}' in str(exc), exc + assert exc.__cause__ is not None + else: + assert False, ('ImportError is not raised', {attr}) + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_module_getattr_warning(self): + """Lazy from import should respect module-level __getattr__.""" + code = textwrap.dedent(""" + import warnings + + with warnings.catch_warnings(record=True) as log: + lazy from test.test_lazy_import.data.module_with_getattr import warning_attr + + assert log == [] + + with warnings.catch_warnings(record=True) as log: + warning_attr + assert warning_attr == 'from_warning_attr', warning_attr + assert len(log) == 1, log + assert isinstance(log[0].message, UserWarning), log + assert str(log[0].message) == 'from_getattr', log + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_imported_module_getattr(self): + """Lazy from import should not shadow an imported module's __getattr__.""" + code = textwrap.dedent(""" + import test.test_lazy_import.data.module_with_getattr as mod + lazy from test.test_lazy_import.data.module_with_getattr import dynamic_attr + assert dynamic_attr == "from_getattr" + assert mod.dynamic_attr == "from_getattr" + """) + assert_python_ok("-c", code) + class GlobalLazyImportModeTests(unittest.TestCase): """Tests for sys.set_lazy_imports() global mode control.""" @@ -281,6 +354,15 @@ def f(): f() self.assertIn("only allowed at module level", str(cm.exception)) + def test_lazy_import_exec_in_class(self): + """lazy import via exec() inside a class should raise SyntaxError.""" + # exec() inside a class body also has non-module-level locals. + with self.assertRaises(SyntaxError) as cm: + class C: + exec("lazy import json") + + self.assertIn("only allowed at module level", str(cm.exception)) + @support.requires_subprocess() def test_lazy_import_exec_at_module_level(self): """lazy import via exec() at module level should work.""" @@ -332,6 +414,50 @@ def test_eager_import_func(self): f = test.test_lazy_import.data.eager_import_func.f self.assertEqual(type(f()), type(sys)) + def test_exec_import_func(self): + """Implicit lazy imports via exec() inside functions should be eager.""" + sys.set_lazy_imports("all") + + def f(): + exec("import test.test_lazy_import.data.basic2") + + f() + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_exec_import_func_with_lazy_modules(self): + """__lazy_modules__ should not make exec() imports lazy inside functions.""" + globals()["__lazy_modules__"] = ["test.test_lazy_import.data.basic2"] + try: + def f(): + exec("import test.test_lazy_import.data.basic2") + + f() + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + finally: + del globals()["__lazy_modules__"] + + def test_exec_import_class(self): + """Implicit lazy imports via exec() inside classes should be eager.""" + sys.set_lazy_imports("all") + + class C: + exec("import test.test_lazy_import.data.basic2") + + self.assertIsNotNone(C) + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + + def test_exec_import_class_with_lazy_modules(self): + """__lazy_modules__ should not make exec() imports lazy inside classes.""" + globals()["__lazy_modules__"] = ["test.test_lazy_import.data.basic2"] + try: + class C: + exec("import test.test_lazy_import.data.basic2") + + self.assertIsNotNone(C) + self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + finally: + del globals()["__lazy_modules__"] + class WithStatementTests(unittest.TestCase): """Tests for lazy imports in with statement context.""" @@ -368,10 +494,22 @@ def tearDown(self): def test_lazy_import_pkg(self): """lazy import of package submodule should load the package.""" - import test.test_lazy_import.data.lazy_import_pkg + out = io.StringIO() + + with contextlib.redirect_stdout(out): + import test.test_lazy_import.data.lazy_import_pkg self.assertIn("test.test_lazy_import.data.pkg", sys.modules) self.assertIn("test.test_lazy_import.data.pkg.bar", sys.modules) + self.assertIn("BAR_MODULE_LOADED", out.getvalue()) + + def test_lazy_submodule_stored_in_parent_dict(self): + """Accessing a lazy submodule should store it in the parent's __dict__.""" + import test.test_lazy_import.data.lazy_import_pkg + + pkg = sys.modules["test.test_lazy_import.data.pkg"] + self.assertIn("bar", pkg.__dict__) + self.assertIs(pkg.__dict__["bar"], sys.modules["test.test_lazy_import.data.pkg.bar"]) def test_lazy_import_pkg_cross_import(self): """Cross-imports within package should preserve lazy imports.""" @@ -385,6 +523,82 @@ def test_lazy_import_pkg_cross_import(self): self.assertEqual(type(g["x"]), int) self.assertEqual(type(g["b"]), types.LazyImportType) + @support.requires_subprocess() + def test_lazy_from_import_does_not_pollute_parent(self): + """Lazy from import should not add the name to the parent module's dict.""" + code = textwrap.dedent(""" + lazy from json import nonexistent_attr + import json + assert "nonexistent_attr" not in json.__dict__, ( + "lazy from import should not publish attributes on the parent module" + ) + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_package_from_import_with_module_getattr_raising(self): + """Lazy from import should respect a package's __getattr__.""" + code = textwrap.dedent(""" + lazy from test.test_lazy_import.data.pkg import raising_attr + + try: + raising_attr + except ValueError as exc: + assert str(exc) == 'from_getattr', exc + else: + assert False, f'ValueError is not raised: {raising_attr}' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_package_from_import_with_module_getattr_missing(self): + """Lazy from import should respect package's __getattr__.""" + for attr in ("missing_attr", "import_error_attr"): + with self.subTest(attr=attr): + code = textwrap.dedent(f""" + lazy from test.test_lazy_import.data.pkg import {attr} + + try: + {attr} + except ImportError as exc: + assert '{attr}' in str(exc), exc + assert exc.__cause__ is not None + else: + assert False, ('ImportError is not raised', {attr}) + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_module_getattr_warning(self): + """Lazy from import should respect package's __getattr__.""" + code = textwrap.dedent(""" + import warnings + + with warnings.catch_warnings(record=True) as log: + lazy from test.test_lazy_import.data.pkg import warning_attr + + assert log == [] + + with warnings.catch_warnings(record=True) as log: + warning_attr + assert warning_attr == 'from_warning_attr', warning_attr + assert len(log) == 1, log + assert isinstance(log[0].message, UserWarning), log + assert str(log[0].message) == 'from_getattr', log + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_package_from_import_with_module_getattr(self): + """Lazy from import should respect a package's __getattr__.""" + code = textwrap.dedent(""" + import test.test_lazy_import.data.pkg as pkg + lazy from test.test_lazy_import.data.pkg import dynamic_attr + assert dynamic_attr == "from_getattr" + assert pkg.dynamic_attr == "from_getattr" + """) + assert_python_ok("-c", code) + class DunderLazyImportTests(unittest.TestCase): """Tests for __lazy_import__ builtin function.""" @@ -485,8 +699,8 @@ def my_filter(name): self.assertIs(sys.get_lazy_imports_filter(), my_filter) def test_lazy_modules_attribute_is_dict(self): - """sys.lazy_modules should be a dict per PEP 810.""" - self.assertIsInstance(sys.lazy_modules, dict) + """sys.lazy_modules should be a set per PEP 810.""" + self.assertIsInstance(sys.lazy_modules, set) @support.requires_subprocess() def test_lazy_modules_tracks_lazy_imports(self): @@ -495,8 +709,7 @@ def test_lazy_modules_tracks_lazy_imports(self): import sys initial_count = len(sys.lazy_modules) import test.test_lazy_import.data.basic_unused - assert "test.test_lazy_import.data" in sys.lazy_modules - assert sys.lazy_modules["test.test_lazy_import.data"] == {"basic2"} + assert "test.test_lazy_import.data.basic2" in sys.lazy_modules assert len(sys.lazy_modules) > initial_count print("OK") """) @@ -526,19 +739,14 @@ def tearDown(self): sys.set_lazy_imports("normal") def test_import_error_shows_chained_traceback(self): - """ImportError during reification should chain to show both definition and access.""" - # Errors at reification must show where the lazy import was defined - # AND where the access happened, per PEP 810 "Reification" section + """Accessing a nonexistent lazy submodule via parent attr raises AttributeError.""" code = textwrap.dedent(""" import sys lazy import test.test_lazy_import.data.nonexistent_module try: x = test.test_lazy_import.data.nonexistent_module - except ImportError as e: - # Should have __cause__ showing the original error - # The exception chain shows both where import was defined and where access happened - assert e.__cause__ is not None, "Expected chained exception" + except AttributeError as e: print("OK") """) result = subprocess.run( @@ -586,7 +794,7 @@ def test_reification_retries_on_failure(self): # First access - should fail try: x = test.test_lazy_import.data.broken_module - except ValueError: + except AttributeError: pass # The lazy object should still be a lazy proxy (not reified) @@ -596,7 +804,7 @@ def test_reification_retries_on_failure(self): # Second access - should also fail (retry the import) try: x = test.test_lazy_import.data.broken_module - except ValueError: + except AttributeError: print("OK - retry worked") """) result = subprocess.run( @@ -609,7 +817,6 @@ def test_reification_retries_on_failure(self): def test_error_during_module_execution_propagates(self): """Errors in module code during reification should propagate correctly.""" - # Module that raises during import should propagate with chaining code = textwrap.dedent(""" import sys lazy import test.test_lazy_import.data.broken_module @@ -617,12 +824,8 @@ def test_error_during_module_execution_propagates(self): try: _ = test.test_lazy_import.data.broken_module print("FAIL - should have raised") - except ValueError as e: - # The ValueError from the module should be the cause - if "always fails" in str(e) or (e.__cause__ and "always fails" in str(e.__cause__)): - print("OK") - else: - print(f"FAIL - wrong error: {e}") + except AttributeError: + print("OK") """) result = subprocess.run( [sys.executable, "-c", code], @@ -945,15 +1148,14 @@ def test_module_added_to_lazy_modules_on_lazy_import(self): lazy import test.test_lazy_import.data.basic2 # Should be in lazy_modules after lazy import - assert "test.test_lazy_import.data" in sys.lazy_modules - assert sys.lazy_modules["test.test_lazy_import.data"] == {"basic2"} + assert "test.test_lazy_import.data.basic2" in sys.lazy_modules assert len(sys.lazy_modules) > initial_count # Trigger reification _ = test.test_lazy_import.data.basic2.x # Module should still be tracked (for diagnostics per PEP 810) - assert "test.test_lazy_import.data" not in sys.lazy_modules + assert "test.test_lazy_import.data.basic2" not in sys.lazy_modules print("OK") """) result = subprocess.run( @@ -966,8 +1168,8 @@ def test_module_added_to_lazy_modules_on_lazy_import(self): def test_lazy_modules_is_per_interpreter(self): """Each interpreter should have independent sys.lazy_modules.""" - # Basic test that sys.lazy_modules exists and is a dict - self.assertIsInstance(sys.lazy_modules, dict) + # Basic test that sys.lazy_modules exists and is a set + self.assertIsInstance(sys.lazy_modules, set) def test_lazy_module_without_children_is_tracked(self): code = textwrap.dedent(""" @@ -976,10 +1178,6 @@ def test_lazy_module_without_children_is_tracked(self): assert "json" in sys.lazy_modules, ( f"expected 'json' in sys.lazy_modules, got {set(sys.lazy_modules)}" ) - assert sys.lazy_modules["json"] == set(), ( - f"expected empty set for sys.lazy_modules['json'], " - f"got {sys.lazy_modules['json']!r}" - ) print("OK") """) assert_python_ok("-c", code) @@ -1848,7 +2046,7 @@ def create_lazy_imports(idx): t.join() assert not errors, f"Errors: {errors}" - assert isinstance(sys.lazy_modules, dict), "sys.lazy_modules is not a dict" + assert isinstance(sys.lazy_modules, set), "sys.lazy_modules is not a dict" print("OK") """) diff --git a/Lib/test/test_lazy_import/__main__.py b/Lib/test/test_lazy_import/__main__.py new file mode 100644 index 000000000000000..d6c94efaf30833e --- /dev/null +++ b/Lib/test/test_lazy_import/__main__.py @@ -0,0 +1,3 @@ +import unittest + +unittest.main('test.test_lazy_import') diff --git a/Lib/test/test_lazy_import/data/module_with_getattr.py b/Lib/test/test_lazy_import/data/module_with_getattr.py new file mode 100644 index 000000000000000..db3a2301075c2ee --- /dev/null +++ b/Lib/test/test_lazy_import/data/module_with_getattr.py @@ -0,0 +1,12 @@ +def __getattr__(name): + if name == "dynamic_attr": + return "from_getattr" + elif name == "raising_attr": + raise ValueError("from_getattr") + elif name == "import_error_attr": + raise ImportError(name) + elif name == "warning_attr": + import warnings + warnings.warn("from_getattr", category=UserWarning) + return "from_warning_attr" + raise AttributeError(name) diff --git a/Lib/test/test_lazy_import/data/pkg/__init__.py b/Lib/test/test_lazy_import/data/pkg/__init__.py index 2d76abaa89f8937..5f7b8662596cac6 100644 --- a/Lib/test/test_lazy_import/data/pkg/__init__.py +++ b/Lib/test/test_lazy_import/data/pkg/__init__.py @@ -1 +1,14 @@ x = 42 + +def __getattr__(name): + if name == "dynamic_attr": + return "from_getattr" + elif name == "raising_attr": + raise ValueError("from_getattr") + elif name == "import_error_attr": + raise ImportError(name) + elif name == "warning_attr": + import warnings + warnings.warn("from_getattr", category=UserWarning) + return "from_warning_attr" + raise AttributeError(name) diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 7e670e5a139d999..6fcf94fc8253852 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -2137,6 +2137,94 @@ def test_mode(self): self.assertEqual(os.stat(path).st_mode & 0o777, 0o555) self.assertEqual(os.stat(parent).st_mode & 0o777, 0o775) + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mode_with_parent_mode(self): + # Test the parent_mode parameter + parent = os.path.join(os_helper.TESTFN, 'dir1') + path = os.path.join(parent, 'dir2') + with os_helper.temp_umask(0o002): + # Specify mode for both leaf and parent directories + os.makedirs(path, 0o770, parent_mode=0o750) + self.assertTrue(os.path.exists(path)) + self.assertTrue(os.path.isdir(path)) + if os.name != 'nt': + # Leaf directory gets the mode parameter + self.assertEqual(os.stat(path).st_mode & 0o777, 0o770) + # Parent directory gets the parent_mode parameter + self.assertEqual(os.stat(parent).st_mode & 0o777, 0o750) + + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_parent_mode_deep_hierarchy(self): + # Test parent_mode with deep directory hierarchy + base = os.path.join(os_helper.TESTFN, 'dir1', 'dir2', 'dir3') + with os_helper.temp_umask(0o002): + os.makedirs(base, 0o755, parent_mode=0o700) + self.assertTrue(os.path.exists(base)) + if os.name != 'nt': + # Check that all parent directories have parent_mode + level1 = os.path.join(os_helper.TESTFN, 'dir1') + level2 = os.path.join(level1, 'dir2') + self.assertEqual(os.stat(level1).st_mode & 0o777, 0o700) + self.assertEqual(os.stat(level2).st_mode & 0o777, 0o700) + # Leaf directory has the regular mode + self.assertEqual(os.stat(base).st_mode & 0o777, 0o755) + + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_parent_mode_same_as_mode(self): + # Test emulating Python 3.6 behavior by setting parent_mode=mode + parent = os.path.join(os_helper.TESTFN, 'dir1') + path = os.path.join(parent, 'dir2') + with os_helper.temp_umask(0o002): + os.makedirs(path, 0o705, parent_mode=0o705) + self.assertTrue(os.path.exists(path)) + if os.name != 'nt': + # Both directories should have the same mode + self.assertEqual(os.stat(path).st_mode & 0o777, 0o705) + self.assertEqual(os.stat(parent).st_mode & 0o777, 0o705) + + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_parent_mode_combined_with_umask(self): + # parent_mode, like mode, is combined with the process umask; it does + # not bypass it. + parent = os.path.join(os_helper.TESTFN, 'dir1') + path = os.path.join(parent, 'dir2') + with os_helper.temp_umask(0o022): + os.makedirs(path, 0o777, parent_mode=0o777) + self.assertTrue(os.path.isdir(path)) + if os.name != 'nt': + # 0o777 is masked down to 0o755 by the 0o022 umask, for both + # the leaf (mode) and the parent (parent_mode). + self.assertEqual(os.stat(path).st_mode & 0o777, 0o755) + self.assertEqual(os.stat(parent).st_mode & 0o777, 0o755) + @unittest.skipIf( support.is_wasi, "WASI's umask is a stub." @@ -2210,15 +2298,9 @@ def test_win32_mkdir_700(self): ) def tearDown(self): - path = os.path.join(os_helper.TESTFN, 'dir1', 'dir2', 'dir3', - 'dir4', 'dir5', 'dir6') - # If the tests failed, the bottom-most directory ('../dir6') - # may not have been created, so we look for the outermost directory - # that exists. - while not os.path.exists(path) and path != os_helper.TESTFN: - path = os.path.dirname(path) - - os.removedirs(path) + # Remove the whole tree regardless of which sub-directories a test + # created and regardless of their permission bits. + os_helper.rmtree(os_helper.TESTFN) @unittest.skipUnless(hasattr(os, "chown"), "requires os.chown()") diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 09d1b5d725e5ba7..2cb4876f5c6400a 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -2492,6 +2492,116 @@ def my_mkdir(path, mode=0o777): self.assertNotIn(str(p12), concurrently_created) self.assertTrue(p.exists()) + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_parents_umask(self): + # Test that parent directories respect umask when parent_mode is not set + p = self.cls(self.base, 'umasktest', 'child') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o002): + p.mkdir(0o755, parents=True) + self.assertTrue(p.exists()) + # Leaf directory gets the specified mode + self.assertEqual(p.stat().st_mode & 0o777, 0o755) + # Parent directory respects umask (0o777 & ~0o002 = 0o775) + self.assertEqual(p.parent.stat().st_mode & 0o777, 0o775) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_with_parent_mode(self): + # Test the parent_mode parameter + p = self.cls(self.base, 'newdirPM', 'subdirPM') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o022): + # Specify different modes for parent and leaf directories + p.mkdir(0o755, parents=True, parent_mode=0o750) + self.assertTrue(p.exists()) + self.assertTrue(p.is_dir()) + # Leaf directory gets the mode parameter + self.assertEqual(p.stat().st_mode & 0o777, 0o755) + # Parent directory gets the parent_mode parameter + self.assertEqual(p.parent.stat().st_mode & 0o777, 0o750) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_parent_mode_deep_hierarchy(self): + # Test parent_mode with deep directory hierarchy + p = self.cls(self.base, 'level1PM', 'level2PM', 'level3PM') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o022): + p.mkdir(0o755, parents=True, parent_mode=0o700) + self.assertTrue(p.exists()) + # Check that all parent directories have parent_mode + level1 = self.cls(self.base, 'level1PM') + level2 = level1 / 'level2PM' + self.assertEqual(level1.stat().st_mode & 0o777, 0o700) + self.assertEqual(level2.stat().st_mode & 0o777, 0o700) + # Leaf directory has the regular mode + self.assertEqual(p.stat().st_mode & 0o777, 0o755) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_parent_mode_combined_with_umask(self): + # parent_mode, like mode, is combined with the process umask; it does + # not bypass it. + p = self.cls(self.base, 'umaskPM', 'child') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o022): + p.mkdir(0o777, parents=True, parent_mode=0o777) + self.assertTrue(p.exists()) + # 0o777 is masked down to 0o755 by the 0o022 umask, for both + # the leaf (mode) and the parent (parent_mode). + self.assertEqual(p.stat().st_mode & 0o777, 0o755) + self.assertEqual(p.parent.stat().st_mode & 0o777, 0o755) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_parent_mode_same_as_mode(self): + # Test setting parent_mode same as mode + p = self.cls(self.base, 'samedirPM', 'subdirPM') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o022): + p.mkdir(0o705, parents=True, parent_mode=0o705) + self.assertTrue(p.exists()) + # Both directories should have the same mode + self.assertEqual(p.stat().st_mode & 0o777, 0o705) + self.assertEqual(p.parent.stat().st_mode & 0o777, 0o705) + @needs_symlinks def test_symlink_to(self): P = self.cls(self.base) diff --git a/Lib/test/test_profiling/test_heatmap.py b/Lib/test/test_profiling/test_heatmap.py index b2acb1cf577341d..ee27fdd3fa3053c 100644 --- a/Lib/test/test_profiling/test_heatmap.py +++ b/Lib/test/test_profiling/test_heatmap.py @@ -345,6 +345,21 @@ def test_process_frames_tracks_edge_samples(self): # Check that edge count is tracked self.assertGreater(len(collector.edge_samples), 0) + def test_process_frames_weight_applies_to_identical_samples(self): + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('callee.py', (5, 5, -1, -1), 'callee', None), + ('caller.py', (10, 10, -1, -1), 'caller', None), + ] + + collector.process_frames(frames, thread_id=1, weight=5) + + edge_key = (('caller.py', 10), ('callee.py', 5)) + self.assertEqual(collector.edge_samples[edge_key], 5) + self.assertEqual(collector.line_samples[('callee.py', 5)], 5) + self.assertEqual(collector.line_samples[('caller.py', 10)], 5) + def test_process_frames_handles_empty_frames(self): """Test that process_frames handles empty frame list.""" collector = HeatmapCollector(sample_interval_usec=100) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py index 9cf706aa2dafeee..1fbb4e2d6c6fbb4 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py @@ -2,6 +2,7 @@ import json import os +import pathlib import random import struct import tempfile @@ -814,6 +815,35 @@ def test_invalid_file_path(self): with BinaryReader("/nonexistent/path/file.bin") as reader: reader.replay_samples(RawCollector()) + def test_path_arguments_round_trip(self): + """Reader and writer accept str, bytes or os.PathLike.""" + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + + for path_arg in (filename, os.fsencode(filename), pathlib.Path(filename)): + with self.subTest(path_type=type(path_arg).__name__): + writer = _remote_debugging.BinaryWriter(path_arg, 1000, 0) + writer.finalize() + reader = _remote_debugging.BinaryReader(path_arg) + info = reader.get_info() + reader.close() + self.assertEqual(info["sample_count"], 0) + + def test_rejects_non_pathlike(self): + """Reader and writer raise TypeError on non-path-like filenames.""" + with self.assertRaises(TypeError): + _remote_debugging.BinaryWriter(123, 1000, 0) + with self.assertRaises(TypeError): + _remote_debugging.BinaryReader(123) + + def test_invalid_path_error_preserves_pathlib(self): + """Missing path: OSError carries the original path object, not a string.""" + missing = pathlib.Path("/i/do/not/exist") + with self.assertRaises(FileNotFoundError) as cm: + _remote_debugging.BinaryReader(missing) + self.assertEqual(os.fspath(cm.exception.filename), os.fspath(missing)) + def test_writer_handles_empty_stack_first_sample(self): """BinaryWriter.write_sample tolerates an empty stack on a fresh thread. diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_children.py b/Lib/test/test_profiling/test_sampling_profiler/test_children.py index bb49faa890f3481..e64d917eedde56b 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_children.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_children.py @@ -109,6 +109,39 @@ def _wait_for_process_ready(proc, timeout): return proc.poll() is None +@unittest.skipIf( + _build_child_profiler_args is None, + "profiling.sampling.cli unavailable", +) +class TestChildProfilerArgBuilder(unittest.TestCase): + """Tests for child profiler CLI argument construction.""" + + def test_build_child_profiler_args_diff_flamegraph(self): + """Test child args use the real --diff-flamegraph flag.""" + args = argparse.Namespace( + sample_interval_usec=1000, + duration=None, + all_threads=False, + realtime_stats=False, + native=False, + gc=True, + opcodes=False, + async_aware=False, + mode="wall", + format="diff_flamegraph", + diff_baseline="baseline.bin", + ) + + child_args = _build_child_profiler_args(args) + + self.assertIn("--diff-flamegraph", child_args) + self.assertNotIn("--diff_flamegraph", child_args) + + flag_index = child_args.index("--diff-flamegraph") + self.assertGreater(len(child_args), flag_index + 1) + self.assertEqual(child_args[flag_index + 1], "baseline.bin") + + @requires_remote_subprocess_debugging() class TestGetChildPids(unittest.TestCase): """Tests for the get_child_pids function.""" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index b42e7aa579f40ca..390a1479fdd2975 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -18,6 +18,7 @@ ) from profiling.sampling.jsonl_collector import JsonlCollector from profiling.sampling.gecko_collector import GeckoCollector + from profiling.sampling.heatmap_collector import _TemplateLoader from profiling.sampling.collector import extract_lineno, normalize_location from profiling.sampling.opcode_utils import get_opcode_info, format_opcode from profiling.sampling.constants import ( @@ -82,6 +83,18 @@ def test_mock_frame_info_with_empty_and_unicode_values(self): self.assertEqual(frame.location.lineno, 999999) self.assertEqual(frame.funcname, long_funcname) + def test_heatmap_navigation_restarts_line_highlight(self): + """Test heatmap navigation can replay target line highlights.""" + loader = _TemplateLoader() + + self.assertIn(".code-line:target", loader.file_css) + self.assertIn("function restartLineHighlight(target)", loader.file_js) + self.assertIn("target.style.animation = 'none'", loader.file_js) + self.assertIn("void target.offsetWidth", loader.file_js) + self.assertIn("url.href === window.location.href", loader.file_js) + self.assertIn("navigateToLine(JSON.parse(navData).link)", loader.file_js) + self.assertIn("navigateToLine(linkData.link)", loader.file_js) + def test_pstats_collector_with_extreme_intervals_and_empty_data(self): """Test PstatsCollector handles zero/large intervals, empty frames, None thread IDs, and duplicate frames.""" # Test with zero interval @@ -1403,6 +1416,39 @@ def test_diff_flamegraph_elided_stacks(self): self.assertGreater(child["baseline"], 0) self.assertAlmostEqual(child["diff"], -child["baseline"]) + def test_diff_flamegraph_elided_top_level_root(self): + """Elided top-level roots do not crash metadata generation.""" + baseline_frames_1 = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 10, "kept_leaf"), + MockFrameInfo("file.py", 20, "kept_root"), + ]) + ]) + ] + baseline_frames_2 = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 30, "old_leaf"), + MockFrameInfo("file.py", 40, "old_root"), + ]) + ]) + ] + + diff = make_diff_collector_with_mock_baseline([ + baseline_frames_1, + baseline_frames_2, + ]) + diff.collect(baseline_frames_1) + + data = diff._convert_to_flamegraph_format() + elided = data["stats"]["elided_flamegraph"] + elided_strings = elided.get("strings", []) + children = elided.get("children", []) + + self.assertEqual(len(children), 1) + self.assertIn("old_root", resolve_name(children[0], elided_strings)) + def test_diff_flamegraph_function_matched_despite_line_change(self): """Functions match by (filename, funcname), ignoring lineno.""" baseline_frames = [ diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py index 68bc59a5414a05c..2f5a5e273286590 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py @@ -198,8 +198,83 @@ def test_sample_profiler_sample_method_timing(self): self.assertIn("samples", result) # Verify collector was called multiple times - self.assertGreaterEqual(mock_collector.collect.call_count, 5) - self.assertLessEqual(mock_collector.collect.call_count, 11) + total_weight = sum( + len(c.kwargs.get("timestamps_us") or [None]) + for c in mock_collector.collect.call_args_list + ) + self.assertGreaterEqual(total_weight, 5) + self.assertLessEqual(total_weight, 11) + + def test_sample_profiler_does_not_buffer_non_aggregating_collectors(self): + """Test that non-aggregating collectors get each sample immediately.""" + + stack_frames = [mock.sentinel.stack_frames] + mock_collector = mock.MagicMock() + mock_collector.aggregating = False + + with self._patched_unwinder() as u: + u.instance.get_stack_trace.return_value = stack_frames + + manager = mock.Mock() + manager.attach_mock(u.instance.get_stack_trace, "unwind") + manager.attach_mock(mock_collector.collect, "collect") + + profiler = SampleProfiler( + pid=12345, sample_interval_usec=10000, all_threads=False + ) + + times = [0.0, 0.01, 0.011, 0.02, 0.03] + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.025) + + self.assertEqual( + manager.mock_calls, + [ + mock.call.unwind(), + mock.call.collect(stack_frames), + mock.call.unwind(), + mock.call.collect(stack_frames), + ], + ) + + def test_sample_profiler_flushes_aggregated_batches_at_limit(self): + """Test that aggregating collectors flush after MAX_PENDING_SAMPLES samples.""" + + stack_frames = [mock.sentinel.stack_frames] + mock_collector = mock.MagicMock() + mock_collector.aggregating = True + + with self._patched_unwinder() as u: + u.instance.get_stack_trace.return_value = stack_frames + + profiler = SampleProfiler( + pid=12345, sample_interval_usec=10000, all_threads=False + ) + + times = [ + 0.0, + 0.01, 0.011, + 0.02, 0.021, + 0.03, 0.031, + 0.04, 0.041, + 0.05, 0.051, + ] + with mock.patch("profiling.sampling.sample.MAX_PENDING_SAMPLES", 2): + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.045) + + batches = [ + (c.args[0], len(c.kwargs["timestamps_us"])) + for c in mock_collector.collect.call_args_list + ] + self.assertEqual( + batches, + [(stack_frames, 2), (stack_frames, 2), (stack_frames, 1)], + ) def test_sample_profiler_error_handling(self): """Test that the sample method handles errors gracefully.""" diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 2e190d1b81be8ec..5cd26923f75c311 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -2172,7 +2172,7 @@ def mock_getline(prompt): def test_keywords(self): self.assertEqual(sorted(pydoc.Helper.keywords), - sorted(keyword.kwlist)) + sorted(keyword.kwlist + ['lazy'])) def test_interact_empty_line_continues(self): # gh-138568: test pressing Enter without input should continue in help session diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index aaa91aca36e3c4c..10dca684accee3c 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -227,8 +227,7 @@ def _verify_parse_output(self, operations): "Character data: '\xb5'", "End element: 'root'", ] - for operation, expected_operation in zip(operations, expected_operations): - self.assertEqual(operation, expected_operation) + self.assertEqual(operations, expected_operations) def test_parse_bytes(self): out = self.Outputter() @@ -276,6 +275,119 @@ def test_parse_again(self): self.assertEqual(expat.ErrorString(cm.exception.code), expat.errors.XML_ERROR_FINISHED) + @support.subTests('encoding', [ + 'utf-8', 'utf-16', 'utf-16be', 'utf-16le', + 'iso8859-1', 'iso8859-2', 'iso8859-3', 'iso8859-4', 'iso8859-5', + 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', + 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', + 'cp437', 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', + 'cp855', 'cp856', 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', + 'cp863', 'cp865', 'cp866', 'cp869', 'cp874', 'cp1006', 'cp1125', + 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255', + 'cp1256', 'cp1257', 'cp1258', + 'mac-cyrillic', 'mac-greek', 'mac-iceland', 'mac-latin2', + 'mac-roman', 'mac-turkish', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'ptcp154', + ]) + def test_supported_encodings(self, encoding): + out = self.Outputter() + parser = expat.ParserCreate() + self._hookup_callbacks(parser, out) + c = 'éπя\u05d0\u060c€'.encode(encoding, 'ignore').decode(encoding)[0] + data = (f'\n' + f'{c}').encode(encoding) + parser.Parse(data, True) + self.assertEqual(out.out, [ + ('XML declaration', ('1.0', encoding, -1)), + "Start element: 'root' {}", + f'Character data: {c!r}', + "End element: 'root'", + ]) + + @support.subTests('encoding', [ + 'UTF-8', 'utf-8', 'utf-16', 'utf-16le', 'utf-16be', + 'koi8-u', 'cp1125', 'cp1251', 'iso8859-5', 'mac-cyrillic', + ]) + def test_supported_encodings2(self, encoding): + out = self.Outputter() + parser = expat.ParserCreate() + self._hookup_callbacks(parser, out) + data = (f'\n' + '' + '<корінь атрибут="значення">зміст').encode(encoding) + parser.Parse(data, True) + self.assertEqual(out.out, [ + ('XML declaration', ('1.0', encoding, -1)), + "Comment: ' коментар '", + "Start element: 'корінь' {'атрибут': 'значення'}", + "Character data: 'зміст'", + "End element: 'корінь'", + ]) + + @support.subTests('encoding', [ + 'UTF-7', + "Big5-HKSCS", "Big5", + "cp932", "cp949", "cp950", + "EUC_JIS-2004", "EUC_JISX0213", "EUC-JP", "EUC-KR", + "GB18030", "GB2312", "GBK", + "ISO-2022-KR", + "johab", + "Shift_JIS", "Shift_JIS-2004", "Shift_JISX0213", + ]) + def test_unsupported_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'\n' + '').encode(encoding) + with self.assertRaises(ValueError): + parser.Parse(data, True) + + parser = expat.ParserCreate() + data = (f'\n' + '').encode() + with self.assertRaises(ValueError): + parser.Parse(data, True) + + @support.subTests('encoding', [ + 'cp037', 'cp273', 'cp424', 'cp500', 'cp864', 'cp875', + 'cp1026', 'cp1140', + 'mac_arabic', 'mac_farsi', + ]) + def test_incompatible_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'\n' + '').encode(encoding) + with self.assertRaises(expat.ExpatError): + parser.Parse(data, True) + + parser = expat.ParserCreate() + data = (f'\n' + '').encode() + with self.assertRaisesRegex(expat.ExpatError, 'unknown encoding'): + parser.Parse(data, True) + + @support.subTests('encoding', [ + 'hex_codec', 'rot_13', + ]) + def test_non_text_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'\n' + '').encode() + with self.assertRaises(LookupError): + parser.Parse(data, True) + + def test_undefined_encoding(self): + parser = expat.ParserCreate() + data = b'\n' + with self.assertRaises(UnicodeError): + parser.Parse(data, True) + + def test_unknown_encoding(self): + parser = expat.ParserCreate() + data = b'\n' + with self.assertRaises(LookupError): + parser.Parse(data, True) + + class NamespaceSeparatorTest(unittest.TestCase): def test_legal(self): # Tests that make sure we get errors when the namespace_separator value @@ -712,6 +824,20 @@ def test_change_size_2(self): parser.Parse(xml2, True) self.assertEqual(self.n, 4) + @support.requires_resource('cpu') + @support.requires_resource('walltime') + @support.bigmemtest(size=2**31, memuse=4, dry_run=False) + def test_large_character_data_does_not_crash(self): + # See https://github.com/python/cpython/issues/148441 + parser = expat.ParserCreate() + parser.buffer_text = True + parser.buffer_size = 2**31 - 1 # INT_MAX + N = 2049 * (1 << 20) - 3 # Character data greater than INT_MAX + self.assertGreater(N, parser.buffer_size) + parser.CharacterDataHandler = lambda text: None + xml_data = b"" + b"A" * N + b"" + self.assertEqual(parser.Parse(xml_data, True), 1) + class ElementDeclHandlerTest(unittest.TestCase): def test_trigger_leak(self): # Unfixed, this test would leak the memory of the so-called diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index a8914953ce9eb48..e6d727d417b2985 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import patch import builtins +import types import rlcompleter from test.support import MISSING_C_DOCSTRINGS @@ -135,6 +136,57 @@ def bar(self): self.assertEqual(completer.complete('f.b', 0), 'f.bar') self.assertFalse(f.property_called) + def test_released_memoryview_completion_works(self): + mv = memoryview(b"abc") + mv.release() + + self.assertIsInstance(type(mv).shape, types.GetSetDescriptorType) + self.assertIsInstance(type(mv).strides, types.GetSetDescriptorType) + + completer = rlcompleter.Completer(dict(mv=mv)) + matches = completer.attr_matches('mv.') + + # These are getset descriptors on memoryview and should be completed + # without evaluating the released-memoryview getters. + self.assertIn('mv.shape', matches) + self.assertIn('mv.strides', matches) + + def test_member_descriptor_not_evaluated(self): + class Foo: + __slots__ = ("boom",) + boom_accesses = 0 + + def __getattribute__(self, name): + if name == "boom": + type(self).boom_accesses += 1 + raise RuntimeError("boom access should be skipped") + return super().__getattribute__(name) + + self.assertIsInstance(Foo.boom, types.MemberDescriptorType) + + completer = rlcompleter.Completer(dict(f=Foo())) + matches = completer.attr_matches('f.') + self.assertIn('f.boom', matches) + self.assertEqual(Foo.boom_accesses, 0) + + def test_raising_descriptor_completion_works(self): + class ExplodingDescriptor: + def __init__(self): + self.instance_get_calls = 0 + + def __get__(self, obj, owner): + if obj is None: + return self + self.instance_get_calls += 1 + raise RuntimeError("descriptor getter exploded") + + class Foo: + boom = ExplodingDescriptor() + + completer = rlcompleter.Completer(dict(f=Foo())) + matches = completer.attr_matches('f.') + self.assertIn('f.boom', matches) + self.assertEqual(Foo.boom.instance_get_calls, 0) def test_uncreated_attr(self): # Attributes like properties and slots should be completed even when diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 3ea0ec66fbfbe9e..cd1477037e94b74 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -646,26 +646,23 @@ def test_group_without_user_agent(self): ) class BaseLocalNetworkTestCase: - def setUp(self): + @classmethod + def setUpClass(cls): # clear _opener global variable - self.addCleanup(urllib.request.urlcleanup) + cls.addClassCleanup(urllib.request.urlcleanup) - self.server = HTTPServer((socket_helper.HOST, 0), self.RobotHandler) + cls.server = HTTPServer((socket_helper.HOST, 0), cls.RobotHandler) + cls.addClassCleanup(cls.server.server_close) - self.t = threading.Thread( + t = threading.Thread( name='HTTPServer serving', - target=self.server.serve_forever, + target=cls.server.serve_forever, # Short poll interval to make the test finish quickly. # Time between requests is short enough that we won't wake # up spuriously too many times. kwargs={'poll_interval':0.01}) - self.t.daemon = True # In case this function raises. - self.t.start() - - def tearDown(self): - self.server.shutdown() - self.t.join() - self.server.server_close() + cls.enterClassContext(threading_helper.start_threads([t])) + cls.addClassCleanup(cls.server.shutdown) SAMPLE_ROBOTS_TXT = b'''\ @@ -687,7 +684,6 @@ def do_GET(self): def log_message(self, format, *args): pass - @threading_helper.reap_threads def testRead(self): # Test that reading a weird robots.txt doesn't fail. addr = self.server.server_address @@ -702,31 +698,79 @@ def testRead(self): self.assertTrue(parser.can_fetch(agent, url + '/utf8/')) self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) self.assertFalse(parser.can_fetch(agent, url + '/utf8/%F0%9F%90%8D')) - self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) self.assertTrue(parser.can_fetch(agent, url + '/non-utf8/')) self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/%F0')) self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/\U0001f40d')) self.assertFalse(parser.can_fetch(agent, url + '/%2F[spam]/path')) -class PasswordProtectedSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase): +class HttpErrorsTestCase(BaseLocalNetworkTestCase, unittest.TestCase): class RobotHandler(BaseHTTPRequestHandler): def do_GET(self): - self.send_error(403, "Forbidden access") + self.send_error(self.server.return_code) def log_message(self, format, *args): pass - @threading_helper.reap_threads - def testPasswordProtectedSite(self): + def setUp(self): + # Make sure that a valid code is set in the test. + self.server.return_code = None + + def testUnauthorized(self): + self.server.return_code = 401 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/some/file.html')) + + def testForbidden(self): + self.server.return_code = 403 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/some/file.html')) + + def testNotFound(self): + self.server.return_code = 404 addr = self.server.server_address - url = 'http://' + socket_helper.HOST + ':' + str(addr[1]) + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertTrue(parser.can_fetch("*", robots_url)) + self.assertTrue(parser.can_fetch("*", url + '/path/file.html')) + + def testTeapot(self): + self.server.return_code = 418 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertTrue(parser.can_fetch("*", robots_url)) + self.assertTrue(parser.can_fetch("*", url + '/pot-1?milk-type=Cream')) + + def testServiceUnavailable(self): + self.server.return_code = 503 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' robots_url = url + "/robots.txt" parser = urllib.robotparser.RobotFileParser() parser.set_url(url) parser.read() self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/path/file.html')) @support.requires_working_socket() @@ -738,6 +782,7 @@ class NetworkTestCase(unittest.TestCase): @classmethod def setUpClass(cls): support.requires('network') + cls.addClassCleanup(urllib.request.urlcleanup) with socket_helper.transient_internet(cls.base_url): cls.parser = urllib.robotparser.RobotFileParser(cls.robots_txt) cls.parser.read() diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index ac69e2cbdbbe547..e2a81b82321ede5 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -196,8 +196,9 @@ def test_addsitedir_explicit_flush(self): pth_file.cleanup(prep=True) with pth_file.create(): # Pass defer_processing_start_files=True to prevent flushing. - site.addsitedir(pth_file.base_dir, set(), - defer_processing_start_files=True) + site.addsitedir( + pth_file.base_dir, set(), + defer_processing_start_files=True) self.assertNotIn(pth_file.imported, sys.modules) site.process_startup_files() self.pth_file_tests(pth_file) @@ -423,15 +424,14 @@ def create(self): Used as a context manager: self.cleanup() is called on exit. """ - FILE = open(self.file_path, 'w') - try: - print("#import @bad module name", file=FILE) - print("\n", file=FILE) - print("import %s" % self.imported, file=FILE) - print(self.good_dirname, file=FILE) - print(self.bad_dirname, file=FILE) - finally: - FILE.close() + with open(self.file_path, 'w') as fp: + print(f"""\ +#import @bad module name +import {self.imported} +{self.good_dirname} +{self.bad_dirname} +""", file=fp) + os.mkdir(self.good_dir_path) try: yield self @@ -456,6 +456,7 @@ def cleanup(self, prep=False): if os.path.exists(self.bad_dir_path): os.rmdir(self.bad_dir_path) + class ImportSideEffectTests(unittest.TestCase): """Test side-effects from importing 'site'.""" @@ -545,7 +546,6 @@ def test_customization_modules_on_startup(self): output = subprocess.check_output([sys.executable, '-s', '-c', '""']) self.assertNotIn(eyecatcher, output.decode('utf-8')) - @unittest.skipUnless(hasattr(urllib.request, "HTTPSHandler"), 'need SSL support to download license') @test.support.requires_resource('network') @@ -915,39 +915,69 @@ class StartFileTests(unittest.TestCase): def setUp(self): self.enterContext(import_helper.DirsOnSysPath()) self.tmpdir = self.sitedir = self.enterContext(os_helper.temp_dir()) - # Save and clear all pending dicts. - self.saved_entrypoints = site._pending_entrypoints.copy() - self.saved_syspaths = site._pending_syspaths.copy() - self.saved_importexecs = site._pending_importexecs.copy() - site._pending_entrypoints.clear() - site._pending_syspaths.clear() - site._pending_importexecs.clear() - - def tearDown(self): - site._pending_entrypoints = self.saved_entrypoints.copy() - site._pending_syspaths = self.saved_syspaths.copy() - site._pending_importexecs = self.saved_importexecs.copy() - - def _make_start(self, content, name='testpkg'): - """Write a .start file and return its basename.""" + # Each test gets its own _StartupState to drive the parser and + # processor methods directly. Defensively clear any _startup_state + # that a prior test may have left set via defer_processing_start_files + # without a corresponding process_startup_files() flush. + self.state = site._StartupState() + site._startup_state = None + self.addCleanup(self._reset_startup_state) + + def _reset_startup_state(self): + site._startup_state = None + + def _make_start(self, content, name='testpkg', basedir=None): + """Write a .start file and return its basename. + + ``basedir`` defaults to ``self.tmpdir``. Pass an explicit directory + when the .start file needs to live somewhere other than the test's + primary tmpdir (e.g. a nested user-site). + """ basename = f"{name}.start" - filepath = os.path.join(self.tmpdir, basename) + filepath = os.path.join(self.tmpdir if basedir is None else basedir, basename) with open(filepath, 'w', encoding='utf-8') as f: f.write(content) return basename - def _make_pth(self, content, name='testpkg'): - """Write a .pth file and return its basename.""" + def _make_pth(self, content, name='testpkg', basedir=None): + """Write a .pth file and return its basename. + + ``basedir`` defaults to ``self.tmpdir``. Pass an explicit directory + when the .pth file needs to live somewhere other than the test's + primary tmpdir (e.g. a nested user-site). + """ basename = f"{name}.pth" - filepath = os.path.join(self.tmpdir, basename) + filepath = os.path.join(self.tmpdir if basedir is None else basedir, basename) with open(filepath, 'w', encoding='utf-8') as f: f.write(content) return basename + def _make_mod(self, contents, name='mod', *, package=False, on_path=False): + """Write an importable module (or package), returning its parent dir.""" + extdir = os.path.join(self.sitedir, 'extdir') + os.makedirs(extdir, exist_ok=True) + + # Put the code in a package's dunder-init or flat module. + if package: + pkgdir = os.path.join(extdir, name) + os.mkdir(pkgdir) + modpath = os.path.join(pkgdir, '__init__.py') + else: + modpath = os.path.join(extdir, f'{name}.py') + + with open(modpath, 'w') as fp: + fp.write(contents) + + self.addCleanup(sys.modules.pop, name, None) + if on_path: + # Don't worry, DirsOnSysPath() in setUp() will clean this up. + sys.path.insert(0, extdir) + return extdir + def _all_entrypoints(self): - """Flatten _pending_entrypoints dict into a list of (filename, entry) tuples.""" + """Flatten state._entrypoints into a list of (filename, entry) tuples.""" result = [] - for filename, entries in site._pending_entrypoints.items(): + for filename, entries in self.state._entrypoints.items(): for entry in entries: result.append((filename, entry)) return result @@ -955,28 +985,42 @@ def _all_entrypoints(self): def _just_entrypoints(self): return [entry for filename, entry in self._all_entrypoints()] - # --- _read_start_file tests --- + # There are two classes of tests here. Tests that start with `test_impl_` + # know details about the implementation and they access non-public methods + # and data structures to perform focused functional tests. + # + # Tests that start with `test_addsitedir_` are end-to-end tests that ensure + # integration semantics and functionality as a caller of the public + # surfaces would see. + + # --- _StartupState.read_start_file tests --- - def test_read_start_file_basic(self): + def test_impl_read_start_file_basic(self): self._make_start("os.path:join\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], ['os.path:join']) + self.assertEqual( + self.state._entrypoints[fullname], ['os.path:join'] + ) - def test_read_start_file_multiple_entries(self): + def test_impl_read_start_file_multiple_entries(self): self._make_start("os.path:join\nos.path:exists\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], - ['os.path:join', 'os.path:exists']) + self.assertEqual( + self.state._entrypoints[fullname], + ['os.path:join', 'os.path:exists'], + ) - def test_read_start_file_comments_and_blanks(self): + def test_impl_read_start_file_comments_and_blanks(self): self._make_start("# a comment\n\nos.path:join\n \n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], ['os.path:join']) + self.assertEqual( + self.state._entrypoints[fullname], ['os.path:join'] + ) - def test_read_start_file_accepts_all_non_blank_lines(self): + def test_impl_read_start_file_accepts_all_non_blank_lines(self): # Syntax validation is deferred to entry-point execution time # (where pkgutil.resolve_name(strict=True) enforces the strict # pkg.mod:callable form), so parsing accepts every non-blank, @@ -989,9 +1033,9 @@ def test_read_start_file_accepts_all_non_blank_lines(self): "os.path:join\n" # valid ) self._make_start(content, name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], [ + self.assertEqual(self.state._entrypoints[fullname], [ 'os.path', 'pkg.mod:', ':callable', @@ -999,155 +1043,169 @@ def test_read_start_file_accepts_all_non_blank_lines(self): 'os.path:join', ]) - def test_read_start_file_empty(self): + def test_impl_read_start_file_empty(self): # PEP 829: an empty .start file is still registered as present - # (with an empty entry-point list) so that it suppresses `import` + # (with an empty entry point list) so that it suppresses `import` # lines in any matching .pth file. self._make_start("", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {fullname: []}) + self.assertEqual(self.state._entrypoints, {fullname: []}) - def test_read_start_file_comments_only(self): + def test_impl_read_start_file_comments_only(self): # As with an empty file, a comments-only .start file is registered # as present so it can suppress matching .pth `import` lines. self._make_start("# just a comment\n# another\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {fullname: []}) + self.assertEqual(self.state._entrypoints, {fullname: []}) - def test_read_start_file_nonexistent(self): + def test_impl_read_start_file_nonexistent(self): with captured_stderr(): - site._read_start_file(self.tmpdir, 'nonexistent.start') - self.assertEqual(site._pending_entrypoints, {}) + self.state.read_start_file(self.tmpdir, 'nonexistent.start') + self.assertEqual(self.state._entrypoints, {}) @unittest.skipUnless(hasattr(os, 'chflags'), 'test needs os.chflags()') - def test_read_start_file_hidden_flags(self): + def test_impl_read_start_file_hidden_flags(self): self._make_start("os.path:join\n", name='foo') filepath = os.path.join(self.tmpdir, 'foo.start') st = os.stat(filepath) os.chflags(filepath, st.st_flags | stat.UF_HIDDEN) - site._read_start_file(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {}) + self.state.read_start_file(self.sitedir, 'foo.start') + self.assertEqual(self.state._entrypoints, {}) - def test_read_start_file_duplicates_not_deduplicated(self): + def test_impl_one_start_file_with_duplicates_not_deduplicated(self): # PEP 829: duplicate entry points are NOT deduplicated. self._make_start("os.path:join\nos.path:join\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], - ['os.path:join', 'os.path:join']) + self.assertEqual( + self.state._entrypoints[fullname], + ['os.path:join', 'os.path:join'], + ) + + def test_impl_two_start_files_with_duplicates_not_deduplicated(self): + self._make_start("os.path:join", name="foo") + self._make_start("os.path:join", name="bar") + self.state.read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'bar.start') + self.assertEqual( + self._just_entrypoints(), + ['os.path:join', 'os.path:join'], + ) - def test_read_start_file_accepts_utf8_bom(self): + def test_impl_read_start_file_accepts_utf8_bom(self): # PEP 829: .start files MUST be utf-8-sig (UTF-8 with optional BOM). filepath = os.path.join(self.tmpdir, 'foo.start') with open(filepath, 'wb') as f: f.write(b'\xef\xbb\xbf' + b'os.path:join\n') - site._read_start_file(self.sitedir, 'foo.start') + self.state.read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') self.assertEqual( - site._pending_entrypoints[fullname], ['os.path:join']) + self.state._entrypoints[fullname], ['os.path:join'] + ) - def test_read_start_file_invalid_utf8_silently_skipped(self): - # PEP 829: .start files MUST be utf-8-sig. Unlike .pth, there is - # no locale-encoding fallback -- a .start file that is not valid + def test_impl_read_start_file_invalid_utf8_silently_skipped(self): + # PEP 829: .start files MUST be utf-8-sig. Unlike .pth files, there + # is no locale-encoding fallback. A .start file that is not valid # UTF-8 is silently skipped, with no key registered in - # _pending_entrypoints and no output to stderr (parsing errors - # are reported only under -v). + # state._entrypoints and no output to stderr (parsing errors are + # reported only under -v). filepath = os.path.join(self.tmpdir, 'foo.start') with open(filepath, 'wb') as f: # Bare continuation byte -- invalid as a UTF-8 start byte. f.write(b'\x80\x80\x80\n') with captured_stderr() as err: - site._read_start_file(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {}) + self.state.read_start_file(self.sitedir, 'foo.start') + self.assertEqual(self.state._entrypoints, {}) self.assertEqual(err.getvalue(), "") - def test_two_start_files_with_duplicates_not_deduplicated(self): - self._make_start("os.path:join", name="foo") - self._make_start("os.path:join", name="bar") - site._read_start_file(self.sitedir, 'foo.start') - site._read_start_file(self.sitedir, 'bar.start') - self.assertEqual(self._just_entrypoints(), - ['os.path:join', 'os.path:join']) - - # --- _read_pth_file tests --- + # --- _StartupState.read_pth_file tests --- - def test_read_pth_file_paths(self): + def test_impl_read_pth_file_paths(self): subdir = os.path.join(self.sitedir, 'mylib') os.mkdir(subdir) self._make_pth("mylib\n", name='foo') - site._read_pth_file(self.sitedir, 'foo.pth', set()) + self.state.read_pth_file(self.sitedir, 'foo.pth', set()) fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths[fullname]) + self.assertIn(subdir, self.state._syspaths[fullname]) - def test_read_pth_file_imports_collected(self): + def test_impl_read_pth_file_imports_collected(self): self._make_pth("import sys\n", name='foo') - site._read_pth_file(self.sitedir, 'foo.pth', set()) + self.state.read_pth_file(self.sitedir, 'foo.pth', set()) fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertEqual(site._pending_importexecs[fullname], ['import sys']) + self.assertEqual( + self.state._importexecs[fullname], ['import sys'] + ) - def test_read_pth_file_comments_and_blanks(self): + def test_impl_read_pth_file_comments_and_blanks(self): self._make_pth("# comment\n\n \n", name='foo') - site._read_pth_file(self.sitedir, 'foo.pth', set()) - self.assertEqual(site._pending_syspaths, {}) - self.assertEqual(site._pending_importexecs, {}) + self.state.read_pth_file(self.sitedir, 'foo.pth', set()) + self.assertEqual(self.state._syspaths, {}) + self.assertEqual(self.state._importexecs, {}) - def test_read_pth_file_deduplication(self): + def test_impl_read_pth_file_deduplication(self): subdir = os.path.join(self.sitedir, 'mylib') os.mkdir(subdir) + # An accumulator acts as a deduplication ledger. known_paths = set() self._make_pth("mylib\n", name='a') self._make_pth("mylib\n", name='b') - site._read_pth_file(self.sitedir, 'a.pth', known_paths) - site._read_pth_file(self.sitedir, 'b.pth', known_paths) - # Only one entry across both files. + self.state.read_pth_file(self.sitedir, 'a.pth', known_paths) + self.state.read_pth_file(self.sitedir, 'b.pth', known_paths) + # There is only one entry across both files. all_dirs = [] - for dirs in site._pending_syspaths.values(): + for dirs in self.state._syspaths.values(): all_dirs.extend(dirs) self.assertEqual(all_dirs, [subdir]) - def test_read_pth_file_bad_line_continues(self): - # PEP 829: errors on individual lines don't abort the file. + def test_impl_read_pth_file_bad_line_continues(self): + # PEP 829: errors on individual lines don't abort processing the file. subdir = os.path.join(self.sitedir, 'goodpath') os.mkdir(subdir) self._make_pth("abc\x00def\ngoodpath\n", name='foo') with captured_stderr(): - site._read_pth_file(self.sitedir, 'foo.pth', set()) + self.state.read_pth_file(self.sitedir, 'foo.pth', set()) fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) + self.assertIn(subdir, self.state._syspaths.get(fullname, [])) def _flags_with_verbose(self, verbose): # Build a sys.flags clone with verbose overridden but every # other field preserved, so unrelated reads like # sys.flags.optimize during io.open_code() continue to work. - attrs = {name: getattr(sys.flags, name) - for name in sys.flags.__match_args__} + attrs = { + name: getattr(sys.flags, name) + for name in sys.flags.__match_args__ + } attrs['verbose'] = verbose return SimpleNamespace(**attrs) - def test_read_pth_file_parse_error_silent_by_default(self): + def test_impl_read_pth_file_parse_error_silent_by_default(self): # PEP 829: parse-time errors are silent unless -v is given. - # Force the error path by making makepath() raise. + # Force the error path by making makepath() raise an exception. self._make_pth("badline\n", name='foo') - with mock.patch('site.makepath', side_effect=ValueError("boom")), \ - mock.patch('sys.flags', self._flags_with_verbose(False)), \ - captured_stderr() as err: - site._read_pth_file(self.sitedir, 'foo.pth', set()) + with ( + mock.patch('site.makepath', side_effect=ValueError("boom")), + mock.patch('sys.flags', self._flags_with_verbose(False)), + captured_stderr() as err, + ): + self.state.read_pth_file(self.sitedir, 'foo.pth', set()) self.assertEqual(err.getvalue(), "") - def test_read_pth_file_parse_error_reported_under_verbose(self): + def test_impl_read_pth_file_parse_error_reported_under_verbose(self): # PEP 829: parse-time errors are reported when -v is given. self._make_pth("badline\n", name='foo') - with mock.patch('site.makepath', side_effect=ValueError("boom")), \ - mock.patch('sys.flags', self._flags_with_verbose(True)), \ - captured_stderr() as err: - site._read_pth_file(self.sitedir, 'foo.pth', set()) + with ( + mock.patch('site.makepath', side_effect=ValueError("boom")), + mock.patch('sys.flags', self._flags_with_verbose(True)), + captured_stderr() as err, + ): + self.state.read_pth_file(self.sitedir, 'foo.pth', set()) out = err.getvalue() self.assertIn('Error in', out) self.assertIn('foo.pth', out) - def test_read_pth_file_locale_fallback(self): + def test_impl_read_pth_file_locale_fallback(self): # PEP 829: .pth files that fail UTF-8 decoding fall back to the # locale encoding for backward compatibility (deprecated in # 3.15, to be removed in 3.20). Mock locale.getencoding() so @@ -1158,186 +1216,236 @@ def test_read_pth_file_locale_fallback(self): # \xe9 is invalid UTF-8 but valid in latin-1. with open(filepath, 'wb') as f: f.write(b'# caf\xe9 comment\nmylib\n') - with mock.patch('locale.getencoding', return_value='latin-1'), \ - captured_stderr(): - site._read_pth_file(self.sitedir, 'foo.pth', set()) + with ( + mock.patch('locale.getencoding', return_value='latin-1'), + captured_stderr(), + ): + self.state.read_pth_file(self.sitedir, 'foo.pth', set()) fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) + self.assertIn(subdir, self.state._syspaths.get(fullname, [])) - # --- _execute_start_entrypoints tests --- + # --- _StartupState._execute_start_entrypoints tests --- - def test_execute_entrypoints_with_callable(self): - # Entrypoint with callable is invoked. - mod_dir = os.path.join(self.sitedir, 'epmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ + def test_impl_execute_entrypoints_with_callable(self): + # An entry point with a callable. + self._make_mod("""\ called = False def startup(): global called called = True -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'epmod', None) +""", name='epmod', package=True, on_path=True) fullname = os.path.join(self.sitedir, 'epmod.start') - site._pending_entrypoints[fullname] = ['epmod:startup'] - site._execute_start_entrypoints() + self.state._entrypoints[fullname] = ['epmod:startup'] + self.state._execute_start_entrypoints() import epmod self.assertTrue(epmod.called) - def test_execute_entrypoints_import_error(self): - # Import error prints traceback but continues. + def test_impl_execute_entrypoints_import_error(self): + # Import errors print a traceback and continue. fullname = os.path.join(self.sitedir, 'bad.start') - site._pending_entrypoints[fullname] = [ - 'nosuchmodule_xyz:func', 'os.path:join'] + self.state._entrypoints[fullname] = [ + 'nosuchmodule_xyz:func', 'os.path:join', + ] with captured_stderr() as err: - site._execute_start_entrypoints() + self.state._execute_start_entrypoints() self.assertIn('nosuchmodule_xyz', err.getvalue()) # os.path:join should still have been called (no exception for it) - def test_execute_entrypoints_strict_syntax_rejection(self): - # PEP 829: only the strict pkg.mod:callable form is valid. - # At entry-point execution, pkgutil.resolve_name(strict=True) - # raises ValueError for invalid syntax; the invalid entry is - # reported and execution continues with the next one. + def test_impl_execute_entrypoints_strict_syntax_rejection(self): + # PEP 829: only the strict pkg.mod:callable form is valid. At entry + # point execution time, pkgutil.resolve_name(strict=True) raises a + # ValueError for the invalid syntax. The invalid entry is reported + # and execution continues with the next one. fullname = os.path.join(self.sitedir, 'bad.start') - site._pending_entrypoints[fullname] = [ + self.state._entrypoints[fullname] = [ 'os.path', # no colon 'pkg.mod:', # empty callable ':callable', # empty module 'pkg.mod:callable:extra', # multiple colons ] with captured_stderr() as err: - site._execute_start_entrypoints() + self.state._execute_start_entrypoints() out = err.getvalue() self.assertIn('Invalid entry point syntax', out) - for bad in ('os.path', 'pkg.mod:', ':callable', - 'pkg.mod:callable:extra'): + for bad in ( + 'os.path', + 'pkg.mod:', + ':callable', + 'pkg.mod:callable:extra', + ): self.assertIn(bad, out) - def test_execute_entrypoints_callable_error(self): - # Callable that raises prints traceback but continues. - mod_dir = os.path.join(self.sitedir, 'badmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ + def test_impl_execute_entrypoints_callable_error(self): + # A callable that errors prints a traceback but continues. + self._make_mod("""\ def fail(): raise RuntimeError("boom") -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'badmod', None) +""", name='badmod', package=True, on_path=True) fullname = os.path.join(self.sitedir, 'badmod.start') - site._pending_entrypoints[fullname] = ['badmod:fail'] + self.state._entrypoints[fullname] = ['badmod:fail'] with captured_stderr() as err: - site._execute_start_entrypoints() + self.state._execute_start_entrypoints() self.assertIn('RuntimeError', err.getvalue()) self.assertIn('boom', err.getvalue()) - def test_execute_entrypoints_duplicates_called_twice(self): + def test_impl_execute_entrypoints_duplicates_called_twice(self): # PEP 829: duplicate entry points execute multiple times. - mod_dir = os.path.join(self.sitedir, 'countmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ + self._make_mod("""\ call_count = 0 def bump(): global call_count call_count += 1 -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'countmod', None) +""", name='countmod', package=False, on_path=True) fullname = os.path.join(self.sitedir, 'countmod.start') - site._pending_entrypoints[fullname] = [ - 'countmod:bump', 'countmod:bump'] - site._execute_start_entrypoints() + self.state._entrypoints[fullname] = [ + 'countmod:bump', 'countmod:bump', + ] + self.state._execute_start_entrypoints() import countmod self.assertEqual(countmod.call_count, 2) - # --- _exec_imports tests --- + # --- _StartupState._exec_imports tests --- - def test_exec_imports_suppressed_by_matching_start(self): + def test_impl_exec_imports_suppressed_by_matching_start(self): # Import lines from foo.pth are suppressed when foo.start exists. + self._make_mod("""\ +call_count = 0 +def bump(): + global call_count + call_count += 1 +""", name='countmod', package=False, on_path=True) pth_fullname = os.path.join(self.sitedir, 'foo.pth') start_fullname = os.path.join(self.sitedir, 'foo.start') - site._pending_importexecs[pth_fullname] = ['import sys'] - site._pending_entrypoints[start_fullname] = ['os.path:join'] - # Should not exec the import line; no error expected. - site._exec_imports() + self.state._importexecs[pth_fullname] = ['import countmod; countmod.bump()'] + self.state._entrypoints[start_fullname] = ['os.path:join'] + self.state._exec_imports() + import countmod + self.assertEqual(countmod.call_count, 0) - def test_exec_imports_not_suppressed_by_different_start(self): + def test_impl_exec_imports_not_suppressed_by_different_start(self): # Import lines from foo.pth are NOT suppressed by bar.start. + self._make_mod("""\ +call_count = 0 +def bump(): + global call_count + call_count += 1 +""", name='countmod', package=False, on_path=True) pth_fullname = os.path.join(self.sitedir, 'foo.pth') start_fullname = os.path.join(self.sitedir, 'bar.start') - site._pending_importexecs[pth_fullname] = ['import sys'] - site._pending_entrypoints[start_fullname] = ['os.path:join'] - # Should execute the import line without error. - site._exec_imports() + self.state._importexecs[pth_fullname] = ['import countmod; countmod.bump()'] + self.state._entrypoints[start_fullname] = ['os.path:join'] + self.state._exec_imports() + import countmod + self.assertEqual(countmod.call_count, 1) - def test_exec_imports_suppressed_by_empty_matching_start(self): + def test_impl_exec_imports_suppressed_by_empty_matching_start(self): self._make_start("", name='foo') self._make_pth("import epmod; epmod.startup()", name='foo') - mod_dir = os.path.join(self.sitedir, 'epmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ + self._make_mod("""\ called = False def startup(): global called called = True -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'epmod', None) - site._read_pth_file(self.sitedir, 'foo.pth', set()) - site._read_start_file(self.sitedir, 'foo.start') - site._exec_imports() +""", name='epmod', package=True, on_path=True) + self.state.read_pth_file(self.sitedir, 'foo.pth', set()) + self.state.read_start_file(self.sitedir, 'foo.start') + self.state._exec_imports() import epmod self.assertFalse(epmod.called) - # --- _extend_syspath tests --- + # --- _StartupState._extend_syspath tests --- - def test_extend_syspath_existing_dir(self): + def test_impl_extend_syspath_existing_dir(self): subdir = os.path.join(self.sitedir, 'extlib') os.mkdir(subdir) - site._pending_syspaths['test.pth'] = [subdir] - site._extend_syspath() + self.state._syspaths['test.pth'] = [subdir] + self.state._extend_syspath() self.assertIn(subdir, sys.path) - def test_extend_syspath_nonexistent_dir(self): - nosuch = os.path.join(self.sitedir, 'nosuchdir') - site._pending_syspaths['test.pth'] = [nosuch] + def test_impl_extend_syspath_nonexistent_dir(self): + nonesuch = os.path.join(self.sitedir, 'nosuchdir') + self.state._syspaths['test.pth'] = [nonesuch] with captured_stderr() as err: - site._extend_syspath() - self.assertNotIn(nosuch, sys.path) + self.state._extend_syspath() + self.assertNotIn(nonesuch, sys.path) self.assertIn('does not exist', err.getvalue()) # --- addsitedir integration tests --- + def test_addsitedir_pth_import_skipped_when_matching_start_exists(self): + # PEP 829: an empty .start file disables the matching .pth's import + # lines, even when the .start has no entry points of its own. + self._make_mod("flag = False\n", name='suppressed', on_path=True) + self._make_start("", name='foo') + self._make_pth( + "import suppressed; suppressed.flag = True\n", + name='foo') + site.addsitedir(self.sitedir, set()) + import suppressed + self.assertFalse( + suppressed.flag, + "import line in foo.pth should be suppressed by foo.start") + + def test_addsitedir_dotfile_start_entrypoint_not_executed(self): + # .start files starting with '.' are skipped, so their entry + # points must not run. + self._make_mod("""\ +called = False +def hook(): + global called + called = True +""", + name='dotted', on_path=True) + self._make_start("dotted:hook\n", name='.hidden') + site.addsitedir(self.sitedir, set()) + import dotted + self.assertFalse(dotted.called) + + def test_addsitedir_dedups_paths_across_pth_files(self): + # PEP 829: when multiple .pth files reference the same path within + # a single addsitedir() invocation, the path is appended to + # sys.path exactly once. + subdir = os.path.join(self.sitedir, 'shared') + os.mkdir(subdir) + self._make_pth("shared\n", name='a') + self._make_pth("shared\n", name='b') + before = sys.path.count(subdir) + site.addsitedir(self.sitedir, set()) + self.assertEqual(sys.path.count(subdir), before + 1) + def test_addsitedir_discovers_start_files(self): # addsitedir() should discover .start files and accumulate entries. + # With defer_processing_start_files=True the preserved state lives on + # site._startup_state and isn't flushed until the caller invokes + # process_startup_files(). self._make_start("os.path:join\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + site.addsitedir( + self.sitedir, set(), + defer_processing_start_files=True, + ) fullname = os.path.join(self.sitedir, 'foo.start') - self.assertIn('os.path:join', site._pending_entrypoints[fullname]) + self.assertIn( + 'os.path:join', site._startup_state._entrypoints[fullname] + ) - def test_addsitedir_start_suppresses_pth_imports(self): + def test_impl_exec_imports_skips_when_matching_start(self): # When foo.start exists, import lines in foo.pth are skipped - # at flush time by _exec_imports(). + # at flush time by _StartupState._exec_imports(). self._make_start("os.path:join\n", name='foo') self._make_pth("import sys\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + site.addsitedir( + self.sitedir, set(), + defer_processing_start_files=True, + ) pth_fullname = os.path.join(self.sitedir, 'foo.pth') start_fullname = os.path.join(self.sitedir, 'foo.start') # Import line was collected... - self.assertIn('import sys', - site._pending_importexecs.get(pth_fullname, [])) + self.assertIn( + 'import sys', + site._startup_state._importexecs.get(pth_fullname, []), + ) # ...but _exec_imports() will skip it because foo.start exists. - site._exec_imports() + site._startup_state._exec_imports() def test_addsitedir_pth_paths_still_work_with_start(self): # Path lines in .pth files still work even when a .start file exists. @@ -1345,17 +1453,26 @@ def test_addsitedir_pth_paths_still_work_with_start(self): os.mkdir(subdir) self._make_start("os.path:join\n", name='foo') self._make_pth("mylib\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + site.addsitedir( + self.sitedir, set(), + defer_processing_start_files=True, + ) fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) + self.assertIn( + subdir, site._startup_state._syspaths.get(fullname, []) + ) def test_addsitedir_start_alphabetical_order(self): # Multiple .start files are discovered alphabetically. + # _all_entrypoints() reads from self.state, so swap in the + # preserved batch state for the duration of the assertion. self._make_start("os.path:join\n", name='zzz') self._make_start("os.path:exists\n", name='aaa') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + site.addsitedir( + self.sitedir, set(), + defer_processing_start_files=True, + ) + self.state = site._startup_state all_entries = self._all_entrypoints() entries = [entry for _, entry in all_entries] idx_a = entries.index('os.path:exists') @@ -1370,49 +1487,65 @@ def test_addsitedir_pth_before_start(self): os.mkdir(subdir) self._make_pth("mylib\n", name='foo') self._make_start("os.path:join\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + site.addsitedir( + self.sitedir, set(), + defer_processing_start_files=True, + ) # Both should be collected. pth_fullname = os.path.join(self.sitedir, 'foo.pth') start_fullname = os.path.join(self.sitedir, 'foo.start') - self.assertIn(subdir, site._pending_syspaths.get(pth_fullname, [])) - self.assertIn('os.path:join', - site._pending_entrypoints.get(start_fullname, [])) + self.assertIn( + subdir, site._startup_state._syspaths.get(pth_fullname, []) + ) + self.assertIn( + 'os.path:join', + site._startup_state._entrypoints.get(start_fullname, []), + ) - def test_addsitedir_dotfile_start_ignored(self): + def test_impl_addsitedir_skips_dotfile_start(self): # .start files starting with '.' are skipped. Defer flushing so - # the assertion against _pending_entrypoints is meaningful; - # otherwise process_startup_files() would clear the dict - # regardless of whether the dotfile was picked up. + # the preserved batch state stays inspectable on + # site._startup_state; otherwise process_startup_files() would + # detach and consume it regardless of whether the dotfile was + # picked up. self._make_start("os.path:join\n", name='.hidden') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) - self.assertEqual(site._pending_entrypoints, {}) + site.addsitedir( + self.sitedir, set(), + defer_processing_start_files=True, + ) + self.assertEqual(site._startup_state._entrypoints, {}) def test_addsitedir_standalone_flushes(self): - # When called with known_paths=None (standalone), addsitedir - # flushes immediately so the caller sees the effect. + # When called with defer_processing_start_files=False (the + # default), addsitedir creates a per-call _StartupState and + # processes it before returning, so the caller sees the effect + # immediately. No batch state is left behind on + # site._startup_state. subdir = os.path.join(self.sitedir, 'flushlib') os.mkdir(subdir) self._make_pth("flushlib\n", name='foo') site.addsitedir(self.sitedir) # known_paths=None self.assertIn(subdir, sys.path) - # Pending dicts should be cleared after flush. - self.assertEqual(site._pending_syspaths, {}) + self.assertIsNone(site._startup_state) def test_addsitedir_defer_does_not_flush(self): # With defer_processing_start_files=True, addsitedir accumulates # pending state but does not flush; sys.path is updated only when - # process_startup_files() is called explicitly. + # process_startup_files() is called explicitly. The accumulated + # state lives on the lazily-promoted site._startup_state. subdir = os.path.join(self.sitedir, 'acclib') os.mkdir(subdir) self._make_pth("acclib\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + site.addsitedir( + self.sitedir, set(), + defer_processing_start_files=True, + ) # Path is pending, not yet on sys.path. self.assertNotIn(subdir, sys.path) fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) + self.assertIn( + subdir, site._startup_state._syspaths.get(fullname, []) + ) def test_pth_path_is_available_to_start_entrypoint(self): # Core PEP 829 invariant: all .pth path extensions are applied to @@ -1420,18 +1553,12 @@ def test_pth_path_is_available_to_start_entrypoint(self): # point may live in a module reachable only via a .pth-extended # path. If the flush phases were inverted, resolving the entry # point would fail with ModuleNotFoundError. - extdir = os.path.join(self.sitedir, 'extdir') - os.mkdir(extdir) - modpath = os.path.join(extdir, 'mod.py') - with open(modpath, 'w') as f: - f.write("""\ + extdir = self._make_mod("""\ called = False def hook(): global called called = True """) - self.addCleanup(sys.modules.pop, 'mod', None) - # extdir is not on sys.path; only the .pth file makes it so. self.assertNotIn(extdir, sys.path) self._make_pth("extdir\n", name='extlib') @@ -1447,6 +1574,156 @@ def hook(): "entry point did not run; .pth path was likely not applied " "before .start entry-point execution") + # --- bugs --- + + # gh-75723 + def test_addsitdir_idempotent_pth(self): + # Adding the same sitedir twice with a known_paths, should not + # process .pth files twice. + extdir = self._make_mod("""\ +_pth_count = 0 +""") + self._make_pth(f"""\ +{extdir} +import mod; mod._pth_count += 1 +""") + dirs = set() + dirs = site.addsitedir(self.sitedir, dirs) + dirs = site.addsitedir(self.sitedir, dirs) + import mod + self.assertEqual(mod._pth_count, 1) + + def test_addsitdir_idempotent_start(self): + # Adding the same sitedir twice with a known_paths, should not + # process .pth files twice. + extdir = self._make_mod("""\ +_pth_count = 0 +def increment(): + global _pth_count + _pth_count += 1 +""") + self._make_pth(f"""\ +{extdir} +""") + self._make_start("""\ +mod:increment +""") + dirs = set() + dirs = site.addsitedir(self.sitedir, dirs) + dirs = site.addsitedir(self.sitedir, dirs) + import mod + self.assertEqual(mod._pth_count, 1) + + # gh-149504 + def test_reentrant_addsitedir_pth(self): + # An import line in a .pth file that calls site.addsitedir() + # must not crash or re-execute outer entries while the outer + # call is still processing its pending startup state. + overlay = self.enterContext(os_helper.temp_dir()) + overlay_pth = os.path.join(overlay, 'overlay.pth') + pkgdir = self.enterContext(os_helper.temp_dir()) + with open(overlay_pth, 'w', encoding='utf-8') as fp: + print(pkgdir, file=fp) + self._make_pth(f"import site; site.addsitedir({overlay!r})\n") + site.addsitedir(self.sitedir, set()) + self.assertIn(overlay, sys.path) + self.assertIn(pkgdir, sys.path) + + # gh-149504 + def test_reentrant_addsitedir_start(self): + # As above, but the re-entry happens from a .start entry point + # instead of a .pth import line. The entry point execution + # phase is vulnerable to the same class of bug. + overlay = self.enterContext(os_helper.temp_dir()) + overlay_pth = os.path.join(overlay, 'overlay.pth') + pkgdir = self.enterContext(os_helper.temp_dir()) + with open(overlay_pth, 'w', encoding='utf-8') as fp: + print(pkgdir, file=fp) + self._make_mod(f"""\ +import site +def bootstrap(): + site.addsitedir({overlay!r}) +""", + name='reenter_helper', on_path=True) + self._make_start("reenter_helper:bootstrap\n") + site.addsitedir(self.sitedir, set()) + self.assertIn(overlay, sys.path) + self.assertIn(pkgdir, sys.path) + + # gh-149819 + @unittest.skipUnless(site.ENABLE_USER_SITE, "requires user-site") + @support.requires_subprocess() + def test_pth_processed_when_sitedir_already_on_path(self): + # A .pth file in a site-packages directory must still be processed by + # site.main() when that directory is already on sys.path at + # interpreter start up, for example in a subprocess that inherits + # PYTHONPATH from its parent. Before the fix, main() seeded + # known_paths with all entries derived from removeduppaths(), and + # addsitedir() then skipped .pth processing for any directory already + # in known_paths. + user_base = self.tmpdir + user_site = site._get_path(user_base) + os.makedirs(user_site) + sentinel = "GH149819_PTH_RAN" + # Writing some text to stderr is the simplest observable side effect. + self._make_pth(f"""\ +import sys; sys.stderr.write({sentinel!r}); sys.stderr.flush() +""", + name='gh149819', + basedir=user_site) + with EnvironmentVarGuard() as env: + # PYTHONUSERBASE points USER_SITE at our temp directory so + # site.main() will call addsitedir() on it, rather than on the + # host interpreter's real user-site. + env['PYTHONUSERBASE'] = user_base + # PYTHONPATH puts that same directory on sys.path before + # site.main() runs in the subprocess. This is what triggers the + # bug: removeduppaths() records it in known_paths, and the unfixed + # addsitedir() then skips .pth processing. + env['PYTHONPATH'] = user_site + result = subprocess.run( + [sys.executable, '-c', ''], + capture_output=True, + check=True, + ) + self.assertIn(sentinel.encode(), result.stderr) + + @unittest.skipUnless(site.ENABLE_USER_SITE, "requires user-site") + @support.requires_subprocess() + def test_start_processed_when_sitedir_already_on_path(self): + # Companion to test_pth_processed_when_sitedir_already_on_path: + # the same dedup-guard skip in addsitedir() suppressed both .pth + # and .start file processing, so verify .start entry points also + # run for a site-packages directory inherited via PYTHONPATH. + user_base = self.tmpdir + user_site = site._get_path(user_base) + os.makedirs(user_site) + sentinel = "GH149819_START_RAN" + # The .start entry point resolves to a callable, so we write a + # tiny importable module that outputs the sentinel text. It lands in + # /extdir. That path is added to PYTHONPATH below so + # the subprocess can import it. + extdir = self._make_mod(f"""\ +import sys +def run(): + sys.stderr.write({sentinel!r}) + sys.stderr.flush() +""", name='gh149819mod') + self._make_start( + 'gh149819mod:run\n', name='gh149819', basedir=user_site + ) + with EnvironmentVarGuard() as env: + # See above for details. + env['PYTHONUSERBASE'] = user_base + env['PYTHONPATH'] = os.pathsep.join([user_site, extdir]) + result = subprocess.run( + [sys.executable, '-c', ''], + capture_output=True, + check=True, + ) + self.assertIn(sentinel.encode(), result.stderr) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 9e03069494345b3..47830d0e9645efc 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -205,6 +205,25 @@ def _have_socket_hyperv(): return True +def _have_udp_lite(): + if not hasattr(socket, "IPPROTO_UDPLITE"): + return False + # Older Android versions block UDPLITE with SELinux. + if support.is_android and platform.android_ver().api_level < 29: + return False + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDPLITE) + except OSError as exc: + # Linux 7.1 removed UDP Lite support + if exc.errno == errno.EPROTONOSUPPORT: + return False + raise + sock.close() + + return True + + @contextlib.contextmanager def socket_setdefaulttimeout(timeout): old_timeout = socket.getdefaulttimeout() @@ -247,10 +266,7 @@ def downgrade_malformed_data_warning(): HAVE_SOCKET_VSOCK = _have_socket_vsock() -# Older Android versions block UDPLITE with SELinux. -HAVE_SOCKET_UDPLITE = ( - hasattr(socket, "IPPROTO_UDPLITE") - and not (support.is_android and platform.android_ver().api_level < 29)) +HAVE_SOCKET_UDPLITE = _have_udp_lite() HAVE_SOCKET_BLUETOOTH = _have_socket_bluetooth() diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index e270cbb22e2d1a9..4be207e8cbf4e60 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -893,10 +893,39 @@ def test_extractall_hardlink_on_symlink(self): self._assert_on_file_content(hardlink_filepath, sha256_regtype) +class GzipReadTestBase: + + def test_read_with_extra_field(self): + with open(self.tarname, 'rb') as f: + data = bytearray(f.read()) + flags = data[3] + self.assertEqual(flags, 8) + data[3] = flags | 4 + data[10:10] = b'\x05\x00extra' + with open(tmpname, 'wb') as f: + f.write(data) + print(self.mode) + with tarfile.open(tmpname, mode=self.mode): + pass + + def test_read_with_file_comment(self): + with open(self.tarname, 'rb') as f: + data = bytearray(f.read()) + flags = data[3] + self.assertEqual(flags, 8) + data[3] = flags | 16 + i = data.index(0, 10) + 1 + data[i:i] = b'comment\x00' + with open(tmpname, 'wb') as f: + f.write(data) + with tarfile.open(tmpname, mode=self.mode): + pass + + class MiscReadTest(MiscReadTestBase, unittest.TestCase): test_fail_comp = None -class GzipMiscReadTest(GzipTest, MiscReadTestBase, unittest.TestCase): +class GzipMiscReadTest(GzipTest, GzipReadTestBase, MiscReadTestBase, unittest.TestCase): pass class Bz2MiscReadTest(Bz2Test, MiscReadTestBase, unittest.TestCase): @@ -970,7 +999,7 @@ def test_compare_members(self): finally: tar1.close() -class GzipStreamReadTest(GzipTest, StreamReadTest): +class GzipStreamReadTest(GzipTest, GzipReadTestBase, StreamReadTest): pass class Bz2StreamReadTest(Bz2Test, StreamReadTest): @@ -3911,10 +3940,19 @@ def test_parent_symlink(self): + "which is outside the destination") with self.check_context(arc.open(), 'data'): - self.expect_exception( - tarfile.LinkOutsideDestinationError, - """'parent' would link to ['"].*outerdir['"], """ - + "which is outside the destination") + if self.dotdot_resolves_early: + # 'current/../..' normalises to '..', which is rejected. + self.expect_exception( + tarfile.LinkOutsideDestinationError, + """'parent' would link to ['"].*outerdir['"], """ + + "which is outside the destination") + else: + # 'current/..' normalises to '.'; the rewritten link is + # created and 'parent/evil' lands harmlessly inside the + # destination. + self.expect_file('current', symlink_to='.') + self.expect_file('parent', symlink_to='.') + self.expect_file('evil') else: # No symlink support. The symlinks are ignored. @@ -4174,6 +4212,76 @@ def test_sly_relative2(self): + """['"].*moo['"], which is outside the """ + "destination") + @symlink_test + @os_helper.skip_unless_symlink + def test_normpath_realpath_mismatch(self): + # The link-target check must validate the value that will actually + # be written to disk (the normalised linkname), not the original. + # Here 'a' is a symlink to a deep nonexistent path, so realpath() + # of 'a/../../...' stays inside the destination while normpath() + # collapses 'a/..' lexically and escapes. + depth = len(self.destdir.parts) + 5 + deep = '/'.join(f'p{i}' for i in range(depth)) + sneaky = 'a/' + '../' * depth + 'flag' + for kind in 'symlink_to', 'hardlink_to': + with self.subTest(kind): + with ArchiveMaker() as arc: + arc.add('a', symlink_to=deep) + arc.add('escape', **{kind: sneaky}) + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.LinkOutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_symlink_trailing_slash(self): + # A trailing slash on a symlink member's name must not cause the + # link target to be resolved relative to the wrong directory. + with ArchiveMaker() as arc: + t = tarfile.TarInfo('x/') + t.type = tarfile.SYMTYPE + t.linkname = '..' + arc.tar_w.addfile(t) + arc.add('x/escaped', content='hi') + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.LinkOutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_link_at_destination(self): + # A link member whose name resolves to the destination directory + # itself must be rejected: otherwise the destination is replaced + # by a symlink and later members can be redirected through it. + for name in '', '.', './': + with ArchiveMaker() as arc: + t = tarfile.TarInfo(name) + t.type = tarfile.SYMTYPE + t.linkname = '.' + arc.tar_w.addfile(t) + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.OutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_empty_name_symlink_chain(self): + # Regression test for a chain of empty-named symlinks that + # incrementally redirects the destination outwards. + with ArchiveMaker() as arc: + for name, target in [('', ''), ('a/', '..'), + ('', 'dummy'), ('', 'a'), + ('b/', '..'), + ('', 'dummy'), ('', 'a/b')]: + t = tarfile.TarInfo(name) + t.type = tarfile.SYMTYPE + t.linkname = target + arc.tar_w.addfile(t) + arc.add('escaped', content='hi') + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.FilterError) + @symlink_test def test_deep_symlink(self): # Test that symlinks and hardlinks inside a directory diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 47450d3fd5976fa..70731d3222ced94 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -54,7 +54,11 @@ def test_eval_null_in_result(self): def test_eval_surrogates_in_result(self): tcl = self.interp - self.assertEqual(tcl.eval(r'set a "<\ud83d\udcbb>"'), '<\U0001f4bb>') + result = tcl.eval(r'set a "<\ud83d\udcbb>"') + if sys.platform == 'win32' and tcl_version >= (9, 0): + self.assertEqual('<\ud83d\udcbb>', result) + else: + self.assertEqual('<\U0001f4bb>', result) def testEvalException(self): tcl = self.interp @@ -289,7 +293,11 @@ def test_evalfile_surrogates_in_result(self): set b "<\\ud83d\\udcbb>" """) tcl.evalfile(filename) - self.assertEqual(tcl.eval('set b'), '<\U0001f4bb>') + result = tcl.eval('set b') + if sys.platform == 'win32' and tcl_version >= (9, 0): + self.assertEqual('<\ud83d\udcbb>', result) + else: + self.assertEqual('<\U0001f4bb>', result) def testEvalFileException(self): tcl = self.interp diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 78461abcd69f337..9d2960664abfad5 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -301,9 +301,9 @@ def test_sysconfig(self): self.assertEqual(out.strip(), expected, err) for attr, expected in ( ('executable', self.envpy()), - # Usually compare to sys.executable, but if we're running in our own - # venv then we really need to compare to our base executable - ('_base_executable', sys._base_executable), + # Usually compare to sys.prefix, but if we're running in our own + # venv then we really need to compare to our base prefix + ('base_prefix', sys.base_prefix), ): with self.subTest(attr): cmd[2] = f'import sys; print(sys.{attr})' @@ -656,6 +656,26 @@ def test_deactivate_with_strict_bash_opts(self): self.assertEqual(out, "".encode()) self.assertEqual(err, "".encode()) + # gh-149701: Test exit code is zero even when hashing is disabled + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') + def test_deactivate_with_strict_bash_opts_and_hashing_disabled(self): + bash = shutil.which("bash") + if bash is None: + self.skipTest("bash required for this test") + rmtree(self.env_dir) + builder = venv.EnvBuilder(clear=True) + builder.create(self.env_dir) + activate = os.path.join(self.env_dir, self.bindir, "activate") + test_script = os.path.join(self.env_dir, "test_hash_disabled.sh") + with open(test_script, "w") as f: + f.write("set -euo pipefail\n" + "set +h\n" # disable hashing + f"source {activate}\n" + "deactivate") + out, err = check_output([bash, test_script]) + self.assertEqual(out, "".encode()) + self.assertEqual(err, "".encode()) + @unittest.skipUnless(sys.platform == 'darwin', 'only relevant on macOS') def test_macos_env(self): @@ -896,10 +916,10 @@ def test_venvwlauncher(self): exename = exename.replace("python", "pythonw") envpyw = os.path.join(self.env_dir, self.bindir, exename) try: - subprocess.check_call([envpyw, "-c", "import sys; " - "assert sys._base_executable.endswith('%s')" % exename]) + subprocess.check_call([envpyw, "-c", "import fnmatch, sys; " + "assert fnmatch.fnmatch(sys._base_executable, '**/pythonw*.exe')"]) except subprocess.CalledProcessError: - self.fail("venvwlauncher.exe did not run %s" % exename) + self.fail("venvwlauncher.exe did not run pythonw.exe") @requireVenvCreate diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 51d627d24c5a8a3..82f14ca968f266b 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -340,6 +340,10 @@ def close(self): @requires_subprocess() class MacOSTest(unittest.TestCase): + def setUp(self): + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env.unset("BROWSER") + def test_default(self): browser = webbrowser.get() self.assertIsInstance(browser, webbrowser.MacOS) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 8f3efe9fc90794b..3a41ea97a2e0a26 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1009,12 +1009,12 @@ def check(encoding, body=''): check("cp437", '\u221a') check("mac-roman", '\u02da') - def xml(encoding): - return "" % encoding - def bxml(encoding): - return xml(encoding).encode(encoding) + def xml(encoding, body=''): + return "%s" % (encoding, body) + def bxml(encoding, body=''): + return xml(encoding, body).encode(encoding) supported_encodings = [ - 'ascii', 'utf-8', 'utf-8-sig', 'utf-16', 'utf-16be', 'utf-16le', + 'utf-8', 'utf-16', 'utf-16be', 'utf-16le', 'iso8859-1', 'iso8859-2', 'iso8859-3', 'iso8859-4', 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', @@ -1025,13 +1025,14 @@ def bxml(encoding): 'cp1256', 'cp1257', 'cp1258', 'mac-cyrillic', 'mac-greek', 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', - 'iso2022-jp', 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', - 'iso2022-jp-3', 'iso2022-jp-ext', - 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', - 'hz', 'ptcp154', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'ptcp154', ] for encoding in supported_encodings: - self.assertEqual(ET.tostring(ET.XML(bxml(encoding))), b'') + with self.subTest(encoding=encoding): + self.assertEqual(ET.tostring(ET.XML(bxml(encoding))), b'') + c = 'éπя\u05d0\u060c€'.encode(encoding, 'ignore').decode(encoding)[0] + self.assertEqual(ET.tostring(ET.XML(bxml(encoding, c))), + ('&#%d;' % ord(c)).encode()) unsupported_ascii_compatible_encodings = [ 'big5', 'big5hkscs', @@ -1043,14 +1044,16 @@ def bxml(encoding): 'utf-7', ] for encoding in unsupported_ascii_compatible_encodings: - self.assertRaises(ValueError, ET.XML, bxml(encoding)) + with self.subTest(encoding=encoding): + self.assertRaises(ValueError, ET.XML, bxml(encoding)) unsupported_ascii_incompatible_encodings = [ 'cp037', 'cp424', 'cp500', 'cp864', 'cp875', 'cp1026', 'cp1140', 'utf_32', 'utf_32_be', 'utf_32_le', ] for encoding in unsupported_ascii_incompatible_encodings: - self.assertRaises(ET.ParseError, ET.XML, bxml(encoding)) + with self.subTest(encoding=encoding): + self.assertRaises(ET.ParseError, ET.XML, bxml(encoding)) self.assertRaises(ValueError, ET.XML, xml('undefined').encode('ascii')) self.assertRaises(LookupError, ET.XML, xml('xxx').encode('ascii')) diff --git a/Lib/test/test_xxlimited.py b/Lib/test/test_xxlimited.py index b52e78bc4fb7e05..c6e9dc375d9a676 100644 --- a/Lib/test/test_xxlimited.py +++ b/Lib/test/test_xxlimited.py @@ -1,19 +1,39 @@ import unittest from test.support import import_helper -import types xxlimited = import_helper.import_module('xxlimited') -xxlimited_35 = import_helper.import_module('xxlimited_35') - -class CommonTests: - module: types.ModuleType - - def test_xxo_new(self): - xxo = self.module.Xxo() - - def test_xxo_attributes(self): - xxo = self.module.Xxo() +# if import of xxlimited succeeded, the other ones should be importable. +import xxlimited_3_13 +import xxlimited_35 + +MODULES = { + (3, 15): xxlimited, + (3, 13): xxlimited_3_13, + (3, 5): xxlimited_35, +} + +def test_with_xxlimited_modules(since=None, until=None): + def _decorator(func): + def _wrapper(self, *args, **kwargs): + for version, module in MODULES.items(): + if since and version < since: + continue + if until and version >= until: + continue + with self.subTest(version=version): + func(self, module, *args, **kwargs) + return _wrapper + return _decorator + +class XXLimitedTests(unittest.TestCase): + @test_with_xxlimited_modules() + def test_xxo_new(self, module): + xxo = module.Xxo() + + @test_with_xxlimited_modules() + def test_xxo_attributes(self, module): + xxo = module.Xxo() with self.assertRaises(AttributeError): xxo.foo with self.assertRaises(AttributeError): @@ -26,40 +46,61 @@ def test_xxo_attributes(self): with self.assertRaises(AttributeError): xxo.foo - def test_foo(self): + @test_with_xxlimited_modules() + def test_foo(self, module): # the foo function adds 2 numbers - self.assertEqual(self.module.foo(1, 2), 3) + self.assertEqual(module.foo(1, 2), 3) - def test_str(self): - self.assertIsSubclass(self.module.Str, str) - self.assertIsNot(self.module.Str, str) + @test_with_xxlimited_modules() + def test_str(self, module): + self.assertIsSubclass(module.Str, str) + self.assertIsNot(module.Str, str) - custom_string = self.module.Str("abcd") + custom_string = module.Str("abcd") self.assertEqual(custom_string, "abcd") self.assertEqual(custom_string.upper(), "ABCD") - def test_new(self): - xxo = self.module.new() + @test_with_xxlimited_modules() + def test_new(self, module): + xxo = module.new() self.assertEqual(xxo.demo("abc"), "abc") - -class TestXXLimited(CommonTests, unittest.TestCase): - module = xxlimited - - def test_xxo_demo(self): - xxo = self.module.Xxo() - other = self.module.Xxo() + @test_with_xxlimited_modules() + def test_xxo_demo(self, module): + xxo = module.Xxo() self.assertEqual(xxo.demo("abc"), "abc") + self.assertEqual(xxo.demo(0), None) + self.assertEqual(xxo.__module__, module.__name__) + with self.assertRaises(TypeError): + module.Xxo('arg') + with self.assertRaises(TypeError): + module.Xxo(kwarg='arg') + + @test_with_xxlimited_modules(since=(3, 13)) + def test_xxo_demo_extra(self, module): + xxo = module.Xxo() + other = module.Xxo() self.assertEqual(xxo.demo(xxo), xxo) self.assertEqual(xxo.demo(other), other) - self.assertEqual(xxo.demo(0), None) - def test_error(self): - with self.assertRaises(self.module.Error): - raise self.module.Error - - def test_buffer(self): - xxo = self.module.Xxo() + @test_with_xxlimited_modules(since=(3, 15)) + def test_xxo_subclass(self, module): + class Sub(module.Xxo): + pass + sub = Sub() + sub.a = 123 + self.assertEqual(sub.a, 123) + with self.assertRaisesRegex(AttributeError, "cannot set 'reserved'"): + sub.reserved = 123 + + @test_with_xxlimited_modules(since=(3, 13)) + def test_error(self, module): + with self.assertRaises(module.Error): + raise module.Error + + @test_with_xxlimited_modules(since=(3, 13)) + def test_buffer(self, module): + xxo = module.Xxo() self.assertEqual(xxo.x_exports, 0) b1 = memoryview(xxo) self.assertEqual(xxo.x_exports, 1) @@ -69,21 +110,13 @@ def test_buffer(self): self.assertEqual(b1[0], 1) self.assertEqual(b2[0], 1) - -class TestXXLimited35(CommonTests, unittest.TestCase): - module = xxlimited_35 - - def test_xxo_demo(self): - xxo = self.module.Xxo() - other = self.module.Xxo() - self.assertEqual(xxo.demo("abc"), "abc") - self.assertEqual(xxo.demo(0), None) - + @test_with_xxlimited_modules(until=(3, 5)) def test_roj(self): # the roj function always fails with self.assertRaises(SystemError): self.module.roj(0) + @test_with_xxlimited_modules(until=(3, 5)) def test_null(self): null1 = self.module.Null() null2 = self.module.Null() diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 0d407371f40a0f7..30550263ad50aab 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -1886,11 +1886,8 @@ def test_write_with_source_date_epoch(self): with zipfile.ZipFile(TESTFN, "r") as zf: zip_info = zf.getinfo("test_source_date_epoch.txt") - get_time = time.localtime(int(os.environ['SOURCE_DATE_EPOCH']))[:6] - # Compare each element of the date_time tuple - # Allow for a 1-second difference - for z_time, g_time in zip(zip_info.date_time, get_time): - self.assertAlmostEqual(z_time, g_time, delta=1) + expected_utc = (2025, 1, 1, 7, 19, 58) + self.assertEqual(zip_info.date_time, expected_utc) def test_write_without_source_date_epoch(self): with os_helper.EnvironmentVarGuard() as env: @@ -1901,9 +1898,13 @@ def test_write_without_source_date_epoch(self): with zipfile.ZipFile(TESTFN, "r") as zf: zip_info = zf.getinfo("test_no_source_date_epoch.txt") - current_time = time.localtime()[:6] - for z_time, c_time in zip(zip_info.date_time, current_time): - self.assertAlmostEqual(z_time, c_time, delta=2) + self.assertTimestampAlmostEqual(time.localtime(), zip_info.date_time, tolerance=2) + + def assertTimestampAlmostEqual(self, time1, time2, tolerance): + import datetime + dt1 = datetime.datetime(*time1[:6]) + dt2 = datetime.datetime(*time2[:6]) + self.assertLessEqual((dt1 - dt2).total_seconds(), tolerance) def test_close(self): """Check that the zipfile is closed after the 'with' block.""" diff --git a/Lib/tomllib/mypy.ini b/Lib/tomllib/mypy.ini index 1761dce45562a60..f7eeffd575c1c76 100644 --- a/Lib/tomllib/mypy.ini +++ b/Lib/tomllib/mypy.ini @@ -12,6 +12,4 @@ pretty = True # Enable most stricter settings enable_error_code = ignore-without-code strict = True -strict_bytes = True -local_partial_types = True warn_unreachable = True diff --git a/Lib/typing.py b/Lib/typing.py index 5b1e223d59641e1..bd1f6448894e8f1 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -5,7 +5,7 @@ * Generic, Protocol, and internal machinery to support generic aliases. All subscripted types like X[int], Union[int, str] are generic aliases. * Various "special forms" that have unique meanings in type annotations: - NoReturn, Never, ClassVar, Self, Concatenate, Unpack, and others. + Any, Never, ClassVar, Self, Concatenate, Unpack, and others. * Classes whose instances can be type arguments to generic classes and functions: TypeVar, ParamSpec, TypeVarTuple. * Public helper functions: get_type_hints, overload, cast, final, and others. @@ -604,12 +604,12 @@ def __repr__(self): class Any(metaclass=_AnyMeta): """Special type indicating an unconstrained type. - - Any is compatible with every type. - - Any assumed to have all methods. - - All values assumed to be instances of Any. + - Any is assignable to every type. + - Any assumed to have all methods and attributes. + - All values are assignable to Any. Note that all the above statements are true from the point of view of - static type checkers. At runtime, Any should not be used with instance + static type checkers. At runtime, Any cannot be used with instance checks. """ @@ -728,7 +728,7 @@ class Starship: ClassVar accepts only types and cannot be further subscribed. - Note that ClassVar is not a class itself, and should not + Note that ClassVar is not a class itself, and cannot be used with isinstance() or issubclass(). """ item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) @@ -758,7 +758,7 @@ class FastConnector(Connection): @_SpecialForm def Optional(self, parameters): - """Optional[X] is equivalent to Union[X, None].""" + """Optional[X] is equivalent to X | None.""" arg = _type_check(parameters, f"{self} requires a single type.") return Union[arg, type(None)] @@ -801,7 +801,7 @@ def open_helper(file: str, mode: MODE) -> str: def TypeAlias(self, parameters): """Special form for marking type aliases. - Use TypeAlias to indicate that an assignment should + TypeAlias can be used to indicate that an assignment should be recognized as a proper type alias definition by type checkers. @@ -1809,7 +1809,7 @@ class Movie(TypedDict): def foo(**kwargs: Unpack[Movie]): ... Note that there is only some runtime checking of this operator. Not - everything the runtime allows may be accepted by static type checkers. + everything the runtime allows is accepted by static type checkers. For more information, see PEPs 646 and 692. """ @@ -2320,7 +2320,7 @@ def runtime_checkable(cls): Such protocol can be used with isinstance() and issubclass(). Raise TypeError if applied to a non-protocol class. This allows a simple-minded structural check very similar to - one trick ponies in collections.abc such as Iterable. + one-trick ponies in collections.abc such as Iterable. For example:: @@ -2390,8 +2390,8 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, *, format=None): """Return type hints for an object. - This is often the same as obj.__annotations__, but it handles - forward references encoded as string literals and recursively replaces all + This is often the same as annotationlib.get_annotations(obj) or obj.__annotations__, + but it handles forward references encoded as string literals and recursively replaces all 'Annotated[T, ...]' with 'T' (unless 'include_extras=True'). The argument may be a module, class, method, or function. The annotations @@ -2603,7 +2603,7 @@ def get_args(tp): def is_typeddict(tp): - """Check if an annotation is a TypedDict class. + """Check if an object is a TypedDict class. For example:: @@ -2700,10 +2700,10 @@ def _overload_dummy(*args, **kwds): def overload(func): """Decorator for overloaded functions/methods. - In a stub file, place two or more stub definitions for the same - function in a row, each decorated with @overload. - - For example:: + In a non-stub file, place two or more stub definitions for the same + function in a row, each decorated with @overload, followed + by an implementation. The implementation should *not* + be decorated with @overload:: @overload def utf8(value: None) -> None: ... @@ -2711,10 +2711,11 @@ def utf8(value: None) -> None: ... def utf8(value: bytes) -> bytes: ... @overload def utf8(value: str) -> bytes: ... + def utf8(value): + ... # implementation goes here - In a non-stub file (i.e. a regular .py file), do the same but - follow it with an implementation. The implementation should *not* - be decorated with @overload:: + In a stub file or in an abstract method (for example, in a Protocol definition), + the implementation may be omitted:: @overload def utf8(value: None) -> None: ... @@ -2722,8 +2723,6 @@ def utf8(value: None) -> None: ... def utf8(value: bytes) -> bytes: ... @overload def utf8(value: str) -> bytes: ... - def utf8(value): - ... # implementation goes here The overloads for a function can be retrieved at runtime using the get_overloads() function. @@ -2759,7 +2758,7 @@ def final(f): """Decorator to indicate final methods and final classes. Use this decorator to indicate to type checkers that the decorated - method cannot be overridden, and decorated class cannot be subclassed. + method cannot be overridden, and the decorated class cannot be subclassed. For example:: @@ -2824,7 +2823,7 @@ class Disjoint3(Disjoint1, Disjoint2): pass # Type checker error V_co = TypeVar('V_co', covariant=True) # Any type covariant containers. VT_co = TypeVar('VT_co', covariant=True) # Value type covariant containers. T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. -# Internal type variable used for Type[]. +# Internal type bound to class object types. CT_co = TypeVar('CT_co', covariant=True, bound=type) @@ -2912,7 +2911,7 @@ class TeamUser(User): ... And a function that takes a class argument that's a subclass of User and returns an instance of the corresponding class:: - def new_user[U](user_class: Type[U]) -> U: + def new_user[U](user_class: type[U]) -> U: user = user_class() # (Here we could write the user object to a database) return user @@ -2925,7 +2924,7 @@ def new_user[U](user_class: Type[U]) -> U: @runtime_checkable class SupportsInt(Protocol): - """An ABC with one abstract method __int__.""" + """A protocol with one abstract method __int__.""" __slots__ = () @@ -2936,7 +2935,7 @@ def __int__(self) -> int: @runtime_checkable class SupportsFloat(Protocol): - """An ABC with one abstract method __float__.""" + """A protocol with one abstract method __float__.""" __slots__ = () @@ -2947,7 +2946,7 @@ def __float__(self) -> float: @runtime_checkable class SupportsComplex(Protocol): - """An ABC with one abstract method __complex__.""" + """A protocol with one abstract method __complex__.""" __slots__ = () @@ -2958,7 +2957,7 @@ def __complex__(self) -> complex: @runtime_checkable class SupportsBytes(Protocol): - """An ABC with one abstract method __bytes__.""" + """A protocol with one abstract method __bytes__.""" __slots__ = () @@ -2969,7 +2968,7 @@ def __bytes__(self) -> bytes: @runtime_checkable class SupportsIndex(Protocol): - """An ABC with one abstract method __index__.""" + """A protocol with one abstract method __index__.""" __slots__ = () @@ -2980,7 +2979,7 @@ def __index__(self) -> int: @runtime_checkable class SupportsAbs[T](Protocol): - """An ABC with one abstract method __abs__ that is covariant in its return type.""" + """A protocol with one abstract method __abs__ that is covariant in its return type.""" __slots__ = () @@ -2991,7 +2990,7 @@ def __abs__(self) -> T: @runtime_checkable class SupportsRound[T](Protocol): - """An ABC with one abstract method __round__ that is covariant in its return type.""" + """A protocol with one abstract method __round__ that is covariant in its return type.""" __slots__ = () @@ -3108,7 +3107,7 @@ def annotate(format): def NamedTuple(typename, fields, /): - """Typed version of namedtuple. + """Typed version of collections.namedtuple. Usage:: @@ -3120,8 +3119,8 @@ class Employee(NamedTuple): Employee = collections.namedtuple('Employee', ['name', 'id']) - The resulting class has an extra __annotations__ attribute, giving a - dict that maps field names to types. (The field names are also in + The types for each field name can be retrieved by calling + annotationlib.get_annotations(Employee). (The field names are also in the _fields attribute, which is part of the namedtuple API.) An alternative equivalent functional syntax is also accepted:: @@ -3174,7 +3173,7 @@ def __new__(cls, name, bases, ns, total=True, closed=None, This method is called when TypedDict is subclassed, or when TypedDict is instantiated. This way - TypedDict supports all three syntax forms described in its docstring. + TypedDict classes can be created through both class-based and functional syntax. Subclasses and instances of TypedDict return actual dictionaries. """ for base in bases: @@ -3328,14 +3327,22 @@ def TypedDict(typename, fields, /, *, total=True, closed=None, >>> Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') True - The type info can be accessed via the Point2D.__annotations__ dict, and - the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. + The type info can be accessed by calling annotationlib.get_annotations(Point2D), and + via the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. TypedDict supports an additional equivalent form:: Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) By default, all keys must be present in a TypedDict. It is possible - to override this by specifying totality:: + to override this by using the NotRequired and Required special forms:: + + class Point2D(TypedDict): + x: int # the "x" key must always be present (Required is the default) + y: NotRequired[int] # the "y" key can be omitted + + This means that a Point2D TypedDict can have the "y" key omitted, but the "x" key must be present. + Items are required by default, so the Required special form is not necessary in this example. + In addition, the total argument to the TypedDict function can be used to make all items not required:: class Point2D(TypedDict, total=False): x: int @@ -3344,16 +3351,8 @@ class Point2D(TypedDict, total=False): This means that a Point2D TypedDict can have any of the keys omitted. A type checker is only expected to support a literal False or True as the value of the total argument. True is the default, and makes all items defined in the - class body be required. - - The Required and NotRequired special forms can also be used to mark - individual keys as being required or not required:: - - class Point2D(TypedDict): - x: int # the "x" key must always be present (Required is the default) - y: NotRequired[int] # the "y" key can be omitted - - See PEP 655 for more details on Required and NotRequired. + class body be required. The Required special form can be used to mark individual + keys as required in a total=False TypedDict. The ReadOnly special form can be used to mark individual keys as immutable for type checkers:: @@ -3387,7 +3386,7 @@ class Point3D(Point2D): by default, and it may not be used with the closed argument at the same time. - See PEP 728 for more information about closed and extra_items. + See PEPs 589, 655, 705, and 728 for more information. """ ns = {'__annotations__': dict(fields)} module = _caller() @@ -3417,7 +3416,7 @@ class Movie(TypedDict, total=False): year: int m = Movie( - title='The Matrix', # typechecker error if key is omitted + title='The Matrix', # type checker error if key is omitted year=1999, ) @@ -3439,7 +3438,7 @@ class Movie(TypedDict): year: NotRequired[int] m = Movie( - title='The Matrix', # typechecker error if key is omitted + title='The Matrix', # type checker error if key is omitted year=1999, ) """ @@ -3459,7 +3458,7 @@ class Movie(TypedDict): def mutate_movie(m: Movie) -> None: m["year"] = 1992 # allowed - m["title"] = "The Matrix" # typechecker error + m["title"] = "The Matrix" # type checker error There is no runtime checking for this property. """ @@ -3546,8 +3545,8 @@ class IO(Generic[AnyStr]): classes (text vs. binary, read vs. write vs. read/write, append-only, unbuffered). The TextIO and BinaryIO subclasses below capture the distinctions between text vs. binary, which is - pervasive in the interface; however we currently do not offer a - way to track the other distinctions in the type system. + pervasive in the interface. For more precise types, define a custom + Protocol. """ __slots__ = () @@ -3637,7 +3636,7 @@ def __exit__(self, type, value, traceback, /) -> None: class BinaryIO(IO[bytes]): - """Typed version of the return of open() in binary mode.""" + """Typed approximation of the return of open() in binary mode.""" __slots__ = () @@ -3651,7 +3650,7 @@ def __enter__(self) -> BinaryIO: class TextIO(IO[str]): - """Typed version of the return of open() in text mode.""" + """Typed approximation of the return of open() in text mode.""" __slots__ = () @@ -3718,7 +3717,7 @@ def dataclass_transform( field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (), **kwargs: Any, ) -> _IdentityCallable: - """Decorator to mark an object as providing dataclass-like behaviour. + """Decorator to mark an object as providing dataclass-like behavior. The decorator can be applied to a function, class, or metaclass. diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 70673a265d41f80..241a8650bda33aa 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -17,7 +17,7 @@ deactivate () { # Call hash to forget past locations. Without forgetting # past locations the $PATH changes we made may not be respected. # See "man bash" for more details. hash is usually a builtin of your shell - hash -r 2> /dev/null + hash -r 2> /dev/null || true if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then PS1="${_OLD_VIRTUAL_PS1:-}" @@ -73,4 +73,4 @@ fi # Call hash to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected -hash -r 2> /dev/null +hash -r 2> /dev/null || true diff --git a/Lib/xml/__init__.py b/Lib/xml/__init__.py index 002d6d3e0e8267c..ecfce1c6ae52cf2 100644 --- a/Lib/xml/__init__.py +++ b/Lib/xml/__init__.py @@ -18,4 +18,4 @@ from .utils import * -__all__ = ["dom", "parsers", "sax", "etree", "is_valid_name"] +__all__ = ["dom", "parsers", "sax", "etree", "is_valid_name", "is_valid_text"] diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 86c3bc36b695c79..c5c6ac03fb7b8cc 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -663,9 +663,12 @@ def _for_archive(self, archive): Return self. """ # gh-91279: Set the SOURCE_DATE_EPOCH to a specific timestamp - epoch = os.environ.get('SOURCE_DATE_EPOCH') - get_time = int(epoch) if epoch else time.time() - self.date_time = time.localtime(get_time)[:6] + source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH') + + if source_date_epoch: + self.date_time = time.gmtime(int(source_date_epoch))[:6] + else: + self.date_time = time.localtime(time.time())[:6] self.compress_type = archive.compression self.compress_level = archive.compresslevel diff --git a/Misc/NEWS.d/next/Build/2026-05-18-16-00-41.gh-issue-148260.UwFiIX.rst b/Misc/NEWS.d/next/Build/2026-05-18-16-00-41.gh-issue-148260.UwFiIX.rst new file mode 100644 index 000000000000000..8248c24cbd511ac --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-05-18-16-00-41.gh-issue-148260.UwFiIX.rst @@ -0,0 +1,3 @@ +On Linux when Python is linked to the musl C library, use a thread stack +size of at least 1 MiB instead of musl default which is 128 kiB. Patch by +Victor Stinner. diff --git a/Misc/NEWS.d/next/Build/2026-05-21-15-14-59.gh-issue-148294.VtFaW4.rst b/Misc/NEWS.d/next/Build/2026-05-21-15-14-59.gh-issue-148294.VtFaW4.rst new file mode 100644 index 000000000000000..861261dd97269f9 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-05-21-15-14-59.gh-issue-148294.VtFaW4.rst @@ -0,0 +1,2 @@ +Corrected the use of ``AC_PATH_TOOL`` in ``configure.ac`` to allow a C++ +compiler to be found on :envvar:`!PATH`. diff --git a/Misc/NEWS.d/next/C_API/2026-02-25-13-37-10.gh-issue-145235.-1ySNR.rst b/Misc/NEWS.d/next/C_API/2026-02-25-13-37-10.gh-issue-145235.-1ySNR.rst new file mode 100644 index 000000000000000..98a8c2687357265 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-02-25-13-37-10.gh-issue-145235.-1ySNR.rst @@ -0,0 +1,3 @@ +Made :c:func:`PyDict_AddWatcher`, :c:func:`PyDict_ClearWatcher`, +:c:func:`PyDict_Watch`, and :c:func:`PyDict_Unwatch` thread-safe on the +:term:`free threaded ` build. diff --git a/Misc/NEWS.d/next/C_API/2026-05-12-16-47-21.gh-issue-149725.HZLBTZ.rst b/Misc/NEWS.d/next/C_API/2026-05-12-16-47-21.gh-issue-149725.HZLBTZ.rst new file mode 100644 index 000000000000000..97721430edbd69d --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-05-12-16-47-21.gh-issue-149725.HZLBTZ.rst @@ -0,0 +1,2 @@ +Add :c:func:`PySentinel_CheckExact` for exact :class:`sentinel` type tests +to accompany the existing :c:func:`PySentinel_Check`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-07-03-18-59.gh-issue-149459.5fhAqP.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-07-03-18-59.gh-issue-149459.5fhAqP.rst new file mode 100644 index 000000000000000..4cd0a148df3c704 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-07-03-18-59.gh-issue-149459.5fhAqP.rst @@ -0,0 +1 @@ +Fix a crash in the JIT optimizer when a specialized ``LOAD_SPECIAL`` guard deoptimized after inserting the synthetic ``NULL`` stack entry. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-09-15-22-32.gh-issue-144957.u1F2aQ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-09-15-22-32.gh-issue-144957.u1F2aQ.rst new file mode 100644 index 000000000000000..3063f1a3c0e6d3e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-09-15-22-32.gh-issue-144957.u1F2aQ.rst @@ -0,0 +1,2 @@ +Fix lazy ``from`` imports of module attributes provided by module-level +``__getattr__``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-07-42-36.gh-issue-149642.6ZksML.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-07-42-36.gh-issue-149642.6ZksML.rst new file mode 100644 index 000000000000000..815a084db69d8d8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-07-42-36.gh-issue-149642.6ZksML.rst @@ -0,0 +1,2 @@ +Allow imports inside ``exec()`` calls within functions under +``PYTHON_LAZY_IMPORTS=all``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-14-48-56.gh-issue-149676.6aTrw1.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-14-48-56.gh-issue-149676.6aTrw1.rst new file mode 100644 index 000000000000000..96f407cf5ad25a1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-14-48-56.gh-issue-149676.6aTrw1.rst @@ -0,0 +1 @@ +Fix ``frozendict | frozendict`` hash. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-12-16-47-23.gh-issue-139808.iIs7_E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-12-16-47-23.gh-issue-139808.iIs7_E.rst new file mode 100644 index 000000000000000..3e9d930bf1de894 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-12-16-47-23.gh-issue-139808.iIs7_E.rst @@ -0,0 +1,2 @@ +Add branch protections for AArch64 (BTI/PAC) in assembly code used by +:option:`-X perf_jit <-X>` (Linux perf profiler integration). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-14-19-41-03.gh-issue-149807.IwGaCo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-14-19-41-03.gh-issue-149807.IwGaCo.rst new file mode 100644 index 000000000000000..a94c737e73619d8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-14-19-41-03.gh-issue-149807.IwGaCo.rst @@ -0,0 +1,2 @@ +Fix ``hash(frozendict)``: compute the hash of each ``(key, value)`` pair +correctly. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-15-11-31-57.gh-issue-149816.ugN2rx.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-15-11-31-57.gh-issue-149816.ugN2rx.rst new file mode 100644 index 000000000000000..016c17dd66b19ea --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-15-11-31-57.gh-issue-149816.ugN2rx.rst @@ -0,0 +1 @@ +Fix a race condition in :class:`memoryview` with free-threading. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-16-11-03-54.gh-issue-149816.X_gqMT.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-16-11-03-54.gh-issue-149816.X_gqMT.rst new file mode 100644 index 000000000000000..d35f0857a1aefe8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-16-11-03-54.gh-issue-149816.X_gqMT.rst @@ -0,0 +1 @@ +Fix a race condition in ``_PyBytes_FromList`` in free-threading mode. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-13-47-17.gh-issue-149590.IPBeQx.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-13-47-17.gh-issue-149590.IPBeQx.rst new file mode 100644 index 000000000000000..8d3b29d69cc8578 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-13-47-17.gh-issue-149590.IPBeQx.rst @@ -0,0 +1 @@ +Fix crash when faulthandler is imported more than once. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-16-54-54.gh-issue-150042.LSr5W8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-16-54-54.gh-issue-150042.LSr5W8.rst new file mode 100644 index 000000000000000..18a4fbd9dadd60f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-16-54-54.gh-issue-150042.LSr5W8.rst @@ -0,0 +1 @@ +Fix refleak in queue.SimpleQueue.put if memory allocation fails. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-18-36-28.gh-issue-148587.-RD3z5.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-18-36-28.gh-issue-148587.-RD3z5.rst new file mode 100644 index 000000000000000..61bfdcdd37362cd --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-18-36-28.gh-issue-148587.-RD3z5.rst @@ -0,0 +1 @@ +``sys.lazy_modules`` is now a set instead of a dict as initially spelled out in PEP 810. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-20-13-06-17.gh-issue-150146.i5m_SL.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-20-13-06-17.gh-issue-150146.i5m_SL.rst new file mode 100644 index 000000000000000..f373f0bee7023ef --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-20-13-06-17.gh-issue-150146.i5m_SL.rst @@ -0,0 +1,5 @@ +Fix a crash on a complex type variable substitution. + +``from typing import TypeVar; memoryview[TypeVar("")][*typing.Mapping[..., +...]]`` used to fail due to missing ``NULL`` check on ``_unpack_args`` C +function call. diff --git a/Misc/NEWS.d/next/Library/2021-10-18-13-46-55.bpo-45509.Upwb60.rst b/Misc/NEWS.d/next/Library/2021-10-18-13-46-55.bpo-45509.Upwb60.rst new file mode 100644 index 000000000000000..80c38c03f8fe787 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-18-13-46-55.bpo-45509.Upwb60.rst @@ -0,0 +1 @@ +Gzip headers are now checked for corrupted NAME, COMMENT and HCRC fields. diff --git a/Misc/NEWS.d/next/Library/2024-11-02-02-02-31.gh-issue-107398.uUtA6Q.rst b/Misc/NEWS.d/next/Library/2024-11-02-02-02-31.gh-issue-107398.uUtA6Q.rst new file mode 100644 index 000000000000000..d5af322d68d309a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-02-02-02-31.gh-issue-107398.uUtA6Q.rst @@ -0,0 +1 @@ +Fix :mod:`tarfile` stream mode exception when process the file with the gzip extra field. diff --git a/Misc/NEWS.d/next/Library/2025-03-01-13-36-02.gh-issue-128110.9wx_G0.rst b/Misc/NEWS.d/next/Library/2025-03-01-13-36-02.gh-issue-128110.9wx_G0.rst new file mode 100644 index 000000000000000..b08b1886cff9cf6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-01-13-36-02.gh-issue-128110.9wx_G0.rst @@ -0,0 +1,5 @@ +Fix bug in the parsing of :mod:`email` address headers that could result in +extraneous spaces in the decoded text when using a modern email policy. +Space between pairs of adjacent :rfc:`2047` encoded-words is now ignored, per +section 6.2 (and consistent with existing parsing of unstructured +headers like *Subject*). diff --git a/Misc/NEWS.d/next/Library/2025-05-19-20-29-35.gh-issue-133998.KmElUw.rst b/Misc/NEWS.d/next/Library/2025-05-19-20-29-35.gh-issue-133998.KmElUw.rst new file mode 100644 index 000000000000000..77d92628beefacd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-19-20-29-35.gh-issue-133998.KmElUw.rst @@ -0,0 +1,5 @@ +Fix :exc:`struct.error` exception when creating a file with +:class:`gzip.GzipFile` or compressing data with :func:`gzip.compress` +if the system time is outside the range 00:00:00 UTC, January 1, 1970 +through 06:28:15 UTC, February 7, 2106, or explicitly passed *mtime* +argument is outside the range ``0`` to ``2**32-1``. diff --git a/Misc/NEWS.d/next/Library/2025-05-19-21-08-25.gh-issue-134261.ravGYm.rst b/Misc/NEWS.d/next/Library/2025-05-19-21-08-25.gh-issue-134261.ravGYm.rst new file mode 100644 index 000000000000000..bf552fee814acbf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-19-21-08-25.gh-issue-134261.ravGYm.rst @@ -0,0 +1 @@ +zip: On reproducible builds, ZipFile uses UTC instead of the local time when writing file datetimes to avoid underflows. diff --git a/Misc/NEWS.d/next/Library/2025-08-30-07-44-30.gh-issue-86533.pathlib.rst b/Misc/NEWS.d/next/Library/2025-08-30-07-44-30.gh-issue-86533.pathlib.rst new file mode 100644 index 000000000000000..9c32671173e0ad2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-30-07-44-30.gh-issue-86533.pathlib.rst @@ -0,0 +1,4 @@ +The :func:`os.makedirs` function and :meth:`pathlib.Path.mkdir` method now have +a *parent_mode* parameter to specify the mode for intermediate directories when +creating parent directories. This allows one to match the behavior from Python +3.6 and earlier for :func:`os.makedirs`. diff --git a/Misc/NEWS.d/next/Library/2026-04-23-12-50-15.gh-issue-148441.zvpCkR.rst b/Misc/NEWS.d/next/Library/2026-04-23-12-50-15.gh-issue-148441.zvpCkR.rst new file mode 100644 index 000000000000000..762815270e4d403 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-23-12-50-15.gh-issue-148441.zvpCkR.rst @@ -0,0 +1,4 @@ +:mod:`xml.parsers.expat`: prevent a crash in +:meth:`~xml.parsers.expat.xmlparser.CharacterDataHandler` +when the character data size exceeds the parser's +:attr:`buffer size `. diff --git a/Misc/NEWS.d/next/Library/2026-05-07-21-58-17.gh-issue-149388.DDBPeA.rst b/Misc/NEWS.d/next/Library/2026-05-07-21-58-17.gh-issue-149388.DDBPeA.rst new file mode 100644 index 000000000000000..4a1c6f3f5b4e579 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-07-21-58-17.gh-issue-149388.DDBPeA.rst @@ -0,0 +1 @@ +Make :class:`!asyncio.windows_utils.PipeHandle` closing idempotent. diff --git a/Misc/NEWS.d/next/Library/2026-05-08-15-08-35.gh-issue-112821.t9T1YD.rst b/Misc/NEWS.d/next/Library/2026-05-08-15-08-35.gh-issue-112821.t9T1YD.rst new file mode 100644 index 000000000000000..cfbcde81493e221 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-08-15-08-35.gh-issue-112821.t9T1YD.rst @@ -0,0 +1,4 @@ +In the REPL, autocompletion might run arbitrary code in the getter of a +descriptor. If that getter raised an exception, autocompletion would fail to +present any options for the entire object. Autocompletion now works as +expected for these objects. diff --git a/Misc/NEWS.d/next/Library/2026-05-09-21-02-08.gh-issue-149614.U4snj3.rst b/Misc/NEWS.d/next/Library/2026-05-09-21-02-08.gh-issue-149614.U4snj3.rst new file mode 100644 index 000000000000000..5169c6c203fc1b3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-09-21-02-08.gh-issue-149614.U4snj3.rst @@ -0,0 +1 @@ +Fix a regression that broke the ability to deepcopy :class:`argparse.ArgumentParser` instances. diff --git a/Misc/NEWS.d/next/Library/2026-05-10-07-21-51.gh-issue-139489.rS7LTA.rst b/Misc/NEWS.d/next/Library/2026-05-10-07-21-51.gh-issue-139489.rS7LTA.rst new file mode 100644 index 000000000000000..40fe7e9fd6a0086 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-10-07-21-51.gh-issue-139489.rS7LTA.rst @@ -0,0 +1 @@ +Add :func:`xml.is_valid_text` to ``xml.__all__``. diff --git a/Misc/NEWS.d/next/Library/2026-05-10-19-26-50.gh-issue-149584.x7Qm9A.rst b/Misc/NEWS.d/next/Library/2026-05-10-19-26-50.gh-issue-149584.x7Qm9A.rst new file mode 100644 index 000000000000000..6734250fdd6af3c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-10-19-26-50.gh-issue-149584.x7Qm9A.rst @@ -0,0 +1,4 @@ +Fix excessive overhead in the Tachyon profiler when inspecting a remote +process by avoiding repeated remote page-cache scans, batching predicted +remote reads, and reusing cached profiler result objects. Patch by Pablo +Galindo and Maurycy Pawłowski-Wieroński. diff --git a/Misc/NEWS.d/next/Library/2026-05-10-23-51-23.gh-issue-149504.pDSCbn.rst b/Misc/NEWS.d/next/Library/2026-05-10-23-51-23.gh-issue-149504.pDSCbn.rst new file mode 100644 index 000000000000000..88bf268123bbecc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-10-23-51-23.gh-issue-149504.pDSCbn.rst @@ -0,0 +1,5 @@ +Fix :func:`site.addsitedir` to allow re-entrant calls from within startup +files. Previously, a ``.pth`` file containing an ``import`` line that +called :func:`site.addsitedir` (or a ``.start`` entry point doing the same) +could crash with ``RuntimeError: dictionary changed size during iteration`` +during site initialization, breaking tools such as ``uv run --with``. diff --git a/Misc/NEWS.d/next/Library/2026-05-12-06-24-54.gh-issue-149701.8v9RTm.rst b/Misc/NEWS.d/next/Library/2026-05-12-06-24-54.gh-issue-149701.8v9RTm.rst new file mode 100644 index 000000000000000..676d788cbce62a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-12-06-24-54.gh-issue-149701.8v9RTm.rst @@ -0,0 +1 @@ +Fix bad return code from Lib/venv/bin/activate if hashing is disabled diff --git a/Misc/NEWS.d/next/Library/2026-05-12-13-03-45.gh-issue-149718.SaM1NJ.rst b/Misc/NEWS.d/next/Library/2026-05-12-13-03-45.gh-issue-149718.SaM1NJ.rst new file mode 100644 index 000000000000000..25344e5a90f022c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-12-13-03-45.gh-issue-149718.SaM1NJ.rst @@ -0,0 +1,4 @@ +Coalesce consecutive identical stack frames in Tachyon, so aggregating +collectors (pstats, collapsed, flamegraph, gecko) receive one collect. +Improves sample rate 3x, error rate and missed rate drop by 70%. Patch by +Maurycy Pawłowski-Wieroński. diff --git a/Misc/NEWS.d/next/Library/2026-05-13-23-18-39.gh-issue-149801.S_FfGr.rst b/Misc/NEWS.d/next/Library/2026-05-13-23-18-39.gh-issue-149801.S_FfGr.rst new file mode 100644 index 000000000000000..f9e8538527d204e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-13-23-18-39.gh-issue-149801.S_FfGr.rst @@ -0,0 +1,2 @@ +Add IANA registered names and aliases with leading zeros before number (like +IBM00858, CP00858, IBM01140, CP01140) for corresponding codecs. diff --git a/Misc/NEWS.d/next/Library/2026-05-14-15-55-28.gh-issue-149816.ZaXQ0q.rst b/Misc/NEWS.d/next/Library/2026-05-14-15-55-28.gh-issue-149816.ZaXQ0q.rst new file mode 100644 index 000000000000000..3ea70071ec3c75d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-14-15-55-28.gh-issue-149816.ZaXQ0q.rst @@ -0,0 +1,2 @@ +Fix a race condition in ``_random.Random.__init__`` method in free-threading +mode. diff --git a/Misc/NEWS.d/next/Library/2026-05-15-16-28-00.gh-issue-149819.fixpth.rst b/Misc/NEWS.d/next/Library/2026-05-15-16-28-00.gh-issue-149819.fixpth.rst new file mode 100644 index 000000000000000..66e6da0ecf0d87c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-15-16-28-00.gh-issue-149819.fixpth.rst @@ -0,0 +1,4 @@ +Fix regression in :func:`site.addsitedir` where ``.pth`` files were no +longer processed in Python subprocesses. This happened because +:func:`site.main` seeded ``known_paths`` with entries inherited from +the parent process, causing ``addsitedir`` to skip ``.pth`` processing. diff --git a/Misc/NEWS.d/next/Library/2026-05-15-18-44-20.gh-issue-142349.fHK3v1.rst b/Misc/NEWS.d/next/Library/2026-05-15-18-44-20.gh-issue-142349.fHK3v1.rst new file mode 100644 index 000000000000000..fa667c4110941e9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-15-18-44-20.gh-issue-142349.fHK3v1.rst @@ -0,0 +1 @@ +Add :keyword:`lazy` to the list of support topic by :func:`help`. diff --git a/Misc/NEWS.d/next/Library/2026-05-16-21-08-33.gh-issue-149921.I1yNML.rst b/Misc/NEWS.d/next/Library/2026-05-16-21-08-33.gh-issue-149921.I1yNML.rst new file mode 100644 index 000000000000000..113bd1a802f7990 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-16-21-08-33.gh-issue-149921.I1yNML.rst @@ -0,0 +1,2 @@ +Fix reference leaks in error paths of the :mod:`!_interpchannels` and +:mod:`!_interpqueues` extension modules. diff --git a/Misc/NEWS.d/next/Library/2026-05-18-07-44-46.gh-issue-149995.vvtFHn.rst b/Misc/NEWS.d/next/Library/2026-05-18-07-44-46.gh-issue-149995.vvtFHn.rst new file mode 100644 index 000000000000000..a8e412b578da378 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-18-07-44-46.gh-issue-149995.vvtFHn.rst @@ -0,0 +1 @@ +Update various docstrings in :mod:`typing`. diff --git a/Misc/NEWS.d/next/Library/2026-05-18-15-30-34.gh-issue-146452.RM0EVJ.rst b/Misc/NEWS.d/next/Library/2026-05-18-15-30-34.gh-issue-146452.RM0EVJ.rst new file mode 100644 index 000000000000000..66f9acf6c710a79 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-18-15-30-34.gh-issue-146452.RM0EVJ.rst @@ -0,0 +1,2 @@ +Fix race condition when pickling dictionaries in free threaded builds. Also +reduce critical section cover. diff --git a/Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst b/Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst new file mode 100644 index 000000000000000..7c69edb683cf80a --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst @@ -0,0 +1,5 @@ +:func:`tarfile.data_filter` now validates link targets using the same +normalised value that is written to disk, strips trailing separators from +the member name when resolving a symlink's directory, and rejects link +members that would replace the destination directory itself. This closes +several path-traversal bypasses of the ``data`` extraction filter. diff --git a/Misc/NEWS.d/next/Security/2026-05-08-02-18-54.gh-issue-149474.ujQ-mu.rst b/Misc/NEWS.d/next/Security/2026-05-08-02-18-54.gh-issue-149474.ujQ-mu.rst new file mode 100644 index 000000000000000..48e718b95ebe3ae --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-08-02-18-54.gh-issue-149474.ujQ-mu.rst @@ -0,0 +1,3 @@ +Fix the binary writer in :mod:`profiling.sampling` not firing the audit +(:pep:`578`) when creating the output file. The writer and the reader now +accept any path-like object. Patch by Maurycy Pawłowski-Wieroński. diff --git a/Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst b/Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst new file mode 100644 index 000000000000000..21a79c3e0e7db74 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst @@ -0,0 +1,6 @@ +The :mod:`ftplib` module's undocumented ``ftpcp`` function no longer trusts +the IPv4 address value returned from the source server in response to the +``PASV`` command by default, completing the fix for CVE-2021-4189. As with +:class:`ftplib.FTP`, the former behavior can be re-enabled by setting the +``trust_server_pasv_ipv4_address`` attribute on the source :class:`ftplib.FTP` +instance to ``True``. Thanks to Qi Deng at Aurascape AI for the report. diff --git a/Misc/NEWS.d/next/Security/2026-05-11-21-15-07.gh-issue-149698.OudOcW.rst b/Misc/NEWS.d/next/Security/2026-05-11-21-15-07.gh-issue-149698.OudOcW.rst new file mode 100644 index 000000000000000..3c8671b9a5adc43 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-11-21-15-07.gh-issue-149698.OudOcW.rst @@ -0,0 +1,2 @@ +Update bundled `libexpat `_ to version 2.8.1 +for the fix for :cve:`2026-45186`. diff --git a/Misc/NEWS.d/next/Tests/2026-05-13-14-53-23.gh-issue-149776.orqgsn.rst b/Misc/NEWS.d/next/Tests/2026-05-13-14-53-23.gh-issue-149776.orqgsn.rst new file mode 100644 index 000000000000000..e86a9130ff9bfb6 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-05-13-14-53-23.gh-issue-149776.orqgsn.rst @@ -0,0 +1,2 @@ +Fix test_socket on Linux kernel 7.1 and newer: skip UDP Lite tests if it's +not supported. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Windows/2026-04-29-14-44-51.gh-issue-138489.234aj6.rst b/Misc/NEWS.d/next/Windows/2026-04-29-14-44-51.gh-issue-138489.234aj6.rst new file mode 100644 index 000000000000000..4afb8f737b692e8 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-04-29-14-44-51.gh-issue-138489.234aj6.rst @@ -0,0 +1,4 @@ +Windows distributions now include a :file:`build-details.json` file (see +:pep:`739`). The legacy installer does not install it, but all other +distributions from python.org and all preset configurations in the +``PC\layout`` script will include one. diff --git a/Misc/NEWS.d/next/Windows/2026-05-06-21-36-53.gh-issue-124111.m4OBX8.rst b/Misc/NEWS.d/next/Windows/2026-05-06-21-36-53.gh-issue-124111.m4OBX8.rst new file mode 100644 index 000000000000000..9a57536f1dc96b6 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-05-06-21-36-53.gh-issue-124111.m4OBX8.rst @@ -0,0 +1 @@ +Updated Windows builds to use Tcl/Tk 9.0.3. diff --git a/Misc/NEWS.d/next/Windows/2026-05-14-22-09-46.gh-issue-149786.UI-HZM.rst b/Misc/NEWS.d/next/Windows/2026-05-14-22-09-46.gh-issue-149786.UI-HZM.rst new file mode 100644 index 000000000000000..64ca91a01f41afc --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-05-14-22-09-46.gh-issue-149786.UI-HZM.rst @@ -0,0 +1 @@ +Fixes virtual environment launchers on Windows free-threaded builds. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 593fa01bf25ed1e..9a571fba732ab4a 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -108,46 +108,46 @@ "versionInfo": "3.50.4.0" }, { - "SPDXID": "SPDXRef-PACKAGE-tcl-core", + "SPDXID": "SPDXRef-PACKAGE-tcl", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "4c23f0dd3efcbe6f3a22c503a68d147617bb30c4f5290f1eb3eaacf0b460440b" + "checksumValue": "7a1d1f3a2b8f4484a9c2a027a157963c18f85a81785e85fcb5d1e3df6b6a4fd4" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.15.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-9.0.3.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.15.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:9.0.3.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", - "name": "tcl-core", + "name": "tcl", "primaryPackagePurpose": "SOURCE", - "versionInfo": "8.6.15.0" + "versionInfo": "9.0.3.0" }, { "SPDXID": "SPDXRef-PACKAGE-tk", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "0ae56d39bca92865f338529557a1e56d110594184b6dc5a91339c5675751e264" + "checksumValue": "54fb59df12c489c6264f5b7d3d7444b150d1e3d6561fd59cdb11483440cec000" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.15.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-9.0.3.1.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.15.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:9.0.3.1:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "tk", "primaryPackagePurpose": "SOURCE", - "versionInfo": "8.6.15.0" + "versionInfo": "9.0.3.1" }, { "SPDXID": "SPDXRef-PACKAGE-xz", diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index aaeffd58e799ede..1eca892fb12acee 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "5343adc95840915b022b1d4524d0acb66b369ba2" + "checksumValue": "58101ef0951568acadd3117033bef084fea24cc1" }, { "algorithm": "SHA256", - "checksumValue": "1ec3bad08b6864c2c479e1fd941038c2dcd24c6d9a16400f4da54912d95aa321" + "checksumValue": "52d756026bf09befdb211c453e2009a646d6c6b519e6885e971b2550396619fb" } ], "fileName": "Modules/expat/expat.h" @@ -174,11 +174,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "cb0af01558ec7b6474d2bd0c9386380c82618e8f" + "checksumValue": "1dad2ab196cdbe37572674c465bd9187fdbe4495" }, { "algorithm": "SHA256", - "checksumValue": "6745a6b8cdd7344d4bd8f27f605363ed746e57ff02d4ebce3eb1806579cd030f" + "checksumValue": "740137e670d2f3b7269364ffb6f60064e6560091850c5d6f2c3bb1b8ca6e3dd1" } ], "fileName": "Modules/expat/xmlparse.c" @@ -1730,14 +1730,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "c7cec5f60ea3a42e7780781c6745255c19aa3dbfeeae58646b7132f88dc24780" + "checksumValue": "a52eb72108be160e190b5cafa5bba8663f1313f2013e26060d1c18e26e31067b" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_0/expat-2.8.0.tar.gz", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_1/expat-2.8.1.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.1:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1745,7 +1745,7 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.8.0" + "versionInfo": "2.8.1" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", diff --git a/Modules/Setup b/Modules/Setup index 33737c21cb4066e..e97a78e628693dc 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -273,6 +273,7 @@ PYTHONPATH=$(COREPYTHONPATH) #xx xxmodule.c #xxlimited xxlimited.c #xxlimited_35 xxlimited_35.c +#xxlimited_3_13 xxlimited_3_13.c #xxsubtype xxsubtype.c # Testing diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 19765bc313555b7..5f8b0cf482472d2 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -190,6 +190,7 @@ # Limited API template modules; must be built as shared modules. @MODULE_XXLIMITED_TRUE@xxlimited xxlimited.c @MODULE_XXLIMITED_35_TRUE@xxlimited_35 xxlimited_35.c +@MODULE_XXLIMITED_3_13_TRUE@xxlimited_3_13 xxlimited_3_13.c # for performance diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 98ac821c525a647..09eae97dd21a366 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -708,7 +708,7 @@ StructUnionType_paramfunc(ctypes_state *st, CDataObject *self) } assert(stginfo); /* Cannot be NULL for structure/union instances */ - parg->tag = 'V'; + parg->tag = "V"; parg->pffi_type = &stginfo->ffi_type_pointer; parg->value.p = ptr; parg->size = self->b_size; @@ -1282,7 +1282,7 @@ PyCPointerType_paramfunc(ctypes_state *st, CDataObject *self) if (parg == NULL) return NULL; - parg->tag = 'P'; + parg->tag = "P"; parg->pffi_type = &ffi_type_pointer; parg->obj = Py_NewRef(self); parg->value.p = *(void **)self->b_ptr; @@ -1703,7 +1703,7 @@ PyCArrayType_paramfunc(ctypes_state *st, CDataObject *self) PyCArgObject *p = PyCArgObject_new(st); if (p == NULL) return NULL; - p->tag = 'P'; + p->tag = "P"; p->pffi_type = &ffi_type_pointer; p->value.p = (char *)self->b_ptr; p->obj = Py_NewRef(self); @@ -1909,7 +1909,7 @@ c_wchar_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'Z'; + parg->tag = "Z"; parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj == NULL) { Py_DECREF(parg); @@ -1998,7 +1998,7 @@ c_char_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'z'; + parg->tag = "z"; parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj == NULL) { Py_DECREF(parg); @@ -2092,7 +2092,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'P'; + parg->tag = "P"; parg->obj = fd->setfunc(&parg->value, value, sizeof(void*)); if (parg->obj == NULL) { Py_DECREF(parg); @@ -2110,7 +2110,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'z'; + parg->tag = "z"; parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj == NULL) { Py_DECREF(parg); @@ -2127,7 +2127,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'Z'; + parg->tag = "Z"; parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj == NULL) { Py_DECREF(parg); @@ -2152,7 +2152,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (PyCArg_CheckExact(st, value)) { /* byref(c_xxx()) */ PyCArgObject *a = (PyCArgObject *)value; - if (a->tag == 'P') { + if (strcmp(a->tag, "P") == 0) { return Py_NewRef(value); } } @@ -2165,7 +2165,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'P'; + parg->tag = "P"; Py_INCREF(value); // Function pointers don't change their contents, no need to lock parg->value.p = *(void **)func->b_ptr; @@ -2191,7 +2191,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'Z'; + parg->tag = "Z"; parg->obj = Py_NewRef(value); /* Remember: b_ptr points to where the pointer is stored! */ Py_BEGIN_CRITICAL_SECTION(value); @@ -2332,7 +2332,8 @@ PyCSimpleType_paramfunc(ctypes_state *st, CDataObject *self) if (parg == NULL) return NULL; - parg->tag = fmt[0]; + assert(strcmp(fd->code, fmt) == 0); + parg->tag = fd->code; parg->pffi_type = fd->pffi_type; parg->obj = Py_NewRef(self); memcpy(&parg->value, self->b_ptr, self->b_size); @@ -2578,7 +2579,8 @@ PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, if (parg == NULL) return NULL; - parg->tag = fmt[0]; + assert(strcmp(fd->code, fmt) == 0); + parg->tag = fd->code; parg->pffi_type = fd->pffi_type; parg->obj = fd->setfunc(&parg->value, value, info->size); if (parg->obj) @@ -2832,7 +2834,7 @@ PyCFuncPtrType_paramfunc(ctypes_state *st, CDataObject *self) if (parg == NULL) return NULL; - parg->tag = 'P'; + parg->tag = "P"; parg->pffi_type = &ffi_type_pointer; parg->obj = Py_NewRef(self); parg->value.p = *(void **)self->b_ptr; @@ -4303,7 +4305,7 @@ _byref(ctypes_state *st, PyObject *obj) return NULL; } - parg->tag = 'P'; + parg->tag = "P"; parg->pffi_type = &ffi_type_pointer; parg->obj = obj; parg->value.p = ((CDataObject *)obj)->b_ptr; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index a0c9d8b70fee469..991ff0d675c2f1c 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -446,7 +446,7 @@ EXPORT(char *)my_strtok(char *token, const char *delim) return strtok(token, delim); } -EXPORT(char *)my_strchr(const char *s, int c) +EXPORT(const char *) my_strchr(const char *s, int c) { return strchr(s, c); } diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index e208e27c5dbed42..e453cfeec9cc8ca 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -468,7 +468,7 @@ PyCArgObject_new(ctypes_state *st) if (p == NULL) return NULL; p->pffi_type = NULL; - p->tag = '\0'; + p->tag = ""; p->obj = NULL; memset(&p->value, 0, sizeof(p->value)); PyObject_GC_Track(p); @@ -512,45 +512,50 @@ static PyObject * PyCArg_repr(PyObject *op) { PyCArgObject *self = _PyCArgObject_CAST(op); - switch(self->tag) { + + if (strlen(self->tag) != 1) { + goto generic; + } + + switch(self->tag[0]) { case 'b': case 'B': - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->tag, self->value.b); case 'h': case 'H': - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->tag, self->value.h); case 'i': case 'I': - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->tag, self->value.i); case 'l': case 'L': - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->tag, self->value.l); case 'q': case 'Q': - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->tag, self->value.q); case 'd': case 'f': { - PyObject *f = PyFloat_FromDouble((self->tag == 'f') ? self->value.f : self->value.d); + PyObject *f = PyFloat_FromDouble((strcmp(self->tag, "f") == 0) ? self->value.f : self->value.d); if (f == NULL) { return NULL; } - PyObject *result = PyUnicode_FromFormat("", self->tag, f); + PyObject *result = PyUnicode_FromFormat("", self->tag, f); Py_DECREF(f); return result; } case 'c': if (is_literal_char((unsigned char)self->value.c)) { - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->tag, self->value.c); } else { - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->tag, (unsigned char)self->value.c); } @@ -561,20 +566,16 @@ PyCArg_repr(PyObject *op) case 'z': case 'Z': case 'P': - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->tag, self->value.p); - break; default: - if (is_literal_char((unsigned char)self->tag)) { - return PyUnicode_FromFormat("", - (unsigned char)self->tag, (void *)self); - } - else { - return PyUnicode_FromFormat("", - (unsigned char)self->tag, (void *)self); - } + break; } + +generic: + return PyUnicode_FromFormat("", + self->tag, (void *)self); } static PyMemberDef PyCArgType_members[] = { @@ -1807,7 +1808,7 @@ _ctypes_byref_impl(PyObject *module, PyObject *obj, Py_ssize_t offset) if (parg == NULL) return NULL; - parg->tag = 'P'; + parg->tag = "P"; parg->pffi_type = &ffi_type_pointer; parg->obj = Py_NewRef(obj); parg->value.p = (char *)((CDataObject *)obj)->b_ptr + offset; diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 7b6b7f08582251b..248559aa364a198 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -494,7 +494,7 @@ PyObject *_ctypes_callproc(ctypes_state *st, struct tagPyCArgObject { PyObject_HEAD ffi_type *pffi_type; - char tag; + const char *tag; union { char c; char b; @@ -511,7 +511,7 @@ struct tagPyCArgObject { long double G[2]; } value; PyObject *obj; - Py_ssize_t size; /* for the 'V' tag */ + Py_ssize_t size; /* for the "V" tag */ }; #define _PyCArgObject_CAST(op) ((PyCArgObject *)(op)) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 19bdf3d47c2fad5..c702eecc700ac80 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -1066,7 +1066,7 @@ _functools.reduce function as func: object iterable as seq: object / - initial as result: object = NULL + initial as result: object(c_default="NULL") = functools._initial_missing Apply a function of two arguments cumulatively to the items of an iterable, from left to right. @@ -1081,7 +1081,7 @@ calculates ((((1 + 2) + 3) + 4) + 5). static PyObject * _functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, PyObject *result) -/*[clinic end generated code: output=30d898fe1267c79d input=4ccfb74548ce5170]*/ +/*[clinic end generated code: output=30d898fe1267c79d input=5c9088c98ffe2793]*/ { PyObject *args, *it; diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index 3c356cb40d2bca6..c6d107d243dda0e 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -2586,6 +2586,7 @@ static PyObject * _channelid_from_xid(_PyXIData_t *data) { struct _channelid_xid *xid = (struct _channelid_xid *)_PyXIData_DATA(data); + PyObject *cidobj = NULL; // It might not be imported yet, so we can't use _get_current_module(). PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR); @@ -2595,11 +2596,10 @@ _channelid_from_xid(_PyXIData_t *data) assert(mod != Py_None); module_state *state = get_module_state(mod); if (state == NULL) { - return NULL; + goto done; } // Note that we do not preserve the "resolve" flag. - PyObject *cidobj = NULL; int err = newchannelid(state->ChannelIDType, xid->cid, xid->end, _global_channels(), 0, 0, (channelid **)&cidobj); diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 777b68547498847..b23aa5f39489d98 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -1363,6 +1363,7 @@ _queueobj_from_xid(_PyXIData_t *data) if (mod == NULL) { mod = PyImport_ImportModule(MODULE_NAME_STR); if (mod == NULL) { + Py_DECREF(qidobj); return NULL; } } diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 677d7e85d4e626f..4a3fc586fa3a147 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -673,12 +673,13 @@ read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) { maxlen += 1; Py_BLOCK_THREADS newbuf = (wchar_t*)PyMem_Realloc(buf, maxlen * sizeof(wchar_t)); - Py_UNBLOCK_THREADS if (!newbuf) { sig = -1; PyErr_NoMemory(); + Py_UNBLOCK_THREADS break; } + Py_UNBLOCK_THREADS buf = newbuf; /* Only advance by n and not BUFSIZ in this case */ off += n; diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 9874f9475ac0296..15d95c658d6f906 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -3450,6 +3450,9 @@ batch_dict(PickleState *state, PicklerObject *self, PyObject *iter, PyObject *or * Returns 0 on success, -1 on error. * * Note that this currently doesn't work for protocol 0. + + * gh-146452: Wrap the dict iteration in a critical sections to prevent + * concurrent mutation from invalidating PyDict_Next() iteration state. */ static int batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) @@ -3466,15 +3469,24 @@ batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) assert(self->proto > 0); dict_size = PyDict_GET_SIZE(obj); - assert(dict_size); /* Write in batches of BATCHSIZE. */ Py_ssize_t total = 0; do { if (dict_size - total == 1) { - PyDict_Next(obj, &ppos, &key, &value); - Py_INCREF(key); - Py_INCREF(value); + int next; + Py_BEGIN_CRITICAL_SECTION(obj); + next = PyDict_Next(obj, &ppos, &key, &value); + if (next) { + Py_INCREF(key); + Py_INCREF(value); + } + Py_END_CRITICAL_SECTION(); + if (!next) { + PyErr_SetString(PyExc_RuntimeError, + "dictionary changed size during iteration"); + goto error; + } if (save(state, self, key, 0) < 0) { goto error; } @@ -3492,9 +3504,18 @@ batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) i = 0; if (_Pickler_Write(self, &mark_op, 1) < 0) return -1; - while (PyDict_Next(obj, &ppos, &key, &value)) { - Py_INCREF(key); - Py_INCREF(value); + int next; + while (1) { + Py_BEGIN_CRITICAL_SECTION(obj); + next = PyDict_Next(obj, &ppos, &key, &value); + if (next) { + Py_INCREF(key); + Py_INCREF(value); + } + Py_END_CRITICAL_SECTION(); + if (!next) { + break; + } if (save(state, self, key, 0) < 0) { goto error; } diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index ed925f3525a9a7d..d5ba36273c82626 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -154,8 +154,6 @@ RingBuf_Get(RingBuf *buf) } // Returns 0 on success or -1 if the buffer failed to grow. -// -// Steals a reference to item. static int RingBuf_Put(RingBuf *buf, PyObject *item) { @@ -165,11 +163,10 @@ RingBuf_Put(RingBuf *buf, PyObject *item) // Buffer is full, grow it. if (resize_ringbuf(buf, buf->items_cap * 2) < 0) { PyErr_NoMemory(); - Py_DECREF(item); return -1; } } - buf->items[buf->put_idx] = item; + buf->items[buf->put_idx] = Py_NewRef(item); buf->put_idx = (buf->put_idx + 1) % buf->items_cap; buf->num_items++; return 0; @@ -276,16 +273,13 @@ maybe_handoff_item(void *arg, void *park_arg, int has_more_waiters) { HandoffData *data = (HandoffData*)arg; PyObject **item = (PyObject**)park_arg; - if (item == NULL) { - // No threads were waiting - data->handed_off = false; - } - else { + data->queue->has_threads_waiting = has_more_waiters; + + data->handed_off = item != NULL; + if (data->handed_off) { // There was at least one waiting thread, hand off the item - *item = data->item; - data->handed_off = true; + *item = Py_NewRef(data->item); } - data->queue->has_threads_waiting = has_more_waiters; } /*[clinic input] @@ -307,21 +301,22 @@ _queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item, int block, PyObject *timeout) /*[clinic end generated code: output=4333136e88f90d8b input=a16dbb33363c0fa8]*/ { - HandoffData data = { - .handed_off = 0, - .item = Py_NewRef(item), - .queue = self, - }; if (self->has_threads_waiting) { + HandoffData data = { + .handed_off = 0, + .item = item, + .queue = self, + }; // Try to hand the item off directly if there are threads waiting _PyParkingLot_Unpark(&self->has_threads_waiting, maybe_handoff_item, &data); - } - if (!data.handed_off) { - if (RingBuf_Put(&self->buf, item) < 0) { - return NULL; + if (data.handed_off) { + Py_RETURN_NONE; } } + if (RingBuf_Put(&self->buf, item) < 0) { + return NULL; + } Py_RETURN_NONE; } diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 0fb734816517485..a06966be23be1ef 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -123,9 +123,9 @@ typedef struct { /*[clinic input] module _random -class _random.Random "RandomObject *" "_randomstate_type(type)->Random_Type" +class _random.Random "RandomObject *" "(PyTypeObject *)_randomstate_type(Py_TYPE(self))->Random_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=70a2c99619474983]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=f04bcbfba61a322e]*/ /* Random methods */ @@ -549,27 +549,20 @@ _random_Random_getrandbits_impl(RandomObject *self, uint64_t k) return result; } -static int -random_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - PyObject *arg = NULL; - _randomstate *state = _randomstate_type(Py_TYPE(self)); - - if ((Py_IS_TYPE(self, (PyTypeObject *)state->Random_Type) || - Py_TYPE(self)->tp_init == ((PyTypeObject*)state->Random_Type)->tp_init) && - !_PyArg_NoKeywords("Random", kwds)) { - return -1; - } - - if (PyTuple_GET_SIZE(args) > 1) { - PyErr_SetString(PyExc_TypeError, "Random() requires 0 or 1 argument"); - return -1; - } +/*[clinic input] +@critical_section +@text_signature "($self, [seed])" +_random.Random.__init__ as random_init - if (PyTuple_GET_SIZE(args) == 1) - arg = PyTuple_GET_ITEM(args, 0); + seed: object = NULL + / +[clinic start generated code]*/ - return random_seed(RandomObject_CAST(self), arg); +static int +random_init_impl(RandomObject *self, PyObject *seed) +/*[clinic end generated code: output=260734a3739c394f input=e516bf32e8a05e28]*/ +{ + return random_seed(self, seed); } diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 7369cd1514c296d..d91ce54a18c813a 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -30,6 +30,7 @@ extern "C" { #include "internal/pycore_llist.h" // struct llist_node #include "internal/pycore_long.h" // _PyLong_GetZero #include "internal/pycore_pyerrors.h" // _PyErr_FormatFromCause +#include "internal/pycore_pyhash.h" // _Py_HashPointerRaw #include "internal/pycore_stackref.h" // Py_TAG_BITS #include "../../Python/remote_debug.h" @@ -215,6 +216,8 @@ typedef struct { PyObject *file_name; int first_lineno; PyObject *linetable; // bytes + PyObject *last_frame_info; + ptrdiff_t last_addrq; uintptr_t addr_code_adaptive; } CachedCodeMetadata; @@ -224,11 +227,41 @@ typedef struct { typedef struct { uint64_t thread_id; // 0 = empty slot + uintptr_t thread_state_addr; uintptr_t addrs[FRAME_CACHE_MAX_FRAMES]; Py_ssize_t num_addrs; + PyObject *thread_id_obj; // owned reference, NULL if empty PyObject *frame_list; // owned reference, NULL if empty } FrameCacheEntry; +#define INTERPRETER_THREAD_CACHE_SIZE 32 +#if (INTERPRETER_THREAD_CACHE_SIZE & (INTERPRETER_THREAD_CACHE_SIZE - 1)) != 0 +# error "INTERPRETER_THREAD_CACHE_SIZE must be a power of two" +#endif + +// The two per-interpreter L2 caches below are split into per-field tables so +// that a writer rebinding one slot cannot leave stale data in a field owned by +// the other when the slot is reused across interpreters. +typedef struct { + uintptr_t interpreter_addr; + uintptr_t thread_state_addr; +} InterpreterTstateCacheEntry; +typedef struct { + uintptr_t interpreter_addr; + uint64_t code_object_generation; +} InterpreterGenerationCacheEntry; + +// Carries already-read thread state and/or frame buffers across helpers so the +// downstream callee can skip a remote read. Address fields are caller-supplied +// inputs; buffer pointers (tstate, frame) are NULL unless a prior batched read +// successfully populated them. +typedef struct { + const char *tstate; + uintptr_t tstate_addr; + const char *frame; + uintptr_t frame_addr; +} RemoteReadPrefetch; + /* Statistics for profiling performance analysis */ typedef struct { uint64_t total_samples; // Total number of get_stack_trace calls @@ -242,14 +275,44 @@ typedef struct { uint64_t code_object_cache_hits; // Code object cache hits uint64_t code_object_cache_misses; // Code object cache misses uint64_t stale_cache_invalidations; // Times stale entries were cleared + uint64_t batched_read_attempts; // Batched remote-read attempts + uint64_t batched_read_successes; // Attempts that read all requested segments + uint64_t batched_read_misses; // Attempts that fell back or partially read + uint64_t batched_read_segments_requested; // Segments requested by batched reads + uint64_t batched_read_segments_completed; // Segments completed by batched reads } UnwinderStats; +#if defined(__GNUC__) || defined(__clang__) +# define REMOTE_DEBUG_UNLIKELY(value) __builtin_expect(!!(value), 0) +#else +# define REMOTE_DEBUG_UNLIKELY(value) (value) +#endif + /* Stats tracking macros - no-op when stats collection is disabled */ #define STATS_INC(unwinder, field) \ - do { if ((unwinder)->collect_stats) (unwinder)->stats.field++; } while(0) + do { if (REMOTE_DEBUG_UNLIKELY((unwinder)->collect_stats)) (unwinder)->stats.field++; } while(0) #define STATS_ADD(unwinder, field, val) \ - do { if ((unwinder)->collect_stats) (unwinder)->stats.field += (val); } while(0) + do { if (REMOTE_DEBUG_UNLIKELY((unwinder)->collect_stats)) (unwinder)->stats.field += (val); } while(0) + +#if HAVE_PROCESS_VM_READV +# define STATS_BATCHED_READ(unwinder, requested, completed) \ + do { \ + if (REMOTE_DEBUG_UNLIKELY((unwinder)->collect_stats)) { \ + (unwinder)->stats.batched_read_attempts++; \ + (unwinder)->stats.batched_read_segments_requested += (uint64_t)(requested); \ + (unwinder)->stats.batched_read_segments_completed += (uint64_t)(completed); \ + if ((completed) == (requested)) { \ + (unwinder)->stats.batched_read_successes++; \ + } \ + else { \ + (unwinder)->stats.batched_read_misses++; \ + } \ + } \ + } while(0) +#else +# define STATS_BATCHED_READ(unwinder, requested, completed) ((void)0) +#endif typedef struct { PyTypeObject *RemoteDebugging_Type; @@ -290,7 +353,6 @@ typedef struct { struct _Py_AsyncioModuleDebugOffsets async_debug_offsets; uintptr_t interpreter_addr; uintptr_t tstate_addr; - uint64_t code_object_generation; _Py_hashtable_t *code_object_cache; int debug; int only_active_thread; @@ -302,9 +364,17 @@ typedef struct { int cache_frames; int collect_stats; // whether to collect statistics uint32_t stale_invalidation_counter; // counter for throttling frame_cache_invalidate_stale + // L1 single-entry shortcut over cached_tstates[]: most workloads sample one + // interpreter, so check these pairs before hashing into the table below. + uintptr_t cached_tstate_interpreter_addr; + uintptr_t cached_tstate_addr; + uintptr_t cached_generation_interpreter_addr; + uint64_t cached_code_object_generation; RemoteDebuggingState *cached_state; FrameCacheEntry *frame_cache; // preallocated array of FRAME_CACHE_MAX_THREADS entries UnwinderStats stats; // statistics for performance analysis + InterpreterTstateCacheEntry cached_tstates[INTERPRETER_THREAD_CACHE_SIZE]; + InterpreterGenerationCacheEntry cached_generations[INTERPRETER_THREAD_CACHE_SIZE]; #ifdef Py_GIL_DISABLED uint32_t tlbc_generation; _Py_hashtable_t *tlbc_cache; @@ -361,11 +431,13 @@ typedef struct { typedef struct { /* Inputs */ uintptr_t frame_addr; // Starting frame address + uintptr_t thread_state_addr; // Owning thread state address uintptr_t base_frame_addr; // Sentinel at bottom (for validation) uintptr_t gc_frame; // GC frame address (0 if not tracking) uintptr_t last_profiled_frame; // Last cached frame (0 if no cache) StackChunkList *chunks; // Pre-copied stack chunks int skip_first_frame; // Skip frame_addr itself (continue from its caller) + RemoteReadPrefetch prefetch; // Optional already-read thread/frame buffers /* Outputs */ PyObject *frame_info; // List to append FrameInfo objects @@ -548,6 +620,7 @@ extern int process_frame_chain( extern int frame_cache_init(RemoteUnwinderObject *unwinder); extern void frame_cache_cleanup(RemoteUnwinderObject *unwinder); extern FrameCacheEntry *frame_cache_find(RemoteUnwinderObject *unwinder, uint64_t thread_id); +extern FrameCacheEntry *frame_cache_find_by_tstate(RemoteUnwinderObject *unwinder, uintptr_t tstate_addr); extern int clear_last_profiled_frames(RemoteUnwinderObject *unwinder); extern void frame_cache_invalidate_stale(RemoteUnwinderObject *unwinder, PyObject *result); extern int frame_cache_lookup_and_extend( @@ -566,6 +639,7 @@ extern int frame_cache_store( PyObject *frame_list, const uintptr_t *addrs, Py_ssize_t num_addrs, + uintptr_t thread_state_addr, uintptr_t base_frame_addr, uintptr_t last_frame_visited); @@ -605,7 +679,8 @@ extern PyObject* unwind_stack_for_thread( uintptr_t *current_tstate, uintptr_t gil_holder_tstate, uintptr_t gc_frame, - uintptr_t main_thread_tstate + uintptr_t main_thread_tstate, + const RemoteReadPrefetch *prefetch ); /* Thread stopping functions (for blocking mode) */ diff --git a/Modules/_remote_debugging/binary_io.h b/Modules/_remote_debugging/binary_io.h index 87a54371c774f1a..d4188335c0b6d0a 100644 --- a/Modules/_remote_debugging/binary_io.h +++ b/Modules/_remote_debugging/binary_io.h @@ -253,7 +253,6 @@ typedef struct { /* Main binary writer structure */ typedef struct { FILE *fp; - char *filename; /* Write buffer for batched I/O */ uint8_t *write_buffer; @@ -311,10 +310,7 @@ typedef struct { /* Main binary reader structure */ typedef struct { - char *filename; - #if USE_MMAP - int fd; uint8_t *mapped_data; size_t mapped_size; #else @@ -522,7 +518,7 @@ grow_array_inplace(void **ptr_addr, size_t count, size_t *capacity, size_t elem_ * Create a new binary writer. * * Arguments: - * filename: Path to output file + * path: Path to output file * sample_interval_us: Sampling interval in microseconds * compression_type: COMPRESSION_NONE or COMPRESSION_ZSTD * start_time_us: Start timestamp in microseconds (from time.monotonic() * 1e6) @@ -531,7 +527,7 @@ grow_array_inplace(void **ptr_addr, size_t count, size_t *capacity, size_t elem_ * New BinaryWriter* on success, NULL on failure (PyErr set) */ BinaryWriter *binary_writer_create( - const char *filename, + PyObject *path, uint64_t sample_interval_us, int compression_type, uint64_t start_time_us @@ -583,12 +579,12 @@ void binary_writer_destroy(BinaryWriter *writer); * Open a binary file for reading. * * Arguments: - * filename: Path to input file + * path: Path to input file * * Returns: * New BinaryReader* on success, NULL on failure (PyErr set) */ -BinaryReader *binary_reader_open(const char *filename); +BinaryReader *binary_reader_open(PyObject *path); /* * Replay samples from binary file through a collector. diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index 551530b519952c0..972b197cfbad861 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -358,7 +358,7 @@ reader_parse_frame_table(BinaryReader *reader, const uint8_t *data, size_t file_ } BinaryReader * -binary_reader_open(const char *filename) +binary_reader_open(PyObject *path) { BinaryReader *reader = PyMem_Calloc(1, sizeof(BinaryReader)); if (!reader) { @@ -366,29 +366,18 @@ binary_reader_open(const char *filename) return NULL; } -#if USE_MMAP - reader->fd = -1; /* Explicit initialization for cleanup safety */ -#endif - - reader->filename = PyMem_Malloc(strlen(filename) + 1); - if (!reader->filename) { - PyMem_Free(reader); - PyErr_NoMemory(); - return NULL; - } - strcpy(reader->filename, filename); - #if USE_MMAP /* Open with mmap on Unix */ - reader->fd = open(filename, O_RDONLY); - if (reader->fd < 0) { - PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename); + FILE *fp = Py_fopen(path, "rb"); + if (!fp) { goto error; } + int fd = fileno(fp); struct stat st; - if (fstat(reader->fd, &st) < 0) { + if (fstat(fd, &st) < 0) { PyErr_SetFromErrno(PyExc_IOError); + Py_fclose(fp); goto error; } reader->mapped_size = st.st_size; @@ -400,14 +389,15 @@ binary_reader_open(const char *filename) */ #ifdef __linux__ reader->mapped_data = mmap(NULL, reader->mapped_size, PROT_READ, - MAP_PRIVATE | MAP_POPULATE, reader->fd, 0); + MAP_PRIVATE | MAP_POPULATE, fd, 0); #else reader->mapped_data = mmap(NULL, reader->mapped_size, PROT_READ, - MAP_PRIVATE, reader->fd, 0); + MAP_PRIVATE, fd, 0); #endif if (reader->mapped_data == MAP_FAILED) { reader->mapped_data = NULL; PyErr_SetFromErrno(PyExc_IOError); + Py_fclose(fp); goto error; } @@ -428,19 +418,20 @@ binary_reader_open(const char *filename) /* Add file descriptor-level hints for better kernel I/O scheduling */ #if defined(__linux__) && defined(POSIX_FADV_SEQUENTIAL) - (void)posix_fadvise(reader->fd, 0, 0, POSIX_FADV_SEQUENTIAL); + (void)posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); if (reader->mapped_size > (64 * 1024 * 1024)) { - (void)posix_fadvise(reader->fd, 0, 0, POSIX_FADV_WILLNEED); + (void)posix_fadvise(fd, 0, 0, POSIX_FADV_WILLNEED); } #endif + (void)Py_fclose(fp); + uint8_t *data = reader->mapped_data; size_t file_size = reader->mapped_size; #else /* Use stdio on Windows */ - reader->fp = fopen(filename, "rb"); + reader->fp = Py_fopen(path, "rb"); if (!reader->fp) { - PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename); goto error; } @@ -1263,8 +1254,6 @@ binary_reader_close(BinaryReader *reader) return; } - PyMem_Free(reader->filename); - #if USE_MMAP if (reader->mapped_data) { munmap(reader->mapped_data, reader->mapped_size); @@ -1274,13 +1263,9 @@ binary_reader_close(BinaryReader *reader) /* Clear sample_data which may point into the now-unmapped region */ reader->sample_data = NULL; reader->sample_data_size = 0; - if (reader->fd >= 0) { - close(reader->fd); - reader->fd = -1; /* Mark as closed */ - } #else if (reader->fp) { - fclose(reader->fp); + Py_fclose(reader->fp); reader->fp = NULL; } if (reader->file_data) { diff --git a/Modules/_remote_debugging/binary_io_writer.c b/Modules/_remote_debugging/binary_io_writer.c index 4cfed7300ac5ab2..c31ed7d746466f5 100644 --- a/Modules/_remote_debugging/binary_io_writer.c +++ b/Modules/_remote_debugging/binary_io_writer.c @@ -717,7 +717,7 @@ write_sample_with_encoding(BinaryWriter *writer, ThreadEntry *entry, } BinaryWriter * -binary_writer_create(const char *filename, uint64_t sample_interval_us, int compression_type, +binary_writer_create(PyObject *path, uint64_t sample_interval_us, int compression_type, uint64_t start_time_us) { BinaryWriter *writer = PyMem_Calloc(1, sizeof(BinaryWriter)); @@ -726,14 +726,6 @@ binary_writer_create(const char *filename, uint64_t sample_interval_us, int comp return NULL; } - writer->filename = PyMem_Malloc(strlen(filename) + 1); - if (!writer->filename) { - PyMem_Free(writer); - PyErr_NoMemory(); - return NULL; - } - strcpy(writer->filename, filename); - writer->start_time_us = start_time_us; writer->sample_interval_us = sample_interval_us; writer->compression_type = compression_type; @@ -799,9 +791,8 @@ binary_writer_create(const char *filename, uint64_t sample_interval_us, int comp } } - writer->fp = fopen(filename, "wb"); + writer->fp = Py_fopen(path, "wb"); if (!writer->fp) { - PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename); goto error; } @@ -1193,7 +1184,7 @@ binary_writer_finalize(BinaryWriter *writer) return -1; } - if (fclose(writer->fp) != 0) { + if (Py_fclose(writer->fp) != 0) { writer->fp = NULL; PyErr_SetFromErrno(PyExc_IOError); return -1; @@ -1211,10 +1202,9 @@ binary_writer_destroy(BinaryWriter *writer) } if (writer->fp) { - fclose(writer->fp); + Py_fclose(writer->fp); } - PyMem_Free(writer->filename); PyMem_Free(writer->write_buffer); #ifdef HAVE_ZSTD diff --git a/Modules/_remote_debugging/clinic/module.c.h b/Modules/_remote_debugging/clinic/module.c.h index 1133db808efaec3..78b1d3e8d80962e 100644 --- a/Modules/_remote_debugging/clinic/module.c.h +++ b/Modules/_remote_debugging/clinic/module.c.h @@ -411,8 +411,15 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_stats__doc__, " - code_object_cache_hits: Code object cache hits\n" " - code_object_cache_misses: Code object cache misses\n" " - stale_cache_invalidations: Times stale cache entries were cleared\n" +" - batched_read_attempts: Batched remote-read attempts\n" +" - batched_read_successes: Attempts that read all requested segments\n" +" - batched_read_misses: Attempts that fell back or partially read\n" +" - batched_read_segments_requested: Segments requested by batched reads\n" +" - batched_read_segments_completed: Segments completed by batched reads\n" " - frame_cache_hit_rate: Percentage of samples that hit the cache\n" " - code_object_cache_hit_rate: Percentage of code object lookups that hit cache\n" +" - batched_read_success_rate: Percentage of batched reads that completed all segments\n" +" - batched_read_segment_completion_rate: Percentage of requested segments read by batched reads\n" "\n" "Raises:\n" " RuntimeError: If stats collection was not enabled (stats=False)"); @@ -688,7 +695,7 @@ PyDoc_STRVAR(_remote_debugging_BinaryWriter___init____doc__, static int _remote_debugging_BinaryWriter___init___impl(BinaryWriterObject *self, - const char *filename, + PyObject *filename, unsigned long long sample_interval_us, unsigned long long start_time_us, int compression); @@ -728,7 +735,7 @@ _remote_debugging_BinaryWriter___init__(PyObject *self, PyObject *args, PyObject PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 3; - const char *filename; + PyObject *filename; unsigned long long sample_interval_us; unsigned long long start_time_us; int compression = 0; @@ -738,19 +745,7 @@ _remote_debugging_BinaryWriter___init__(PyObject *self, PyObject *args, PyObject if (!fastargs) { goto exit; } - if (!PyUnicode_Check(fastargs[0])) { - _PyArg_BadArgument("BinaryWriter", "argument 'filename'", "str", fastargs[0]); - goto exit; - } - Py_ssize_t filename_length; - filename = PyUnicode_AsUTF8AndSize(fastargs[0], &filename_length); - if (filename == NULL) { - goto exit; - } - if (strlen(filename) != (size_t)filename_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } + filename = fastargs[0]; if (!_PyLong_UnsignedLongLong_Converter(fastargs[1], &sample_interval_us)) { goto exit; } @@ -1009,7 +1004,7 @@ PyDoc_STRVAR(_remote_debugging_BinaryReader___init____doc__, static int _remote_debugging_BinaryReader___init___impl(BinaryReaderObject *self, - const char *filename); + PyObject *filename); static int _remote_debugging_BinaryReader___init__(PyObject *self, PyObject *args, PyObject *kwargs) @@ -1045,26 +1040,14 @@ _remote_debugging_BinaryReader___init__(PyObject *self, PyObject *args, PyObject PyObject *argsbuf[1]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); - const char *filename; + PyObject *filename; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } - if (!PyUnicode_Check(fastargs[0])) { - _PyArg_BadArgument("BinaryReader", "argument 'filename'", "str", fastargs[0]); - goto exit; - } - Py_ssize_t filename_length; - filename = PyUnicode_AsUTF8AndSize(fastargs[0], &filename_length); - if (filename == NULL) { - goto exit; - } - if (strlen(filename) != (size_t)filename_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } + filename = fastargs[0]; return_value = _remote_debugging_BinaryReader___init___impl((BinaryReaderObject *)self, filename); exit: @@ -1564,4 +1547,4 @@ _remote_debugging_get_gc_stats(PyObject *module, PyObject *const *args, Py_ssize exit: return return_value; } -/*[clinic end generated code: output=36674f4cb8a653f3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=884914b100e9c90c input=a9049054013a1b77]*/ diff --git a/Modules/_remote_debugging/code_objects.c b/Modules/_remote_debugging/code_objects.c index 7b95c0f2d4fa8da..3af58f2b3c379ec 100644 --- a/Modules/_remote_debugging/code_objects.c +++ b/Modules/_remote_debugging/code_objects.c @@ -405,6 +405,8 @@ parse_code_object(RemoteUnwinderObject *unwinder, meta->func_name = func; meta->file_name = file; meta->linetable = linetable; + meta->last_frame_info = NULL; + meta->last_addrq = -1; meta->first_lineno = GET_MEMBER(int, code_object, unwinder->debug_offsets.code_object.firstlineno); meta->addr_code_adaptive = real_address + (uintptr_t)unwinder->debug_offsets.code_object.co_code_adaptive; @@ -432,7 +434,7 @@ parse_code_object(RemoteUnwinderObject *unwinder, #ifdef Py_GIL_DISABLED // Handle thread-local bytecode (TLBC) in free threading builds - if (ctx->tlbc_index == 0 || unwinder->debug_offsets.code_object.co_tlbc == 0 || unwinder == NULL) { + if (ctx->tlbc_index == 0 || unwinder == NULL || unwinder->debug_offsets.code_object.co_tlbc == 0) { // No TLBC or no unwinder - use main bytecode directly addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; goto done_tlbc; @@ -482,6 +484,12 @@ parse_code_object(RemoteUnwinderObject *unwinder, addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; #endif ; // Empty statement to avoid C23 extension warning + + if (!unwinder->opcodes && meta->last_frame_info != NULL && meta->last_addrq == addrq) { + *result = Py_NewRef(meta->last_frame_info); + return 0; + } + LocationInfo info = {0}; bool ok = parse_linetable(addrq, PyBytes_AS_STRING(meta->linetable), PyBytes_GET_SIZE(meta->linetable), @@ -529,6 +537,11 @@ parse_code_object(RemoteUnwinderObject *unwinder, goto error; } + if (!unwinder->opcodes) { + Py_XSETREF(meta->last_frame_info, Py_NewRef(tuple)); + meta->last_addrq = addrq; + } + *result = tuple; return 0; diff --git a/Modules/_remote_debugging/frame_cache.c b/Modules/_remote_debugging/frame_cache.c index b6566d7cff7b543..19fc406bca9ac96 100644 --- a/Modules/_remote_debugging/frame_cache.c +++ b/Modules/_remote_debugging/frame_cache.c @@ -30,6 +30,7 @@ frame_cache_cleanup(RemoteUnwinderObject *unwinder) return; } for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + Py_CLEAR(unwinder->frame_cache[i].thread_id_obj); Py_CLEAR(unwinder->frame_cache[i].frame_list); } PyMem_Free(unwinder->frame_cache); @@ -53,6 +54,21 @@ frame_cache_find(RemoteUnwinderObject *unwinder, uint64_t thread_id) return NULL; } +FrameCacheEntry * +frame_cache_find_by_tstate(RemoteUnwinderObject *unwinder, uintptr_t tstate_addr) +{ + if (!unwinder->frame_cache || tstate_addr == 0) { + return NULL; + } + for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + if (unwinder->frame_cache[i].thread_state_addr == tstate_addr) { + assert(unwinder->frame_cache[i].num_addrs <= FRAME_CACHE_MAX_FRAMES); + return &unwinder->frame_cache[i]; + } + } + return NULL; +} + // Allocate a cache slot for a thread // Returns NULL if cache is full (graceful degradation) static FrameCacheEntry * @@ -127,8 +143,10 @@ frame_cache_invalidate_stale(RemoteUnwinderObject *unwinder, PyObject *result) } if (!found) { // Clear this entry + Py_CLEAR(unwinder->frame_cache[i].thread_id_obj); Py_CLEAR(unwinder->frame_cache[i].frame_list); unwinder->frame_cache[i].thread_id = 0; + unwinder->frame_cache[i].thread_state_addr = 0; unwinder->frame_cache[i].num_addrs = 0; STATS_INC(unwinder, stale_cache_invalidations); } @@ -216,6 +234,7 @@ frame_cache_store( PyObject *frame_list, const uintptr_t *addrs, Py_ssize_t num_addrs, + uintptr_t thread_state_addr, uintptr_t base_frame_addr, uintptr_t last_frame_visited) { @@ -257,6 +276,13 @@ frame_cache_store( return -1; } entry->thread_id = thread_id; + entry->thread_state_addr = thread_state_addr; + if (entry->thread_id_obj == NULL) { + entry->thread_id_obj = PyLong_FromUnsignedLongLong(thread_id); + if (entry->thread_id_obj == NULL) { + return -1; + } + } memcpy(entry->addrs, addrs, num_addrs * sizeof(uintptr_t)); entry->num_addrs = num_addrs; assert(entry->num_addrs == num_addrs); diff --git a/Modules/_remote_debugging/frames.c b/Modules/_remote_debugging/frames.c index bbdfce3f7201d9d..8d8019396b3e31a 100644 --- a/Modules/_remote_debugging/frames.c +++ b/Modules/_remote_debugging/frames.c @@ -186,30 +186,16 @@ is_frame_valid( return 1; } -int -parse_frame_object( +static int +parse_frame_buffer( RemoteUnwinderObject *unwinder, PyObject** result, - uintptr_t address, + const char *frame, uintptr_t* address_of_code_object, uintptr_t* previous_frame ) { - char frame[SIZEOF_INTERP_FRAME]; *address_of_code_object = 0; - Py_ssize_t bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - address, - SIZEOF_INTERP_FRAME, - frame - ); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter frame"); - return -1; - } - STATS_INC(unwinder, memory_reads); - STATS_ADD(unwinder, memory_bytes_read, SIZEOF_INTERP_FRAME); - *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); uintptr_t code_object = GET_MEMBER_NO_TAG(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.executable); int frame_valid = is_frame_valid(unwinder, (uintptr_t)frame, code_object); @@ -237,6 +223,31 @@ parse_frame_object( return parse_code_object(unwinder, result, &code_ctx); } +int +parse_frame_object( + RemoteUnwinderObject *unwinder, + PyObject** result, + uintptr_t address, + uintptr_t* address_of_code_object, + uintptr_t* previous_frame +) { + char frame[SIZEOF_INTERP_FRAME]; + Py_ssize_t bytes_read = _Py_RemoteDebug_ReadRemoteMemory( + &unwinder->handle, + address, + SIZEOF_INTERP_FRAME, + frame + ); + if (bytes_read < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter frame"); + return -1; + } + STATS_INC(unwinder, memory_reads); + STATS_ADD(unwinder, memory_bytes_read, SIZEOF_INTERP_FRAME); + + return parse_frame_buffer(unwinder, result, frame, address_of_code_object, previous_frame); +} + int parse_frame_from_chunks( RemoteUnwinderObject *unwinder, @@ -312,15 +323,32 @@ process_frame_chain( } assert(frame_count <= MAX_FRAMES); - if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, &stackpointer, ctx->chunks) < 0) { + if (ctx->chunks && ctx->chunks->count > 0) { + if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, &stackpointer, ctx->chunks) == 0) { + goto parsed_frame; + } PyErr_Clear(); + } + { uintptr_t address_of_code_object = 0; - if (parse_frame_object(unwinder, &frame, frame_addr, &address_of_code_object, &next_frame_addr) < 0) { + int parse_result; + if (ctx->prefetch.frame && ctx->prefetch.frame_addr == frame_addr) { + parse_result = parse_frame_buffer( + unwinder, &frame, ctx->prefetch.frame, + &address_of_code_object, &next_frame_addr); + } + else { + parse_result = parse_frame_object( + unwinder, &frame, frame_addr, + &address_of_code_object, &next_frame_addr); + } + if (parse_result < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in chain"); return -1; } } +parsed_frame: // Skip first frame if requested (used for cache miss continuation) if (ctx->skip_first_frame && frame_count == 1) { Py_XDECREF(frame); @@ -501,41 +529,37 @@ try_full_cache_hit( PyObject *current_frame = NULL; uintptr_t code_object_addr = 0; uintptr_t previous_frame = 0; - int parse_result = parse_frame_object(unwinder, ¤t_frame, ctx->frame_addr, + int parse_result; + if (ctx->prefetch.frame && ctx->prefetch.frame_addr == ctx->frame_addr) { + parse_result = parse_frame_buffer(unwinder, ¤t_frame, + ctx->prefetch.frame, &code_object_addr, &previous_frame); + } + else { + parse_result = parse_frame_object(unwinder, ¤t_frame, ctx->frame_addr, + &code_object_addr, &previous_frame); + } if (parse_result < 0) { return -1; } - Py_ssize_t cached_size = PyList_GET_SIZE(entry->frame_list); - PyObject *parent_slice = NULL; - if (cached_size > 1) { - parent_slice = PyList_GetSlice(entry->frame_list, 1, cached_size); - if (!parent_slice) { - Py_XDECREF(current_frame); - return -1; - } - } - if (current_frame != NULL) { if (PyList_Append(ctx->frame_info, current_frame) < 0) { Py_DECREF(current_frame); - Py_XDECREF(parent_slice); return -1; } Py_DECREF(current_frame); STATS_ADD(unwinder, frames_read_from_memory, 1); } - if (parent_slice) { - Py_ssize_t cur_size = PyList_GET_SIZE(ctx->frame_info); - int result = PyList_SetSlice(ctx->frame_info, cur_size, cur_size, parent_slice); - Py_DECREF(parent_slice); - if (result < 0) { + Py_ssize_t cached_size = PyList_GET_SIZE(entry->frame_list); + for (Py_ssize_t i = 1; i < cached_size; i++) { + PyObject *cached_frame = PyList_GET_ITEM(entry->frame_list, i); + if (PyList_Append(ctx->frame_info, cached_frame) < 0) { return -1; } - STATS_ADD(unwinder, frames_read_from_cache, cached_size - 1); } + STATS_ADD(unwinder, frames_read_from_cache, cached_size > 1 ? cached_size - 1 : 0); STATS_INC(unwinder, frame_cache_hits); return 1; @@ -606,7 +630,8 @@ collect_frames_with_cache( } if (frame_cache_store(unwinder, thread_id, ctx->frame_info, ctx->frame_addrs, ctx->num_addrs, - ctx->base_frame_addr, ctx->last_frame_visited) < 0) { + ctx->thread_state_addr, ctx->base_frame_addr, + ctx->last_frame_visited) < 0) { return -1; } diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index 172f8711a2a2a08..ae2f7e7f31ba779 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -166,6 +166,7 @@ cached_code_metadata_destroy(void *ptr) Py_DECREF(meta->func_name); Py_DECREF(meta->file_name); Py_DECREF(meta->linetable); + Py_XDECREF(meta->last_frame_info); PyMem_RawFree(meta); } @@ -360,6 +361,10 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, self->cache_frames = cache_frames; self->collect_stats = stats; self->stale_invalidation_counter = 0; + self->cached_tstate_interpreter_addr = 0; + self->cached_tstate_addr = 0; + memset(self->cached_tstates, 0, sizeof(self->cached_tstates)); + memset(self->cached_generations, 0, sizeof(self->cached_generations)); self->debug = debug; self->only_active_thread = only_active_thread; self->mode = mode; @@ -473,6 +478,172 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, return 0; } +static inline size_t +interpreter_thread_cache_index(uintptr_t interpreter_addr) +{ + // Direct-mapped table indexed by the remote interpreter address. Each entry + // stores the full address and verifies it on lookup, so hash collisions + // degrade to misses and cannot return a value from the wrong interpreter. + return (size_t)_Py_HashPointerRaw((const void *)interpreter_addr) + & (INTERPRETER_THREAD_CACHE_SIZE - 1); +} + +static inline uintptr_t +get_cached_tstate_for_interpreter( + RemoteUnwinderObject *self, + uintptr_t interpreter_addr) +{ + if (interpreter_addr == 0) { + return 0; + } + + if (self->cached_tstate_interpreter_addr == interpreter_addr) { + return self->cached_tstate_addr; + } + + InterpreterTstateCacheEntry *entry = + &self->cached_tstates[interpreter_thread_cache_index(interpreter_addr)]; + if (entry->interpreter_addr == interpreter_addr) { + self->cached_tstate_interpreter_addr = interpreter_addr; + self->cached_tstate_addr = entry->thread_state_addr; + return entry->thread_state_addr; + } + return 0; +} + +static inline void +set_cached_tstate_for_interpreter( + RemoteUnwinderObject *self, + uintptr_t interpreter_addr, + uintptr_t thread_state_addr) +{ + if (interpreter_addr == 0 || thread_state_addr == 0) { + return; + } + + self->cached_tstate_interpreter_addr = interpreter_addr; + self->cached_tstate_addr = thread_state_addr; + + InterpreterTstateCacheEntry *entry = + &self->cached_tstates[interpreter_thread_cache_index(interpreter_addr)]; + entry->interpreter_addr = interpreter_addr; + entry->thread_state_addr = thread_state_addr; +} + +static void +refresh_generation_caches_from_interp_state( + RemoteUnwinderObject *self, + uintptr_t interpreter_addr, + const char *interp_state_buffer) +{ + uint64_t code_object_generation = GET_MEMBER(uint64_t, interp_state_buffer, + self->debug_offsets.interpreter_state.code_object_generation); + + if (self->cached_generation_interpreter_addr == interpreter_addr) { + if (code_object_generation != self->cached_code_object_generation) { + self->cached_code_object_generation = code_object_generation; + _Py_hashtable_clear(self->code_object_cache); + } + } + else { + InterpreterGenerationCacheEntry *entry = + &self->cached_generations[interpreter_thread_cache_index(interpreter_addr)]; + // A slot rebound from another interpreter must be treated as changed: + // the code_object_cache is global, so even if the new generation + // numerically matches what the previous occupant had, stale entries + // from that occupant could still be served. + int changed = entry->interpreter_addr != interpreter_addr + || entry->code_object_generation != code_object_generation; + entry->interpreter_addr = interpreter_addr; + entry->code_object_generation = code_object_generation; + if (changed) { + _Py_hashtable_clear(self->code_object_cache); + } + self->cached_generation_interpreter_addr = interpreter_addr; + self->cached_code_object_generation = code_object_generation; + } + +#ifdef Py_GIL_DISABLED + uint32_t current_tlbc_generation = GET_MEMBER(uint32_t, interp_state_buffer, + self->debug_offsets.interpreter_state.tlbc_generation); + if (current_tlbc_generation != self->tlbc_generation) { + self->tlbc_generation = current_tlbc_generation; + _Py_hashtable_clear(self->tlbc_cache); + } +#endif +} + +static int +refresh_generation_caches_for_interpreter( + RemoteUnwinderObject *self, + uintptr_t interpreter_addr) +{ + char interp_state_buffer[INTERP_STATE_BUFFER_SIZE]; + if (_Py_RemoteDebug_ReadRemoteMemory( + &self->handle, + interpreter_addr, + INTERP_STATE_BUFFER_SIZE, + interp_state_buffer) < 0) { + set_exception_cause(self, PyExc_RuntimeError, + "Failed to read interpreter state buffer"); + return -1; + } + refresh_generation_caches_from_interp_state(self, interpreter_addr, interp_state_buffer); + return 0; +} + +static int +read_interp_state_and_maybe_thread_frame( + RemoteUnwinderObject *unwinder, + uintptr_t interpreter_addr, + char *interp_state_buffer, + char *tstate_buffer, + char *frame_buffer, + RemoteReadPrefetch *prefetch) +{ + prefetch->tstate = NULL; + prefetch->frame = NULL; + if (prefetch->tstate_addr != 0) { + size_t tstate_size = (size_t)unwinder->debug_offsets.thread_state.size; + _Py_RemoteReadSegment segments[3] = { + {interpreter_addr, interp_state_buffer, INTERP_STATE_BUFFER_SIZE}, + {prefetch->tstate_addr, tstate_buffer, tstate_size}, + {prefetch->frame_addr, frame_buffer, SIZEOF_INTERP_FRAME}, + }; + int nsegs = prefetch->frame_addr != 0 ? 3 : 2; + Py_ssize_t nread = _Py_RemoteDebug_BatchedReadRemoteMemory( + &unwinder->handle, segments, nsegs); + int completed = 0; + if (nread >= (Py_ssize_t)INTERP_STATE_BUFFER_SIZE) { + completed = 1; + Py_ssize_t with_tstate = (Py_ssize_t)INTERP_STATE_BUFFER_SIZE + + (Py_ssize_t)tstate_size; + if (nread >= with_tstate) { + completed = 2; + } + if (nsegs == 3 + && nread == with_tstate + (Py_ssize_t)SIZEOF_INTERP_FRAME) { + completed = 3; + } + } + STATS_BATCHED_READ(unwinder, nsegs, completed); + if (completed >= 1) { + if (completed >= 2) { + prefetch->tstate = tstate_buffer; + } + if (completed >= 3) { + prefetch->frame = frame_buffer; + } + return 0; + } + } + return _Py_RemoteDebug_ReadRemoteMemory( + &unwinder->handle, + interpreter_addr, + INTERP_STATE_BUFFER_SIZE, + interp_state_buffer); +} + /*[clinic input] @permit_long_docstring_body @critical_section @@ -537,15 +708,32 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self while (current_interpreter != 0) { // Read interpreter state to get the interpreter ID char interp_state_buffer[INTERP_STATE_BUFFER_SIZE]; - if (_Py_RemoteDebug_PagedReadRemoteMemory( - &self->handle, + char prefetched_tstate[SIZEOF_THREAD_STATE]; + char prefetched_frame[SIZEOF_INTERP_FRAME]; + RemoteReadPrefetch prefetch = {0}; + if (self->cache_frames) { + prefetch.tstate_addr = get_cached_tstate_for_interpreter( + self, current_interpreter); + } + if (prefetch.tstate_addr != 0) { + FrameCacheEntry *entry = frame_cache_find_by_tstate(self, prefetch.tstate_addr); + if (entry && entry->num_addrs > 0) { + prefetch.frame_addr = entry->addrs[0]; + } + } + + if (read_interp_state_and_maybe_thread_frame( + self, current_interpreter, - INTERP_STATE_BUFFER_SIZE, - interp_state_buffer) < 0) { + interp_state_buffer, + prefetched_tstate, + prefetched_frame, + &prefetch) < 0) { set_exception_cause(self, PyExc_RuntimeError, "Failed to read interpreter state buffer"); Py_CLEAR(result); goto exit; } + refresh_generation_caches_from_interp_state(self, current_interpreter, interp_state_buffer); uintptr_t gc_frame = 0; if (self->gc) { @@ -557,25 +745,6 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self int64_t interpreter_id = GET_MEMBER(int64_t, interp_state_buffer, self->debug_offsets.interpreter_state.id); - // Get code object generation from buffer - uint64_t code_object_generation = GET_MEMBER(uint64_t, interp_state_buffer, - self->debug_offsets.interpreter_state.code_object_generation); - - if (code_object_generation != self->code_object_generation) { - self->code_object_generation = code_object_generation; - _Py_hashtable_clear(self->code_object_cache); - } - -#ifdef Py_GIL_DISABLED - // Check TLBC generation and invalidate cache if needed - uint32_t current_tlbc_generation = GET_MEMBER(uint32_t, interp_state_buffer, - self->debug_offsets.interpreter_state.tlbc_generation); - if (current_tlbc_generation != self->tlbc_generation) { - self->tlbc_generation = current_tlbc_generation; - _Py_hashtable_clear(self->tlbc_cache); - } -#endif - // Create a list to hold threads for this interpreter PyObject *interpreter_threads = PyList_New(0); if (!interpreter_threads) { @@ -611,6 +780,9 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self // Target specific thread (only process first interpreter) current_tstate = self->tstate_addr; } + if (current_tstate != 0 && self->cache_frames) { + set_cached_tstate_for_interpreter(self, current_interpreter, current_tstate); + } // Acquire main thread state information uintptr_t main_thread_tstate = GET_MEMBER(uintptr_t, interp_state_buffer, @@ -621,7 +793,8 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self PyObject* frame_info = unwind_stack_for_thread(self, ¤t_tstate, gil_holder_tstate, gc_frame, - main_thread_tstate); + main_thread_tstate, + &prefetch); if (!frame_info) { // Check if this was an intentional skip due to mode-based filtering if ((self->mode == PROFILING_MODE_CPU || self->mode == PROFILING_MODE_GIL || @@ -771,6 +944,9 @@ _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *s if (ensure_async_debug_offsets(self) < 0) { return NULL; } + if (refresh_generation_caches_for_interpreter(self, self->interpreter_addr) < 0) { + return NULL; + } PyObject *result = PyList_New(0); if (result == NULL) { @@ -860,6 +1036,9 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject if (ensure_async_debug_offsets(self) < 0) { return NULL; } + if (refresh_generation_caches_for_interpreter(self, self->interpreter_addr) < 0) { + return NULL; + } PyObject *result = PyList_New(0); if (result == NULL) { @@ -904,8 +1083,15 @@ RemoteUnwinder was created with stats=True. - code_object_cache_hits: Code object cache hits - code_object_cache_misses: Code object cache misses - stale_cache_invalidations: Times stale cache entries were cleared + - batched_read_attempts: Batched remote-read attempts + - batched_read_successes: Attempts that read all requested segments + - batched_read_misses: Attempts that fell back or partially read + - batched_read_segments_requested: Segments requested by batched reads + - batched_read_segments_completed: Segments completed by batched reads - frame_cache_hit_rate: Percentage of samples that hit the cache - code_object_cache_hit_rate: Percentage of code object lookups that hit cache + - batched_read_success_rate: Percentage of batched reads that completed all segments + - batched_read_segment_completion_rate: Percentage of requested segments read by batched reads Raises: RuntimeError: If stats collection was not enabled (stats=False) @@ -913,7 +1099,7 @@ RemoteUnwinder was created with stats=True. static PyObject * _remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=21e36477122be2a0 input=75fef4134c12a8c9]*/ +/*[clinic end generated code: output=21e36477122be2a0 input=0392d62b278e9c35]*/ { if (!self->collect_stats) { PyErr_SetString(PyExc_RuntimeError, @@ -948,9 +1134,24 @@ _remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self) ADD_STAT(code_object_cache_hits); ADD_STAT(code_object_cache_misses); ADD_STAT(stale_cache_invalidations); + ADD_STAT(batched_read_attempts); + ADD_STAT(batched_read_successes); + ADD_STAT(batched_read_misses); + ADD_STAT(batched_read_segments_requested); + ADD_STAT(batched_read_segments_completed); #undef ADD_STAT +#define ADD_DERIVED_STAT(name, value) do { \ + PyObject *val = PyFloat_FromDouble(value); \ + if (!val || PyDict_SetItemString(result, name, val) < 0) { \ + Py_XDECREF(val); \ + Py_DECREF(result); \ + return NULL; \ + } \ + Py_DECREF(val); \ +} while(0) + // Calculate and add derived statistics // Hit rate is calculated as (hits + partial_hits) / total_cache_lookups double frame_cache_hit_rate = 0.0; @@ -959,26 +1160,33 @@ _remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self) frame_cache_hit_rate = 100.0 * (double)(self->stats.frame_cache_hits + self->stats.frame_cache_partial_hits) / (double)total_cache_lookups; } - PyObject *hit_rate = PyFloat_FromDouble(frame_cache_hit_rate); - if (!hit_rate || PyDict_SetItemString(result, "frame_cache_hit_rate", hit_rate) < 0) { - Py_XDECREF(hit_rate); - Py_DECREF(result); - return NULL; - } - Py_DECREF(hit_rate); + ADD_DERIVED_STAT("frame_cache_hit_rate", frame_cache_hit_rate); double code_object_hit_rate = 0.0; uint64_t total_code_lookups = self->stats.code_object_cache_hits + self->stats.code_object_cache_misses; if (total_code_lookups > 0) { code_object_hit_rate = 100.0 * (double)self->stats.code_object_cache_hits / (double)total_code_lookups; } - PyObject *code_hit_rate = PyFloat_FromDouble(code_object_hit_rate); - if (!code_hit_rate || PyDict_SetItemString(result, "code_object_cache_hit_rate", code_hit_rate) < 0) { - Py_XDECREF(code_hit_rate); - Py_DECREF(result); - return NULL; + ADD_DERIVED_STAT("code_object_cache_hit_rate", code_object_hit_rate); + + double batched_read_success_rate = 0.0; + if (self->stats.batched_read_attempts > 0) { + batched_read_success_rate = + 100.0 * (double)self->stats.batched_read_successes + / (double)self->stats.batched_read_attempts; } - Py_DECREF(code_hit_rate); + ADD_DERIVED_STAT("batched_read_success_rate", batched_read_success_rate); + + double batched_read_segment_completion_rate = 0.0; + if (self->stats.batched_read_segments_requested > 0) { + batched_read_segment_completion_rate = + 100.0 * (double)self->stats.batched_read_segments_completed + / (double)self->stats.batched_read_segments_requested; + } + ADD_DERIVED_STAT("batched_read_segment_completion_rate", + batched_read_segment_completion_rate); + +#undef ADD_DERIVED_STAT return result; } @@ -1476,7 +1684,7 @@ class _remote_debugging.BinaryWriter "BinaryWriterObject *" "&PyBinaryWriter_Typ /*[clinic input] @permit_long_docstring_body _remote_debugging.BinaryWriter.__init__ - filename: str + filename: object sample_interval_us: unsigned_long_long start_time_us: unsigned_long_long * @@ -1495,11 +1703,11 @@ Use as a context manager or call finalize() when done. static int _remote_debugging_BinaryWriter___init___impl(BinaryWriterObject *self, - const char *filename, + PyObject *filename, unsigned long long sample_interval_us, unsigned long long start_time_us, int compression) -/*[clinic end generated code: output=014c0306f1bacf4b input=3bdf01c1cc2f5a1d]*/ +/*[clinic end generated code: output=00446656ea2e5986 input=b92f0c77ba4cd274]*/ { if (self->writer) { binary_writer_destroy(self->writer); @@ -1742,7 +1950,7 @@ class _remote_debugging.BinaryReader "BinaryReaderObject *" "&PyBinaryReader_Typ /*[clinic input] _remote_debugging.BinaryReader.__init__ - filename: str + filename: object High-performance binary reader for profiling data. @@ -1754,8 +1962,8 @@ Use as a context manager or call close() when done. static int _remote_debugging_BinaryReader___init___impl(BinaryReaderObject *self, - const char *filename) -/*[clinic end generated code: output=9699226f7ae052bb input=4201f9cc500ef2f6]*/ + PyObject *filename) +/*[clinic end generated code: output=f04b33ee5c5e6dbf input=9d7cbe8b4f1a97c9]*/ { if (self->reader) { binary_reader_close(self->reader); diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index d775234b8d78d72..ae120a26d5f4ece 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -289,28 +289,110 @@ typedef struct { unsigned int :24; } _thread_status; +static int +read_thread_state_and_maybe_frame( + RemoteUnwinderObject *unwinder, + uintptr_t tstate_addr, + size_t tstate_size, + char *tstate_buffer, + uintptr_t predicted_frame_addr, + char *frame_buffer, + int *frame_read) +{ + *frame_read = 0; + if (predicted_frame_addr != 0) { + _Py_RemoteReadSegment segments[2] = { + {tstate_addr, tstate_buffer, tstate_size}, + {predicted_frame_addr, frame_buffer, SIZEOF_INTERP_FRAME}, + }; + Py_ssize_t nread = _Py_RemoteDebug_BatchedReadRemoteMemory( + &unwinder->handle, segments, 2); + int completed = 0; + if (nread >= (Py_ssize_t)tstate_size) { + completed = 1; + if (nread == (Py_ssize_t)(tstate_size + SIZEOF_INTERP_FRAME)) { + completed = 2; + } + } + STATS_BATCHED_READ(unwinder, 2, completed); + if (completed >= 1) { + *frame_read = completed == 2; + return 0; + } + } + return _Py_RemoteDebug_ReadRemoteMemory( + &unwinder->handle, tstate_addr, tstate_size, tstate_buffer); +} + PyObject* unwind_stack_for_thread( RemoteUnwinderObject *unwinder, uintptr_t *current_tstate, uintptr_t gil_holder_tstate, uintptr_t gc_frame, - uintptr_t main_thread_tstate + uintptr_t main_thread_tstate, + const RemoteReadPrefetch *prefetch ) { PyObject *frame_info = NULL; PyObject *thread_id = NULL; PyObject *result = NULL; StackChunkList chunks = {0}; - char ts[SIZEOF_THREAD_STATE]; - int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, *current_tstate, (size_t)unwinder->debug_offsets.thread_state.size, ts); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); - goto error; + char local_ts[SIZEOF_THREAD_STATE]; + char local_prefetched_frame[SIZEOF_INTERP_FRAME]; + const char *ts; + RemoteReadPrefetch ctx_prefetch = {0}; + if (prefetch->tstate && prefetch->tstate_addr == *current_tstate) { + ts = prefetch->tstate; + if (prefetch->frame) { + ctx_prefetch.frame = prefetch->frame; + ctx_prefetch.frame_addr = prefetch->frame_addr; + } + } + else if (unwinder->cache_frames) { + uintptr_t predicted_frame_addr = 0; + int have_prefetched_frame = 0; + FrameCacheEntry *entry = frame_cache_find_by_tstate(unwinder, *current_tstate); + if (entry && entry->num_addrs > 0) { + predicted_frame_addr = entry->addrs[0]; + } + + int rc = read_thread_state_and_maybe_frame( + unwinder, + *current_tstate, + (size_t)unwinder->debug_offsets.thread_state.size, + local_ts, + predicted_frame_addr, + local_prefetched_frame, + &have_prefetched_frame); + if (rc < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); + goto error; + } + ts = local_ts; + if (have_prefetched_frame) { + ctx_prefetch.frame = local_prefetched_frame; + ctx_prefetch.frame_addr = predicted_frame_addr; + } + } + else { + int rc = _Py_RemoteDebug_ReadRemoteMemory( + &unwinder->handle, + *current_tstate, + (size_t)unwinder->debug_offsets.thread_state.size, + local_ts); + if (rc < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); + goto error; + } + ts = local_ts; } STATS_INC(unwinder, memory_reads); STATS_ADD(unwinder, memory_bytes_read, unwinder->debug_offsets.thread_state.size); + if (ctx_prefetch.frame) { + STATS_INC(unwinder, memory_reads); + STATS_ADD(unwinder, memory_bytes_read, SIZEOF_INTERP_FRAME); + } long tid = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.native_thread_id); @@ -432,9 +514,11 @@ unwind_stack_for_thread( uintptr_t addrs[FRAME_CACHE_MAX_FRAMES]; FrameWalkContext ctx = { .frame_addr = frame_addr, + .thread_state_addr = *current_tstate, .base_frame_addr = base_frame_addr, .gc_frame = gc_frame, .chunks = &chunks, + .prefetch = ctx_prefetch, .frame_info = frame_info, .frame_addrs = addrs, .num_addrs = 0, @@ -467,10 +551,18 @@ unwind_stack_for_thread( *current_tstate = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.next); - thread_id = PyLong_FromLongLong(tid); + if (unwinder->cache_frames) { + FrameCacheEntry *entry = frame_cache_find(unwinder, (uint64_t)tid); + if (entry && entry->thread_id_obj) { + thread_id = Py_NewRef(entry->thread_id_obj); + } + } if (thread_id == NULL) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID"); - goto error; + thread_id = PyLong_FromLongLong(tid); + if (thread_id == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID"); + goto error; + } } RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 6e5c8dcbb725fa5..c62dc1144df6881 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -572,6 +572,12 @@ pysentinel_check(PyObject *self, PyObject *obj) return PyBool_FromLong(PySentinel_Check(obj)); } +static PyObject * +pysentinel_checkexact(PyObject *self, PyObject *obj) +{ + return PyBool_FromLong(PySentinel_CheckExact(obj)); +} + static PyMethodDef test_methods[] = { {"call_pyobject_print", call_pyobject_print, METH_VARARGS}, @@ -604,6 +610,7 @@ static PyMethodDef test_methods[] = { {"pyobject_dump", pyobject_dump, METH_VARARGS}, {"pysentinel_new", pysentinel_new, METH_VARARGS}, {"pysentinel_check", pysentinel_check, METH_O}, + {"pysentinel_checkexact", pysentinel_checkexact, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index e0abf6b1845d8ef..71cdc54009017a7 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -9,6 +9,7 @@ #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_interp_structs.h" // CODE_MAX_WATCHERS #include "pycore_context.h" // CONTEXT_MAX_WATCHERS +#include "pycore_lock.h" // _PyOnceFlag /*[clinic input] module _testcapi @@ -18,6 +19,14 @@ module _testcapi // Test dict watching static PyObject *g_dict_watch_events = NULL; static int g_dict_watchers_installed = 0; +static _PyOnceFlag g_dict_watch_once = {0}; + +static int +_init_dict_watch_events(void *arg) +{ + g_dict_watch_events = PyList_New(0); + return g_dict_watch_events ? 0 : -1; +} static int dict_watch_callback(PyDict_WatchEvent event, @@ -106,13 +115,10 @@ add_dict_watcher(PyObject *self, PyObject *kind) if (watcher_id < 0) { return NULL; } - if (!g_dict_watchers_installed) { - assert(!g_dict_watch_events); - if (!(g_dict_watch_events = PyList_New(0))) { - return NULL; - } + if (_PyOnceFlag_CallOnce(&g_dict_watch_once, _init_dict_watch_events, NULL) < 0) { + return NULL; } - g_dict_watchers_installed++; + _Py_atomic_add_int(&g_dict_watchers_installed, 1); return PyLong_FromLong(watcher_id); } @@ -122,10 +128,8 @@ clear_dict_watcher(PyObject *self, PyObject *watcher_id) if (PyDict_ClearWatcher(PyLong_AsLong(watcher_id))) { return NULL; } - g_dict_watchers_installed--; - if (!g_dict_watchers_installed) { - assert(g_dict_watch_events); - Py_CLEAR(g_dict_watch_events); + if (_Py_atomic_add_int(&g_dict_watchers_installed, -1) == 1) { + PyList_Clear(g_dict_watch_events); } Py_RETURN_NONE; } @@ -164,7 +168,7 @@ _testcapi_unwatch_dict_impl(PyObject *module, int watcher_id, PyObject *dict) static PyObject * get_dict_watcher_events(PyObject *self, PyObject *Py_UNUSED(args)) { - if (!g_dict_watch_events) { + if (_Py_atomic_load_int(&g_dict_watchers_installed) <= 0) { PyErr_SetString(PyExc_RuntimeError, "no watchers active"); return NULL; } diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 238e17bea303d35..a2506524f0bb6dc 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -7946,8 +7946,9 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif @@ -13056,8 +13057,9 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif diff --git a/Modules/binascii.c b/Modules/binascii.c index 673dca6ee134bd8..0e7af135a6f6ce4 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -1057,7 +1057,8 @@ binascii.a2b_ascii85 foldspaces: bool = False Allow 'y' as a short form encoding four spaces. adobe: bool = False - Expect data to be wrapped in '<~' and '~>' as in Adobe Ascii85. + Expect data to be terminated with '~>' as in Adobe Ascii85, and + optionally accept leading '<~'. ignorechars: Py_buffer = b'' A byte string containing characters to ignore from the input. canonical: bool = False @@ -1069,7 +1070,7 @@ Decode Ascii85 data. static PyObject * binascii_a2b_ascii85_impl(PyObject *module, Py_buffer *data, int foldspaces, int adobe, Py_buffer *ignorechars, int canonical) -/*[clinic end generated code: output=09b35f1eac531357 input=dd050604ed30199e]*/ +/*[clinic end generated code: output=09b35f1eac531357 input=08eab2e53c62f1a8]*/ { const unsigned char *ascii_data = data->buf; Py_ssize_t ascii_len = data->len; @@ -1264,7 +1265,7 @@ binascii.b2a_ascii85 wrapcol: size_t = 0 Split result into lines of provided width. pad: bool = False - Pad input to a multiple of 4 before encoding. + Retain zero-padding bytes at end of output. adobe: bool = False Wrap result in '<~' and '~>' as in Adobe Ascii85. @@ -1274,7 +1275,7 @@ Ascii85-encode data. static PyObject * binascii_b2a_ascii85_impl(PyObject *module, Py_buffer *data, int foldspaces, size_t wrapcol, int pad, int adobe) -/*[clinic end generated code: output=5ce8fdee843073f4 input=791da754508c7d17]*/ +/*[clinic end generated code: output=5ce8fdee843073f4 input=a77e31d63517bf19]*/ { const unsigned char *bin_data = data->buf; Py_ssize_t bin_len = data->len; @@ -1539,7 +1540,7 @@ binascii.b2a_base85 / * pad: bool = False - Pad input to a multiple of 4 before encoding. + Retain zero-padding bytes at end of output. wrapcol: size_t = 0 alphabet: Py_buffer(c_default="{NULL, NULL}") = BASE85_ALPHABET @@ -1549,7 +1550,7 @@ Base85-code line of data. static PyObject * binascii_b2a_base85_impl(PyObject *module, Py_buffer *data, int pad, size_t wrapcol, Py_buffer *alphabet) -/*[clinic end generated code: output=98b962ed52c776a4 input=1b20b0bd6572691b]*/ +/*[clinic end generated code: output=98b962ed52c776a4 input=54886d05128d41a8]*/ { const unsigned char *bin_data = data->buf; Py_ssize_t bin_len = data->len; diff --git a/Modules/clinic/_functoolsmodule.c.h b/Modules/clinic/_functoolsmodule.c.h index 23f666310850312..87cdef2ad3cff3b 100644 --- a/Modules/clinic/_functoolsmodule.c.h +++ b/Modules/clinic/_functoolsmodule.c.h @@ -71,7 +71,8 @@ _functools_cmp_to_key(PyObject *module, PyObject *const *args, Py_ssize_t nargs, } PyDoc_STRVAR(_functools_reduce__doc__, -"reduce($module, function, iterable, /, initial=)\n" +"reduce($module, function, iterable, /,\n" +" initial=functools._initial_missing)\n" "--\n" "\n" "Apply a function of two arguments cumulatively to the items of an iterable, from left to right.\n" @@ -192,4 +193,4 @@ _functools__lru_cache_wrapper_cache_clear(PyObject *self, PyObject *Py_UNUSED(ig return return_value; } -/*[clinic end generated code: output=7f2abc718fcc35d5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ac9e26d0a5a23d40 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h index 2563a16aea0b6f9..ca9cad7a572dadf 100644 --- a/Modules/clinic/_randommodule.c.h +++ b/Modules/clinic/_randommodule.c.h @@ -143,4 +143,35 @@ _random_Random_getrandbits(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=7ce97b2194eecaf7 input=a9049054013a1b77]*/ + +static int +random_init_impl(RandomObject *self, PyObject *seed); + +static int +random_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = (PyTypeObject *)_randomstate_type(Py_TYPE(self))->Random_Type; + PyObject *seed = NULL; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Random", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("Random", PyTuple_GET_SIZE(args), 0, 1)) { + goto exit; + } + if (PyTuple_GET_SIZE(args) < 1) { + goto skip_optional; + } + seed = PyTuple_GET_ITEM(args, 0); +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = random_init_impl((RandomObject *)self, seed); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} +/*[clinic end generated code: output=ec95f7df0c3f3c19 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index ed695758ef998c9..29fa9e87de87c7a 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -372,7 +372,8 @@ PyDoc_STRVAR(binascii_a2b_ascii85__doc__, " foldspaces\n" " Allow \'y\' as a short form encoding four spaces.\n" " adobe\n" -" Expect data to be wrapped in \'<~\' and \'~>\' as in Adobe Ascii85.\n" +" Expect data to be terminated with \'~>\' as in Adobe Ascii85, and\n" +" optionally accept leading \'<~\'.\n" " ignorechars\n" " A byte string containing characters to ignore from the input.\n" " canonical\n" @@ -492,7 +493,7 @@ PyDoc_STRVAR(binascii_b2a_ascii85__doc__, " wrapcol\n" " Split result into lines of provided width.\n" " pad\n" -" Pad input to a multiple of 4 before encoding.\n" +" Retain zero-padding bytes at end of output.\n" " adobe\n" " Wrap result in \'<~\' and \'~>\' as in Adobe Ascii85."); @@ -709,7 +710,7 @@ PyDoc_STRVAR(binascii_b2a_base85__doc__, "Base85-code line of data.\n" "\n" " pad\n" -" Pad input to a multiple of 4 before encoding."); +" Retain zero-padding bytes at end of output."); #define BINASCII_B2A_BASE85_METHODDEF \ {"b2a_base85", _PyCFunction_CAST(binascii_b2a_base85), METH_FASTCALL|METH_KEYWORDS, binascii_b2a_base85__doc__}, @@ -1684,4 +1685,4 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -/*[clinic end generated code: output=b41544f39b0ef681 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=42dd48f323cbb118 input=a9049054013a1b77]*/ diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 79c609f19aa4cff..ec3f58544cb00d5 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -1094,7 +1094,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ # define XML_MAJOR_VERSION 2 # define XML_MINOR_VERSION 8 -# define XML_MICRO_VERSION 0 +# define XML_MICRO_VERSION 1 # ifdef __cplusplus } diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh index 774e0b89d94c0ec..fa3692f9379510e 100755 --- a/Modules/expat/refresh.sh +++ b/Modules/expat/refresh.sh @@ -12,9 +12,9 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. These values are used for verifying the SBOM, too. -expected_libexpat_tag="R_2_8_0" -expected_libexpat_version="2.8.0" -expected_libexpat_sha256="c7cec5f60ea3a42e7780781c6745255c19aa3dbfeeae58646b7132f88dc24780" +expected_libexpat_tag="R_2_8_1" +expected_libexpat_version="2.8.1" +expected_libexpat_sha256="a52eb72108be160e190b5cafa5bba8663f1313f2013e26060d1c18e26e31067b" expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" cd ${expat_dir} diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index e6842f3f0bf750b..95d346758563ab7 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* a5d18f6a50f536615ac1c70304f87d94f99cc85a86b502188952440610ccf0f8 (2.8.0+) +/* 75ef4224f81c052e9e5aeea2ac7de75357d2169ff9908e39edc08b9dc3052513 (2.8.1+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -387,6 +387,7 @@ typedef struct { int nDefaultAtts; int allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; + HASH_TABLE defaultAttsNames; } ELEMENT_TYPE; typedef struct { @@ -3769,6 +3770,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sizeof(ELEMENT_TYPE)); if (! elementType) return XML_ERROR_NO_MEMORY; + if (! elementType->defaultAttsNames.parser) + hashTableInit(&(elementType->defaultAttsNames), parser); if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } @@ -7102,10 +7105,10 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, if (value || isId) { /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ - int i; - for (i = 0; i < type->nDefaultAtts; i++) - if (attId == type->defaultAtts[i].id) - return 1; + NAMED *const nameFound + = (NAMED *)lookup(parser, &(type->defaultAttsNames), attId->name, 0); + if (nameFound) + return 1; if (isId && ! type->idAtt && ! attId->xmlns) type->idAtt = attId; } @@ -7152,6 +7155,12 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, att->isCdata = isCdata; if (! isCdata) attId->maybeTokenized = XML_TRUE; + + NAMED *const nameAddedOrFound = (NAMED *)lookup( + parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); + if (! nameAddedOrFound) + return 0; + type->nDefaultAtts += 1; return 1; } @@ -7477,6 +7486,7 @@ dtdReset(DTD *p, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; + hashTableDestroy(&(e->defaultAttsNames)); if (e->allocDefaultAtts != 0) FREE(parser, e->defaultAtts); } @@ -7518,6 +7528,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; + hashTableDestroy(&(e->defaultAttsNames)); if (e->allocDefaultAtts != 0) FREE(parser, e->defaultAtts); } @@ -7611,6 +7622,10 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, sizeof(ELEMENT_TYPE)); if (! newE) return 0; + + if (! newE->defaultAttsNames.parser) + hashTableInit(&(newE->defaultAttsNames), parser); + if (oldE->nDefaultAtts) { /* Detect and prevent integer overflow. * The preprocessor guard addresses the "always false" warning @@ -7635,8 +7650,9 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, newE->prefix = (PREFIX *)lookup(oldParser, &(newDtd->prefixes), oldE->prefix->name, 0); for (i = 0; i < newE->nDefaultAtts; i++) { + const XML_Char *const attributeName = oldE->defaultAtts[i].id->name; newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup( - oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0); + oldParser, &(newDtd->attributeIds), attributeName, 0); newE->defaultAtts[i].isCdata = oldE->defaultAtts[i].isCdata; if (oldE->defaultAtts[i].value) { newE->defaultAtts[i].value @@ -7645,6 +7661,12 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, return 0; } else newE->defaultAtts[i].value = NULL; + + NAMED *const nameAddedOrFound = (NAMED *)lookup( + parser, &(newE->defaultAttsNames), attributeName, sizeof(NAMED)); + if (! nameAddedOrFound) { + return 0; + } } } @@ -8391,6 +8413,8 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, sizeof(ELEMENT_TYPE)); if (! ret) return NULL; + if (! ret->defaultAttsNames.parser) + hashTableInit(&(ret->defaultAttsNames), getRootParserOf(parser, NULL)); if (ret->name != name) poolDiscard(&dtd->pool); else { diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 1b4f0c2302daae2..fa7fb7085d7e8b9 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1349,21 +1349,6 @@ faulthandler__stack_overflow_impl(PyObject *module) #endif /* defined(FAULTHANDLER_USE_ALT_STACK) && defined(HAVE_SIGACTION) */ -static int -faulthandler_traverse(PyObject *module, visitproc visit, void *arg) -{ - Py_VISIT(thread.file); -#ifdef FAULTHANDLER_USER - if (user_signals != NULL) { - for (size_t signum=0; signum < Py_NSIG; signum++) - Py_VISIT(user_signals[signum].file); - } -#endif - Py_VISIT(fatal_error.file); - return 0; -} - - #ifdef MS_WINDOWS /*[clinic input] faulthandler._raise_exception @@ -1459,7 +1444,6 @@ static struct PyModuleDef module_def = { .m_name = "faulthandler", .m_doc = module_doc, .m_methods = module_methods, - .m_traverse = faulthandler_traverse, .m_slots = faulthandler_slots }; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 0f0afe17513ef1c..c01f7babe745279 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -393,7 +393,7 @@ my_CharacterDataHandler(void *userData, const XML_Char *data, int len) if (self->buffer == NULL) call_character_handler(self, data, len); else { - if ((self->buffer_used + len) > self->buffer_size) { + if (len > (self->buffer_size - self->buffer_used)) { if (flush_character_buffer(self) < 0) return; /* handler might have changed; drop the rest on the floor diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 09c8d9487f54266..96454ee5e83eab7 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -11,7 +11,13 @@ other files, you'll have to create a file "foobarobject.h"; see floatobject.h for an example. - This module roughly corresponds to:: + This module uses Limited API 3.15. + See ``xxlimited_3_13.c`` if you want to support older CPython versions. + + This module roughly corresponds to the following. + (All underscore-prefixed attributes are not accessible from Python.) + + :: class Xxo: """A class that explicitly stores attributes in an internal dict @@ -27,6 +33,8 @@ return self._x_attr[name] def __setattr__(self, name, value): + if name == "reserved": + raise AttributeError("cannot set 'reserved'") self._x_attr[name] = value def __delattr__(self, name): @@ -64,11 +72,13 @@ pass */ -// Need limited C API version 3.13 for Py_mod_gil -#include "pyconfig.h" // Py_GIL_DISABLED -#ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030d0000 -#endif +// Target both flavors of the Stable ABI. +// Both are set to version 3.15, which adds PyModExport +// (When using a build tool, check if it has an option to set these +// so they do not need to be defined in the source.) +#define Py_LIMITED_API 0x030f0000 // abi3 (GIL-enabled builds) +#define Py_TARGET_ABI3T 0x030f0000 // abi3t (free-threaded builds) + #include "Python.h" #include @@ -77,43 +87,135 @@ // Module state typedef struct { - PyObject *Xxo_Type; // Xxo class + PyTypeObject *Xxo_Type; // Xxo class PyObject *Error_Type; // Error class } xx_state; -/* Xxo objects */ +/* Xxo objects. + * + * A non-trivial extension type, intentionally showing a number of features + * that aren't easy to implement in the Limited API. + */ + +// Forward declaration +static PyType_Spec Xxo_Type_spec; + +// Get the module state (xx_state*) from a given type object 'type', which +// must be a subclass of Xxo (the type we're defining). +// This is complicated by the fact that the Xxo type is dynamically allocated, +// and there may be several such types in a given Python process -- for +// example, in different subinterpreters, or through loading this +// extension module several times. +// So, we don't have a "global" pointer to the type, or to the module, etc.; +// instead we search based on `Xxo_Type_spec` (which is static, immutable, +// and process-global). +// +// When possible, it's better to avoid `PyType_GetBaseByToken` -- for an +// example, see the `demo` method (Xxo_demo C function), which uses a +// "defining class". But, in many cases it's the best solution. +static xx_state * +Xxo_state_from_type(PyTypeObject *type) +{ + PyTypeObject *base; + // Search all superclasses of 'type' for one that was defined using + // "Xxo_Type_spec". That must be our 'Xxo' class. + if (PyType_GetBaseByToken(type, &Xxo_Type_spec, &base) < 0) { + return NULL; + } + if (base == NULL) { + PyErr_SetString(PyExc_TypeError, "need Xxo subclass"); + return NULL; + } + // From this type, get the associated module. That must be the + // relevant `xxlimited` module. + xx_state *state = PyType_GetModuleState(base); + Py_DECREF(base); + return state; +} -// Instance state +// Structure for data needed by the XxoObject type. +// Since the object may be shared across threads, access to the fields +// usually needs to be synchronized (using Py_BEGIN_CRITICAL_SECTION). typedef struct { - PyObject_HEAD - PyObject *x_attr; /* Attributes dictionary. - * May be NULL, which acts as an - * empty dict. - */ - char x_buffer[BUFSIZE]; /* buffer for Py_buffer */ - Py_ssize_t x_exports; /* how many buffer are exported */ -} XxoObject; - -#define XxoObject_CAST(op) ((XxoObject *)(op)) -// TODO: full support for type-checking was added in 3.14 (Py_tp_token) -// #define XxoObject_Check(v) Py_IS_TYPE(v, Xxo_Type) - -static XxoObject * -newXxoObject(PyObject *module) + PyObject *x_attr; /* Attributes dictionary. + * May be NULL, which acts as an + * empty dict. + */ + Py_ssize_t x_exports; /* how many buffers are exported */ + char x_buffer[BUFSIZE]; /* buffer for Py_buffer (for simplicity, + * this is constant, so does not need + * synchronization) + */ +} XxoObject_Data; + +// Get the `XxoObject_Data` structure for a given instance of our type. +static XxoObject_Data * +Xxo_get_data(PyObject *self) { - xx_state *state = PyModule_GetState(module); + xx_state *state = Xxo_state_from_type(Py_TYPE(self)); + if (!state) { + return NULL; + } + XxoObject_Data *data = PyObject_GetTypeData(self, state->Xxo_Type); + return data; +} + +// A variant of Xxo_get_data to be used in the tp_traverse handler. +// This function cannot have side effects (including reference count +// manipulation, creating objects, and raising exceptions), and must not +// call API functions that might have side effects. +// See: https://docs.python.org/3.15/c-api/gcsupport.html#traversal +static XxoObject_Data * +Xxo_get_data_DuringGC(PyObject *self) +{ + PyTypeObject *base; + PyType_GetBaseByToken_DuringGC(Py_TYPE(self), &Xxo_Type_spec, &base); + if (base == NULL) { + return NULL; + } + xx_state *state = PyType_GetModuleState_DuringGC(base); if (state == NULL) { return NULL; } - XxoObject *self; - self = PyObject_GC_New(XxoObject, (PyTypeObject*)state->Xxo_Type); + XxoObject_Data *data = PyObject_GetTypeData_DuringGC(self, state->Xxo_Type); + return data; +} + +// Xxo initialization +// This is the implementation of Xxo.__new__ +static PyObject * +Xxo_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + // Validate that we did not get any arguments. + if ((args != NULL && PyObject_Length(args)) + || (kwargs != NULL && PyObject_Length(kwargs))) + { + PyErr_SetString(PyExc_TypeError, "Xxo.__new__() takes no arguments"); + return NULL; + } + // Create an instance of *type* (which may be a subclass) + allocfunc alloc = PyType_GetSlot(type, Py_tp_alloc); + PyObject *self = alloc(type, 0); if (self == NULL) { return NULL; } - self->x_attr = NULL; - memset(self->x_buffer, 0, BUFSIZE); - self->x_exports = 0; + + // Initialize the C members on the instance. + // This is only included for the sake of example. The default alloc + // function zeroes instance memory; we don't need to do it again. + // Note that we during initialization (and finalization), we hold the only + // reference to the object, so we don't need to synchronize with + // other threads. + XxoObject_Data *xxo_data = Xxo_get_data(self); + if (xxo_data == NULL) { + Py_DECREF(self); + return NULL; + } + + xxo_data->x_attr = NULL; + memset(xxo_data->x_buffer, 0, BUFSIZE); + xxo_data->x_exports = 0; return self; } @@ -125,45 +227,63 @@ newXxoObject(PyObject *module) // traverse: Visit all references from an object, including its type static int -Xxo_traverse(PyObject *op, visitproc visit, void *arg) +Xxo_traverse(PyObject *self, visitproc visit, void *arg) { // Visit the type - Py_VISIT(Py_TYPE(op)); + Py_VISIT(Py_TYPE(self)); // Visit the attribute dict - XxoObject *self = XxoObject_CAST(op); - Py_VISIT(self->x_attr); + XxoObject_Data *data = Xxo_get_data_DuringGC(self); + if (data == NULL) { + return 0; + } + Py_VISIT(data->x_attr); return 0; } // clear: drop references in order to break all reference cycles static int -Xxo_clear(PyObject *op) +Xxo_clear(PyObject *self) { - XxoObject *self = XxoObject_CAST(op); - Py_CLEAR(self->x_attr); + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return 0; + } + Py_CLEAR(data->x_attr); return 0; } // finalize: like clear, but should leave the object in a consistent state. // Equivalent to `__del__` in Python. static void -Xxo_finalize(PyObject *op) +Xxo_finalize(PyObject *self) { - XxoObject *self = XxoObject_CAST(op); - Py_CLEAR(self->x_attr); + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return; + } + Py_CLEAR(data->x_attr); } // dealloc: drop all remaining references and free memory static void Xxo_dealloc(PyObject *self) { + // This function must preserve currently raised exception, if any. + PyObject *exc = PyErr_GetRaisedException(); + PyObject_GC_UnTrack(self); Xxo_finalize(self); + PyTypeObject *tp = Py_TYPE(self); freefunc free = PyType_GetSlot(tp, Py_tp_free); free(self); Py_DECREF(tp); + + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(NULL); + } + PyErr_SetRaisedException(exc); } @@ -171,11 +291,20 @@ Xxo_dealloc(PyObject *self) // Get an attribute. static PyObject * -Xxo_getattro(PyObject *op, PyObject *name) +Xxo_getattro(PyObject *self, PyObject *name) { - XxoObject *self = XxoObject_CAST(op); - if (self->x_attr != NULL) { - PyObject *v = PyDict_GetItemWithError(self->x_attr, name); + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return 0; + } + + PyObject *x_attr; + Py_BEGIN_CRITICAL_SECTION(self); + x_attr = data->x_attr; + Py_END_CRITICAL_SECTION(); + + if (x_attr != NULL) { + PyObject *v = PyDict_GetItemWithError(x_attr, name); if (v != NULL) { return Py_NewRef(v); } @@ -185,24 +314,42 @@ Xxo_getattro(PyObject *op, PyObject *name) } // Fall back to generic implementation (this handles special attributes, // raising AttributeError, etc.) - return PyObject_GenericGetAttr(op, name); + return PyObject_GenericGetAttr(self, name); } // Set or delete an attribute. static int -Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) +Xxo_setattro(PyObject *self, PyObject *name, PyObject *v) { - XxoObject *self = XxoObject_CAST(op); - if (self->x_attr == NULL) { + // filter a specific attribute name + if (PyUnicode_Check(name) && PyUnicode_EqualToUTF8(name, "reserved")) { + PyErr_Format(PyExc_AttributeError, "cannot set %R", name); + return -1; + } + + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return -1; + } + + // If the attribute dict is not created yet, make one. + // This needs to be protected by a critical section to avoid another thread + // creating a duplicate dict. + PyObject *x_attr; + Py_BEGIN_CRITICAL_SECTION(self); + x_attr = data->x_attr; + if (x_attr == NULL) { // prepare the attribute dict - self->x_attr = PyDict_New(); - if (self->x_attr == NULL) { - return -1; - } + data->x_attr = x_attr = PyDict_New(); } + Py_END_CRITICAL_SECTION(); + if (x_attr == NULL) { + return -1; + } + if (v == NULL) { // delete an attribute - int rv = PyDict_DelItem(self->x_attr, name); + int rv = PyDict_DelItem(x_attr, name); if (rv < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_SetString(PyExc_AttributeError, "delete non-existing Xxo attribute"); @@ -212,7 +359,7 @@ Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) } else { // set an attribute - return PyDict_SetItem(self->x_attr, name, v); + return PyDict_SetItem(x_attr, name, v); } } @@ -221,7 +368,7 @@ Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) */ static PyObject * -Xxo_demo(PyObject *op, PyTypeObject *defining_class, +Xxo_demo(PyObject *self, PyTypeObject *defining_class, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (kwnames != NULL && PyObject_Length(kwnames)) { @@ -260,30 +407,49 @@ static PyMethodDef Xxo_methods[] = { */ static int -Xxo_getbuffer(PyObject *op, Py_buffer *view, int flags) +Xxo_getbuffer(PyObject *self, Py_buffer *view, int flags) { - XxoObject *self = XxoObject_CAST(op); - int res = PyBuffer_FillInfo(view, op, - (void *)self->x_buffer, BUFSIZE, + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return -1; + } + int res = PyBuffer_FillInfo(view, self, + (void *)data->x_buffer, BUFSIZE, 0, flags); if (res == 0) { - self->x_exports++; + Py_BEGIN_CRITICAL_SECTION(self); + data->x_exports++; + Py_END_CRITICAL_SECTION(); } return res; } static void -Xxo_releasebuffer(PyObject *op, Py_buffer *Py_UNUSED(view)) +Xxo_releasebuffer(PyObject *self, Py_buffer *Py_UNUSED(view)) { - XxoObject *self = XxoObject_CAST(op); - self->x_exports--; + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return; + } + Py_BEGIN_CRITICAL_SECTION(self); + data->x_exports--; + Py_END_CRITICAL_SECTION(); } static PyObject * -Xxo_get_x_exports(PyObject *op, void *Py_UNUSED(closure)) +Xxo_get_x_exports(PyObject *self, void *Py_UNUSED(closure)) { - XxoObject *self = XxoObject_CAST(op); - return PyLong_FromSsize_t(self->x_exports); + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return NULL; + } + Py_ssize_t result; + + Py_BEGIN_CRITICAL_SECTION(self); + result = data->x_exports; + Py_END_CRITICAL_SECTION(); + + return PyLong_FromSsize_t(result); } /* Xxo type definition */ @@ -299,6 +465,7 @@ static PyGetSetDef Xxo_getsetlist[] = { static PyType_Slot Xxo_Type_slots[] = { {Py_tp_doc, (char *)Xxo_doc}, + {Py_tp_new, Xxo_new}, {Py_tp_traverse, Xxo_traverse}, {Py_tp_clear, Xxo_clear}, {Py_tp_finalize, Xxo_finalize}, @@ -309,13 +476,14 @@ static PyType_Slot Xxo_Type_slots[] = { {Py_bf_getbuffer, Xxo_getbuffer}, {Py_bf_releasebuffer, Xxo_releasebuffer}, {Py_tp_getset, Xxo_getsetlist}, + {Py_tp_token, Py_TP_USE_SPEC}, {0, 0}, /* sentinel */ }; static PyType_Spec Xxo_Type_spec = { .name = "xxlimited.Xxo", - .basicsize = sizeof(XxoObject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .basicsize = -(Py_ssize_t)sizeof(XxoObject_Data), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, .slots = Xxo_Type_slots, }; @@ -354,17 +522,17 @@ xx_foo(PyObject *module, PyObject *args) } -/* Function of no arguments returning new Xxo object */ +/* Function of no arguments returning new Xxo object. + * Note that a function exposed to Python with METH_NOARGS requires an unused + * second argument, so we cannot use newXxoObject directly. + */ static PyObject * xx_new(PyObject *module, PyObject *Py_UNUSED(unused)) { - XxoObject *rv; + xx_state *state = PyModule_GetState(module); - rv = newXxoObject(module); - if (rv == NULL) - return NULL; - return (PyObject *)rv; + return Xxo_new(state->Xxo_Type, NULL, NULL); } @@ -398,11 +566,12 @@ xx_modexec(PyObject *m) return -1; } - state->Xxo_Type = PyType_FromModuleAndSpec(m, &Xxo_Type_spec, NULL); + state->Xxo_Type = (PyTypeObject*)PyType_FromModuleAndSpec( + m, &Xxo_Type_spec, NULL); if (state->Xxo_Type == NULL) { return -1; } - if (PyModule_AddType(m, (PyTypeObject*)state->Xxo_Type) < 0) { + if (PyModule_AddType(m, state->Xxo_Type) < 0) { return -1; } @@ -410,12 +579,13 @@ xx_modexec(PyObject *m) // added to the module dict. // It does not inherit from "object" (PyObject_Type), but from "str" // (PyUnincode_Type). - PyObject *Str_Type = PyType_FromModuleAndSpec( + PyTypeObject *Str_Type = (PyTypeObject*)PyType_FromModuleAndSpec( m, &Str_Type_spec, (PyObject *)&PyUnicode_Type); if (Str_Type == NULL) { return -1; } - if (PyModule_AddType(m, (PyTypeObject*)Str_Type) < 0) { + if (PyModule_AddType(m, Str_Type) < 0) { + Py_DECREF(Str_Type); return -1; } Py_DECREF(Str_Type); @@ -423,29 +593,6 @@ xx_modexec(PyObject *m) return 0; } -static PyModuleDef_Slot xx_slots[] = { - - /* exec function to initialize the module (called as part of import - * after the object was added to sys.modules) - */ - {Py_mod_exec, xx_modexec}, - - /* Signal that this module supports being loaded in multiple interpreters - * with separate GILs (global interpreter locks). - * See "Isolating Extension Modules" on how to prepare a module for this: - * https://docs.python.org/3/howto/isolating-extensions.html - */ - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - - /* Signal that this module does not rely on the GIL for its own needs. - * Without this slot, free-threaded builds of CPython will enable - * the GIL when this module is loaded. - */ - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - - {0, NULL} -}; - // Module finalization: modules that hold references in their module state // need to implement the fullowing GC hooks. They're similar to the ones for // types (see "Xxo finalization"). @@ -453,7 +600,10 @@ static PyModuleDef_Slot xx_slots[] = { static int xx_traverse(PyObject *module, visitproc visit, void *arg) { - xx_state *state = PyModule_GetState(module); + xx_state *state = PyModule_GetState_DuringGC(module); + if (state == NULL) { + return 0; + } Py_VISIT(state->Xxo_Type); Py_VISIT(state->Error_Type); return 0; @@ -463,6 +613,9 @@ static int xx_clear(PyObject *module) { xx_state *state = PyModule_GetState(module); + if (state == NULL) { + return 0; + } Py_CLEAR(state->Xxo_Type); Py_CLEAR(state->Error_Type); return 0; @@ -473,27 +626,59 @@ xx_free(void *module) { // allow xx_modexec to omit calling xx_clear on error (void)xx_clear((PyObject *)module); + + xx_state *state = PyModule_GetState(module); + if (state == NULL) { + return; + } } -static struct PyModuleDef xxmodule = { - PyModuleDef_HEAD_INIT, - .m_name = "xxlimited", - .m_doc = module_doc, - .m_size = sizeof(xx_state), - .m_methods = xx_methods, - .m_slots = xx_slots, - .m_traverse = xx_traverse, - .m_clear = xx_clear, - .m_free = xx_free, +// Information that CPython uses to prevent loading incompatible extenstions +PyABIInfo_VAR(abi_info); + +static PySlot xx_slots[] = { + /* Basic metadata */ + PySlot_STATIC_DATA(Py_mod_name, "xxlimited"), + PySlot_STATIC_DATA(Py_mod_doc, (void*)module_doc), + PySlot_DATA(Py_mod_abi, &abi_info), + + /* The method table */ + PySlot_STATIC_DATA(Py_mod_methods, xx_methods), + + /* exec function to initialize the module (called as part of import + * after the object was added to sys.modules) + */ + PySlot_FUNC(Py_mod_exec, xx_modexec), + + /* Module state and associated functions */ + PySlot_SIZE(Py_mod_state_size, sizeof(xx_state)), + PySlot_FUNC(Py_mod_state_traverse, xx_traverse), + PySlot_FUNC(Py_mod_state_clear, xx_clear), + PySlot_FUNC(Py_mod_state_free, xx_free), + + /* Signal that this module supports being loaded in multiple interpreters + * with separate GILs (global interpreter locks). + * See "Isolating Extension Modules" on how to prepare a module for this: + * https://docs.python.org/3/howto/isolating-extensions.html + */ + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + + /* Signal that this module does not rely on the GIL for its own needs. + * Without this slot, free-threaded builds of CPython will enable + * the GIL when this module is loaded. + */ + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + + PySlot_END }; -/* Export function for the module. *Must* be called PyInit_xx; usually it is - * the only non-`static` object in a module definition. +/* Export function for the module. *Must* be called PyModExport_xx; usually + * it is the only non-`static` object in a module definition. */ -PyMODINIT_FUNC -PyInit_xxlimited(void) +PyMODEXPORT_FUNC +PyModExport_xxlimited(void) { - return PyModuleDef_Init(&xxmodule); + return xx_slots; } diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index b0a682ac4e6bb69..9ef0eac9a924e6c 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -305,7 +305,7 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, #ifdef Py_GIL_DISABLED - // These definitions are in the limited API, but not until 3.13. + // In a free-threaded build, we don't use Limited API. {Py_mod_gil, Py_MOD_GIL_NOT_USED}, #endif {0, NULL} diff --git a/Modules/xxlimited_3_13.c b/Modules/xxlimited_3_13.c new file mode 100644 index 000000000000000..4f100f9150fc2a3 --- /dev/null +++ b/Modules/xxlimited_3_13.c @@ -0,0 +1,499 @@ +/* Use this file as a template to start implementing a module that + also declares object types. All occurrences of 'Xxo' should be changed + to something reasonable for your objects. After that, all other + occurrences of 'xx' should be changed to something reasonable for your + module. If your module is named foo your source file should be named + foo.c or foomodule.c. + + You will probably want to delete all references to 'x_attr' and add + your own types of attributes instead. Maybe you want to name your + local variables other than 'self'. If your object type is needed in + other files, you'll have to create a file "foobarobject.h"; see + floatobject.h for an example. + + This module roughly corresponds to:: + + class Xxo: + """A class that explicitly stores attributes in an internal dict + (to simulate custom attribute handling). + """ + + def __init__(self): + # In the C class, "_x_attr" is not accessible from Python code + self._x_attr = {} + self._x_exports = 0 + + def __getattr__(self, name): + return self._x_attr[name] + + def __setattr__(self, name, value): + self._x_attr[name] = value + + def __delattr__(self, name): + del self._x_attr[name] + + @property + def x_exports(self): + """Return the number of times an internal buffer is exported.""" + # Each Xxo instance has a 10-byte buffer that can be + # accessed via the buffer interface (e.g. `memoryview`). + return self._x_exports + + def demo(o, /): + if isinstance(o, str): + return o + elif isinstance(o, Xxo): + return o + else: + raise Error('argument must be str or Xxo') + + class Error(Exception): + """Exception raised by the xxlimited module""" + + def foo(i: int, j: int, /): + """Return the sum of i and j.""" + # Unlike this pseudocode, the C function will *only* work with + # integers and perform C long int arithmetic + return i + j + + def new(): + return Xxo() + + def Str(str): + # A trivial subclass of a built-in type + pass + */ + +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + +#include "Python.h" +#include + +#define BUFSIZE 10 + +// Module state +typedef struct { + PyObject *Xxo_Type; // Xxo class + PyObject *Error_Type; // Error class +} xx_state; + + +/* Xxo objects */ + +// Instance state +typedef struct { + PyObject_HEAD + PyObject *x_attr; /* Attributes dictionary. + * May be NULL, which acts as an + * empty dict. + */ + char x_buffer[BUFSIZE]; /* buffer for Py_buffer */ + Py_ssize_t x_exports; /* how many buffer are exported */ +} XxoObject; + +#define XxoObject_CAST(op) ((XxoObject *)(op)) +// TODO: full support for type-checking was added in 3.14 (Py_tp_token) +// #define XxoObject_Check(v) Py_IS_TYPE(v, Xxo_Type) + +static XxoObject * +newXxoObject(PyObject *module) +{ + xx_state *state = PyModule_GetState(module); + if (state == NULL) { + return NULL; + } + XxoObject *self; + self = PyObject_GC_New(XxoObject, (PyTypeObject*)state->Xxo_Type); + if (self == NULL) { + return NULL; + } + self->x_attr = NULL; + memset(self->x_buffer, 0, BUFSIZE); + self->x_exports = 0; + return self; +} + +/* Xxo finalization. + * + * Types that store references to other PyObjects generally need to implement + * the GC slots: traverse, clear, dealloc, and (optionally) finalize. + */ + +// traverse: Visit all references from an object, including its type +static int +Xxo_traverse(PyObject *op, visitproc visit, void *arg) +{ + // Visit the type + Py_VISIT(Py_TYPE(op)); + + // Visit the attribute dict + XxoObject *self = XxoObject_CAST(op); + Py_VISIT(self->x_attr); + return 0; +} + +// clear: drop references in order to break all reference cycles +static int +Xxo_clear(PyObject *op) +{ + XxoObject *self = XxoObject_CAST(op); + Py_CLEAR(self->x_attr); + return 0; +} + +// finalize: like clear, but should leave the object in a consistent state. +// Equivalent to `__del__` in Python. +static void +Xxo_finalize(PyObject *op) +{ + XxoObject *self = XxoObject_CAST(op); + Py_CLEAR(self->x_attr); +} + +// dealloc: drop all remaining references and free memory +static void +Xxo_dealloc(PyObject *self) +{ + PyObject_GC_UnTrack(self); + Xxo_finalize(self); + PyTypeObject *tp = Py_TYPE(self); + freefunc free = PyType_GetSlot(tp, Py_tp_free); + free(self); + Py_DECREF(tp); +} + + +/* Xxo attribute handling */ + +// Get an attribute. +static PyObject * +Xxo_getattro(PyObject *op, PyObject *name) +{ + XxoObject *self = XxoObject_CAST(op); + if (self->x_attr != NULL) { + PyObject *v = PyDict_GetItemWithError(self->x_attr, name); + if (v != NULL) { + return Py_NewRef(v); + } + else if (PyErr_Occurred()) { + return NULL; + } + } + // Fall back to generic implementation (this handles special attributes, + // raising AttributeError, etc.) + return PyObject_GenericGetAttr(op, name); +} + +// Set or delete an attribute. +static int +Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) +{ + XxoObject *self = XxoObject_CAST(op); + if (self->x_attr == NULL) { + // prepare the attribute dict + self->x_attr = PyDict_New(); + if (self->x_attr == NULL) { + return -1; + } + } + if (v == NULL) { + // delete an attribute + int rv = PyDict_DelItem(self->x_attr, name); + if (rv < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_SetString(PyExc_AttributeError, + "delete non-existing Xxo attribute"); + return -1; + } + return rv; + } + else { + // set an attribute + return PyDict_SetItem(self->x_attr, name, v); + } +} + +/* Xxo methods: C functions plus a PyMethodDef array that lists them and + * specifies metadata. + */ + +static PyObject * +Xxo_demo(PyObject *op, PyTypeObject *defining_class, + PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (kwnames != NULL && PyObject_Length(kwnames)) { + PyErr_SetString(PyExc_TypeError, "demo() takes no keyword arguments"); + return NULL; + } + if (nargs != 1) { + PyErr_SetString(PyExc_TypeError, "demo() takes exactly 1 argument"); + return NULL; + } + + PyObject *o = args[0]; + + /* Test if the argument is "str" */ + if (PyUnicode_Check(o)) { + return Py_NewRef(o); + } + + /* test if the argument is of the Xxo class */ + if (PyObject_TypeCheck(o, defining_class)) { + return Py_NewRef(o); + } + + return Py_NewRef(Py_None); +} + +static PyMethodDef Xxo_methods[] = { + {"demo", _PyCFunction_CAST(Xxo_demo), + METH_METHOD | METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("demo(o) -> o")}, + {NULL, NULL} /* sentinel */ +}; + +/* Xxo buffer interface: C functions later referenced from PyType_Slot array. + * Other interfaces (e.g. for sequence-like or number-like types) are defined + * similarly. + */ + +static int +Xxo_getbuffer(PyObject *op, Py_buffer *view, int flags) +{ + XxoObject *self = XxoObject_CAST(op); + int res = PyBuffer_FillInfo(view, op, + (void *)self->x_buffer, BUFSIZE, + 0, flags); + if (res == 0) { + self->x_exports++; + } + return res; +} + +static void +Xxo_releasebuffer(PyObject *op, Py_buffer *Py_UNUSED(view)) +{ + XxoObject *self = XxoObject_CAST(op); + self->x_exports--; +} + +static PyObject * +Xxo_get_x_exports(PyObject *op, void *Py_UNUSED(closure)) +{ + XxoObject *self = XxoObject_CAST(op); + return PyLong_FromSsize_t(self->x_exports); +} + +/* Xxo type definition */ + +PyDoc_STRVAR(Xxo_doc, + "A class that explicitly stores attributes in an internal dict"); + +static PyGetSetDef Xxo_getsetlist[] = { + {"x_exports", Xxo_get_x_exports, NULL, NULL}, + {NULL}, +}; + + +static PyType_Slot Xxo_Type_slots[] = { + {Py_tp_doc, (char *)Xxo_doc}, + {Py_tp_traverse, Xxo_traverse}, + {Py_tp_clear, Xxo_clear}, + {Py_tp_finalize, Xxo_finalize}, + {Py_tp_dealloc, Xxo_dealloc}, + {Py_tp_getattro, Xxo_getattro}, + {Py_tp_setattro, Xxo_setattro}, + {Py_tp_methods, Xxo_methods}, + {Py_bf_getbuffer, Xxo_getbuffer}, + {Py_bf_releasebuffer, Xxo_releasebuffer}, + {Py_tp_getset, Xxo_getsetlist}, + {0, 0}, /* sentinel */ +}; + +static PyType_Spec Xxo_Type_spec = { + .name = "xxlimited_3_13.Xxo", + .basicsize = sizeof(XxoObject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .slots = Xxo_Type_slots, +}; + + +/* Str type definition*/ + +static PyType_Slot Str_Type_slots[] = { + // slots array intentionally kept empty + {0, 0}, /* sentinel */ +}; + +static PyType_Spec Str_Type_spec = { + .name = "xxlimited_3_13.Str", + .basicsize = 0, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = Str_Type_slots, +}; + + +/* Function of two integers returning integer (with C "long int" arithmetic) */ + +PyDoc_STRVAR(xx_foo_doc, +"foo(i,j)\n\ +\n\ +Return the sum of i and j."); + +static PyObject * +xx_foo(PyObject *module, PyObject *args) +{ + long i, j; + long res; + if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) + return NULL; + res = i+j; /* XXX Do something here */ + return PyLong_FromLong(res); +} + + +/* Function of no arguments returning new Xxo object */ + +static PyObject * +xx_new(PyObject *module, PyObject *Py_UNUSED(unused)) +{ + XxoObject *rv; + + rv = newXxoObject(module); + if (rv == NULL) + return NULL; + return (PyObject *)rv; +} + + + +/* List of functions defined in the module */ + +static PyMethodDef xx_methods[] = { + {"foo", xx_foo, METH_VARARGS, + xx_foo_doc}, + {"new", xx_new, METH_NOARGS, + PyDoc_STR("new() -> new Xx object")}, + {NULL, NULL} /* sentinel */ +}; + + +/* The module itself */ + +PyDoc_STRVAR(module_doc, +"This is a template module just for instruction."); + +static int +xx_modexec(PyObject *m) +{ + xx_state *state = PyModule_GetState(m); + + state->Error_Type = PyErr_NewException("xxlimited_3_13.Error", NULL, NULL); + if (state->Error_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject*)state->Error_Type) < 0) { + return -1; + } + + state->Xxo_Type = PyType_FromModuleAndSpec(m, &Xxo_Type_spec, NULL); + if (state->Xxo_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject*)state->Xxo_Type) < 0) { + return -1; + } + + // Add the Str type. It is not needed from C code, so it is only + // added to the module dict. + // It does not inherit from "object" (PyObject_Type), but from "str" + // (PyUnincode_Type). + PyObject *Str_Type = PyType_FromModuleAndSpec( + m, &Str_Type_spec, (PyObject *)&PyUnicode_Type); + if (Str_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject*)Str_Type) < 0) { + return -1; + } + Py_DECREF(Str_Type); + + return 0; +} + +static PyModuleDef_Slot xx_slots[] = { + + /* exec function to initialize the module (called as part of import + * after the object was added to sys.modules) + */ + {Py_mod_exec, xx_modexec}, + + /* Signal that this module supports being loaded in multiple interpreters + * with separate GILs (global interpreter locks). + * See "Isolating Extension Modules" on how to prepare a module for this: + * https://docs.python.org/3/howto/isolating-extensions.html + */ + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + + /* Signal that this module does not rely on the GIL for its own needs. + * Without this slot, free-threaded builds of CPython will enable + * the GIL when this module is loaded. + */ + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + + {0, NULL} +}; + +// Module finalization: modules that hold references in their module state +// need to implement the fullowing GC hooks. They're similar to the ones for +// types (see "Xxo finalization"). + +static int +xx_traverse(PyObject *module, visitproc visit, void *arg) +{ + xx_state *state = PyModule_GetState(module); + Py_VISIT(state->Xxo_Type); + Py_VISIT(state->Error_Type); + return 0; +} + +static int +xx_clear(PyObject *module) +{ + xx_state *state = PyModule_GetState(module); + Py_CLEAR(state->Xxo_Type); + Py_CLEAR(state->Error_Type); + return 0; +} + +static void +xx_free(void *module) +{ + // allow xx_modexec to omit calling xx_clear on error + (void)xx_clear((PyObject *)module); +} + +static struct PyModuleDef xxmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "xxlimited_3_13", + .m_doc = module_doc, + .m_size = sizeof(xx_state), + .m_methods = xx_methods, + .m_slots = xx_slots, + .m_traverse = xx_traverse, + .m_clear = xx_clear, + .m_free = xx_free, +}; + + +/* Export function for the module. *Must* be called PyInit_xx; usually it is + * the only non-`static` object in a module definition. + */ + +PyMODINIT_FUNC +PyInit_xxlimited_3_13(void) +{ + return PyModuleDef_Init(&xxmodule); +} diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 8a9d1b133affb3e..2d694922557429a 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -11,6 +11,7 @@ #include "pycore_global_objects.h"// _Py_GET_GLOBAL_OBJECT() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_long.h" // _PyLong_DigitValue +#include "pycore_list.h" // _PyList_GetItemRef #include "pycore_object.h" // _PyObject_GC_TRACK #include "pycore_pymem.h" // PYMEM_CLEANBYTE #include "pycore_strhex.h" // _Py_strhex_with_sep() @@ -2991,8 +2992,10 @@ _PyBytes_FromList(PyObject *x) size = _PyBytesWriter_GetAllocated(writer); for (Py_ssize_t i = 0; i < PyList_GET_SIZE(x); i++) { - PyObject *item = PyList_GET_ITEM(x, i); - Py_INCREF(item); + PyObject *item = _PyList_GetItemRef((PyListObject *)x, i); + if (item == NULL) { + goto error; + } Py_ssize_t value = PyNumber_AsSsize_t(item, NULL); Py_DECREF(item); if (value == -1 && PyErr_Occurred()) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 42bc63acd9049ce..3830fedd42bd273 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -900,7 +900,7 @@ free_values(PyDictValues *values, bool use_qsbr) static inline PyObject * new_dict_impl(PyDictObject *mp, PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, - int free_values_on_failure) + int free_values_on_failure, int frozendict) { assert(keys != NULL); if (mp == NULL) { @@ -915,6 +915,9 @@ new_dict_impl(PyDictObject *mp, PyDictKeysObject *keys, mp->ma_values = values; mp->ma_used = used; mp->_ma_watcher_tag = 0; + if (frozendict) { + ((PyFrozenDictObject *)mp)->ma_hash = -1; + } ASSERT_CONSISTENT(mp); _PyObject_GC_TRACK(mp); return (PyObject *)mp; @@ -931,7 +934,7 @@ new_dict(PyDictKeysObject *keys, PyDictValues *values, } assert(mp == NULL || Py_IS_TYPE(mp, &PyDict_Type)); - return new_dict_impl(mp, keys, values, used, free_values_on_failure); + return new_dict_impl(mp, keys, values, used, free_values_on_failure, 0); } /* Consumes a reference to the keys object */ @@ -940,7 +943,7 @@ new_frozendict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free_values_on_failure) { PyDictObject *mp = PyObject_GC_New(PyDictObject, &PyFrozenDict_Type); - return new_dict_impl(mp, keys, values, used, free_values_on_failure); + return new_dict_impl(mp, keys, values, used, free_values_on_failure, 1); } static PyObject * @@ -3080,10 +3083,12 @@ clear_lock_held(PyObject *op) set_keys(mp, Py_EMPTY_KEYS); n = oldkeys->dk_nentries; for (i = 0; i < n; i++) { - Py_CLEAR(oldvalues->values[i]); + PyObject *tmp = oldvalues->values[i]; + FT_ATOMIC_STORE_PTR_RELEASE(oldvalues->values[i], NULL); + Py_XDECREF(tmp); } free_values(oldvalues, IS_DICT_SHARED(mp)); - dictkeys_decref(oldkeys, false); + dictkeys_decref(oldkeys, IS_DICT_SHARED(mp)); } ASSERT_CONSISTENT(mp); } @@ -8015,13 +8020,19 @@ validate_watcher_id(PyInterpreterState *interp, int watcher_id) PyErr_Format(PyExc_ValueError, "Invalid dict watcher ID %d", watcher_id); return -1; } - if (!interp->dict_state.watchers[watcher_id]) { + PyDict_WatchCallback cb = FT_ATOMIC_LOAD_PTR_RELAXED( + interp->dict_state.watchers[watcher_id]); + if (cb == NULL) { PyErr_Format(PyExc_ValueError, "No dict watcher set for ID %d", watcher_id); return -1; } return 0; } +// In free-threaded builds, Add/Clear serialize on watcher_mutex and publish +// callbacks with release stores. SendEvent reads them lock-free using +// acquire loads. + int PyDict_Watch(int watcher_id, PyObject* dict) { @@ -8033,7 +8044,8 @@ PyDict_Watch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - FT_ATOMIC_OR_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, (1LL << watcher_id)); + FT_ATOMIC_OR_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, + 1ULL << watcher_id); return 0; } @@ -8048,36 +8060,48 @@ PyDict_Unwatch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - FT_ATOMIC_AND_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, ~(1LL << watcher_id)); + FT_ATOMIC_AND_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, + ~(1ULL << watcher_id)); return 0; } int PyDict_AddWatcher(PyDict_WatchCallback callback) { + int watcher_id = -1; PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_MUTEX_LOCK_FLAGS(&interp->dict_state.watcher_mutex, + _Py_LOCK_DONT_DETACH); /* Some watchers are reserved for CPython, start at the first available one */ for (int i = FIRST_AVAILABLE_WATCHER; i < DICT_MAX_WATCHERS; i++) { if (!interp->dict_state.watchers[i]) { - interp->dict_state.watchers[i] = callback; - return i; + FT_ATOMIC_STORE_PTR_RELEASE(interp->dict_state.watchers[i], callback); + watcher_id = i; + goto done; } } - PyErr_SetString(PyExc_RuntimeError, "no more dict watcher IDs available"); - return -1; +done: + FT_MUTEX_UNLOCK(&interp->dict_state.watcher_mutex); + return watcher_id; } int PyDict_ClearWatcher(int watcher_id) { + int res = 0; PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_MUTEX_LOCK_FLAGS(&interp->dict_state.watcher_mutex, + _Py_LOCK_DONT_DETACH); if (validate_watcher_id(interp, watcher_id)) { - return -1; + res = -1; + goto done; } - interp->dict_state.watchers[watcher_id] = NULL; - return 0; + FT_ATOMIC_STORE_PTR_RELEASE(interp->dict_state.watchers[watcher_id], NULL); +done: + FT_MUTEX_UNLOCK(&interp->dict_state.watcher_mutex); + return res; } static const char * @@ -8102,7 +8126,8 @@ _PyDict_SendEvent(int watcher_bits, PyInterpreterState *interp = _PyInterpreterState_GET(); for (int i = 0; i < DICT_MAX_WATCHERS; i++) { if (watcher_bits & 1) { - PyDict_WatchCallback cb = interp->dict_state.watchers[i]; + PyDict_WatchCallback cb = FT_ATOMIC_LOAD_PTR_ACQUIRE( + interp->dict_state.watchers[i]); if (cb && (cb(event, (PyObject*)mp, key, value) < 0)) { // We don't want to resurrect the dict by potentially having an // unraisablehook keep a reference to it, so we don't pass the @@ -8205,6 +8230,39 @@ _shuffle_bits(Py_uhash_t h) return ((h ^ 89869747UL) ^ (h << 16)) * 3644798167UL; } +// Compute hash((key, value)). +// Code copied from tuple_hash(). +static Py_hash_t +frozendict_pair_hash(Py_hash_t key_hash, PyObject *value) +{ + assert(key_hash != -1); + + const Py_ssize_t len = 2; + Py_uhash_t acc = _PyTuple_HASH_XXPRIME_5; + + Py_uhash_t lane = key_hash; + acc += lane * _PyTuple_HASH_XXPRIME_2; + acc = _PyTuple_HASH_XXROTATE(acc); + acc *= _PyTuple_HASH_XXPRIME_1; + + lane = PyObject_Hash(value); + if (lane == (Py_uhash_t)-1) { + return -1; + } + acc += lane * _PyTuple_HASH_XXPRIME_2; + acc = _PyTuple_HASH_XXROTATE(acc); + acc *= _PyTuple_HASH_XXPRIME_1; + + /* Add input length, mangled to keep the historical value of hash(()). */ + acc += len ^ (_PyTuple_HASH_XXPRIME_5 ^ 3527539UL); + + if (acc == (Py_uhash_t)-1) { + acc = 1546275796; + } + return acc; +} + + // Code copied from frozenset_hash() static Py_hash_t frozendict_hash(PyObject *op) @@ -8218,20 +8276,15 @@ frozendict_hash(PyObject *op) PyDictObject *mp = _PyAnyDict_CAST(op); Py_uhash_t hash = 0; - PyObject *key, *value; // borrowed refs + PyObject *value; // borrowed ref Py_ssize_t pos = 0; - while (PyDict_Next(op, &pos, &key, &value)) { - Py_hash_t key_hash = PyObject_Hash(key); - if (key_hash == -1) { - return -1; - } - hash ^= _shuffle_bits(key_hash); - - Py_hash_t value_hash = PyObject_Hash(value); - if (value_hash == -1) { + Py_hash_t key_hash; + while (_PyDict_Next(op, &pos, NULL, &value, &key_hash)) { + Py_hash_t pair_hash = frozendict_pair_hash(key_hash, value); + if (pair_hash == -1) { return -1; } - hash ^= _shuffle_bits(value_hash); + hash ^= _shuffle_bits(pair_hash); } /* Factor in the number of active entries */ diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index e3bc8eb2739e3fa..9c797e8dd6fd2cc 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -412,6 +412,9 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje self); } item = _unpack_args(item); + if (item == NULL) { + return NULL; + } for (Py_ssize_t i = 0; i < nparams; i++) { PyObject *param = PyTuple_GET_ITEM(parameters, i); PyObject *prepare, *tmp; diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index 451f335e033f16b..fa1eb25047d9617 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -135,7 +135,7 @@ PyDoc_STRVAR(lazy_import_doc, "lazy_import(builtins, name, fromlist=None, /)\n" "--\n" "\n" -"Represents a deferred import that will be resolved on first use.\n" +"Represents a lazy import that will be resolved on first use.\n" "\n" "Instances of this object accessed from the global scope will be\n" "automatically imported based upon their name and then replaced with\n" diff --git a/Objects/listobject.c b/Objects/listobject.c index 10e25bbdcdcb6c5..c76721c5d2ac9ea 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3793,16 +3793,13 @@ list_ass_subscript_lock_held(PyObject *_self, PyObject *item, PyObject *value) lim = Py_SIZE(self) - cur - 1; } - memmove(self->ob_item + cur - i, - self->ob_item + cur + 1, - lim * sizeof(PyObject *)); + ptr_wise_atomic_memmove(self, self->ob_item + cur - i, + self->ob_item + cur + 1, lim); } cur = start + (size_t)slicelength * step; if (cur < (size_t)Py_SIZE(self)) { - memmove(self->ob_item + cur - slicelength, - self->ob_item + cur, - (Py_SIZE(self) - cur) * - sizeof(PyObject *)); + ptr_wise_atomic_memmove(self, self->ob_item + cur - slicelength, + self->ob_item + cur, Py_SIZE(self) - cur); } Py_SET_SIZE(self, Py_SIZE(self) - slicelength); diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 900db864621a84c..9d1ca633780f92c 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1629,11 +1629,7 @@ memory_getbuf(PyObject *_self, Py_buffer *view, int flags) view->obj = Py_NewRef(self); -#ifdef Py_GIL_DISABLED - _Py_atomic_add_ssize(&self->exports, 1); -#else - self->exports++; -#endif + FT_ATOMIC_ADD_SSIZE(self->exports, 1); return 0; } @@ -1642,11 +1638,7 @@ static void memory_releasebuf(PyObject *_self, Py_buffer *view) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; -#ifdef Py_GIL_DISABLED - _Py_atomic_add_ssize(&self->exports, -1); -#else - self->exports--; -#endif + FT_ATOMIC_ADD_SSIZE(self->exports, -1); return; /* PyBuffer_Release() decrements view->obj after this function returns. */ } @@ -2434,9 +2426,9 @@ memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, // Prevent 'self' from being freed if computing len(sep) mutates 'self' // in _Py_strhex_with_sep(). // See: https://github.com/python/cpython/issues/143195. - self->exports++; + FT_ATOMIC_ADD_SSIZE(self->exports, 1); PyObject *ret = _Py_strhex_with_sep(src->buf, src->len, sep, bytes_per_sep); - self->exports--; + FT_ATOMIC_ADD_SSIZE(self->exports, -1); return ret; } @@ -3363,9 +3355,9 @@ memory_hash(PyObject *_self) if (view->obj != NULL) { // Prevent 'self' from being freed when computing the item's hash. // See https://github.com/python/cpython/issues/142664. - self->exports++; + FT_ATOMIC_ADD_SSIZE(self->exports, 1); Py_hash_t h = PyObject_Hash(view->obj); - self->exports--; + FT_ATOMIC_ADD_SSIZE(self->exports, -1); if (h == -1) { /* Keep the original error message */ return -1; diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index b7d2e5ffde4fe7d..f447403ef31b43a 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1299,6 +1299,33 @@ _PyModule_IsPossiblyShadowing(PyObject *origin) return result; } +// Check if `name` is a lazily pending submodule of module `m`. +// Returns a new reference on success, or NULL with no error set. +static PyObject * +try_load_lazy_submodule(PyModuleObject *m, PyObject *name) +{ + PyObject *mod_name; + int rc = PyDict_GetItemRef(m->md_dict, &_Py_ID(__name__), &mod_name); + if (rc <= 0) { + return NULL; + } + if (!PyUnicode_Check(mod_name)) { + Py_DECREF(mod_name); + return NULL; + } + PyObject *result = _PyImport_TryLoadLazySubmodule(mod_name, name); + Py_DECREF(mod_name); + if (result == NULL) { + PyErr_Clear(); + return NULL; + } + if (PyDict_SetItem(m->md_dict, name, result) < 0) { + Py_DECREF(result); + return NULL; + } + return result; +} + PyObject* _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) { @@ -1307,6 +1334,25 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) attr = _PyObject_GenericGetAttrWithDict((PyObject *)m, name, NULL, suppress); if (attr) { if (PyLazyImport_CheckExact(attr)) { + // gh-144957: Module __getattr__ should get a chance to provide + // the attribute before resolving a lazy import placeholder. + if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { + Py_DECREF(attr); + return NULL; + } + if (getattr) { + PyObject *result = PyObject_CallOneArg(getattr, name); + Py_DECREF(getattr); + if (result != NULL) { + Py_DECREF(attr); + return result; + } + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + Py_DECREF(attr); + return NULL; + } + PyErr_Clear(); + } PyObject *new_value = _PyImport_LoadLazyImportTstate( PyThreadState_GET(), attr); if (new_value == NULL) { @@ -1344,6 +1390,13 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) PyErr_Clear(); } assert(m->md_dict != NULL); + attr = try_load_lazy_submodule(m, name); + if (attr != NULL) { + return attr; + } + if (PyErr_Occurred()) { + return NULL; + } if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { return NULL; } diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 753c270f525976f..6de9487432a3984 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -363,6 +363,9 @@ tuple_repr(PyObject *self) https://github.com/Cyan4973/xxHash/blob/master/doc/xxhash_spec.md The constants for the hash function are defined in pycore_tuple.h. + + If you update this code, update also frozendict_pair_hash() which copied + this code. */ static Py_hash_t diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4f43747ba83fd9d..7cca137f74be58f 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4841,6 +4841,18 @@ type_new_set_attrs(const type_new_ctx *ctx, PyTypeObject *type) if (type_new_set_classdictcell(dict) < 0) { return -1; } + +#ifdef Py_GIL_DISABLED + // enable deferred reference counting on functions and descriptors + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(dict, &pos, &key, &value)) { + if (PyFunction_Check(value) || Py_TYPE(value)->tp_descr_get != NULL) { + PyUnstable_Object_EnableDeferredRefcount(value); + } + } +#endif + return 0; } @@ -6746,12 +6758,11 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_MANAGED_DICT)); #ifdef Py_GIL_DISABLED - // gh-139103: Enable deferred refcounting for functions assigned - // to type objects. This is important for `dataclass.__init__`, - // which is generated dynamically. - if (value != NULL && - PyFunction_Check(value) && - !_PyObject_HasDeferredRefcount(value)) + // gh-139103: Enable deferred refcounting for functions and descriptors + // assigned to type objects. This is important for `dataclass.__init__`, + // which is generated dynamically, and for descriptor scaling on + // free-threaded builds. + if (value != NULL && (PyFunction_Check(value) || Py_TYPE(value)->tp_descr_get != NULL)) { PyUnstable_Object_EnableDeferredRefcount(value); } @@ -11079,14 +11090,22 @@ slot_tp_iternext(PyObject *self) return vectorcall_method(&_Py_ID(__next__), stack, 1); } +int +_PyType_HasSlotTpIternext(PyTypeObject *type) +{ + return type->tp_iternext == slot_tp_iternext; +} + static PyObject * slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) { PyTypeObject *tp = Py_TYPE(self); - PyObject *get; - - get = _PyType_LookupRef(tp, &_Py_ID(__get__)); - if (get == NULL) { + PyThreadState *tstate = _PyThreadState_GET(); + _PyCStackRef cref; + _PyThreadState_PushCStackRef(tstate, &cref); + _PyType_LookupStackRefAndVersion(tp, &_Py_ID(__get__), &cref.ref); + if (PyStackRef_IsNull(cref.ref)) { + _PyThreadState_PopCStackRef(tstate, &cref); #ifndef Py_GIL_DISABLED /* Avoid further slowdowns */ if (tp->tp_descr_get == slot_tp_descr_get) @@ -11098,9 +11117,10 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) obj = Py_None; if (type == NULL) type = Py_None; + PyObject *get = PyStackRef_AsPyObjectBorrow(cref.ref); PyObject *stack[3] = {self, obj, type}; PyObject *res = PyObject_Vectorcall(get, stack, 3, NULL); - Py_DECREF(get); + _PyThreadState_PopCStackRef(tstate, &cref); return res; } diff --git a/PC/layout/main.py b/PC/layout/main.py index 3566b8bd873874c..f70a26b2b296591 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -22,6 +22,7 @@ __path__ = [str(Path(__file__).resolve().parent)] from .support.appxmanifest import * +from .support.builddetails import * from .support.catalog import * from .support.constants import * from .support.filesets import * @@ -32,7 +33,8 @@ from .support.pymanager import * from .support.nuspec import * -TEST_PYDS_ONLY = FileStemSet("xxlimited", "xxlimited_35", "_ctypes_test", "_test*") +TEST_PYDS_ONLY = FileStemSet("xxlimited", "xxlimited_3_13", "xxlimited_35", + "_ctypes_test", "_test*") TEST_DLLS_ONLY = set() TEST_DIRS_ONLY = FileNameSet("test", "tests") @@ -316,6 +318,9 @@ def _c(d): for dest, src in get_appx_layout(ns): yield dest, src + for dest, src in get_builddetails(ns): + yield dest, src + if ns.include_cat: if ns.flat_dlls: yield ns.include_cat.name, ns.include_cat diff --git a/PC/layout/support/builddetails.py b/PC/layout/support/builddetails.py new file mode 100644 index 000000000000000..6ef860eeb043545 --- /dev/null +++ b/PC/layout/support/builddetails.py @@ -0,0 +1,119 @@ +import io +import json +from . import constants + +_LEVELS = { + 0xA0: "alpha", + 0xB0: "beta", + 0xC0: "candidate", + 0xF0: "final", +} + + +_TEMPLATE = { + "schema_version": "1.0", + "base_prefix": ".", + "base_interpreter": "python.exe", + "platform": None, # Set later + "language": { + "version": f"{constants.VER_MAJOR}.{constants.VER_MINOR}", + "version_info": { + "major": constants.VER_MAJOR, + "minor": constants.VER_MINOR, + "micro": constants.VER_MICRO, + "releaselevel": _LEVELS.get(constants.VER_FIELD4 & 0xF0, "final"), + "serial": constants.VER_FIELD4 & 0x0F, + }, + }, + "implementation": { + "name": "cpython", + "cache_tag": f"cpython-{constants.VER_MAJOR}{constants.VER_MINOR}", + "version": { + "major": constants.VER_MAJOR, + "minor": constants.VER_MINOR, + "micro": constants.VER_MICRO, + "releaselevel": _LEVELS.get(constants.VER_FIELD4 & 0xF0, "final"), + "serial": constants.VER_FIELD4 & 0x0F, + }, + "hexversion": constants.VER_HEXVERSION, + }, + "abi": { + "flags": [], + "extension_suffix": ".pyd", + "stable_abi_suffix": ".pyd", + }, + "suffixes": { + "source": [".py", ".pyw"], + "bytecode": [".pyc"], + "extensions": [".pyd"], + }, + "libpython": { + "dynamic": constants.PYTHON_DLL_NAME, + "dynamic_stableabi": constants.PYTHON_STABLE_DLL_NAME, + "link_extensions": True, + }, + "c_api": { + }, +} + + +def _with_d(path): + pre, sep, post = path.partition(".") + return pre + "_d" + sep + post + + +def _add_d(data, *args): + for a in args[:-1]: + data = data[a] + a = args[-1] + v = data[a] + if isinstance(v, list): + data[a] = [_with_d(i) for i in data[a]] + else: + data[a] = _with_d(data[a]) + + +def get_builddetails(ns): + if not ns.include_builddetails_json: + return + + details = dict(_TEMPLATE) + + plat = { + "win32": "win32", + "amd64": "win-amd64", + "arm64": "win-arm64", + }.get(ns.arch, ns.arch) + + pyd_abi_flags = "" + if ns.include_freethreaded: + details["abi"]["flags"].append("t") + pyd_abi_flags += "t" + if ns.debug: + details["abi"]["flags"].append("d") + + norm_plat = plat.replace("-", "_") + ext_suffix = f".cp{constants.VER_MAJOR}{constants.VER_MINOR}{pyd_abi_flags}-{norm_plat}.pyd" + details["abi"]["extension_suffix"] = ext_suffix + details["suffixes"]["extensions"].insert(0, ext_suffix) + + details["platform"] = plat + + if ns.include_dev: + details["c_api"]["headers"] = "Include" + + if ns.include_freethreaded: + details["libpython"]["dynamic"] = constants.FREETHREADED_PYTHON_DLL_NAME + details["libpython"]["dynamic_stableabi"] = constants.FREETHREADED_PYTHON_STABLE_DLL_NAME + + if ns.debug: + _add_d(details, "base_interpreter") + _add_d(details, "abi", "stable_abi_suffix") + _add_d(details, "abi", "extension_suffix") + _add_d(details, "suffixes", "extensions") + _add_d(details, "libpython", "dynamic") + _add_d(details, "libpython", "dynamic_stableabi") + + buffer = io.StringIO() + json.dump(details, buffer, indent=2) + yield "build-details.json", ("build-details.json", buffer.getvalue().encode()) diff --git a/PC/layout/support/constants.py b/PC/layout/support/constants.py index 6b8c915e519743f..cb16f534685c8f8 100644 --- a/PC/layout/support/constants.py +++ b/PC/layout/support/constants.py @@ -23,6 +23,14 @@ def _unpack_hexversion(): return _read_patchlevel_version(pathlib.Path(os.getenv("PYTHONINCLUDE"))) except OSError: pass + # Manual search for a '-s ` arument + try: + src = sys.argv[sys.argv.index("-s") + 1] + return _read_patchlevel_version(pathlib.Path(src) / "Include") + except (IndexError, ValueError): + pass + except OSError: + pass return struct.pack(">i", sys.hexversion) @@ -68,6 +76,7 @@ def check_patchlevel_version(sources): VER_MAJOR, VER_MINOR, VER_MICRO, VER_FIELD4 = _unpack_hexversion() +VER_HEXVERSION = (VER_MAJOR << 24) | (VER_MINOR << 16) | (VER_MICRO << 8) | (VER_FIELD4) VER_SUFFIX = _get_suffix(VER_FIELD4) VER_FIELD3 = VER_MICRO << 8 | VER_FIELD4 VER_DOT = "{}.{}".format(VER_MAJOR, VER_MINOR) diff --git a/PC/layout/support/options.py b/PC/layout/support/options.py index e8c393385425e72..f67d8ba04d90703 100644 --- a/PC/layout/support/options.py +++ b/PC/layout/support/options.py @@ -39,6 +39,7 @@ def public(f): "install-json": {"help": "a PyManager __install__.json file"}, "install-embed-json": {"help": "a PyManager __install__.json file for embeddable distro"}, "install-test-json": {"help": "a PyManager __install__.json for the test distro"}, + "builddetails-json": {"help": "a PEP 739 build-details.json"}, } @@ -69,6 +70,7 @@ def public(f): "props", "nuspec", "alias", + "builddetails-json", ], }, "iot": {"help": "Windows IoT Core", "options": ["alias", "stable", "pip"]}, @@ -85,6 +87,7 @@ def public(f): "symbols", "html-doc", "alias", + "builddetails-json", ], }, "embed": { @@ -96,6 +99,7 @@ def public(f): "flat-dlls", "underpth", "precompile", + "builddetails-json", ], }, "pymanager": { @@ -108,7 +112,9 @@ def public(f): "venv", "dev", "html-doc", + "alias", "install-json", + "builddetails-json", ], }, "pymanager-test": { @@ -123,7 +129,9 @@ def public(f): "html-doc", "symbols", "tests", + "alias", "install-test-json", + "builddetails-json", ], }, } diff --git a/PC/layout/support/pymanager.py b/PC/layout/support/pymanager.py index 831d49ea3f9b46f..f6316e0295c74af 100644 --- a/PC/layout/support/pymanager.py +++ b/PC/layout/support/pymanager.py @@ -66,8 +66,9 @@ def calculate_install_json(ns, *, for_embed=False, for_test=False): if ns.include_freethreaded: # Free-threaded distro comes with a tag suffix TAG_SUFFIX = "t" - TARGET = f"python{VER_MAJOR}.{VER_MINOR}t.exe" - TARGETW = f"pythonw{VER_MAJOR}.{VER_MINOR}t.exe" + if not ns.include_alias: + TARGET = f"python{VER_MAJOR}.{VER_MINOR}t.exe" + TARGETW = f"pythonw{VER_MAJOR}.{VER_MINOR}t.exe" DISPLAY_TAGS.append("free-threaded") FILE_SUFFIX = f"t-{ns.arch}" diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 405285b65dd270a..368bc489bfa9680 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -57,8 +57,8 @@ if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.5.6 set libraries=%libraries% mpdecimal-4.0.0 set libraries=%libraries% sqlite-3.50.4.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.15.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-9.0.3.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-9.0.3.1 set libraries=%libraries% xz-5.8.1.1 set libraries=%libraries% zlib-ng-2.2.4 set libraries=%libraries% zstd-1.5.7 @@ -80,7 +80,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.5.6 -if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 +if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-9.0.3.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-21.1.4.0 diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index bb7d8042176d8f1..9d077bbd3f0ba27 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -84,6 +84,7 @@ + false diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index 70dabaa3c8bc027..417ede34c54af3a 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -135,6 +135,14 @@ set PYTHONPATH=$(PySourcePath)Lib "$(OutDir)$(PyExeName)$(PyDebugExt).exe" "$(PySourcePath)PC\validate_ucrtbase.py" $(UcrtName)' ContinueOnError="true" /> + + + + <_Content>@rem This script invokes the most recently built Python with all arguments diff --git a/PCbuild/pythonw.vcxproj b/PCbuild/pythonw.vcxproj index c6a5b8ce90a0d9b..244cdf622ad915c 100644 --- a/PCbuild/pythonw.vcxproj +++ b/PCbuild/pythonw.vcxproj @@ -115,4 +115,12 @@ + + + + \ No newline at end of file diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index c291b7f86325f2f..6aecbfff182dcb4 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -168,8 +168,9 @@ xxlimited builds an example module that makes use of the PEP 384 Stable ABI, see Modules\xxlimited.c xxlimited_35 - ditto for testing the Python 3.5 stable ABI, see - Modules\xxlimited_35.c +xxlimited_3_13 + ditto for testing older Limited API, see + Modules\xxlimited_*.c The following sub-projects are for individual modules of the standard library which are implemented in C; each one builds a DLL (renamed to @@ -246,7 +247,7 @@ _sqlite3 https://www.sqlite.org/ _tkinter - Wraps version 8.6.15 of the Tk windowing system, which is downloaded + Wraps version 9.0.3 of the Tk windowing system, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index a1da1155b881fd7..28e8c0db4d1eafd 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -2,7 +2,7 @@ - 8.6.15.0 + 9.0.3.0 $(TclVersion) $([System.Version]::Parse($(TclVersion)).Major) $([System.Version]::Parse($(TclVersion)).Minor) @@ -12,7 +12,9 @@ $([System.Version]::Parse($(TkVersion)).Minor) $([System.Version]::Parse($(TkVersion)).Build) $([System.Version]::Parse($(TkVersion)).Revision) - $(ExternalsDir)tcl-core-$(TclVersion)\ + + $(ExternalsDir)tcl-core-$(TclVersion)\ + $(ExternalsDir)tcl-$(TclVersion)\ $(ExternalsDir)tk-$(TkVersion)\ $(ExternalsDir)tcltk-$(TclVersion)\$(ArchName)\ t diff --git a/PCbuild/venvlauncher.vcxproj b/PCbuild/venvlauncher.vcxproj index abaf3a979af2681..a2e8ffa82b10eb7 100644 --- a/PCbuild/venvlauncher.vcxproj +++ b/PCbuild/venvlauncher.vcxproj @@ -89,10 +89,13 @@ - + + $(PyExeName)$(PyDebugExt).exe + $(PyExeName)$(MajorVersionNumber).$(MinorVersionNumber)t$(PyDebugExt).exe + - EXENAME=L"$(PyExeName)$(PyDebugExt).exe";_CONSOLE;%(PreprocessorDefinitions) + EXENAME=L"$(ExeName)";_CONSOLE;%(PreprocessorDefinitions) MultiThreaded diff --git a/PCbuild/venvwlauncher.vcxproj b/PCbuild/venvwlauncher.vcxproj index c58280deb8abeb3..f2aaf83fe2b3785 100644 --- a/PCbuild/venvwlauncher.vcxproj +++ b/PCbuild/venvwlauncher.vcxproj @@ -89,10 +89,13 @@ - + + $(PyWExeName)$(PyDebugExt).exe + $(PyWExeName)$(MajorVersionNumber).$(MinorVersionNumber)t$(PyDebugExt).exe + - EXENAME=L"$(PyWExeName)$(PyDebugExt).exe";_WINDOWS;%(PreprocessorDefinitions) + EXENAME=L"$(ExeName)";_WINDOWS;%(PreprocessorDefinitions) MultiThreaded diff --git a/PCbuild/xxlimited_3_13.vcxproj b/PCbuild/xxlimited_3_13.vcxproj new file mode 100644 index 000000000000000..7a9760fd43121ef --- /dev/null +++ b/PCbuild/xxlimited_3_13.vcxproj @@ -0,0 +1,111 @@ + + + + + Debug + ARM + + + Debug + ARM64 + + + Debug + Win32 + + + Debug + x64 + + + PGInstrument + ARM + + + PGInstrument + ARM64 + + + PGInstrument + Win32 + + + PGInstrument + x64 + + + PGUpdate + ARM + + + PGUpdate + ARM64 + + + PGUpdate + Win32 + + + PGUpdate + x64 + + + Release + ARM + + + Release + ARM64 + + + Release + Win32 + + + Release + x64 + + + + {fb868ea7-f93a-4d9b-be78-ca4e9ba14fff} + xxlimited_3_13 + Win32Proj + + + + + DynamicLibrary + NotSet + false + + + + $(PyStdlibPydExt) + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + + + + wsock32.lib;%(AdditionalDependencies) + + + + + + + + {885d4898-d08d-4091-9c40-c700cfe3fc5a} + + + + + + diff --git a/PCbuild/xxlimited_3_13.vcxproj.filters b/PCbuild/xxlimited_3_13.vcxproj.filters new file mode 100644 index 000000000000000..3dfb7800edc4419 --- /dev/null +++ b/PCbuild/xxlimited_3_13.vcxproj.filters @@ -0,0 +1,13 @@ + + + + + {5be27194-6530-452d-8d86-3767b991fa83} + + + + + Source Files + + + diff --git a/Platforms/WASI/_build.py b/Platforms/WASI/_build.py index 76d2853163baa9e..c1a91a9c833b8e8 100644 --- a/Platforms/WASI/_build.py +++ b/Platforms/WASI/_build.py @@ -222,10 +222,8 @@ def wasi_sdk(context): if wasi_sdk_path := context.wasi_sdk_path: if not wasi_sdk_path.exists(): raise ValueError( - "WASI SDK not found; " - "download from " - "https://github.com/WebAssembly/wasi-sdk and/or " - "specify via $WASI_SDK_PATH or --wasi-sdk" + "WASI SDK not found at " + f"{os.fsdecode(wasi_sdk_path)!r} (via --wasi-sdk)" ) return wasi_sdk_path @@ -237,7 +235,8 @@ def wasi_sdk(context): wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var) if not wasi_sdk_path.exists(): raise ValueError( - f"WASI SDK not found at $WASI_SDK_PATH ({wasi_sdk_path})" + f"WASI SDK not found at {os.fsdecode(wasi_sdk_path)!r} " + "(via $WASI_SDK_PATH)" ) else: opt_path = pathlib.Path("/opt") @@ -272,6 +271,14 @@ def wasi_sdk(context): f" Found WASI SDK {major_version}, " f"but WASI SDK {wasi_sdk_version} is the supported version", ) + elif not wasi_sdk_path: + raise ValueError( + f"WASI SDK {wasi_sdk_version} not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and install in " + f"{os.fsdecode(opt_path)!r} or specify the SDK via " + "$WASI_SDK_PATH or --wasi-sdk" + ) # Cache the result. context.wasi_sdk_path = wasi_sdk_path diff --git a/Python/asm_trampoline.S b/Python/asm_trampoline.S index 93adae3d99038f8..9f3ca909ab7d852 100644 --- a/Python/asm_trampoline.S +++ b/Python/asm_trampoline.S @@ -1,3 +1,5 @@ +#include "asm_trampoline_aarch64.h" + .text #if defined(__APPLE__) .globl __Py_trampoline_func_start @@ -29,10 +31,12 @@ _Py_trampoline_func_start: #if defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) // ARM64 little endian, 64bit ABI // generate with aarch64-linux-gnu-gcc 12.1 + SIGN_LR stp x29, x30, [sp, -16]! mov x29, sp blr x3 ldp x29, x30, [sp], 16 + VERIFY_LR ret #endif #ifdef __riscv diff --git a/Python/asm_trampoline_aarch64.h b/Python/asm_trampoline_aarch64.h new file mode 100644 index 000000000000000..bc83aa460b6860d --- /dev/null +++ b/Python/asm_trampoline_aarch64.h @@ -0,0 +1,56 @@ +#ifndef ASM_TRAMPOLINE_AARCH_64_H_ +#define ASM_TRAMPOLINE_AARCH_64_H_ + +/* + * References: + * - https://developer.arm.com/documentation/101028/0012/5--Feature-test-macros + * - https://github.com/ARM-software/abi-aa/blob/main/aaelf64/aaelf64.rst + */ + +#if defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1 + #define BTI_J hint 36 /* bti j: for jumps, IE br instructions */ + #define BTI_C hint 34 /* bti c: for calls, IE bl instructions */ + #define GNU_PROPERTY_AARCH64_BTI 1 /* bit 0 GNU Notes is for BTI support */ +#else + #define BTI_J + #define BTI_C + #define GNU_PROPERTY_AARCH64_BTI 0 +#endif + +#if defined(__ARM_FEATURE_PAC_DEFAULT) + #if __ARM_FEATURE_PAC_DEFAULT & 1 + #define SIGN_LR hint 25 /* paciasp: sign with the A key */ + #define VERIFY_LR hint 29 /* autiasp: verify with the A key */ + #elif __ARM_FEATURE_PAC_DEFAULT & 2 + #define SIGN_LR hint 27 /* pacibsp: sign with the b key */ + #define VERIFY_LR hint 31 /* autibsp: verify with the b key */ + #endif + #define GNU_PROPERTY_AARCH64_POINTER_AUTH 2 /* bit 1 GNU Notes is for PAC support */ +#else + #define SIGN_LR BTI_C + #define VERIFY_LR + #define GNU_PROPERTY_AARCH64_POINTER_AUTH 0 +#endif + +#if defined(__ARM_FEATURE_GCS_DEFAULT) && __ARM_FEATURE_GCS_DEFAULT == 1 + #define GNU_PROPERTY_AARCH64_GCS 4 /* bit 2 GNU Notes is for GCS support */ +#else + #define GNU_PROPERTY_AARCH64_GCS 0 +#endif + +/* Add the BTI, PAC and GCS support to GNU Notes section */ +#if GNU_PROPERTY_AARCH64_BTI != 0 || GNU_PROPERTY_AARCH64_POINTER_AUTH != 0 || GNU_PROPERTY_AARCH64_GCS != 0 + .pushsection .note.gnu.property, "a"; /* Start a new allocatable section */ + .balign 8; /* align it on a byte boundry */ + .long 4; /* size of "GNU\0" */ + .long 0x10; /* size of descriptor */ + .long 0x5; /* NT_GNU_PROPERTY_TYPE_0 */ + .asciz "GNU"; + .long 0xc0000000; /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */ + .long 4; /* Four bytes of data */ + .long (GNU_PROPERTY_AARCH64_BTI|GNU_PROPERTY_AARCH64_POINTER_AUTH|GNU_PROPERTY_AARCH64_GCS); /* BTI, PAC or GCS is enabled */ + .long 0; /* padding for 8 byte alignment */ + .popsection; /* end the section */ +#endif + +#endif diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3bd489122da9d42..f7487c7136962f1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1867,8 +1867,9 @@ dummy_func( assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif diff --git a/Python/ceval.c b/Python/ceval.c index 060e948e6b01c9f..a080ae42b937667 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3059,25 +3059,35 @@ check_lazy_import_compatibility(PyThreadState *tstate, PyObject *globals, return res; } +static int +is_lazy_import_module_level(void) +{ + _PyInterpreterFrame *frame = _PyEval_GetFrame(); + return frame != NULL && frame->f_globals == frame->f_locals; +} + PyObject * _PyEval_LazyImportName(PyThreadState *tstate, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *name, PyObject *fromlist, PyObject *level, int lazy) { PyObject *res = NULL; + PyImport_LazyImportsMode mode = PyImport_GetLazyImportsMode(); // Check if global policy overrides the local syntax - switch (PyImport_GetLazyImportsMode()) { + switch (mode) { case PyImport_LAZY_NONE: lazy = 0; break; case PyImport_LAZY_ALL: - lazy = 1; + if (!lazy) { + lazy = is_lazy_import_module_level(); + } break; case PyImport_LAZY_NORMAL: break; } - if (!lazy && PyImport_GetLazyImportsMode() != PyImport_LAZY_NONE) { + if (!lazy && mode != PyImport_LAZY_NONE && is_lazy_import_module_level()) { // See if __lazy_modules__ forces this to be lazy. lazy = check_lazy_import_compatibility(tstate, globals, name, level); if (lazy < 0) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b6a2821db3007ef..efa61d7de74e88c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -9346,8 +9346,9 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2623105656c90c3..53e09a8f4523c7c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7945,8 +7945,9 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif @@ -13053,8 +13054,9 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif diff --git a/Python/import.c b/Python/import.c index 60a5ee6e770f598..352941a836ef21a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -94,6 +94,8 @@ static struct _inittab *inittab_copy = NULL; (interp)->imports.modules_by_index #define LAZY_MODULES(interp) \ (interp)->imports.lazy_modules +#define LAZY_PENDING_SUBMODULES(interp) \ + (interp)->imports.lazy_pending_submodules #define IMPORTLIB(interp) \ (interp)->imports.importlib #define OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp) \ @@ -271,8 +273,11 @@ import_get_module(PyThreadState *tstate, PyObject *name) PyObject * _PyImport_InitLazyModules(PyInterpreterState *interp) { - assert(LAZY_MODULES(interp) == NULL); - LAZY_MODULES(interp) = PyDict_New(); + assert(LAZY_MODULES(interp) == NULL && + LAZY_PENDING_SUBMODULES(interp) == NULL); + + LAZY_PENDING_SUBMODULES(interp) = PyDict_New(); + LAZY_MODULES(interp) = PySet_New(0); return LAZY_MODULES(interp); } @@ -280,6 +285,7 @@ void _PyImport_ClearLazyModules(PyInterpreterState *interp) { Py_CLEAR(LAZY_MODULES(interp)); + Py_CLEAR(LAZY_PENDING_SUBMODULES(interp)); } static int @@ -4049,7 +4055,7 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) // Create a cause exception showing where the lazy import was declared. PyObject *msg = PyUnicode_FromFormat( - "deferred import of '%U' raised an exception during resolution", + "lazy import of '%U' raised an exception during resolution", import_name ); Py_DECREF(import_name); // Done with import_name. @@ -4326,20 +4332,10 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, return final_mod; } -static PyObject * -get_mod_dict(PyObject *module) -{ - if (PyModule_Check(module)) { - return Py_NewRef(_PyModule_GetDict(module)); - } - - return PyObject_GetAttr(module, &_Py_ID(__dict__)); -} - // ensure we have the set for the parent module name in sys.lazy_modules. // Returns a new reference. static PyObject * -ensure_lazy_submodules(PyDictObject *lazy_modules, PyObject *parent) +ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) { PyObject *lazy_submodules; Py_BEGIN_CRITICAL_SECTION(lazy_modules); @@ -4358,27 +4354,28 @@ ensure_lazy_submodules(PyDictObject *lazy_modules, PyObject *parent) return lazy_submodules; } +// Records all parent-child relationships in lazy_pending_submodules +// for a lazily imported module name. When a parent module's attribute +// is accessed, _Py_module_getattro_impl will check lazy_pending_submodules +// and trigger the import. static int -register_lazy_on_parent(PyThreadState *tstate, PyObject *name, - PyObject *builtins) +register_lazy_on_parent(PyThreadState *tstate, PyObject *name) { int ret = -1; PyObject *parent = NULL; PyObject *child = NULL; - PyObject *parent_module = NULL; - PyObject *parent_dict = NULL; PyInterpreterState *interp = tstate->interp; - PyObject *lazy_modules = LAZY_MODULES(interp); - assert(lazy_modules != NULL); + PyObject *lazy_pending_submodules = LAZY_PENDING_SUBMODULES(interp); + assert(lazy_pending_submodules != NULL); Py_INCREF(name); while (true) { Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, PyUnicode_GET_LENGTH(name), -1); if (dot < 0) { - PyObject *lazy_submodules = ensure_lazy_submodules( - (PyDictObject *)lazy_modules, name); + PyObject *lazy_submodules = ensure_lazy_pending_submodules( + (PyDictObject *)lazy_pending_submodules, name); if (lazy_submodules == NULL) { goto done; } @@ -4387,9 +4384,6 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, goto done; } parent = PyUnicode_Substring(name, 0, dot); - // If `parent` is NULL then this has hit the end of the import, no - // more "parent.child" in the import name. The entire import will be - // resolved lazily. if (parent == NULL) { goto done; } @@ -4399,9 +4393,8 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, goto done; } - // Record the child as being lazily imported from the parent. - PyObject *lazy_submodules = ensure_lazy_submodules( - (PyDictObject *)lazy_modules, parent); + PyObject *lazy_submodules = ensure_lazy_pending_submodules( + (PyDictObject *)lazy_pending_submodules, parent); if (lazy_submodules == NULL) { goto done; } @@ -4412,44 +4405,11 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, } Py_DECREF(lazy_submodules); - // Add the lazy import for the child to the parent. - Py_XSETREF(parent_module, PyImport_GetModule(parent)); - if (parent_module != NULL) { - Py_XSETREF(parent_dict, get_mod_dict(parent_module)); - if (parent_dict == NULL) { - goto done; - } - if (PyDict_CheckExact(parent_dict)) { - int contains = PyDict_Contains(parent_dict, child); - if (contains < 0) { - goto done; - } - if (!contains) { - PyObject *lazy_module_attr = _PyLazyImport_New( - tstate->current_frame, builtins, parent, child - ); - if (lazy_module_attr == NULL) { - goto done; - } - if (PyDict_SetItem(parent_dict, child, - lazy_module_attr) < 0) { - Py_DECREF(lazy_module_attr); - goto done; - } - Py_DECREF(lazy_module_attr); - } - } - ret = 0; - goto done; - } - Py_SETREF(name, parent); parent = NULL; } done: - Py_XDECREF(parent_dict); - Py_XDECREF(parent_module); Py_XDECREF(child); Py_XDECREF(parent); Py_XDECREF(name); @@ -4458,17 +4418,73 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, static int register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, - PyObject *from, PyObject *builtins) + PyObject *from) { PyObject *fromname = PyUnicode_FromFormat("%U.%U", abs_name, from); if (fromname == NULL) { return -1; } - int res = register_lazy_on_parent(tstate, fromname, builtins); + + // Add the module name to sys.lazy_modules set (PEP 810). + PyObject *lazy_modules = LAZY_MODULES(tstate->interp); + if (PySet_Add(lazy_modules, fromname) < 0) { + Py_DECREF(fromname); + return -1; + } + + int res = register_lazy_on_parent(tstate, fromname); Py_DECREF(fromname); return res; } +PyObject * +_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(interp); + if (lazy_pending == NULL) { + return NULL; + } + + PyObject *pending_set; + int rc = PyDict_GetItemRef(lazy_pending, mod_name, &pending_set); + if (rc <= 0) { + return NULL; + } + + int contains = PySet_Contains(pending_set, attr_name); + if (contains <= 0) { + Py_DECREF(pending_set); + return NULL; + } + + PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); + if (full_name == NULL) { + Py_DECREF(pending_set); + return NULL; + } + + PyObject *mod = PyImport_ImportModuleLevelObject( + full_name, NULL, NULL, NULL, 0); + if (mod == NULL) { + Py_DECREF(pending_set); + Py_DECREF(full_name); + return NULL; + } + Py_DECREF(mod); + + if (PySet_Discard(pending_set, attr_name) < 0) { + Py_DECREF(pending_set); + Py_DECREF(full_name); + return NULL; + } + Py_DECREF(pending_set); + + PyObject *submod = PyImport_GetModule(full_name); + Py_DECREF(full_name); + return submod; +} + PyObject * _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, PyObject *name, PyObject *builtins, @@ -4555,9 +4571,15 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, Py_DECREF(abs_name); return NULL; } + + // Add the module name to sys.lazy_modules set (PEP 810). + PyObject *lazy_modules = LAZY_MODULES(tstate->interp); + if (PySet_Add(lazy_modules, abs_name) < 0) { + goto error; + } + if (fromlist && PyUnicode_Check(fromlist)) { - if (register_from_lazy_on_parent(tstate, abs_name, fromlist, - builtins) < 0) { + if (register_from_lazy_on_parent(tstate, abs_name, fromlist) < 0) { goto error; } } @@ -4565,14 +4587,13 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, PyTuple_GET_SIZE(fromlist)) { for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(fromlist); i++) { if (register_from_lazy_on_parent(tstate, abs_name, - PyTuple_GET_ITEM(fromlist, i), - builtins) < 0) + PyTuple_GET_ITEM(fromlist, i)) < 0) { goto error; } } } - else if (register_lazy_on_parent(tstate, abs_name, builtins) < 0) { + else if (register_lazy_on_parent(tstate, abs_name) < 0) { goto error; } @@ -4791,6 +4812,7 @@ _PyImport_ClearCore(PyInterpreterState *interp) Py_CLEAR(IMPORTLIB(interp)); Py_CLEAR(IMPORT_FUNC(interp)); Py_CLEAR(LAZY_IMPORT_FUNC(interp)); + Py_CLEAR(interp->imports.lazy_pending_submodules); Py_CLEAR(interp->imports.lazy_modules); Py_CLEAR(interp->imports.lazy_importing_modules); Py_CLEAR(interp->imports.lazy_imports_filter); @@ -5580,46 +5602,6 @@ _imp_source_hash_impl(PyObject *module, long key, Py_buffer *source) return PyBytes_FromStringAndSize(hash.data, sizeof(hash.data)); } -static int -publish_lazy_imports_on_module(PyThreadState *tstate, - PyObject *lazy_submodules, - PyObject *name, - PyObject *module_dict) -{ - PyObject *builtins = _PyEval_GetBuiltins(tstate); - PyObject *attr_name; - Py_ssize_t pos = 0; - Py_hash_t hash; - - // Enumerate the set of lazy submodules which have been imported from the - // parent module. - while (_PySet_NextEntryRef(lazy_submodules, &pos, &attr_name, &hash)) { - if (_PyDict_Contains_KnownHash(module_dict, attr_name, hash)) { - Py_DECREF(attr_name); - continue; - } - // Create a new lazy module attr for the subpackage which was - // previously lazily imported. - PyObject *lazy_module_attr = _PyLazyImport_New(tstate->current_frame, builtins, - name, attr_name); - if (lazy_module_attr == NULL) { - Py_DECREF(attr_name); - return -1; - } - - // Publish on the module that was just imported. - if (PyDict_SetItem(module_dict, attr_name, - lazy_module_attr) < 0) { - Py_DECREF(lazy_module_attr); - Py_DECREF(attr_name); - return -1; - } - Py_DECREF(lazy_module_attr); - Py_DECREF(attr_name); - } - return 0; -} - /*[clinic input] _imp._set_lazy_attributes modobj: object @@ -5633,43 +5615,11 @@ _imp__set_lazy_attributes_impl(PyObject *module, PyObject *modobj, PyObject *name) /*[clinic end generated code: output=3369bb3242b1f043 input=38ea6f30956dd7d6]*/ { - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *module_dict = NULL; - PyObject *ret = NULL; - PyObject *lazy_modules = LAZY_MODULES(tstate->interp); - assert(lazy_modules != NULL); - - PyObject *lazy_submodules; - if (PyDict_GetItemRef(lazy_modules, name, &lazy_submodules) < 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (PySet_Discard(LAZY_MODULES(interp), name) < 0) { return NULL; } - else if (lazy_submodules == NULL) { - Py_RETURN_NONE; - } - - module_dict = get_mod_dict(modobj); - if (module_dict == NULL || !PyDict_CheckExact(module_dict)) { - Py_DECREF(lazy_submodules); - goto done; - } - - assert(PyAnySet_CheckExact(lazy_submodules)); - Py_BEGIN_CRITICAL_SECTION(lazy_submodules); - publish_lazy_imports_on_module(tstate, lazy_submodules, name, module_dict); - Py_END_CRITICAL_SECTION(); - Py_DECREF(lazy_submodules); - - // once a module is imported it is removed from sys.lazy_modules - if (PyDict_DelItem(lazy_modules, name) < 0) { - goto error; - } - -done: - ret = Py_NewRef(Py_None); - -error: - Py_XDECREF(module_dict); - return ret; + Py_RETURN_NONE; } PyDoc_STRVAR(doc_imp, diff --git a/Python/jit_unwind.c b/Python/jit_unwind.c index 646106f0a9655c0..0941ed593ff7d14 100644 --- a/Python/jit_unwind.c +++ b/Python/jit_unwind.c @@ -60,6 +60,9 @@ enum { DWRF_CFA_offset_extended_sf = 0x11, // Extended signed offset DWRF_CFA_advance_loc = 0x40, // Advance location counter DWRF_CFA_offset = 0x80, // Simple offset instruction +#if defined(__aarch64__) + DWRF_CFA_AARCH64_negate_ra_state = 0x2d, // Toggle return address signing state +#endif DWRF_CFA_restore = 0xc0 // Restore register }; @@ -562,6 +565,13 @@ static void elf_init_ehframe_perf(ELFObjectContext* ctx) { DWRF_UV(8); // New offset: SP + 8 #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) /* AArch64 calling convention unwinding rules */ +#if defined(__ARM_FEATURE_PAC_DEFAULT) || \ + (defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1) + DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past SIGN_LR (4 bytes) +#endif +#if defined(__ARM_FEATURE_PAC_DEFAULT) + DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // Saved LR is PAC-signed from here +#endif DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance by 1 instruction (4 bytes) DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 DWRF_UV(16); // Stack pointer moved by 16 bytes @@ -570,6 +580,9 @@ static void elf_init_ehframe_perf(ELFObjectContext* ctx) { DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // x30 (link register) saved DWRF_UV(1); // At CFA-8 (1 * 8 = 8 bytes from CFA) DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (12 bytes) +#if defined(__ARM_FEATURE_PAC_DEFAULT) + DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // LR is authenticated, no longer PAC-signed +#endif DWRF_U8(DWRF_CFA_def_cfa_register); // CFA = FP (x29) + 16 DWRF_UV(DWRF_REG_FP); DWRF_U8(DWRF_CFA_restore | DWRF_REG_RA); // Restore x30 - NO DWRF_UV() after this! diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 1dc3a248f45f0c8..e726dc0e6fd1114 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -18,6 +18,7 @@ #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_uop_metadata.h" #include "pycore_long.h" @@ -127,7 +128,7 @@ static void increment_mutations(PyObject* dict) { assert(PyDict_CheckExact(dict)); PyDictObject *d = (PyDictObject *)dict; - FT_ATOMIC_ADD_UINT64(d->_ma_watcher_tag, (1 << DICT_MAX_WATCHERS)); + FT_ATOMIC_ADD_UINT64(d->_ma_watcher_tag, 1ULL << DICT_MAX_WATCHERS); } /* The first two dict watcher IDs are reserved for CPython, @@ -156,6 +157,17 @@ type_watcher_callback(PyTypeObject* type) return 0; } +static int +_setup_optimizer_watchers(void *Py_UNUSED(arg)) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_ATOMIC_STORE_PTR_RELEASE( + interp->dict_state.watchers[GLOBALS_WATCHER_ID], + globals_watcher_callback); + interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; + return 0; +} + static void watch_type(PyTypeObject *type, _PyBloomFilter *filter) { @@ -580,10 +592,8 @@ optimize_uops( // Make sure that watchers are set up PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { - interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback; - interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; - } + _PyOnceFlag_CallOnce(&interp->dict_state.watcher_setup_once, + _setup_optimizer_watchers, NULL); _Py_uop_abstractcontext_init(ctx, dependencies); _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, (PyCodeObject *)func->func_code, NULL, 0); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e10a096baa33188..96dbaea5a5797ef 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -2,6 +2,7 @@ #include "pycore_long.h" #include "pycore_opcode_utils.h" #include "pycore_optimizer.h" +#include "pycore_typeobject.h" #include "pycore_uops.h" #include "pycore_uop_ids.h" #include "internal/pycore_moduleobject.h" @@ -1459,7 +1460,8 @@ dummy_func(void) { type = sym_get_probable_type(iter); definite = false; } - if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL) { + if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL + && !_PyType_HasSlotTpIternext(type)) { PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); if (!definite) { @@ -2041,7 +2043,16 @@ dummy_func(void) { PyObject *name = _Py_SpecialMethods[oparg].name; PyObject *descr = _PyType_Lookup(type, name); if (descr != NULL && (Py_TYPE(descr)->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR)) { - ADD_OP(_GUARD_TYPE_VERSION, 0, type->tp_version_tag); + /* LOAD_SPECIAL expands to _RECORD_TOS_TYPE + _INSERT_NULL + + * _LOAD_SPECIAL. Insert _GUARD_TYPE_VERSION before the + * already-emitted _INSERT_NULL so deopt sees the original + * stack shape.*/ + _PyUOpInstruction *insert_null = uop_buffer_last(&ctx->out_buffer); + assert(insert_null->opcode == _INSERT_NULL); + assert(insert_null->target == this_instr->target); + REPLACE_OP(insert_null, _GUARD_TYPE_VERSION, 0, type->tp_version_tag); + ADD_OP(_INSERT_NULL, 0, 0); + bool immortal = _Py_IsImmortal(descr) || (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE); ADD_OP(immortal ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, 0, (uintptr_t)descr); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 01ecb3790aa2cdb..f336549d2ed2440 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3706,7 +3706,8 @@ type = sym_get_probable_type(iter); definite = false; } - if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL) { + if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL + && !_PyType_HasSlotTpIternext(type)) { PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); if (!definite) { @@ -3895,7 +3896,11 @@ PyObject *name = _Py_SpecialMethods[oparg].name; PyObject *descr = _PyType_Lookup(type, name); if (descr != NULL && (Py_TYPE(descr)->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR)) { - ADD_OP(_GUARD_TYPE_VERSION, 0, type->tp_version_tag); + _PyUOpInstruction *insert_null = uop_buffer_last(&ctx->out_buffer); + assert(insert_null->opcode == _INSERT_NULL); + assert(insert_null->target == this_instr->target); + REPLACE_OP(insert_null, _GUARD_TYPE_VERSION, 0, type->tp_version_tag); + ADD_OP(_INSERT_NULL, 0, 0); bool immortal = _Py_IsImmortal(descr) || (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE); ADD_OP(immortal ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, 0, (uintptr_t)descr); diff --git a/Python/pystate.c b/Python/pystate.c index bf2616a49148a74..ff712019affbf9e 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -320,6 +320,7 @@ _Py_COMP_DIAG_POP &(runtime)->allocators.mutex, \ &(runtime)->_main_interpreter.types.mutex, \ &(runtime)->_main_interpreter.code_state.mutex, \ + &(runtime)->_main_interpreter.dict_state.watcher_mutex, \ } static void diff --git a/Python/pystrhex.c b/Python/pystrhex.c index 645bb013581288e..8fb1fa36f85e739 100644 --- a/Python/pystrhex.c +++ b/Python/pystrhex.c @@ -36,7 +36,7 @@ _Py_hexlify_scalar(const unsigned char *src, Py_UCS1 *dst, Py_ssize_t len) adds a ton of complication. Who ever really hexes huge data? The 16-64 byte boosts align nicely with md5 - sha512 hexdigests. */ -#ifdef HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR +#ifdef _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR /* 128-bit vector of 16 unsigned bytes */ typedef unsigned char v16u8 __attribute__((vector_size(16))); @@ -110,7 +110,7 @@ _Py_hexlify_simd(const unsigned char *src, Py_UCS1 *dst, Py_ssize_t len) _Py_hexlify_scalar(src + i, dst, len - i); } -#endif /* HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR */ +#endif /* _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR */ static PyObject * _Py_strhex_impl(const char* argbuf, Py_ssize_t arglen, @@ -191,7 +191,7 @@ _Py_strhex_impl(const char* argbuf, Py_ssize_t arglen, unsigned char c; if (bytes_per_sep_group == 0) { -#ifdef HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR +#ifdef _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR if (arglen >= 16) { _Py_hexlify_simd((const unsigned char *)argbuf, retbuf, arglen); } diff --git a/Python/remote_debug.h b/Python/remote_debug.h index 6c089a834dcd40d..7b2c4f3bcb8077a 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -147,6 +147,7 @@ typedef struct { int memfd; #endif page_cache_entry_t pages[MAX_PAGES]; + int page_cache_count; Py_ssize_t page_size; } proc_handle_t; @@ -185,14 +186,16 @@ _Py_RemoteDebug_FreePageCache(proc_handle_t *handle) handle->pages[i].data = NULL; handle->pages[i].valid = 0; } + handle->page_cache_count = 0; } UNUSED static void _Py_RemoteDebug_ClearCache(proc_handle_t *handle) { - for (int i = 0; i < MAX_PAGES; i++) { + for (int i = 0; i < handle->page_cache_count; i++) { handle->pages[i].valid = 0; } + handle->page_cache_count = 0; } #if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX @@ -222,6 +225,7 @@ _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { handle->memfd = -1; #endif handle->page_size = get_page_size(); + handle->page_cache_count = 0; for (int i = 0; i < MAX_PAGES; i++) { handle->pages[i].data = NULL; handle->pages[i].valid = 0; @@ -1287,8 +1291,9 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); } - // Search for valid cached page - for (int i = 0; i < MAX_PAGES; i++) { + // Search only the pages used since the last clear. The cache is cleared + // between profiler samples, so entries are packed at the front. + for (int i = 0; i < handle->page_cache_count; i++) { page_cache_entry_t *entry = &handle->pages[i]; if (entry->valid && entry->page_addr == page_base) { memcpy(out, entry->data + offset_in_page, size); @@ -1296,33 +1301,31 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, } } - // Find reusable slot - for (int i = 0; i < MAX_PAGES; i++) { - page_cache_entry_t *entry = &handle->pages[i]; - if (!entry->valid) { + if (handle->page_cache_count < MAX_PAGES) { + page_cache_entry_t *entry = &handle->pages[handle->page_cache_count]; + if (entry->data == NULL) { + entry->data = PyMem_RawMalloc(page_size); if (entry->data == NULL) { - entry->data = PyMem_RawMalloc(page_size); - if (entry->data == NULL) { - PyErr_NoMemory(); - _set_debug_exception_cause(PyExc_MemoryError, - "Cannot allocate %zu bytes for page cache entry " - "during read from PID %d at address 0x%lx", - page_size, handle->pid, addr); - return -1; - } - } - - if (_Py_RemoteDebug_ReadRemoteMemory(handle, page_base, page_size, entry->data) < 0) { - // Try to just copy the exact amount as a fallback - PyErr_Clear(); - goto fallback; + PyErr_NoMemory(); + _set_debug_exception_cause(PyExc_MemoryError, + "Cannot allocate %zu bytes for page cache entry " + "during read from PID %d at address 0x%lx", + page_size, handle->pid, addr); + return -1; } + } - entry->page_addr = page_base; - entry->valid = 1; - memcpy(out, entry->data + offset_in_page, size); - return 0; + if (_Py_RemoteDebug_ReadRemoteMemory(handle, page_base, page_size, entry->data) < 0) { + // Try to just copy the exact amount as a fallback + PyErr_Clear(); + goto fallback; } + + entry->page_addr = page_base; + entry->valid = 1; + handle->page_cache_count++; + memcpy(out, entry->data + offset_in_page, size); + return 0; } fallback: @@ -1330,6 +1333,49 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); } +typedef struct { + uintptr_t remote_addr; + void *local_buf; + size_t size; +} _Py_RemoteReadSegment; + +#define _PY_REMOTE_DEBUG_MAX_BATCHED_SEGMENTS 4 + +// Batched read of multiple remote regions in a single syscall when supported. +// Returns total bytes read (>= 0) on success, -1 if batched reads are +// unavailable or the syscall failed. Callers compare the return value against +// cumulative segment sizes to determine which segments were fully populated. +UNUSED static Py_ssize_t +_Py_RemoteDebug_BatchedReadRemoteMemory( + proc_handle_t *handle, + const _Py_RemoteReadSegment *segments, + int nsegs) +{ +#if defined(__linux__) && HAVE_PROCESS_VM_READV + if (handle->memfd == -1 + && nsegs > 0 + && nsegs <= _PY_REMOTE_DEBUG_MAX_BATCHED_SEGMENTS) { + struct iovec local[_PY_REMOTE_DEBUG_MAX_BATCHED_SEGMENTS]; + struct iovec remote[_PY_REMOTE_DEBUG_MAX_BATCHED_SEGMENTS]; + for (int i = 0; i < nsegs; i++) { + local[i].iov_base = segments[i].local_buf; + local[i].iov_len = segments[i].size; + remote[i].iov_base = (void *)segments[i].remote_addr; + remote[i].iov_len = segments[i].size; + } + ssize_t nread = process_vm_readv(handle->pid, local, nsegs, remote, nsegs, 0); + if (nread >= 0) { + return (Py_ssize_t)nread; + } + } +#else + (void)handle; + (void)segments; + (void)nsegs; +#endif + return -1; +} + UNUSED static int _Py_RemoteDebug_ReadDebugOffsets( proc_handle_t *handle, diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index f23c1d5286f92af..c619a9a0c1c5a1b 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -463,7 +463,7 @@ def get_location(self, modinfo: ModuleInfo) -> pathlib.Path | None: def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec) -> None: """Check that the module file is present and not empty""" if spec.loader is BuiltinImporter: # type: ignore[comparison-overlap] - return + return # type: ignore[unreachable] try: assert spec.origin is not None st = os.stat(spec.origin) diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index bda725396406118..f8828a56b4c7da7 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -42,6 +42,7 @@ 'test', 'xxlimited', 'xxlimited_35', + 'xxlimited_3_13', 'xxsubtype', } diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index 5465e2d4b6171f1..485c9314cf70015 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -24,8 +24,6 @@ python_version = 3.10 # ...And be strict: strict = True -strict_bytes = True -local_partial_types = True extra_checks = True enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined warn_unreachable = True diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 4a55a1a24ee1bed..92134bc1321e1b5 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -11,6 +11,7 @@ '_testclinic_limited.c', 'xxlimited.c', 'xxlimited_35.c', + 'xxlimited_3_13.c', )) # C files in the fhe following directories must not be built with diff --git a/Tools/c-analyzer/cpython/_analyzer.py b/Tools/c-analyzer/cpython/_analyzer.py index 43ed552fcf75d90..404a81af11e39ff 100644 --- a/Tools/c-analyzer/cpython/_analyzer.py +++ b/Tools/c-analyzer/cpython/_analyzer.py @@ -77,6 +77,7 @@ 'PyStructSequence_Field[]', 'PyStructSequence_Desc', 'PyABIInfo', + 'PySlot[]', } # XXX We should normalize all cases to a single name, diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 7af64ed017ba73d..ddfb93a424c0185 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -467,6 +467,7 @@ Modules/_testcapi/object.c - MyObject_dealloc_called - Modules/_testcapi/object.c - MyType - Modules/_testcapi/structmember.c - test_structmembersType_OldAPI - Modules/_testcapi/watchers.c - g_dict_watch_events - +Modules/_testcapi/watchers.c - g_dict_watch_once - Modules/_testcapi/watchers.c - g_dict_watchers_installed - Modules/_testcapi/watchers.c - g_type_modified_events - Modules/_testcapi/watchers.c - g_type_watchers_installed - diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py index 60f43b99c0f69dd..c8a914c22a9e137 100644 --- a/Tools/ftscalingbench/ftscalingbench.py +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -279,6 +279,23 @@ def staticmethod_call(): for _ in range(1000 * WORK_SCALE): obj.my_staticmethod() + +class MyDescriptor: + def __get__(self, obj, objtype=None): + return 42 + + def __set__(self, obj, value): + pass + +class MyClassWithDescriptor: + attr = MyDescriptor() + +@register_benchmark +def descriptor(): + obj = MyClassWithDescriptor() + for _ in range(1000 * WORK_SCALE): + obj.attr + @register_benchmark def deepcopy(): x = {'list': [1, 2], 'tuple': (1, None)} diff --git a/Tools/inspection/benchmark_external_inspection.py b/Tools/inspection/benchmark_external_inspection.py index fee3435496da0bd..8e367422a961da2 100644 --- a/Tools/inspection/benchmark_external_inspection.py +++ b/Tools/inspection/benchmark_external_inspection.py @@ -151,6 +151,45 @@ def create_threads(n): time.sleep(0.05) ''' +ASYNC_CODE = '''\ +import asyncio +import contextlib +import math + +def compute_slice(seed): + result = 0.0 + for i in range(2000): + result += math.sin(seed + i) * math.sqrt(i + 1) + return result + +async def leaf_task(seed): + total = 0.0 + while True: + total += compute_slice(seed) + await asyncio.sleep(0) + +async def parent_task(seed): + child = asyncio.create_task(leaf_task(seed + 1000), name=f"leaf-{seed}") + try: + while True: + compute_slice(seed) + await asyncio.sleep(0.001) + finally: + child.cancel() + with contextlib.suppress(asyncio.CancelledError): + await child + +async def main(): + tasks = [ + asyncio.create_task(parent_task(i), name=f"parent-{i}") + for i in range(8) + ] + await asyncio.gather(*tasks) + +if __name__ == "__main__": + asyncio.run(main()) +''' + CODE_EXAMPLES = { "basic": { "code": CODE, @@ -164,10 +203,29 @@ def create_threads(n): "code": CODE_WITH_TONS_OF_THREADS, "description": "Tons of threads doing mixed CPU/IO work", }, + "asyncio": { + "code": ASYNC_CODE, + "description": "Asyncio tasks with active and awaited coroutine chains", + }, +} + +OPERATIONS = { + "stack_trace": { + "method": "get_stack_trace", + "label": "get_stack_trace()", + }, + "async_stack_trace": { + "method": "get_async_stack_trace", + "label": "get_async_stack_trace()", + }, + "all_awaited_by": { + "method": "get_all_awaited_by", + "label": "get_all_awaited_by()", + }, } -def benchmark(unwinder, duration_seconds=10, blocking=False): +def benchmark(unwinder, duration_seconds=10, blocking=False, operation="stack_trace"): """Benchmark mode - measure raw sampling speed for specified duration""" sample_count = 0 fail_count = 0 @@ -175,11 +233,14 @@ def benchmark(unwinder, duration_seconds=10, blocking=False): start_time = time.perf_counter() end_time = start_time + duration_seconds total_attempts = 0 + operation_info = OPERATIONS[operation] + operation_method = getattr(unwinder, operation_info["method"]) colors = get_colors(can_colorize()) print( - f"{colors.BOLD_BLUE}Benchmarking sampling speed for {duration_seconds} seconds...{colors.RESET}" + f"{colors.BOLD_BLUE}Benchmarking {operation_info['label']} speed " + f"for {duration_seconds} seconds...{colors.RESET}" ) try: @@ -190,8 +251,8 @@ def benchmark(unwinder, duration_seconds=10, blocking=False): if blocking: unwinder.pause_threads() try: - stack_trace = unwinder.get_stack_trace() - if stack_trace: + sample = operation_method() + if sample: sample_count += 1 finally: if blocking: @@ -239,6 +300,7 @@ def benchmark(unwinder, duration_seconds=10, blocking=False): (sample_count / total_attempts) * 100 if total_attempts > 0 else 0 ), "total_work_time": total_work_time, + "operation": operation_info["label"], "avg_work_time_us": ( (total_work_time / total_attempts) * 1e6 if total_attempts > 0 else 0 ), @@ -252,7 +314,7 @@ def print_benchmark_results(results): colors = get_colors(can_colorize()) print(f"\n{colors.BOLD_GREEN}{'='*60}{colors.RESET}") - print(f"{colors.BOLD_GREEN}get_stack_trace() Benchmark Results{colors.RESET}") + print(f"{colors.BOLD_GREEN}{results['operation']} Benchmark Results{colors.RESET}") print(f"{colors.BOLD_GREEN}{'='*60}{colors.RESET}") # Basic statistics @@ -329,6 +391,8 @@ def parse_arguments(): %(prog)s -d 60 # Run basic benchmark for 60 seconds %(prog)s --code deep_static # Run deep static call stack benchmark %(prog)s --code deep_static -d 30 # Run deep static benchmark for 30 seconds + %(prog)s --operation async_stack_trace + %(prog)s --operation all_awaited_by Available code examples: {examples_desc} @@ -348,8 +412,15 @@ def parse_arguments(): "--code", "-c", choices=list(CODE_EXAMPLES.keys()), - default="basic", - help="Code example to benchmark (default: basic)", + default=None, + help="Code example to benchmark (default: basic, or asyncio for async operations)", + ) + + parser.add_argument( + "--operation", + choices=list(OPERATIONS.keys()), + default="stack_trace", + help="Remote unwinder operation to benchmark (default: stack_trace)", ) parser.add_argument( @@ -365,7 +436,10 @@ def parse_arguments(): help="Stop all threads before sampling for consistent snapshots", ) - return parser.parse_args() + args = parser.parse_args() + if args.code is None: + args.code = "asyncio" if args.operation != "stack_trace" else "basic" + return args def create_target_process(temp_file, code_example="basic"): @@ -420,6 +494,9 @@ def main(): print( f"{colors.CYAN}Benchmark Duration:{colors.RESET} {colors.YELLOW}{args.duration}{colors.RESET} seconds" ) + print( + f"{colors.CYAN}Operation:{colors.RESET} {colors.GREEN}{OPERATIONS[args.operation]['label']}{colors.RESET}" + ) print( f"{colors.CYAN}Blocking Mode:{colors.RESET} {colors.GREEN if args.blocking else colors.YELLOW}{'enabled' if args.blocking else 'disabled'}{colors.RESET}" ) @@ -451,7 +528,12 @@ def main(): unwinder = _remote_debugging.RemoteUnwinder( process.pid, cache_frames=True, **kwargs ) - results = benchmark(unwinder, duration_seconds=args.duration, blocking=args.blocking) + results = benchmark( + unwinder, + duration_seconds=args.duration, + blocking=args.blocking, + operation=args.operation, + ) finally: cleanup_process(process, temp_file_path) diff --git a/Tools/msi/dev/dev_files.wxs b/Tools/msi/dev/dev_files.wxs index 21f9c848cc6be58..a9039d03f5f6fa1 100644 --- a/Tools/msi/dev/dev_files.wxs +++ b/Tools/msi/dev/dev_files.wxs @@ -13,6 +13,9 @@ + + + @@ -24,6 +27,9 @@ + + + diff --git a/Tools/msi/tcltk/tcltk_files.wxs b/Tools/msi/tcltk/tcltk_files.wxs index 5dad7c98d4f048a..7c7784741d9178d 100644 --- a/Tools/msi/tcltk/tcltk_files.wxs +++ b/Tools/msi/tcltk/tcltk_files.wxs @@ -10,11 +10,14 @@ - - + + - - + + + + + diff --git a/Tools/msi/testrelease.bat b/Tools/msi/testrelease.bat index 02bcca943cf79b4..db98f690151196c 100644 --- a/Tools/msi/testrelease.bat +++ b/Tools/msi/testrelease.bat @@ -88,9 +88,7 @@ exit /B 0 ) @if not errorlevel 1 ( @echo Testing Tcl/tk - @set TCL_LIBRARY=%~2\Python\tcl\tcl8.6 "%~2\Python\python.exe" -m test -uall -v test_ttk_guionly test_tk test_idle > "%~2\tcltk.txt" 2>&1 - @set TCL_LIBRARY= ) @set EXITCODE=%ERRORLEVEL% diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index af5cbaa7689f33d..46381ea58a12382 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,7 +1,7 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.19.1 +mypy==2.1.0 # needed for peg_generator: -types-psutil==7.2.2.20260130 -types-setuptools==82.0.0.20260210 +types-psutil==7.2.2.20260508 +types-setuptools==82.0.0.20260508 diff --git a/configure b/configure index cff7dfbfba8b9ad..9ad2171460f7ace 100755 --- a/configure +++ b/configure @@ -647,6 +647,8 @@ MODULE_BLOCK JIT_SHIM_BUILD_O JIT_SHIM_O JIT_STENCILS_H +MODULE_XXLIMITED_3_13_FALSE +MODULE_XXLIMITED_3_13_TRUE MODULE_XXLIMITED_35_FALSE MODULE_XXLIMITED_35_TRUE MODULE_XXLIMITED_FALSE @@ -6643,7 +6645,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6692,7 +6694,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6724,7 +6726,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="g++" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -6753,7 +6755,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6802,7 +6804,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6834,7 +6836,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="c++" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -6863,7 +6865,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6912,7 +6914,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6944,7 +6946,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="clang++" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -6973,7 +6975,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -7022,7 +7024,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -7054,7 +7056,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="icpc" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -9864,6 +9866,61 @@ fi ;; esac +if test "$ac_sys_system" = "Linux" -a "$cross_compiling" = no; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for thread stack size" >&5 +printf %s "checking for thread stack size... " >&6; } +if test ${ac_cv_thread_stack_size+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + cat > conftest.c < + +int main() +{ + pthread_attr_t attrs; + size_t size; + + int rc = pthread_attr_init(&attrs); + if (rc != 0) { + return 2; + } + + rc = pthread_attr_getstacksize(&attrs, &size); + if (rc != 0) { + return 2; + } + + if (size < 1024 * 1024) { + return 1; + } + return 0; +} +EOF + + ac_cv_thread_stack_size=unknown + if $CC -pthread $CFLAGS conftest.c -o conftest &>/dev/null; then + ./conftest &>/dev/null + exitcode=$? + if test $exitcode -eq 1; then + ac_cv_thread_stack_size=1048576 + elif test $exitcode -eq 0; then + ac_cv_thread_stack_size="default" + fi + fi + rm -f conftest.c conftest + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_thread_stack_size" >&5 +printf "%s\n" "$ac_cv_thread_stack_size" >&6; } + + if test "$ac_cv_thread_stack_size" != "default" -a "$ac_cv_thread_stack_size" != "unknown"; then + LDFLAGS="$LDFLAGS -Wl,-z,stack-size=$ac_cv_thread_stack_size" + fi +fi + case $enable_wasm_dynamic_linking in #( yes) : ac_cv_func_dlopen=yes ;; #( @@ -14619,7 +14676,7 @@ if test "x$ac_cv_have_libgcc_eh_frame_registration" = xyes then : -printf "%s\n" "#define HAVE_LIBGCC_EH_FRAME_REGISTRATION 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION 1" >>confdefs.h fi @@ -19483,7 +19540,7 @@ if test "x$ac_cv_efficient_builtin_shufflevector" = xyes then : -printf "%s\n" "#define HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR 1" >>confdefs.h fi @@ -24582,7 +24639,7 @@ printf "%s\n" "#define HAVE_DECL_PR_SET_VMA_ANON_NAME $ac_have_decl" >>confdefs. if test $ac_have_decl = 1 then : -printf "%s\n" "#define HAVE_PR_SET_VMA_ANON_NAME 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_PR_SET_VMA_ANON_NAME 1" >>confdefs.h fi @@ -31944,6 +32001,7 @@ case $ac_sys_system in #( py_cv_module_termios=n/a py_cv_module_xxlimited=n/a py_cv_module_xxlimited_35=n/a + py_cv_module_xxlimited_3_13=n/a py_cv_module_=n/a ;; #( @@ -35005,6 +35063,46 @@ fi printf "%s\n" "$py_cv_module_xxlimited_35" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module xxlimited_3_13" >&5 +printf %s "checking for stdlib extension module xxlimited_3_13... " >&6; } + if test "$py_cv_module_xxlimited_3_13" != "n/a" +then : + + if test "$TEST_MODULES" = yes +then : + if test "$ac_cv_func_dlopen" = yes +then : + py_cv_module_xxlimited_3_13=yes +else case e in #( + e) py_cv_module_xxlimited_3_13=missing ;; +esac +fi +else case e in #( + e) py_cv_module_xxlimited_3_13=disabled ;; +esac +fi + +fi + as_fn_append MODULE_BLOCK "MODULE_XXLIMITED_3_13_STATE=$py_cv_module_xxlimited_3_13$as_nl" + if test "x$py_cv_module_xxlimited_3_13" = xyes +then : + + + + +fi + if test "$py_cv_module_xxlimited_3_13" = yes; then + MODULE_XXLIMITED_3_13_TRUE= + MODULE_XXLIMITED_3_13_FALSE='#' +else + MODULE_XXLIMITED_3_13_TRUE='#' + MODULE_XXLIMITED_3_13_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module_xxlimited_3_13" >&5 +printf "%s\n" "$py_cv_module_xxlimited_3_13" >&6; } + + # Determine JIT stencils header files based on target platform JIT_STENCILS_H="" JIT_SHIM_O="" @@ -35518,6 +35616,10 @@ if test -z "${MODULE_XXLIMITED_35_TRUE}" && test -z "${MODULE_XXLIMITED_35_FALSE as_fn_error $? "conditional \"MODULE_XXLIMITED_35\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE_XXLIMITED_3_13_TRUE}" && test -z "${MODULE_XXLIMITED_3_13_FALSE}"; then + as_fn_error $? "conditional \"MODULE_XXLIMITED_3_13\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi : "${CONFIG_STATUS=./config.status}" ac_write_fail=0 diff --git a/configure.ac b/configure.ac index ac3269ab765c0df..a51e173e5293f2d 100644 --- a/configure.ac +++ b/configure.ac @@ -1137,10 +1137,10 @@ preset_cxx="$CXX" if test -z "$CXX" then case "$ac_cv_cc_name" in - gcc) AC_PATH_TOOL([CXX], [g++], [g++], [notfound]) ;; - cc) AC_PATH_TOOL([CXX], [c++], [c++], [notfound]) ;; - clang) AC_PATH_TOOL([CXX], [clang++], [clang++], [notfound]) ;; - icc) AC_PATH_TOOL([CXX], [icpc], [icpc], [notfound]) ;; + gcc) AC_PATH_TOOL([CXX], [g++], [notfound]) ;; + cc) AC_PATH_TOOL([CXX], [c++], [notfound]) ;; + clang) AC_PATH_TOOL([CXX], [clang++], [notfound]) ;; + icc) AC_PATH_TOOL([CXX], [icpc], [notfound]) ;; esac if test "$CXX" = "notfound" then @@ -2462,6 +2462,54 @@ AS_CASE([$ac_sys_system], ] ) +dnl On Linux, check the thread stack size. musl (ex: Alpine Linux) uses +dnl a default thread stack size of 128 kB, whereas the glibc uses 8 MiB. +dnl Python needs at least 1 MiB. +if test "$ac_sys_system" = "Linux" -a "$cross_compiling" = no; then + AC_CACHE_CHECK([for thread stack size], [ac_cv_thread_stack_size], [ + cat > conftest.c < + +int main() +{ + pthread_attr_t attrs; + size_t size; + + int rc = pthread_attr_init(&attrs); + if (rc != 0) { + return 2; + } + + rc = pthread_attr_getstacksize(&attrs, &size); + if (rc != 0) { + return 2; + } + + if (size < 1024 * 1024) { + return 1; + } + return 0; +} +EOF + + ac_cv_thread_stack_size=unknown + if $CC -pthread $CFLAGS conftest.c -o conftest &>/dev/null; then + ./conftest &>/dev/null + exitcode=$? + if test $exitcode -eq 1; then + ac_cv_thread_stack_size=1048576 + elif test $exitcode -eq 0; then + ac_cv_thread_stack_size="default" + fi + fi + rm -f conftest.c conftest + ]) + + if test "$ac_cv_thread_stack_size" != "default" -a "$ac_cv_thread_stack_size" != "unknown"; then + LDFLAGS="$LDFLAGS -Wl,-z,stack-size=$ac_cv_thread_stack_size" + fi +fi + AS_CASE([$enable_wasm_dynamic_linking], [yes], [ac_cv_func_dlopen=yes], [no], [ac_cv_func_dlopen=no], @@ -3863,7 +3911,7 @@ __deregister_frame(0); [ac_cv_have_libgcc_eh_frame_registration=no]) ]) AS_VAR_IF([ac_cv_have_libgcc_eh_frame_registration], [yes], [ - AC_DEFINE([HAVE_LIBGCC_EH_FRAME_REGISTRATION], [1], + AC_DEFINE([_Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION], [1], [Define to 1 if libgcc __register_frame and __deregister_frame are linkable.]) ]) @@ -5163,7 +5211,7 @@ AC_LINK_IFELSE([ ]) AS_VAR_IF([ac_cv_efficient_builtin_shufflevector], [yes], [ - AC_DEFINE([HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR], [1], + AC_DEFINE([_Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR], [1], [Define if compiler supports __builtin_shufflevector with 128-bit vectors AND the target architecture has native SIMD (not just API availability)]) @@ -5788,7 +5836,7 @@ AC_CHECK_DECLS([UT_NAMESIZE], # musl libc redefines struct prctl_mm_map and conflicts with linux/prctl.h AS_IF([test "$ac_cv_libc" != musl], [ AC_CHECK_DECLS([PR_SET_VMA_ANON_NAME], - [AC_DEFINE([HAVE_PR_SET_VMA_ANON_NAME], [1], + [AC_DEFINE([_Py_HAVE_PR_SET_VMA_ANON_NAME], [1], [Define if you have the 'PR_SET_VMA_ANON_NAME' constant.])], [], [@%:@include @@ -8047,6 +8095,7 @@ AS_CASE([$ac_sys_system], [termios], [xxlimited], [xxlimited_35], + [xxlimited_3_13], ) ], [PY_STDLIB_MOD_SET_NA([_scproxy])] @@ -8438,6 +8487,7 @@ dnl Limited API template modules. dnl Emscripten does not support shared libraries yet. PY_STDLIB_MOD([xxlimited], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([xxlimited_35], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited_3_13], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) # Determine JIT stencils header files based on target platform JIT_STENCILS_H="" diff --git a/pyconfig.h.in b/pyconfig.h.in index ad372255445d138..7ef83fcd0b9e0bf 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -320,10 +320,6 @@ /* Define to 1 if you have the header file. */ #undef HAVE_EDITLINE_READLINE_H -/* Define if compiler supports __builtin_shufflevector with 128-bit vectors - AND the target architecture has native SIMD (not just API availability) */ -#undef HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR - /* Define to 1 if you have the header file. */ #undef HAVE_ENDIAN_H @@ -701,10 +697,6 @@ /* Define to 1 if you have the 'dld' library (-ldld). */ #undef HAVE_LIBDLD -/* Define to 1 if libgcc __register_frame and __deregister_frame are linkable. - */ -#undef HAVE_LIBGCC_EH_FRAME_REGISTRATION - /* Define to 1 if you have the 'ieee' library (-lieee). */ #undef HAVE_LIBIEEE @@ -1007,9 +999,6 @@ /* Define if your compiler supports function prototype */ #undef HAVE_PROTOTYPES -/* Define if you have the 'PR_SET_VMA_ANON_NAME' constant. */ -#undef HAVE_PR_SET_VMA_ANON_NAME - /* Define to 1 if you have the 'pthread_condattr_setclock' function. */ #undef HAVE_PTHREAD_CONDATTR_SETCLOCK @@ -2067,6 +2056,17 @@ /* HACL* library can compile SIMD256 implementations */ #undef _Py_HACL_CAN_COMPILE_VEC256 +/* Define if compiler supports __builtin_shufflevector with 128-bit vectors + AND the target architecture has native SIMD (not just API availability) */ +#undef _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR + +/* Define to 1 if libgcc __register_frame and __deregister_frame are linkable. + */ +#undef _Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION + +/* Define if you have the 'PR_SET_VMA_ANON_NAME' constant. */ +#undef _Py_HAVE_PR_SET_VMA_ANON_NAME + /* Define to 1 if the machine stack grows down (default); 0 if it grows up. */ #undef _Py_STACK_GROWS_DOWN