Description: Updates from the 3.14 branch (until 2026-03-11). We pick the latest updates from the maintainance branch, and carry them in a patch, rather than creating and uploading uploading a new .orig tarball. # git diff --no-renames 323c59a5e348347be2ce2b7ea55fcb30bf68b2d3 0a80015ac26d60bff54f57ce9f73ffc5386249b1 | filterdiff -x ?/.hgignore -x ?/.hgeol -x ?/.hgtags -x ?/.hgtouch -x ?/.gitignore -x ?/.gitattributes -x '?/.github/*' -x '?/.git*' -x ?/.codecov.yml -x ?/.travis.yml -x ?/configure --remove-timestamps diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ed88e9ca81b..1d09596671a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.10 + rev: a27a2e47c7751b639d2b5badf0ef6ff11fee893f # frozen: v0.15.4 hooks: - id: ruff-check name: Run Ruff (lint) on Apple/ @@ -52,20 +52,20 @@ repos: files: ^Tools/wasm/ - repo: https://github.com/psf/black-pre-commit-mirror - rev: 25.12.0 + rev: ea488cebbfd88a5f50b8bd95d5c829d0bb76feb8 # frozen: 26.1.0 hooks: - id: black name: Run Black on Tools/jit/ files: ^Tools/jit/ - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: v1.5.5 + rev: ad1b27d73581aa16cca06fc4a0761fc563ffe8e8 # frozen: v1.5.6 hooks: - id: remove-tabs types: [python] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v6.0.0 + rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0 hooks: - id: check-case-conflict - id: check-merge-conflict @@ -83,24 +83,24 @@ repos: files: '^\.github/CODEOWNERS|\.(gram)$' - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.36.0 + rev: 9f48a48aa91a6040d749ad68ec70907d907a5a7f # frozen: 0.37.0 hooks: - id: check-dependabot - id: check-github-workflows - id: check-readthedocs - repo: https://github.com/rhysd/actionlint - rev: v1.7.9 + rev: 393031adb9afb225ee52ae2ccd7a5af5525e03e8 # frozen: v1.7.11 hooks: - id: actionlint - - repo: https://github.com/woodruffw/zizmor-pre-commit - rev: v1.19.0 + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: b546b77c44c466a54a42af5499dcc0dcc1a3193f # frozen: v1.22.0 hooks: - id: zizmor - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v1.0.2 + rev: c883505f64b59c3c5c9375191e4ad9f98e727ccd # frozen: v1.0.2 hooks: - id: sphinx-lint args: [--enable=default-role] diff --git a/Android/android.py b/Android/android.py index 629696be3db..0a894a958a0 100755 --- a/Android/android.py +++ b/Android/android.py @@ -208,7 +208,7 @@ def make_build_python(context): def unpack_deps(host, prefix_dir): os.chdir(prefix_dir) deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" - for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.18-0", + for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.19-1", "sqlite-3.50.4-0", "xz-5.4.6-1", "zstd-1.5.7-1"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") diff --git a/Apple/__main__.py b/Apple/__main__.py index 256966e76c2..253bdfaab55 100644 --- a/Apple/__main__.py +++ b/Apple/__main__.py @@ -316,7 +316,7 @@ def unpack_deps( for name_ver in [ "BZip2-1.0.8-2", "libFFI-3.4.7-2", - "OpenSSL-3.0.18-1", + "OpenSSL-3.0.19-1", "XZ-5.6.4-2", "mpdecimal-4.0.0-2", "zstd-1.5.7-1", diff --git a/Doc/.ruff.toml b/Doc/.ruff.toml index 3e676e13c3f..6b573fd58d0 100644 --- a/Doc/.ruff.toml +++ b/Doc/.ruff.toml @@ -32,6 +32,9 @@ ignore = [ "E501", # Ignore line length errors (we use auto-formatting) ] +[lint.per-file-ignores] +"tools/check-html-ids.py" = ["I001"] # Unsorted imports + [format] preview = true quote-style = "preserve" diff --git a/Doc/Makefile b/Doc/Makefile index 4d605980a62..5b7fdf8ec08 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -58,7 +58,7 @@ build: @if [ -f ../Misc/NEWS ] ; then \ echo "Using existing Misc/NEWS file"; \ cp ../Misc/NEWS build/NEWS; \ - elif $(BLURB) help >/dev/null 2>&1 && $(SPHINXBUILD) --version >/dev/null 2>&1; then \ + elif $(BLURB) --version && $(SPHINXBUILD) --version ; then \ if [ -d ../Misc/NEWS.d ]; then \ echo "Building NEWS from Misc/NEWS.d with blurb"; \ $(BLURB) merge -f build/NEWS; \ @@ -336,3 +336,9 @@ autobuild-stable-html: exit 1;; \ esac @$(MAKE) autobuild-dev-html + +# Collect HTML IDs to a JSON document +.PHONY: html-ids +html-ids: + $(PYTHON) tools/check-html-ids.py collect build/html \ + -o build/html/html-ids.json.gz diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 1d27579e53f..9f2b9876ba5 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -9,7 +9,7 @@ stability. In order to maintain this reputation, the developers would like to know of any deficiencies you find in Python. It can be sometimes faster to fix bugs yourself and contribute patches to -Python as it streamlines the process and involves less people. Learn how to +Python as it streamlines the process and involves fewer people. Learn how to :ref:`contribute `. Documentation bugs diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index d7fe9e2c9ec..3a7fa0d1ff6 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -673,28 +673,46 @@ Signal Handling single: SIGINT (C macro) single: KeyboardInterrupt (built-in exception) - This function interacts with Python's signal handling. + Handle external interruptions, such as signals or activating a debugger, + whose processing has been delayed until it is safe + to run Python code and/or raise exceptions. - If the function is called from the main thread and under the main Python - interpreter, it checks whether a signal has been sent to the processes - and if so, invokes the corresponding signal handler. If the :mod:`signal` - module is supported, this can invoke a signal handler written in Python. + For example, pressing :kbd:`Ctrl-C` causes a terminal to send the + :py:data:`signal.SIGINT` signal. + This function executes the corresponding Python signal handler, which, + by default, raises the :exc:`KeyboardInterrupt` exception. - The function attempts to handle all pending signals, and then returns ``0``. - However, if a Python signal handler raises an exception, the error - indicator is set and the function returns ``-1`` immediately (such that - other pending signals may not have been handled yet: they will be on the - next :c:func:`PyErr_CheckSignals()` invocation). + :c:func:`!PyErr_CheckSignals` should be called by long-running C code + frequently enough so that the response appears immediate to humans. - If the function is called from a non-main thread, or under a non-main - Python interpreter, it does nothing and returns ``0``. + Handlers invoked by this function currently include: - This function can be called by long-running C code that wants to - be interruptible by user requests (such as by pressing Ctrl-C). + - Signal handlers, including Python functions registered using + the :mod:`signal` module. - .. note:: - The default Python signal handler for :c:macro:`!SIGINT` raises the - :exc:`KeyboardInterrupt` exception. + Signal handlers are only run in the main thread of the main interpreter. + + (This is where the function got the name: originally, signals + were the only way to interrupt the interpreter.) + + - Running the garbage collector, if necessary. + + - Executing a pending :ref:`remote debugger ` script. + + If any handler raises an exception, immediately return ``-1`` with that + exception set. + Any remaining interruptions are left to be processed on the next + :c:func:`PyErr_CheckSignals()` invocation, if appropriate. + + If all handlers finish successfully, or there are no handlers to run, + return ``0``. + + .. versionchanged:: 3.12 + This function may now invoke the garbage collector. + + .. versionchanged:: 3.14 + This function may now execute a remote debugger script, if remote + debugging is enabled. .. c:function:: void PyErr_SetInterrupt() diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 51540004c93..75ea3d819d3 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -80,7 +80,7 @@ Floating-Point Objects .. c:macro:: Py_INFINITY - This macro expands a to constant expression of type :c:expr:`double`, that + This macro expands to a constant expression of type :c:expr:`double`, that represents the positive infinity. On most platforms, this is equivalent to the :c:macro:`!INFINITY` macro from @@ -89,7 +89,7 @@ Floating-Point Objects .. c:macro:: Py_NAN - This macro expands a to constant expression of type :c:expr:`double`, that + This macro expands to a constant expression of type :c:expr:`double`, that represents a quiet not-a-number (qNaN) value. On most platforms, this is equivalent to the :c:macro:`!NAN` macro from diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index fb17cf7f1da..967cfc72765 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -50,6 +50,7 @@ See also :ref:`Reflection `. Return a :term:`strong reference`, or ``NULL`` if *frame* has no outer frame. + This raises no exceptions. .. versionadded:: 3.9 diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index e9df2a304d9..eabe00f4004 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -1,7 +1,7 @@ .. _c-api-index: ################################## - Python/C API Reference Manual + Python/C API reference manual ################################## This manual documents the API used by C and C++ programmers who want to write @@ -21,7 +21,12 @@ document the API functions in detail. utilities.rst abstract.rst concrete.rst - init.rst + interp-lifecycle.rst + threads.rst + synchronization.rst + tls.rst + subinterpreters.rst + profiling.rst init_config.rst memory.rst objimpl.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 1559f1c8fcd..e56c67f9534 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1,2927 +1,13 @@ -.. highlight:: c +:orphan: +Initialization, finalization, and threads +========================================= -.. _initialization: +This page has been split up into the following: -***************************************** -Initialization, Finalization, and Threads -***************************************** - -See :ref:`Python Initialization Configuration ` for details -on how to configure the interpreter prior to initialization. - -.. _pre-init-safe: - -Before Python Initialization -============================ - -In an application embedding Python, the :c:func:`Py_Initialize` function must -be called before using any other Python/C API functions; with the exception of -a few functions and the :ref:`global configuration variables -`. - -The following functions can be safely called before Python is initialized: - -* Functions that initialize the interpreter: - - * :c:func:`Py_Initialize` - * :c:func:`Py_InitializeEx` - * :c:func:`Py_InitializeFromConfig` - * :c:func:`Py_BytesMain` - * :c:func:`Py_Main` - * the runtime pre-initialization functions covered in :ref:`init-config` - -* Configuration functions: - - * :c:func:`PyImport_AppendInittab` - * :c:func:`PyImport_ExtendInittab` - * :c:func:`!PyInitFrozenExtensions` - * :c:func:`PyMem_SetAllocator` - * :c:func:`PyMem_SetupDebugHooks` - * :c:func:`PyObject_SetArenaAllocator` - * :c:func:`Py_SetProgramName` - * :c:func:`Py_SetPythonHome` - * :c:func:`PySys_ResetWarnOptions` - * the configuration functions covered in :ref:`init-config` - -* Informative functions: - - * :c:func:`Py_IsInitialized` - * :c:func:`PyMem_GetAllocator` - * :c:func:`PyObject_GetArenaAllocator` - * :c:func:`Py_GetBuildInfo` - * :c:func:`Py_GetCompiler` - * :c:func:`Py_GetCopyright` - * :c:func:`Py_GetPlatform` - * :c:func:`Py_GetVersion` - * :c:func:`Py_IsInitialized` - -* Utilities: - - * :c:func:`Py_DecodeLocale` - * the status reporting and utility functions covered in :ref:`init-config` - -* Memory allocators: - - * :c:func:`PyMem_RawMalloc` - * :c:func:`PyMem_RawRealloc` - * :c:func:`PyMem_RawCalloc` - * :c:func:`PyMem_RawFree` - -* Synchronization: - - * :c:func:`PyMutex_Lock` - * :c:func:`PyMutex_Unlock` - -.. note:: - - Despite their apparent similarity to some of the functions listed above, - the following functions **should not be called** before the interpreter has - been initialized: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, - :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, - :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome`, - :c:func:`Py_GetProgramName`, :c:func:`PyEval_InitThreads`, and - :c:func:`Py_RunMain`. - - -.. _global-conf-vars: - -Global configuration variables -============================== - -Python has variables for the global configuration to control different features -and options. By default, these flags are controlled by :ref:`command line -options `. - -When a flag is set by an option, the value of the flag is the number of times -that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` -to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. - -.. c:var:: int Py_BytesWarningFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.bytes_warning` should be used instead, see :ref:`Python - Initialization Configuration `. - - Issue a warning when comparing :class:`bytes` or :class:`bytearray` with - :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater - or equal to ``2``. - - Set by the :option:`-b` option. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_DebugFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.parser_debug` should be used instead, see :ref:`Python - Initialization Configuration `. - - Turn on parser debugging output (for expert only, depending on compilation - options). - - Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_DontWriteBytecodeFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.write_bytecode` should be used instead, see :ref:`Python - Initialization Configuration `. - - If set to non-zero, Python won't try to write ``.pyc`` files on the - import of source modules. - - Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` - environment variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_FrozenFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.pathconfig_warnings` should be used instead, see - :ref:`Python Initialization Configuration `. - - Suppress error messages when calculating the module search path in - :c:func:`Py_GetPath`. - - Private flag used by ``_freeze_module`` and ``frozenmain`` programs. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_HashRandomizationFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.hash_seed` and :c:member:`PyConfig.use_hash_seed` should - be used instead, see :ref:`Python Initialization Configuration - `. - - Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to - a non-empty string. - - If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment - variable to initialize the secret hash seed. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_IgnoreEnvironmentFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.use_environment` should be used instead, see - :ref:`Python Initialization Configuration `. - - Ignore all :envvar:`!PYTHON*` environment variables, e.g. - :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. - - Set by the :option:`-E` and :option:`-I` options. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_InspectFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.inspect` should be used instead, see - :ref:`Python Initialization Configuration `. - - When a script is passed as first argument or the :option:`-c` option is used, - enter interactive mode after executing the script or the command, even when - :data:`sys.stdin` does not appear to be a terminal. - - Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_InteractiveFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.interactive` should be used instead, see - :ref:`Python Initialization Configuration `. - - Set by the :option:`-i` option. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_IsolatedFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.isolated` should be used instead, see - :ref:`Python Initialization Configuration `. - - Run Python in isolated mode. In isolated mode :data:`sys.path` contains - neither the script's directory nor the user's site-packages directory. - - Set by the :option:`-I` option. - - .. versionadded:: 3.4 - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_LegacyWindowsFSEncodingFlag - - This API is kept for backward compatibility: setting - :c:member:`PyPreConfig.legacy_windows_fs_encoding` should be used instead, see - :ref:`Python Initialization Configuration `. - - If the flag is non-zero, use the ``mbcs`` encoding with ``replace`` error - handler, instead of the UTF-8 encoding with ``surrogatepass`` error handler, - for the :term:`filesystem encoding and error handler`. - - Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment - variable is set to a non-empty string. - - See :pep:`529` for more details. - - .. availability:: Windows. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_LegacyWindowsStdioFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.legacy_windows_stdio` should be used instead, see - :ref:`Python Initialization Configuration `. - - If the flag is non-zero, use :class:`io.FileIO` instead of - :class:`!io._WindowsConsoleIO` for :mod:`sys` standard streams. - - Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment - variable is set to a non-empty string. - - See :pep:`528` for more details. - - .. availability:: Windows. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_NoSiteFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.site_import` should be used instead, see - :ref:`Python Initialization Configuration `. - - Disable the import of the module :mod:`site` and the site-dependent - manipulations of :data:`sys.path` that it entails. Also disable these - manipulations if :mod:`site` is explicitly imported later (call - :func:`site.main` if you want them to be triggered). - - Set by the :option:`-S` option. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_NoUserSiteDirectory - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.user_site_directory` should be used instead, see - :ref:`Python Initialization Configuration `. - - Don't add the :data:`user site-packages directory ` to - :data:`sys.path`. - - Set by the :option:`-s` and :option:`-I` options, and the - :envvar:`PYTHONNOUSERSITE` environment variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_OptimizeFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.optimization_level` should be used instead, see - :ref:`Python Initialization Configuration `. - - Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_QuietFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.quiet` should be used instead, see :ref:`Python - Initialization Configuration `. - - Don't display the copyright and version messages even in interactive mode. - - Set by the :option:`-q` option. - - .. versionadded:: 3.2 - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_UnbufferedStdioFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.buffered_stdio` should be used instead, see :ref:`Python - Initialization Configuration `. - - Force the stdout and stderr streams to be unbuffered. - - Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` - environment variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_VerboseFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.verbose` should be used instead, see :ref:`Python - Initialization Configuration `. - - Print a message each time a module is initialized, showing the place - (filename or built-in module) from which it is loaded. If greater or equal - to ``2``, print a message for each file that is checked for when - searching for a module. Also provides information on module cleanup at exit. - - Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - - -Initializing and finalizing the interpreter -=========================================== - - -.. c:function:: void Py_Initialize() - - .. index:: - single: PyEval_InitThreads() - single: modules (in module sys) - single: path (in module sys) - pair: module; builtins - pair: module; __main__ - pair: module; sys - triple: module; search; path - single: Py_FinalizeEx (C function) - - Initialize the Python interpreter. In an application embedding Python, - this should be called before using any other Python/C API functions; see - :ref:`Before Python Initialization ` for the few exceptions. - - This initializes the table of loaded modules (``sys.modules``), and creates - the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. - It also initializes the module search path (``sys.path``). It does not set - ``sys.argv``; use the :ref:`Python Initialization Configuration ` - API for that. This is a no-op when called for a second time (without calling - :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal - error if the initialization fails. - - Use :c:func:`Py_InitializeFromConfig` to customize the - :ref:`Python Initialization Configuration `. - - .. note:: - On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, - which will also affect non-Python uses of the console using the C Runtime. - - -.. c:function:: void Py_InitializeEx(int initsigs) - - This function works like :c:func:`Py_Initialize` if *initsigs* is ``1``. If - *initsigs* is ``0``, it skips initialization registration of signal handlers, - which may be useful when CPython is embedded as part of a larger application. - - Use :c:func:`Py_InitializeFromConfig` to customize the - :ref:`Python Initialization Configuration `. - - -.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) - - Initialize Python from *config* configuration, as described in - :ref:`init-from-config`. - - See the :ref:`init-config` section for details on pre-initializing the - interpreter, populating the runtime configuration structure, and querying - the returned status structure. - - -.. c:function:: int Py_IsInitialized() - - Return true (nonzero) when the Python interpreter has been initialized, false - (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until - :c:func:`Py_Initialize` is called again. - - -.. c:function:: int Py_IsFinalizing() - - Return true (non-zero) if the main Python interpreter is - :term:`shutting down `. Return false (zero) otherwise. - - .. versionadded:: 3.13 - - -.. c:function:: int Py_FinalizeEx() - - Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of - Python/C API functions, and destroy all sub-interpreters (see - :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since - the last call to :c:func:`Py_Initialize`. This is a no-op when called for a second - time (without calling :c:func:`Py_Initialize` again first). - - Since this is the reverse of :c:func:`Py_Initialize`, it should be called - in the same thread with the same interpreter active. That means - the main thread and the main interpreter. - This should never be called while :c:func:`Py_RunMain` is running. - - Normally the return value is ``0``. - If there were errors during finalization (flushing buffered data), - ``-1`` is returned. - - Note that Python will do a best effort at freeing all memory allocated by the Python - interpreter. Therefore, any C-Extension should make sure to correctly clean up all - of the preveiously allocated PyObjects before using them in subsequent calls to - :c:func:`Py_Initialize`. Otherwise it could introduce vulnerabilities and incorrect - behavior. - - This function is provided for a number of reasons. An embedding application - might want to restart Python without having to restart the application itself. - An application that has loaded the Python interpreter from a dynamically - loadable library (or DLL) might want to free all memory allocated by Python - before unloading the DLL. During a hunt for memory leaks in an application a - developer might want to free all memory allocated by Python before exiting from - the application. - - **Bugs and caveats:** The destruction of modules and objects in modules is done - in random order; this may cause destructors (:meth:`~object.__del__` methods) to fail - when they depend on other objects (even functions) or modules. Dynamically - loaded extension modules loaded by Python are not unloaded. Small amounts of - memory allocated by the Python interpreter may not be freed (if you find a leak, - please report it). Memory tied up in circular references between objects is not - freed. Interned strings will all be deallocated regardless of their reference count. - Some memory allocated by extension modules may not be freed. Some extensions may not - work properly if their initialization routine is called more than once; this can - happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` - more than once. :c:func:`Py_FinalizeEx` must not be called recursively from - within itself. Therefore, it must not be called by any code that may be run - as part of the interpreter shutdown process, such as :py:mod:`atexit` - handlers, object finalizers, or any code that may be run while flushing the - stdout and stderr files. - - .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx - - .. versionadded:: 3.6 - - -.. c:function:: void Py_Finalize() - - This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that - disregards the return value. - - -.. c:function:: int Py_BytesMain(int argc, char **argv) - - Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings, - allowing the calling application to delegate the text decoding step to - the CPython runtime. - - .. versionadded:: 3.8 - - -.. c:function:: int Py_Main(int argc, wchar_t **argv) - - The main program for the standard interpreter, encapsulating a full - initialization/finalization cycle, as well as additional - behaviour to implement reading configurations settings from the environment - and command line, and then executing ``__main__`` in accordance with - :ref:`using-on-cmdline`. - - This is made available for programs which wish to support the full CPython - command line interface, rather than just embedding a Python runtime in a - larger application. - - The *argc* and *argv* parameters are similar to those which are passed to a - C program's :c:func:`main` function, except that the *argv* entries are first - converted to ``wchar_t`` using :c:func:`Py_DecodeLocale`. It is also - important to note that the argument list entries may be modified to point to - strings other than those passed in (however, the contents of the strings - pointed to by the argument list are not modified). - - The return value is ``2`` if the argument list does not represent a valid - Python command line, and otherwise the same as :c:func:`Py_RunMain`. - - In terms of the CPython runtime configuration APIs documented in the - :ref:`runtime configuration ` section (and without accounting - for error handling), ``Py_Main`` is approximately equivalent to:: - - PyConfig config; - PyConfig_InitPythonConfig(&config); - PyConfig_SetArgv(&config, argc, argv); - Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - - Py_RunMain(); - - In normal usage, an embedding application will call this function - *instead* of calling :c:func:`Py_Initialize`, :c:func:`Py_InitializeEx` or - :c:func:`Py_InitializeFromConfig` directly, and all settings will be applied - as described elsewhere in this documentation. If this function is instead - called *after* a preceding runtime initialization API call, then exactly - which environmental and command line configuration settings will be updated - is version dependent (as it depends on which settings correctly support - being modified after they have already been set once when the runtime was - first initialized). - - -.. c:function:: int Py_RunMain(void) - - Executes the main module in a fully configured CPython runtime. - - Executes the command (:c:member:`PyConfig.run_command`), the script - (:c:member:`PyConfig.run_filename`) or the module - (:c:member:`PyConfig.run_module`) specified on the command line or in the - configuration. If none of these values are set, runs the interactive Python - prompt (REPL) using the ``__main__`` module's global namespace. - - If :c:member:`PyConfig.inspect` is not set (the default), the return value - will be ``0`` if the interpreter exits normally (that is, without raising - an exception), the exit status of an unhandled :exc:`SystemExit`, or ``1`` - for any other unhandled exception. - - If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option - is used), rather than returning when the interpreter exits, execution will - instead resume in an interactive Python prompt (REPL) using the ``__main__`` - module's global namespace. If the interpreter exited with an exception, it - is immediately raised in the REPL session. The function return value is - then determined by the way the *REPL session* terminates: ``0``, ``1``, or - the status of a :exc:`SystemExit`, as specified above. - - This function always finalizes the Python interpreter before it returns. - - See :ref:`Python Configuration ` for an example of a - customized Python that always runs in isolated mode using - :c:func:`Py_RunMain`. - -.. c:function:: int PyUnstable_AtExit(PyInterpreterState *interp, void (*func)(void *), void *data) - - Register an :mod:`atexit` callback for the target interpreter *interp*. - This is similar to :c:func:`Py_AtExit`, but takes an explicit interpreter and - data pointer for the callback. - - There must be an :term:`attached thread state` for *interp*. - - .. versionadded:: 3.13 - -Process-wide parameters -======================= - - -.. c:function:: void Py_SetProgramName(const wchar_t *name) - - .. index:: - single: Py_Initialize() - single: main() - single: Py_GetPath() - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python - Initialization Configuration `. - - This function should be called before :c:func:`Py_Initialize` is called for - the first time, if it is called at all. It tells the interpreter the value - of the ``argv[0]`` argument to the :c:func:`main` function of the program - (converted to wide characters). - This is used by :c:func:`Py_GetPath` and some other functions below to find - the Python run-time libraries relative to the interpreter executable. The - default value is ``'python'``. The argument should point to a - zero-terminated wide character string in static storage whose contents will not - change for the duration of the program's execution. No code in the Python - interpreter will change the contents of this storage. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: wchar_t* Py_GetProgramName() - - Return the program name set with :c:member:`PyConfig.program_name`, or the default. - The returned string points into static storage; the caller should not modify its - value. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("executable") ` - (:data:`sys.executable`) instead. - - -.. c:function:: wchar_t* Py_GetPrefix() - - Return the *prefix* for installed platform-independent files. This is derived - through a number of complicated rules from the program name set with - :c:member:`PyConfig.program_name` and some environment variables; for example, if the - program name is ``'/usr/local/bin/python'``, the prefix is ``'/usr/local'``. The - returned string points into static storage; the caller should not modify its - value. This corresponds to the :makevar:`prefix` variable in the top-level - :file:`Makefile` and the :option:`--prefix` argument to the :program:`configure` - script at build time. The value is available to Python code as ``sys.base_prefix``. - It is only useful on Unix. See also the next function. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("base_prefix") ` - (:data:`sys.base_prefix`) instead. Use :c:func:`PyConfig_Get("prefix") - ` (:data:`sys.prefix`) if :ref:`virtual environments - ` need to be handled. - - -.. c:function:: wchar_t* Py_GetExecPrefix() - - Return the *exec-prefix* for installed platform-*dependent* files. This is - derived through a number of complicated rules from the program name set with - :c:member:`PyConfig.program_name` and some environment variables; for example, if the - program name is ``'/usr/local/bin/python'``, the exec-prefix is - ``'/usr/local'``. The returned string points into static storage; the caller - should not modify its value. This corresponds to the :makevar:`exec_prefix` - variable in the top-level :file:`Makefile` and the ``--exec-prefix`` - argument to the :program:`configure` script at build time. The value is - available to Python code as ``sys.base_exec_prefix``. It is only useful on - Unix. - - Background: The exec-prefix differs from the prefix when platform dependent - files (such as executables and shared libraries) are installed in a different - directory tree. In a typical installation, platform dependent files may be - installed in the :file:`/usr/local/plat` subtree while platform independent may - be installed in :file:`/usr/local`. - - Generally speaking, a platform is a combination of hardware and software - families, e.g. Sparc machines running the Solaris 2.x operating system are - considered the same platform, but Intel machines running Solaris 2.x are another - platform, and Intel machines running Linux are yet another platform. Different - major revisions of the same operating system generally also form different - platforms. Non-Unix operating systems are a different story; the installation - strategies on those systems are so different that the prefix and exec-prefix are - meaningless, and set to the empty string. Note that compiled Python bytecode - files are platform independent (but not independent from the Python version by - which they were compiled!). - - System administrators will know how to configure the :program:`mount` or - :program:`automount` programs to share :file:`/usr/local` between platforms - while having :file:`/usr/local/plat` be a different filesystem for each - platform. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("base_exec_prefix") ` - (:data:`sys.base_exec_prefix`) instead. Use - :c:func:`PyConfig_Get("exec_prefix") ` - (:data:`sys.exec_prefix`) if :ref:`virtual environments ` need - to be handled. - -.. c:function:: wchar_t* Py_GetProgramFullPath() - - .. index:: - single: executable (in module sys) - - Return the full program name of the Python executable; this is computed as a - side-effect of deriving the default module search path from the program name - (set by :c:member:`PyConfig.program_name`). The returned string points into - static storage; the caller should not modify its value. The value is available - to Python code as ``sys.executable``. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("executable") ` - (:data:`sys.executable`) instead. - - -.. c:function:: wchar_t* Py_GetPath() - - .. index:: - triple: module; search; path - single: path (in module sys) - - Return the default module search path; this is computed from the program name - (set by :c:member:`PyConfig.program_name`) and some environment variables. - The returned string consists of a series of directory names separated by a - platform dependent delimiter character. The delimiter character is ``':'`` - on Unix and macOS, ``';'`` on Windows. The returned string points into - static storage; the caller should not modify its value. The list - :data:`sys.path` is initialized with this value on interpreter startup; it - can be (and usually is) modified later to change the search path for loading - modules. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. XXX should give the exact rules - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("module_search_paths") ` - (:data:`sys.path`) instead. - -.. c:function:: const char* Py_GetVersion() - - Return the version of this Python interpreter. This is a string that looks - something like :: - - "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]" - - .. index:: single: version (in module sys) - - The first word (up to the first space character) is the current Python version; - the first characters are the major and minor version separated by a - period. The returned string points into static storage; the caller should not - modify its value. The value is available to Python code as :data:`sys.version`. - - See also the :c:var:`Py_Version` constant. - - -.. c:function:: const char* Py_GetPlatform() - - .. index:: single: platform (in module sys) - - Return the platform identifier for the current platform. On Unix, this is - formed from the "official" name of the operating system, converted to lower - case, followed by the major revision number; e.g., for Solaris 2.x, which is - also known as SunOS 5.x, the value is ``'sunos5'``. On macOS, it is - ``'darwin'``. On Windows, it is ``'win'``. The returned string points into - static storage; the caller should not modify its value. The value is available - to Python code as ``sys.platform``. - - -.. c:function:: const char* Py_GetCopyright() - - Return the official copyright string for the current Python version, for example - - ``'Copyright 1991-1995 Stichting Mathematisch Centrum, Amsterdam'`` - - .. index:: single: copyright (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as ``sys.copyright``. - - -.. c:function:: const char* Py_GetCompiler() - - Return an indication of the compiler used to build the current Python version, - in square brackets, for example:: - - "[GCC 2.7.2.2]" - - .. index:: single: version (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as part of the variable - ``sys.version``. - - -.. c:function:: const char* Py_GetBuildInfo() - - Return information about the sequence number and build date and time of the - current Python interpreter instance, for example :: - - "#67, Aug 1 1997, 22:34:28" - - .. index:: single: version (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as part of the variable - ``sys.version``. - - -.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) - - .. index:: - single: main() - single: Py_FatalError() - single: argv (in module sys) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and - :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python - Initialization Configuration `. - - Set :data:`sys.argv` based on *argc* and *argv*. These parameters are - similar to those passed to the program's :c:func:`main` function with the - difference that the first entry should refer to the script file to be - executed rather than the executable hosting the Python interpreter. If there - isn't a script that will be run, the first entry in *argv* can be an empty - string. If this function fails to initialize :data:`sys.argv`, a fatal - condition is signalled using :c:func:`Py_FatalError`. - - If *updatepath* is zero, this is all the function does. If *updatepath* - is non-zero, the function also modifies :data:`sys.path` according to the - following algorithm: - - - If the name of an existing script is passed in ``argv[0]``, the absolute - path of the directory where the script is located is prepended to - :data:`sys.path`. - - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point - to an existing file name), an empty string is prepended to - :data:`sys.path`, which is the same as prepending the current working - directory (``"."``). - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` - members of the :ref:`Python Initialization Configuration `. - - .. note:: - It is recommended that applications embedding the Python interpreter - for purposes other than executing a single script pass ``0`` as *updatepath*, - and update :data:`sys.path` themselves if desired. - See :cve:`2008-5983`. - - On versions before 3.1.3, you can achieve the same effect by manually - popping the first :data:`sys.path` element after having called - :c:func:`PySys_SetArgv`, for example using:: - - PyRun_SimpleString("import sys; sys.path.pop(0)\n"); - - .. versionadded:: 3.1.3 - - .. XXX impl. doesn't seem consistent in allowing ``0``/``NULL`` for the params; - check w/ Guido. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used - instead, see :ref:`Python Initialization Configuration `. - - This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set - to ``1`` unless the :program:`python` interpreter was started with the - :option:`-I`. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` - members of the :ref:`Python Initialization Configuration `. - - .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: void Py_SetPythonHome(const wchar_t *home) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.home` should be used instead, see :ref:`Python - Initialization Configuration `. - - Set the default "home" directory, that is, the location of the standard - Python libraries. See :envvar:`PYTHONHOME` for the meaning of the - argument string. - - The argument should point to a zero-terminated character string in static - storage whose contents will not change for the duration of the program's - execution. No code in the Python interpreter will change the contents of - this storage. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: wchar_t* Py_GetPythonHome() - - Return the default "home", that is, the value set by - :c:member:`PyConfig.home`, or the value of the :envvar:`PYTHONHOME` - environment variable if it is set. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("home") ` or the - :envvar:`PYTHONHOME` environment variable instead. - - -.. _threads: - -Thread State and the Global Interpreter Lock -============================================ - -.. index:: - single: global interpreter lock - single: interpreter lock - single: lock, interpreter - -Unless on a :term:`free-threaded ` build of :term:`CPython`, -the Python interpreter is not fully thread-safe. In order to support -multi-threaded Python programs, there's a global lock, called the :term:`global -interpreter lock` or :term:`GIL`, that must be held by the current thread before -it can safely access Python objects. Without the lock, even the simplest -operations could cause problems in a multi-threaded program: for example, when -two threads simultaneously increment the reference count of the same object, the -reference count could end up being incremented only once instead of twice. - -.. index:: single: setswitchinterval (in module sys) - -Therefore, the rule exists that only the thread that has acquired the -:term:`GIL` may operate on Python objects or call Python/C API functions. -In order to emulate concurrency of execution, the interpreter regularly -tries to switch threads (see :func:`sys.setswitchinterval`). The lock is also -released around potentially blocking I/O operations like reading or writing -a file, so that other Python threads can run in the meantime. - -.. index:: - single: PyThreadState (C type) - -The Python interpreter keeps some thread-specific bookkeeping information -inside a data structure called :c:type:`PyThreadState`, known as a :term:`thread state`. -Each OS thread has a thread-local pointer to a :c:type:`PyThreadState`; a thread state -referenced by this pointer is considered to be :term:`attached `. - -A thread can only have one :term:`attached thread state` at a time. An attached -thread state is typically analogous with holding the :term:`GIL`, except on -:term:`free-threaded ` builds. On builds with the :term:`GIL` enabled, -:term:`attaching ` a thread state will block until the :term:`GIL` -can be acquired. However, even on builds with the :term:`GIL` disabled, it is still required -to have an attached thread state to call most of the C API. - -In general, there will always be an :term:`attached thread state` when using Python's C API. -Only in some specific cases (such as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block) will the -thread not have an attached thread state. If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns -``NULL``. - -Detaching the thread state from extension code ----------------------------------------------- - -Most extension code manipulating the :term:`thread state` has the following simple -structure:: - - Save the thread state in a local variable. - ... Do some blocking I/O operation ... - Restore the thread state from the local variable. - -This is so common that a pair of macros exists to simplify it:: - - Py_BEGIN_ALLOW_THREADS - ... Do some blocking I/O operation ... - Py_END_ALLOW_THREADS - -.. index:: - single: Py_BEGIN_ALLOW_THREADS (C macro) - single: Py_END_ALLOW_THREADS (C macro) - -The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a -hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the -block. - -The block above expands to the following code:: - - PyThreadState *_save; - - _save = PyEval_SaveThread(); - ... Do some blocking I/O operation ... - PyEval_RestoreThread(_save); - -.. index:: - single: PyEval_RestoreThread (C function) - single: PyEval_SaveThread (C function) - -Here is how these functions work: - -The :term:`attached thread state` holds the :term:`GIL` for the entire interpreter. When detaching -the :term:`attached thread state`, the :term:`GIL` is released, allowing other threads to attach -a thread state to their own thread, thus getting the :term:`GIL` and can start executing. -The pointer to the prior :term:`attached thread state` is stored as a local variable. -Upon reaching :c:macro:`Py_END_ALLOW_THREADS`, the thread state that was -previously :term:`attached ` is passed to :c:func:`PyEval_RestoreThread`. -This function will block until another releases its :term:`thread state `, -thus allowing the old :term:`thread state ` to get re-attached and the -C API can be called again. - -For :term:`free-threaded ` builds, the :term:`GIL` is normally -out of the question, but detaching the :term:`thread state ` is still required -for blocking I/O and long operations. The difference is that threads don't have to wait for the :term:`GIL` -to be released to attach their thread state, allowing true multi-core parallelism. - -.. note:: - Calling system I/O functions is the most common use case for detaching - the :term:`thread state `, but it can also be useful before calling - long-running computations which don't need access to Python objects, such - as compression or cryptographic functions operating over memory buffers. - For example, the standard :mod:`zlib` and :mod:`hashlib` modules detach the - :term:`thread state ` when compressing or hashing data. - - -.. _gilstate: - -Non-Python created threads --------------------------- - -When threads are created using the dedicated Python APIs (such as the -:mod:`threading` module), a thread state is automatically associated to them -and the code showed above is therefore correct. However, when threads are -created from C (for example by a third-party library with its own thread -management), they don't hold the :term:`GIL`, because they don't have an -:term:`attached thread state`. - -If you need to call Python code from these threads (often this will be part -of a callback API provided by the aforementioned third-party library), -you must first register these threads with the interpreter by -creating an :term:`attached thread state` before you can start using the Python/C -API. When you are done, you should detach the :term:`thread state `, and -finally free it. - -The :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` functions do -all of the above automatically. The typical idiom for calling into Python -from a C thread is:: - - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); - - /* Perform Python actions here. */ - result = CallSomeFunction(); - /* evaluate result or handle exception */ - - /* Release the thread. No Python API allowed beyond this point. */ - PyGILState_Release(gstate); - -Note that the ``PyGILState_*`` functions assume there is only one global -interpreter (created automatically by :c:func:`Py_Initialize`). Python -supports the creation of additional interpreters (using -:c:func:`Py_NewInterpreter`), but mixing multiple interpreters and the -``PyGILState_*`` API is unsupported. This is because :c:func:`PyGILState_Ensure` -and similar functions default to :term:`attaching ` a -:term:`thread state` for the main interpreter, meaning that the thread can't safely -interact with the calling subinterpreter. - -Supporting subinterpreters in non-Python threads ------------------------------------------------- - -If you would like to support subinterpreters with non-Python created threads, you -must use the ``PyThreadState_*`` API instead of the traditional ``PyGILState_*`` -API. - -In particular, you must store the interpreter state from the calling -function and pass it to :c:func:`PyThreadState_New`, which will ensure that -the :term:`thread state` is targeting the correct interpreter:: - - /* The return value of PyInterpreterState_Get() from the - function that created this thread. */ - PyInterpreterState *interp = ThreadData->interp; - PyThreadState *tstate = PyThreadState_New(interp); - PyThreadState_Swap(tstate); - - /* GIL of the subinterpreter is now held. - Perform Python actions here. */ - result = CallSomeFunction(); - /* evaluate result or handle exception */ - - /* Destroy the thread state. No Python API allowed beyond this point. */ - PyThreadState_Clear(tstate); - PyThreadState_DeleteCurrent(); - -.. _fork-and-threads: - -Cautions about fork() ---------------------- - -Another important thing to note about threads is their behaviour in the face -of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a -process forks only the thread that issued the fork will exist. This has a -concrete impact both on how locks must be handled and on all stored state -in CPython's runtime. - -The fact that only the "current" thread remains -means any locks held by other threads will never be released. Python solves -this for :func:`os.fork` by acquiring the locks it uses internally before -the fork, and releasing them afterwards. In addition, it resets any -:ref:`lock-objects` in the child. When extending or embedding Python, there -is no way to inform Python of additional (non-Python) locks that need to be -acquired before or reset after a fork. OS facilities such as -:c:func:`!pthread_atfork` would need to be used to accomplish the same thing. -Additionally, when extending or embedding Python, calling :c:func:`fork` -directly rather than through :func:`os.fork` (and returning to or calling -into Python) may result in a deadlock by one of Python's internal locks -being held by a thread that is defunct after the fork. -:c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not -always able to. - -The fact that all other threads go away also means that CPython's -runtime state there must be cleaned up properly, which :func:`os.fork` -does. This means finalizing all other :c:type:`PyThreadState` objects -belonging to the current interpreter and all other -:c:type:`PyInterpreterState` objects. Due to this and the special -nature of the :ref:`"main" interpreter `, -:c:func:`fork` should only be called in that interpreter's "main" -thread, where the CPython global runtime was originally initialized. -The only exception is if :c:func:`exec` will be called immediately -after. - -.. _cautions-regarding-runtime-finalization: - -Cautions regarding runtime finalization ---------------------------------------- - -In the late stage of :term:`interpreter shutdown`, after attempting to wait for -non-daemon threads to exit (though this can be interrupted by -:class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime -is marked as *finalizing*: :c:func:`Py_IsFinalizing` and -:func:`sys.is_finalizing` return true. At this point, only the *finalization -thread* that initiated finalization (typically the main thread) is allowed to -acquire the :term:`GIL`. - -If any thread, other than the finalization thread, attempts to attach a :term:`thread state` -during finalization, either explicitly or -implicitly, the thread enters **a permanently blocked state** -where it remains until the program exits. In most cases this is harmless, but this can result -in deadlock if a later stage of finalization attempts to acquire a lock owned by the -blocked thread, or otherwise waits on the blocked thread. - -Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++ -finalizations further up the call stack when such threads were forcibly exited -here in CPython 3.13 and earlier. The CPython runtime :term:`thread state` C APIs -have never had any error reporting or handling expectations at :term:`thread state` -attachment time that would've allowed for graceful exit from this situation. Changing that -would require new stable C APIs and rewriting the majority of C code in the -CPython ecosystem to use those with error handling. - - -High-level API --------------- - -These are the most commonly used types and functions when writing C extension -code, or when embedding the Python interpreter: - -.. c:type:: PyInterpreterState - - This data structure represents the state shared by a number of cooperating - threads. Threads belonging to the same interpreter share their module - administration and a few other internal items. There are no public members in - this structure. - - Threads belonging to different interpreters initially share nothing, except - process state like available memory, open file descriptors and such. The global - interpreter lock is also shared by all threads, regardless of to which - interpreter they belong. - - .. versionchanged:: 3.12 - - :pep:`684` introduced the possibility - of a :ref:`per-interpreter GIL `. - See :c:func:`Py_NewInterpreterFromConfig`. - - -.. c:type:: PyThreadState - - This data structure represents the state of a single thread. The only public - data member is: - - .. c:member:: PyInterpreterState *interp - - This thread's interpreter state. - - -.. c:function:: void PyEval_InitThreads() - - .. index:: - single: PyEval_AcquireThread() - single: PyEval_ReleaseThread() - single: PyEval_SaveThread() - single: PyEval_RestoreThread() - - Deprecated function which does nothing. - - In Python 3.6 and older, this function created the GIL if it didn't exist. - - .. versionchanged:: 3.9 - The function now does nothing. - - .. versionchanged:: 3.7 - This function is now called by :c:func:`Py_Initialize()`, so you don't - have to call it yourself anymore. - - .. versionchanged:: 3.2 - This function cannot be called before :c:func:`Py_Initialize()` anymore. - - .. deprecated:: 3.9 - - .. index:: pair: module; _thread - - -.. c:function:: PyThreadState* PyEval_SaveThread() - - Detach the :term:`attached thread state` and return it. - The thread will have no :term:`thread state` upon returning. - - -.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) - - Set the :term:`attached thread state` to *tstate*. - The passed :term:`thread state` **should not** be :term:`attached `, - otherwise deadlock ensues. *tstate* will be attached upon returning. - - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - -.. c:function:: PyThreadState* PyThreadState_Get() - - Return the :term:`attached thread state`. If the thread has no attached - thread state, (such as when inside of :c:macro:`Py_BEGIN_ALLOW_THREADS` - block), then this issues a fatal error (so that the caller needn't check - for ``NULL``). - - See also :c:func:`PyThreadState_GetUnchecked`. - -.. c:function:: PyThreadState* PyThreadState_GetUnchecked() - - Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a - fatal error if it is NULL. The caller is responsible to check if the result - is NULL. - - .. versionadded:: 3.13 - In Python 3.5 to 3.12, the function was private and known as - ``_PyThreadState_UncheckedGet()``. - - -.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) - - Set the :term:`attached thread state` to *tstate*, and return the - :term:`thread state` that was attached prior to calling. - - This function is safe to call without an :term:`attached thread state`; it - will simply return ``NULL`` indicating that there was no prior thread state. - - .. seealso:: - :c:func:`PyEval_ReleaseThread` - - .. note:: - Similar to :c:func:`PyGILState_Ensure`, this function will hang the - thread if the runtime is finalizing. - - -The following functions use thread-local storage, and are not compatible -with sub-interpreters: - -.. c:type:: PyGILState_STATE - - The type of the value returned by :c:func:`PyGILState_Ensure` and passed to - :c:func:`PyGILState_Release`. - - .. c:enumerator:: PyGILState_LOCKED - - The GIL was already held when :c:func:`PyGILState_Ensure` was called. - - .. c:enumerator:: PyGILState_UNLOCKED - - The GIL was not held when :c:func:`PyGILState_Ensure` was called. - -.. c:function:: PyGILState_STATE PyGILState_Ensure() - - Ensure that the current thread is ready to call the Python C API regardless - of the current state of Python, or of the :term:`attached thread state`. This may - be called as many times as desired by a thread as long as each call is - matched with a call to :c:func:`PyGILState_Release`. In general, other - thread-related APIs may be used between :c:func:`PyGILState_Ensure` and - :c:func:`PyGILState_Release` calls as long as the thread state is restored to - its previous state before the Release(). For example, normal usage of the - :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is - acceptable. - - The return value is an opaque "handle" to the :term:`attached thread state` when - :c:func:`PyGILState_Ensure` was called, and must be passed to - :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even - though recursive calls are allowed, these handles *cannot* be shared - each - unique call to :c:func:`PyGILState_Ensure` must save the handle for its call - to :c:func:`PyGILState_Release`. - - When the function returns, there will be an :term:`attached thread state` - and the thread will be able to call arbitrary Python code. Failure is a fatal error. - - .. warning:: - Calling this function when the runtime is finalizing is unsafe. Doing - so will either hang the thread until the program ends, or fully crash - the interpreter in rare cases. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - -.. c:function:: void PyGILState_Release(PyGILState_STATE) - - Release any resources previously acquired. After this call, Python's state will - be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call - (but generally this state will be unknown to the caller, hence the use of the - GILState API). - - Every call to :c:func:`PyGILState_Ensure` must be matched by a call to - :c:func:`PyGILState_Release` on the same thread. - -.. c:function:: PyThreadState* PyGILState_GetThisThreadState() - - Get the :term:`attached thread state` for this thread. May return ``NULL`` if no - GILState API has been used on the current thread. Note that the main thread - always has such a thread-state, even if no auto-thread-state call has been - made on the main thread. This is mainly a helper/diagnostic function. - - .. note:: - This function may return non-``NULL`` even when the :term:`thread state` - is detached. - Prefer :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` - for most cases. - - .. seealso:: :c:func:`PyThreadState_Get` - -.. c:function:: int PyGILState_Check() - - Return ``1`` if the current thread is holding the :term:`GIL` and ``0`` otherwise. - This function can be called from any thread at any time. - Only if it has had its :term:`thread state ` initialized - via :c:func:`PyGILState_Ensure` will it return ``1``. - This is mainly a helper/diagnostic function. It can be useful - for example in callback contexts or memory allocation functions when - knowing that the :term:`GIL` is locked can allow the caller to perform sensitive - actions or otherwise behave differently. - - .. note:: - If the current Python process has ever created a subinterpreter, this - function will *always* return ``1``. Prefer :c:func:`PyThreadState_GetUnchecked` - for most cases. - - .. versionadded:: 3.4 - - -The following macros are normally used without a trailing semicolon; look for -example usage in the Python source distribution. - - -.. c:macro:: Py_BEGIN_ALLOW_THREADS - - This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. - Note that it contains an opening brace; it must be matched with a following - :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this - macro. - - -.. c:macro:: Py_END_ALLOW_THREADS - - This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains - a closing brace; it must be matched with an earlier - :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of - this macro. - - -.. c:macro:: Py_BLOCK_THREADS - - This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to - :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. - - -.. c:macro:: Py_UNBLOCK_THREADS - - This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to - :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable - declaration. - - -Low-level API -------------- - -All of the following functions must be called after :c:func:`Py_Initialize`. - -.. versionchanged:: 3.7 - :c:func:`Py_Initialize()` now initializes the :term:`GIL` - and sets an :term:`attached thread state`. - - -.. c:function:: PyInterpreterState* PyInterpreterState_New() - - Create a new interpreter state object. An :term:`attached thread state` is not needed, - but may optionally exist if it is necessary to serialize calls to this - function. - - .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New - - -.. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) - - Reset all information in an interpreter state object. There must be - an :term:`attached thread state` for the interpreter. - - .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear - - -.. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) - - Destroy an interpreter state object. There **should not** be an - :term:`attached thread state` for the target interpreter. The interpreter - state must have been reset with a previous call to :c:func:`PyInterpreterState_Clear`. - - -.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) - - Create a new thread state object belonging to the given interpreter object. - An :term:`attached thread state` is not needed. - -.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) - - Reset all information in a :term:`thread state` object. *tstate* - must be :term:`attached ` - - .. versionchanged:: 3.9 - This function now calls the :c:member:`!PyThreadState.on_delete` callback. - Previously, that happened in :c:func:`PyThreadState_Delete`. - - .. versionchanged:: 3.13 - The :c:member:`!PyThreadState.on_delete` callback was removed. - - -.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) - - Destroy a :term:`thread state` object. *tstate* should not - be :term:`attached ` to any thread. - *tstate* must have been reset with a previous call to - :c:func:`PyThreadState_Clear`. - - -.. c:function:: void PyThreadState_DeleteCurrent(void) - - Detach the :term:`attached thread state` (which must have been reset - with a previous call to :c:func:`PyThreadState_Clear`) and then destroy it. - - No :term:`thread state` will be :term:`attached ` upon - returning. - -.. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) - - Get the current frame of the Python thread state *tstate*. - - Return a :term:`strong reference`. Return ``NULL`` if no frame is currently - executing. - - See also :c:func:`PyEval_GetFrame`. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) - - Get the unique :term:`thread state` identifier of the Python thread state *tstate*. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: PyInterpreterState* PyThreadState_GetInterpreter(PyThreadState *tstate) - - Get the interpreter of the Python thread state *tstate*. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) - - Suspend tracing and profiling in the Python thread state *tstate*. - - Resume them using the :c:func:`PyThreadState_LeaveTracing` function. - - .. versionadded:: 3.11 - - -.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) - - Resume tracing and profiling in the Python thread state *tstate* suspended - by the :c:func:`PyThreadState_EnterTracing` function. - - See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` - functions. - - .. versionadded:: 3.11 - - -.. c:function:: int PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, void *stack_start_addr, size_t stack_size) - - Set the stack protection start address and stack protection size - of a Python thread state. - - On success, return ``0``. - On failure, set an exception and return ``-1``. - - CPython implements :ref:`recursion control ` for C code by raising - :py:exc:`RecursionError` when it notices that the machine execution stack is close - to overflow. See for example the :c:func:`Py_EnterRecursiveCall` function. - For this, it needs to know the location of the current thread's stack, which it - normally gets from the operating system. - When the stack is changed, for example using context switching techniques like the - Boost library's ``boost::context``, you must call - :c:func:`~PyUnstable_ThreadState_SetStackProtection` to inform CPython of the change. - - Call :c:func:`~PyUnstable_ThreadState_SetStackProtection` either before - or after changing the stack. - Do not call any other Python C API between the call and the stack - change. - - See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. - - .. versionadded:: 3.14.1 - - .. warning:: - - This function was added in a bugfix release, and - extensions that use it will be incompatible with Python 3.14.0. - Most packaging tools for Python are not able to handle this - incompatibility automatically, and will need explicit configuration. - When using PyPA standards (wheels and source distributions), - specify ``Requires-Python: != 3.14.0.*`` in - `core metadata `_. - - -.. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) - - Reset the stack protection start address and stack protection size - of a Python thread state to the operating system defaults. - - See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. - - .. versionadded:: 3.14.1 - - .. warning:: - - This function was added in a bugfix release, and - extensions that use it will be incompatible with Python 3.14.0. - Most packaging tools for Python are not able to handle this - incompatibility automatically, and will need explicit configuration. - When using PyPA standards (wheels and source distributions), - specify ``Requires-Python: != 3.14.0.*`` in - `core metadata `_. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Get(void) - - Get the current interpreter. - - Issue a fatal error if there no :term:`attached thread state`. - It cannot return NULL. - - .. versionadded:: 3.9 - - -.. c:function:: int64_t PyInterpreterState_GetID(PyInterpreterState *interp) - - Return the interpreter's unique ID. If there was any error in doing - so then ``-1`` is returned and an error is set. - - The caller must have an :term:`attached thread state`. - - .. versionadded:: 3.7 - - -.. c:function:: PyObject* PyInterpreterState_GetDict(PyInterpreterState *interp) - - Return a dictionary in which interpreter-specific data may be stored. - If this function returns ``NULL`` then no exception has been raised and - the caller should assume no interpreter-specific dict is available. - - This is not a replacement for :c:func:`PyModule_GetState()`, which - extensions should use to store interpreter-specific state information. - - The returned dictionary is borrowed from the interpreter and is valid until - interpreter shutdown. - - .. versionadded:: 3.8 - - -.. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) - - Type of a frame evaluation function. - - The *throwflag* parameter is used by the ``throw()`` method of generators: - if non-zero, handle the current exception. - - .. versionchanged:: 3.9 - The function now takes a *tstate* parameter. - - .. versionchanged:: 3.11 - The *frame* parameter changed from ``PyFrameObject*`` to ``_PyInterpreterFrame*``. - -.. c:function:: _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) - - Get the frame evaluation function. - - See the :pep:`523` "Adding a frame evaluation API to CPython". - - .. versionadded:: 3.9 - -.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) - - Set the frame evaluation function. - - See the :pep:`523` "Adding a frame evaluation API to CPython". - - .. versionadded:: 3.9 - - -.. c:function:: PyObject* PyThreadState_GetDict() - - Return a dictionary in which extensions can store thread-specific state - information. Each extension should use a unique key to use to store state in - the dictionary. It is okay to call this function when no :term:`thread state` - is :term:`attached `. If this function returns - ``NULL``, no exception has been raised and the caller should assume no - thread state is attached. - - -.. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) - - Asynchronously raise an exception in a thread. The *id* argument is the thread - id of the target thread; *exc* is the exception object to be raised. This - function does not steal any references to *exc*. To prevent naive misuse, you - must write your own C extension to call this. Must be called with an :term:`attached thread state`. - Returns the number of thread states modified; this is normally one, but will be - zero if the thread id isn't found. If *exc* is ``NULL``, the pending - exception (if any) for the thread is cleared. This raises no exceptions. - - .. versionchanged:: 3.7 - The type of the *id* parameter changed from :c:expr:`long` to - :c:expr:`unsigned long`. - -.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) - - :term:`Attach ` *tstate* to the current thread, - which must not be ``NULL`` or already :term:`attached `. - - The calling thread must not already have an :term:`attached thread state`. - - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.8 - Updated to be consistent with :c:func:`PyEval_RestoreThread`, - :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, - and terminate the current thread if called while the interpreter is finalizing. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - - :c:func:`PyEval_RestoreThread` is a higher-level function which is always - available (even when threads have not been initialized). - - -.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) - - Detach the :term:`attached thread state`. - The *tstate* argument, which must not be ``NULL``, is only used to check - that it represents the :term:`attached thread state` --- if it isn't, a fatal error is - reported. - - :c:func:`PyEval_SaveThread` is a higher-level function which is always - available (even when threads have not been initialized). - - -.. _sub-interpreter-support: - -Sub-interpreter support -======================= - -While in most uses, you will only embed a single Python interpreter, there -are cases where you need to create several independent interpreters in the -same process and perhaps even in the same thread. Sub-interpreters allow -you to do that. - -The "main" interpreter is the first one created when the runtime initializes. -It is usually the only Python interpreter in a process. Unlike sub-interpreters, -the main interpreter has unique process-global responsibilities like signal -handling. It is also responsible for execution during runtime initialization and -is usually the active interpreter during runtime finalization. The -:c:func:`PyInterpreterState_Main` function returns a pointer to its state. - -You can switch between sub-interpreters using the :c:func:`PyThreadState_Swap` -function. You can create and destroy them using the following functions: - - -.. c:type:: PyInterpreterConfig - - Structure containing most parameters to configure a sub-interpreter. - Its values are used only in :c:func:`Py_NewInterpreterFromConfig` and - never modified by the runtime. - - .. versionadded:: 3.12 - - Structure fields: - - .. c:member:: int use_main_obmalloc - - If this is ``0`` then the sub-interpreter will use its own - "object" allocator state. - Otherwise it will use (share) the main interpreter's. - - If this is ``0`` then - :c:member:`~PyInterpreterConfig.check_multi_interp_extensions` - must be ``1`` (non-zero). - If this is ``1`` then :c:member:`~PyInterpreterConfig.gil` - must not be :c:macro:`PyInterpreterConfig_OWN_GIL`. - - .. c:member:: int allow_fork - - If this is ``0`` then the runtime will not support forking the - process in any thread where the sub-interpreter is currently active. - Otherwise fork is unrestricted. - - Note that the :mod:`subprocess` module still works - when fork is disallowed. - - .. c:member:: int allow_exec - - If this is ``0`` then the runtime will not support replacing the - current process via exec (e.g. :func:`os.execv`) in any thread - where the sub-interpreter is currently active. - Otherwise exec is unrestricted. - - Note that the :mod:`subprocess` module still works - when exec is disallowed. - - .. c:member:: int allow_threads - - If this is ``0`` then the sub-interpreter's :mod:`threading` module - won't create threads. - Otherwise threads are allowed. - - .. c:member:: int allow_daemon_threads - - If this is ``0`` then the sub-interpreter's :mod:`threading` module - won't create daemon threads. - Otherwise daemon threads are allowed (as long as - :c:member:`~PyInterpreterConfig.allow_threads` is non-zero). - - .. c:member:: int check_multi_interp_extensions - - If this is ``0`` then all extension modules may be imported, - including legacy (single-phase init) modules, - in any thread where the sub-interpreter is currently active. - Otherwise only multi-phase init extension modules - (see :pep:`489`) may be imported. - (Also see :c:macro:`Py_mod_multiple_interpreters`.) - - This must be ``1`` (non-zero) if - :c:member:`~PyInterpreterConfig.use_main_obmalloc` is ``0``. - - .. c:member:: int gil - - This determines the operation of the GIL for the sub-interpreter. - It may be one of the following: - - .. c:namespace:: NULL - - .. c:macro:: PyInterpreterConfig_DEFAULT_GIL - - Use the default selection (:c:macro:`PyInterpreterConfig_SHARED_GIL`). - - .. c:macro:: PyInterpreterConfig_SHARED_GIL - - Use (share) the main interpreter's GIL. - - .. c:macro:: PyInterpreterConfig_OWN_GIL - - Use the sub-interpreter's own GIL. - - If this is :c:macro:`PyInterpreterConfig_OWN_GIL` then - :c:member:`PyInterpreterConfig.use_main_obmalloc` must be ``0``. - - -.. c:function:: PyStatus Py_NewInterpreterFromConfig(PyThreadState **tstate_p, const PyInterpreterConfig *config) - - .. index:: - pair: module; builtins - pair: module; __main__ - pair: module; sys - single: stdout (in module sys) - single: stderr (in module sys) - single: stdin (in module sys) - - Create a new sub-interpreter. This is an (almost) totally separate environment - for the execution of Python code. In particular, the new interpreter has - separate, independent versions of all imported modules, including the - fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. The - table of loaded modules (``sys.modules``) and the module search path - (``sys.path``) are also separate. The new environment has no ``sys.argv`` - variable. It has new standard I/O stream file objects ``sys.stdin``, - ``sys.stdout`` and ``sys.stderr`` (however these refer to the same underlying - file descriptors). - - The given *config* controls the options with which the interpreter - is initialized. - - Upon success, *tstate_p* will be set to the first :term:`thread state` - created in the new sub-interpreter. This thread state is - :term:`attached `. - Note that no actual thread is created; see the discussion of thread states - below. If creation of the new interpreter is unsuccessful, - *tstate_p* is set to ``NULL``; - no exception is set since the exception state is stored in the - :term:`attached thread state`, which might not exist. - - Like all other Python/C API functions, an :term:`attached thread state` - must be present before calling this function, but it might be detached upon - returning. On success, the returned thread state will be :term:`attached `. - If the sub-interpreter is created with its own :term:`GIL` then the - :term:`attached thread state` of the calling interpreter will be detached. - When the function returns, the new interpreter's :term:`thread state` - will be :term:`attached ` to the current thread and - the previous interpreter's :term:`attached thread state` will remain detached. - - .. versionadded:: 3.12 - - Sub-interpreters are most effective when isolated from each other, - with certain functionality restricted:: - - PyInterpreterConfig config = { - .use_main_obmalloc = 0, - .allow_fork = 0, - .allow_exec = 0, - .allow_threads = 1, - .allow_daemon_threads = 0, - .check_multi_interp_extensions = 1, - .gil = PyInterpreterConfig_OWN_GIL, - }; - PyThreadState *tstate = NULL; - PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - - Note that the config is used only briefly and does not get modified. - During initialization the config's values are converted into various - :c:type:`PyInterpreterState` values. A read-only copy of the config - may be stored internally on the :c:type:`PyInterpreterState`. - - .. index:: - single: Py_FinalizeEx (C function) - single: Py_Initialize (C function) - - Extension modules are shared between (sub-)interpreters as follows: - - * For modules using multi-phase initialization, - e.g. :c:func:`PyModule_FromDefAndSpec`, a separate module object is - created and initialized for each interpreter. - Only C-level static and global variables are shared between these - module objects. - - * For modules using single-phase initialization, - e.g. :c:func:`PyModule_Create`, the first time a particular extension - is imported, it is initialized normally, and a (shallow) copy of its - module's dictionary is squirreled away. - When the same extension is imported by another (sub-)interpreter, a new - module is initialized and filled with the contents of this copy; the - extension's ``init`` function is not called. - Objects in the module's dictionary thus end up shared across - (sub-)interpreters, which might cause unwanted behavior (see - `Bugs and caveats`_ below). - - Note that this is different from what happens when an extension is - imported after the interpreter has been completely re-initialized by - calling :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that - case, the extension's ``initmodule`` function *is* called again. - As with multi-phase initialization, this means that only C-level static - and global variables are shared between these modules. - - .. index:: single: close (in module os) - - -.. c:function:: PyThreadState* Py_NewInterpreter(void) - - .. index:: - pair: module; builtins - pair: module; __main__ - pair: module; sys - single: stdout (in module sys) - single: stderr (in module sys) - single: stdin (in module sys) - - Create a new sub-interpreter. This is essentially just a wrapper - around :c:func:`Py_NewInterpreterFromConfig` with a config that - preserves the existing behavior. The result is an unisolated - sub-interpreter that shares the main interpreter's GIL, allows - fork/exec, allows daemon threads, and allows single-phase init - modules. - - -.. c:function:: void Py_EndInterpreter(PyThreadState *tstate) - - .. index:: single: Py_FinalizeEx (C function) - - Destroy the (sub-)interpreter represented by the given :term:`thread state`. - The given thread state must be :term:`attached `. - When the call returns, there will be no :term:`attached thread state`. - All thread states associated with this interpreter are destroyed. - - :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that - haven't been explicitly destroyed at that point. - - -.. _per-interpreter-gil: - -A Per-Interpreter GIL ---------------------- - -Using :c:func:`Py_NewInterpreterFromConfig` you can create -a sub-interpreter that is completely isolated from other interpreters, -including having its own GIL. The most important benefit of this -isolation is that such an interpreter can execute Python code without -being blocked by other interpreters or blocking any others. Thus a -single Python process can truly take advantage of multiple CPU cores -when running Python code. The isolation also encourages a different -approach to concurrency than that of just using threads. -(See :pep:`554` and :pep:`684`.) - -Using an isolated interpreter requires vigilance in preserving that -isolation. That especially means not sharing any objects or mutable -state without guarantees about thread-safety. Even objects that are -otherwise immutable (e.g. ``None``, ``(1, 5)``) can't normally be shared -because of the refcount. One simple but less-efficient approach around -this is to use a global lock around all use of some state (or object). -Alternately, effectively immutable objects (like integers or strings) -can be made safe in spite of their refcounts by making them :term:`immortal`. -In fact, this has been done for the builtin singletons, small integers, -and a number of other builtin objects. - -If you preserve isolation then you will have access to proper multi-core -computing without the complications that come with free-threading. -Failure to preserve isolation will expose you to the full consequences -of free-threading, including races and hard-to-debug crashes. - -Aside from that, one of the main challenges of using multiple isolated -interpreters is how to communicate between them safely (not break -isolation) and efficiently. The runtime and stdlib do not provide -any standard approach to this yet. A future stdlib module would help -mitigate the effort of preserving isolation and expose effective tools -for communicating (and sharing) data between interpreters. - -.. versionadded:: 3.12 - - -Bugs and caveats ----------------- - -Because sub-interpreters (and the main interpreter) are part of the same -process, the insulation between them isn't perfect --- for example, using -low-level file operations like :func:`os.close` they can -(accidentally or maliciously) affect each other's open files. Because of the -way extensions are shared between (sub-)interpreters, some extensions may not -work properly; this is especially likely when using single-phase initialization -or (static) global variables. -It is possible to insert objects created in one sub-interpreter into -a namespace of another (sub-)interpreter; this should be avoided if possible. - -Special care should be taken to avoid sharing user-defined functions, -methods, instances or classes between sub-interpreters, since import -operations executed by such objects may affect the wrong (sub-)interpreter's -dictionary of loaded modules. It is equally important to avoid sharing -objects from which the above are reachable. - -Also note that combining this functionality with ``PyGILState_*`` APIs -is delicate, because these APIs assume a bijection between Python thread states -and OS-level threads, an assumption broken by the presence of sub-interpreters. -It is highly recommended that you don't switch sub-interpreters between a pair -of matching :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` calls. -Furthermore, extensions (such as :mod:`ctypes`) using these APIs to allow calling -of Python code from non-Python created threads will probably be broken when using -sub-interpreters. - - -Asynchronous Notifications -========================== - -A mechanism is provided to make asynchronous notifications to the main -interpreter thread. These notifications take the form of a function -pointer and a void pointer argument. - - -.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) - - Schedule a function to be called from the main interpreter thread. On - success, ``0`` is returned and *func* is queued for being called in the - main thread. On failure, ``-1`` is returned without setting any exception. - - When successfully queued, *func* will be *eventually* called from the - main interpreter thread with the argument *arg*. It will be called - asynchronously with respect to normally running Python code, but with - both these conditions met: - - * on a :term:`bytecode` boundary; - * with the main thread holding an :term:`attached thread state` - (*func* can therefore use the full C API). - - *func* must return ``0`` on success, or ``-1`` on failure with an exception - set. *func* won't be interrupted to perform another asynchronous - notification recursively, but it can still be interrupted to switch - threads if the :term:`thread state ` is detached. - - This function doesn't need an :term:`attached thread state`. However, to call this - function in a subinterpreter, the caller must have an :term:`attached thread state`. - Otherwise, the function *func* can be scheduled to be called from the wrong interpreter. - - .. warning:: - This is a low-level function, only useful for very special cases. - There is no guarantee that *func* will be called as quick as - possible. If the main thread is busy executing a system call, - *func* won't be called before the system call returns. This - function is generally **not** suitable for calling Python code from - arbitrary C threads. Instead, use the :ref:`PyGILState API`. - - .. versionadded:: 3.1 - - .. versionchanged:: 3.9 - If this function is called in a subinterpreter, the function *func* is - now scheduled to be called from the subinterpreter, rather than being - called from the main interpreter. Each subinterpreter now has its own - list of scheduled calls. - - .. versionchanged:: 3.12 - This function now always schedules *func* to be run in the main - interpreter. - - -.. c:function:: int Py_MakePendingCalls(void) - - Execute all pending calls. This is usually executed automatically by the - interpreter. - - This function returns ``0`` on success, and returns ``-1`` with an exception - set on failure. - - If this is not called in the main thread of the main - interpreter, this function does nothing and returns ``0``. - The caller must hold an :term:`attached thread state`. - - .. versionadded:: 3.1 - - .. versionchanged:: 3.12 - This function only runs pending calls in the main interpreter. - - -.. _profiling: - -Profiling and Tracing -===================== - -.. sectionauthor:: Fred L. Drake, Jr. - - -The Python interpreter provides some low-level support for attaching profiling -and execution tracing facilities. These are used for profiling, debugging, and -coverage analysis tools. - -This C interface allows the profiling or tracing code to avoid the overhead of -calling through Python-level callable objects, making a direct C function call -instead. The essential attributes of the facility have not changed; the -interface allows trace functions to be installed per-thread, and the basic -events reported to the trace function are the same as had been reported to the -Python-level trace functions in previous versions. - - -.. c:type:: int (*Py_tracefunc)(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) - - The type of the trace function registered using :c:func:`PyEval_SetProfile` and - :c:func:`PyEval_SetTrace`. The first parameter is the object passed to the - registration function as *obj*, *frame* is the frame object to which the event - pertains, *what* is one of the constants :c:data:`PyTrace_CALL`, - :c:data:`PyTrace_EXCEPTION`, :c:data:`PyTrace_LINE`, :c:data:`PyTrace_RETURN`, - :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION`, :c:data:`PyTrace_C_RETURN`, - or :c:data:`PyTrace_OPCODE`, and *arg* depends on the value of *what*: - - +-------------------------------+----------------------------------------+ - | Value of *what* | Meaning of *arg* | - +===============================+========================================+ - | :c:data:`PyTrace_CALL` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_EXCEPTION` | Exception information as returned by | - | | :func:`sys.exc_info`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_LINE` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_RETURN` | Value being returned to the caller, | - | | or ``NULL`` if caused by an exception. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_CALL` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_EXCEPTION` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_RETURN` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - -.. c:var:: int PyTrace_CALL - - The value of the *what* parameter to a :c:type:`Py_tracefunc` function when a new - call to a function or method is being reported, or a new entry into a generator. - Note that the creation of the iterator for a generator function is not reported - as there is no control transfer to the Python bytecode in the corresponding - frame. - - -.. c:var:: int PyTrace_EXCEPTION - - The value of the *what* parameter to a :c:type:`Py_tracefunc` function when an - exception has been raised. The callback function is called with this value for - *what* when after any bytecode is processed after which the exception becomes - set within the frame being executed. The effect of this is that as exception - propagation causes the Python stack to unwind, the callback is called upon - return to each frame as the exception propagates. Only trace functions receives - these events; they are not needed by the profiler. - - -.. c:var:: int PyTrace_LINE - - The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function - (but not a profiling function) when a line-number event is being reported. - It may be disabled for a frame by setting :attr:`~frame.f_trace_lines` to - *0* on that frame. - - -.. c:var:: int PyTrace_RETURN - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a - call is about to return. - - -.. c:var:: int PyTrace_C_CALL - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function is about to be called. - - -.. c:var:: int PyTrace_C_EXCEPTION - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function has raised an exception. - - -.. c:var:: int PyTrace_C_RETURN - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function has returned. - - -.. c:var:: int PyTrace_OPCODE - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not - profiling functions) when a new opcode is about to be executed. This event is - not emitted by default: it must be explicitly requested by setting - :attr:`~frame.f_trace_opcodes` to *1* on the frame. - - -.. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) - - Set the profiler function to *func*. The *obj* parameter is passed to the - function as its first parameter, and may be any Python object, or ``NULL``. If - the profile function needs to maintain state, using a different value for *obj* - for each thread provides a convenient and thread-safe place to store it. The - profile function is called for all monitored events except :c:data:`PyTrace_LINE` - :c:data:`PyTrace_OPCODE` and :c:data:`PyTrace_EXCEPTION`. - - See also the :func:`sys.setprofile` function. - - The caller must have an :term:`attached thread state`. - -.. c:function:: void PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *obj) - - Like :c:func:`PyEval_SetProfile` but sets the profile function in all running threads - belonging to the current interpreter instead of the setting it only on the current thread. - - The caller must have an :term:`attached thread state`. - - As :c:func:`PyEval_SetProfile`, this function ignores any exceptions raised while - setting the profile functions in all threads. - -.. versionadded:: 3.12 - - -.. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj) - - Set the tracing function to *func*. This is similar to - :c:func:`PyEval_SetProfile`, except the tracing function does receive line-number - events and per-opcode events, but does not receive any event related to C function - objects being called. Any trace function registered using :c:func:`PyEval_SetTrace` - will not receive :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION` or - :c:data:`PyTrace_C_RETURN` as a value for the *what* parameter. - - See also the :func:`sys.settrace` function. - - The caller must have an :term:`attached thread state`. - -.. c:function:: void PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *obj) - - Like :c:func:`PyEval_SetTrace` but sets the tracing function in all running threads - belonging to the current interpreter instead of the setting it only on the current thread. - - The caller must have an :term:`attached thread state`. - - As :c:func:`PyEval_SetTrace`, this function ignores any exceptions raised while - setting the trace functions in all threads. - -.. versionadded:: 3.12 - -Reference tracing -================= - -.. versionadded:: 3.13 - -.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) - - The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. - The first parameter is a Python object that has been just created (when **event** - is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** - is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer - that was provided when :c:func:`PyRefTracer_SetTracer` was called. - -.. versionadded:: 3.13 - -.. c:var:: int PyRefTracer_CREATE - - The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python - object has been created. - -.. c:var:: int PyRefTracer_DESTROY - - The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python - object has been destroyed. - -.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) - - Register a reference tracer function. The function will be called when a new - Python has been created or when an object is going to be destroyed. If - **data** is provided it must be an opaque pointer that will be provided when - the tracer function is called. Return ``0`` on success. Set an exception and - return ``-1`` on error. - - Not that tracer functions **must not** create Python objects inside or - otherwise the call will be re-entrant. The tracer also **must not** clear - any existing exception or set an exception. A :term:`thread state` will be active - every time the tracer function is called. - - There must be an :term:`attached thread state` when calling this function. - -.. versionadded:: 3.13 - -.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) - - Get the registered reference tracer function and the value of the opaque data - pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. - If no tracer was registered this function will return NULL and will set the - **data** pointer to NULL. - - There must be an :term:`attached thread state` when calling this function. - -.. versionadded:: 3.13 - -.. _advanced-debugging: - -Advanced Debugger Support -========================= - -.. sectionauthor:: Fred L. Drake, Jr. - - -These functions are only intended to be used by advanced debugging tools. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Head() - - Return the interpreter state object at the head of the list of all such objects. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Main() - - Return the main interpreter state object. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Next(PyInterpreterState *interp) - - Return the next interpreter state object after *interp* from the list of all - such objects. - - -.. c:function:: PyThreadState * PyInterpreterState_ThreadHead(PyInterpreterState *interp) - - Return the pointer to the first :c:type:`PyThreadState` object in the list of - threads associated with the interpreter *interp*. - - -.. c:function:: PyThreadState* PyThreadState_Next(PyThreadState *tstate) - - Return the next thread state object after *tstate* from the list of all such - objects belonging to the same :c:type:`PyInterpreterState` object. - - -.. _thread-local-storage: - -Thread Local Storage Support -============================ - -.. sectionauthor:: Masayuki Yamamoto - -The Python interpreter provides low-level support for thread-local storage -(TLS) which wraps the underlying native TLS implementation to support the -Python-level thread local storage API (:class:`threading.local`). The -CPython C level APIs are similar to those offered by pthreads and Windows: -use a thread key and functions to associate a :c:expr:`void*` value per -thread. - -A :term:`thread state` does *not* need to be :term:`attached ` -when calling these functions; they suppl their own locking. - -Note that :file:`Python.h` does not include the declaration of the TLS APIs, -you need to include :file:`pythread.h` to use thread-local storage. - -.. note:: - None of these API functions handle memory management on behalf of the - :c:expr:`void*` values. You need to allocate and deallocate them yourself. - If the :c:expr:`void*` values happen to be :c:expr:`PyObject*`, these - functions don't do refcount operations on them either. - -.. _thread-specific-storage-api: - -Thread Specific Storage (TSS) API ---------------------------------- - -TSS API is introduced to supersede the use of the existing TLS API within the -CPython interpreter. This API uses a new type :c:type:`Py_tss_t` instead of -:c:expr:`int` to represent thread keys. - -.. versionadded:: 3.7 - -.. seealso:: "A New C-API for Thread-Local Storage in CPython" (:pep:`539`) - - -.. c:type:: Py_tss_t - - This data structure represents the state of a thread key, the definition of - which may depend on the underlying TLS implementation, and it has an - internal field representing the key's initialization state. There are no - public members in this structure. - - When :ref:`Py_LIMITED_API ` is not defined, static allocation of - this type by :c:macro:`Py_tss_NEEDS_INIT` is allowed. - - -.. c:macro:: Py_tss_NEEDS_INIT - - This macro expands to the initializer for :c:type:`Py_tss_t` variables. - Note that this macro won't be defined with :ref:`Py_LIMITED_API `. - - -Dynamic Allocation -~~~~~~~~~~~~~~~~~~ - -Dynamic allocation of the :c:type:`Py_tss_t`, required in extension modules -built with :ref:`Py_LIMITED_API `, where static allocation of this type -is not possible due to its implementation being opaque at build time. - - -.. c:function:: Py_tss_t* PyThread_tss_alloc() - - Return a value which is the same state as a value initialized with - :c:macro:`Py_tss_NEEDS_INIT`, or ``NULL`` in the case of dynamic allocation - failure. - - -.. c:function:: void PyThread_tss_free(Py_tss_t *key) - - Free the given *key* allocated by :c:func:`PyThread_tss_alloc`, after - first calling :c:func:`PyThread_tss_delete` to ensure any associated - thread locals have been unassigned. This is a no-op if the *key* - argument is ``NULL``. - - .. note:: - A freed key becomes a dangling pointer. You should reset the key to - ``NULL``. - - -Methods -~~~~~~~ - -The parameter *key* of these functions must not be ``NULL``. Moreover, the -behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are -undefined if the given :c:type:`Py_tss_t` has not been initialized by -:c:func:`PyThread_tss_create`. - - -.. c:function:: int PyThread_tss_is_created(Py_tss_t *key) - - Return a non-zero value if the given :c:type:`Py_tss_t` has been initialized - by :c:func:`PyThread_tss_create`. - - -.. c:function:: int PyThread_tss_create(Py_tss_t *key) - - Return a zero value on successful initialization of a TSS key. The behavior - is undefined if the value pointed to by the *key* argument is not - initialized by :c:macro:`Py_tss_NEEDS_INIT`. This function can be called - repeatedly on the same key -- calling it on an already initialized key is a - no-op and immediately returns success. - - -.. c:function:: void PyThread_tss_delete(Py_tss_t *key) - - Destroy a TSS key to forget the values associated with the key across all - threads, and change the key's initialization state to uninitialized. A - destroyed key is able to be initialized again by - :c:func:`PyThread_tss_create`. This function can be called repeatedly on - the same key -- calling it on an already destroyed key is a no-op. - - -.. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) - - Return a zero value to indicate successfully associating a :c:expr:`void*` - value with a TSS key in the current thread. Each thread has a distinct - mapping of the key to a :c:expr:`void*` value. - - -.. c:function:: void* PyThread_tss_get(Py_tss_t *key) - - Return the :c:expr:`void*` value associated with a TSS key in the current - thread. This returns ``NULL`` if no value is associated with the key in the - current thread. - - -.. _thread-local-storage-api: - -Thread Local Storage (TLS) API ------------------------------- - -.. deprecated:: 3.7 - This API is superseded by - :ref:`Thread Specific Storage (TSS) API `. - -.. note:: - This version of the API does not support platforms where the native TLS key - is defined in a way that cannot be safely cast to ``int``. On such platforms, - :c:func:`PyThread_create_key` will return immediately with a failure status, - and the other TLS functions will all be no-ops on such platforms. - -Due to the compatibility problem noted above, this version of the API should not -be used in new code. - -.. c:function:: int PyThread_create_key() -.. c:function:: void PyThread_delete_key(int key) -.. c:function:: int PyThread_set_key_value(int key, void *value) -.. c:function:: void* PyThread_get_key_value(int key) -.. c:function:: void PyThread_delete_key_value(int key) -.. c:function:: void PyThread_ReInitTLS() - -Synchronization Primitives -========================== - -The C-API provides a basic mutual exclusion lock. - -.. c:type:: PyMutex - - A mutual exclusion lock. The :c:type:`!PyMutex` should be initialized to - zero to represent the unlocked state. For example:: - - PyMutex mutex = {0}; - - Instances of :c:type:`!PyMutex` should not be copied or moved. Both the - contents and address of a :c:type:`!PyMutex` are meaningful, and it must - remain at a fixed, writable location in memory. - - .. note:: - - A :c:type:`!PyMutex` currently occupies one byte, but the size should be - considered unstable. The size may change in future Python releases - without a deprecation period. - - .. versionadded:: 3.13 - -.. c:function:: void PyMutex_Lock(PyMutex *m) - - Lock mutex *m*. If another thread has already locked it, the calling - thread will block until the mutex is unlocked. While blocked, the thread - will temporarily detach the :term:`thread state ` if one exists. - - .. versionadded:: 3.13 - -.. c:function:: void PyMutex_Unlock(PyMutex *m) - - Unlock mutex *m*. The mutex must be locked --- otherwise, the function will - issue a fatal error. - - .. versionadded:: 3.13 - -.. c:function:: int PyMutex_IsLocked(PyMutex *m) - - Returns non-zero if the mutex *m* is currently locked, zero otherwise. - - .. note:: - - This function is intended for use in assertions and debugging only and - should not be used to make concurrency control decisions, as the lock - state may change immediately after the check. - - .. versionadded:: 3.14 - -.. _python-critical-section-api: - -Python Critical Section API ---------------------------- - -The critical section API provides a deadlock avoidance layer on top of -per-object locks for :term:`free-threaded ` CPython. They are -intended to replace reliance on the :term:`global interpreter lock`, and are -no-ops in versions of Python with the global interpreter lock. - -Critical sections are intended to be used for custom types implemented -in C-API extensions. They should generally not be used with built-in types like -:class:`list` and :class:`dict` because their public C-APIs -already use critical sections internally, with the notable -exception of :c:func:`PyDict_Next`, which requires critical section -to be acquired externally. - -Critical sections avoid deadlocks by implicitly suspending active critical -sections, hence, they do not provide exclusive access such as provided by -traditional locks like :c:type:`PyMutex`. When a critical section is started, -the per-object lock for the object is acquired. If the code executed inside the -critical section calls C-API functions then it can suspend the critical section thereby -releasing the per-object lock, so other threads can acquire the per-object lock -for the same object. - -Variants that accept :c:type:`PyMutex` pointers rather than Python objects are also -available. Use these variants to start a critical section in a situation where -there is no :c:type:`PyObject` -- for example, when working with a C type that -does not extend or wrap :c:type:`PyObject` but still needs to call into the C -API in a manner that might lead to deadlocks. - -The functions and structs used by the macros are exposed for cases -where C macros are not available. They should only be used as in the -given macro expansions. Note that the sizes and contents of the structures may -change in future Python versions. - -.. note:: - - Operations that need to lock two objects at once must use - :c:macro:`Py_BEGIN_CRITICAL_SECTION2`. You *cannot* use nested critical - sections to lock more than one object at once, because the inner critical - section may suspend the outer critical sections. This API does not provide - a way to lock more than two objects at once. - -Example usage:: - - static PyObject * - set_field(MyObject *self, PyObject *value) - { - Py_BEGIN_CRITICAL_SECTION(self); - Py_SETREF(self->field, Py_XNewRef(value)); - Py_END_CRITICAL_SECTION(); - Py_RETURN_NONE; - } - -In the above example, :c:macro:`Py_SETREF` calls :c:macro:`Py_DECREF`, which -can call arbitrary code through an object's deallocation function. The critical -section API avoids potential deadlocks due to reentrancy and lock ordering -by allowing the runtime to temporarily suspend the critical section if the -code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION(op) - - Acquires the per-object lock for the object *op* and begins a - critical section. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection _py_cs; - PyCriticalSection_Begin(&_py_cs, (PyObject*)(op)) - - In the default build, this macro expands to ``{``. - - .. versionadded:: 3.13 - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION_MUTEX(m) - - Locks the mutex *m* and begins a critical section. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection _py_cs; - PyCriticalSection_BeginMutex(&_py_cs, m) - - Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION`, there is no cast for - the argument of the macro - it must be a :c:type:`PyMutex` pointer. - - On the default build, this macro expands to ``{``. - - .. versionadded:: 3.14 - -.. c:macro:: Py_END_CRITICAL_SECTION() - - Ends the critical section and releases the per-object lock. - - In the free-threaded build, this macro expands to:: - - PyCriticalSection_End(&_py_cs); - } - - In the default build, this macro expands to ``}``. - - .. versionadded:: 3.13 - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION2(a, b) - - Acquires the per-objects locks for the objects *a* and *b* and begins a - critical section. The locks are acquired in a consistent order (lowest - address first) to avoid lock ordering deadlocks. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection2 _py_cs2; - PyCriticalSection2_Begin(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) - - In the default build, this macro expands to ``{``. - - .. versionadded:: 3.13 - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) - - Locks the mutexes *m1* and *m2* and begins a critical section. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection2 _py_cs2; - PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) - - Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION2`, there is no cast for - the arguments of the macro - they must be :c:type:`PyMutex` pointers. - - On the default build, this macro expands to ``{``. - - .. versionadded:: 3.14 - -.. c:macro:: Py_END_CRITICAL_SECTION2() - - Ends the critical section and releases the per-object locks. - - In the free-threaded build, this macro expands to:: - - PyCriticalSection2_End(&_py_cs2); - } - - In the default build, this macro expands to ``}``. - - .. versionadded:: 3.13 - - -Legacy Locking APIs -------------------- - -These APIs are obsolete since Python 3.13 with the introduction of -:c:type:`PyMutex`. - -.. versionchanged:: 3.15 - These APIs are now a simple wrapper around ``PyMutex``. - - -.. c:type:: PyThread_type_lock - - A pointer to a mutual exclusion lock. - - -.. c:type:: PyLockStatus - - The result of acquiring a lock with a timeout. - - .. c:namespace:: NULL - - .. c:enumerator:: PY_LOCK_FAILURE - - Failed to acquire the lock. - - .. c:enumerator:: PY_LOCK_ACQUIRED - - The lock was successfully acquired. - - .. c:enumerator:: PY_LOCK_INTR - - The lock was interrupted by a signal. - - -.. c:function:: PyThread_type_lock PyThread_allocate_lock(void) - - Allocate a new lock. - - On success, this function returns a lock; on failure, this - function returns ``0`` without an exception set. - - The caller does not need to hold an :term:`attached thread state`. - - .. versionchanged:: 3.15 - This function now always uses :c:type:`PyMutex`. In prior versions, this - would use a lock provided by the operating system. - - -.. c:function:: void PyThread_free_lock(PyThread_type_lock lock) - - Destroy *lock*. The lock should not be held by any thread when calling - this. - - The caller does not need to hold an :term:`attached thread state`. - - -.. c:function:: PyLockStatus PyThread_acquire_lock_timed(PyThread_type_lock lock, long long microseconds, int intr_flag) - - Acquire *lock* with a timeout. - - This will wait for *microseconds* microseconds to acquire the lock. If the - timeout expires, this function returns :c:enumerator:`PY_LOCK_FAILURE`. - If *microseconds* is ``-1``, this will wait indefinitely until the lock has - been released. - - If *intr_flag* is ``1``, acquiring the lock may be interrupted by a signal, - in which case this function returns :c:enumerator:`PY_LOCK_INTR`. Upon - interruption, it's generally expected that the caller makes a call to - :c:func:`Py_MakePendingCalls` to propagate an exception to Python code. - - If the lock is successfully acquired, this function returns - :c:enumerator:`PY_LOCK_ACQUIRED`. - - The caller does not need to hold an :term:`attached thread state`. - - -.. c:function:: int PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) - - Acquire *lock*. - - If *waitflag* is ``1`` and another thread currently holds the lock, this - function will wait until the lock can be acquired and will always return - ``1``. - - If *waitflag* is ``0`` and another thread holds the lock, this function will - not wait and instead return ``0``. If the lock is not held by any other - thread, then this function will acquire it and return ``1``. - - Unlike :c:func:`PyThread_acquire_lock_timed`, acquiring the lock cannot be - interrupted by a signal. - - The caller does not need to hold an :term:`attached thread state`. - - -.. c:function:: int PyThread_release_lock(PyThread_type_lock lock) - - Release *lock*. If *lock* is not held, then this function issues a - fatal error. - - The caller does not need to hold an :term:`attached thread state`. - - -Operating System Thread APIs -============================ - -.. c:macro:: PYTHREAD_INVALID_THREAD_ID - - Sentinel value for an invalid thread ID. - - This is currently equivalent to ``(unsigned long)-1``. - - -.. c:function:: unsigned long PyThread_start_new_thread(void (*func)(void *), void *arg) - - Start function *func* in a new thread with argument *arg*. - The resulting thread is not intended to be joined. - - *func* must not be ``NULL``, but *arg* may be ``NULL``. - - On success, this function returns the identifier of the new thread; on failure, - this returns :c:macro:`PYTHREAD_INVALID_THREAD_ID`. - - The caller does not need to hold an :term:`attached thread state`. - - -.. c:function:: unsigned long PyThread_get_thread_ident(void) - - Return the identifier of the current thread, which will never be zero. - - This function cannot fail, and the caller does not need to hold an - :term:`attached thread state`. - - .. seealso:: - :py:func:`threading.get_ident` - - -.. c:function:: PyObject *PyThread_GetInfo(void) - - Get general information about the current thread in the form of a - :ref:`struct sequence ` object. This information is - accessible as :py:attr:`sys.thread_info` in Python. - - On success, this returns a new :term:`strong reference` to the thread - information; on failure, this returns ``NULL`` with an exception set. - - The caller must hold an :term:`attached thread state`. - - -.. c:macro:: PY_HAVE_THREAD_NATIVE_ID - - This macro is defined when the system supports native thread IDs. - - -.. c:function:: unsigned long PyThread_get_thread_native_id(void) - - Get the native identifier of the current thread as it was assigned by the operating - system's kernel, which will never be less than zero. - - This function is only available when :c:macro:`PY_HAVE_THREAD_NATIVE_ID` is - defined. - - This function cannot fail, and the caller does not need to hold an - :term:`attached thread state`. - - .. seealso:: - :py:func:`threading.get_native_id` - - -.. c:function:: void PyThread_exit_thread(void) - - Terminate the current thread. This function is generally considered unsafe - and should be avoided. It is kept solely for backwards compatibility. - - This function is only safe to call if all functions in the full call - stack are written to safely allow it. - - .. warning:: - - If the current system uses POSIX threads (also known as "pthreads"), - this calls :manpage:`pthread_exit(3)`, which attempts to unwind the stack - and call C++ destructors on some libc implementations. However, if a - ``noexcept`` function is reached, it may terminate the process. - Other systems, such as macOS, do unwinding. - - On Windows, this function calls ``_endthreadex()``, which kills the thread - without calling C++ destructors. - - In any case, there is a risk of corruption on the thread's stack. - - .. deprecated:: 3.14 - - -.. c:function:: void PyThread_init_thread(void) - - Initialize ``PyThread*`` APIs. Python executes this function automatically, - so there's little need to call it from an extension module. - - -.. c:function:: int PyThread_set_stacksize(size_t size) - - Set the stack size of the current thread to *size* bytes. - - This function returns ``0`` on success, ``-1`` if *size* is invalid, or - ``-2`` if the system does not support changing the stack size. This function - does not set exceptions. - - The caller does not need to hold an :term:`attached thread state`. - - -.. c:function:: size_t PyThread_get_stacksize(void) - - Return the stack size of the current thread in bytes, or ``0`` if the system's - default stack size is in use. - - The caller does not need to hold an :term:`attached thread state`. +- :ref:`initialization` +- :ref:`threads` +- :ref:`synchronization` +- :ref:`thread-local-storage` +- :ref:`sub-interpreter-support` +- :ref:`profiling` diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 424d7de7089..9b1b88b2f8d 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -544,9 +544,9 @@ Configuration Options Visibility: -* Public: Can by get by :c:func:`PyConfig_Get` and set by +* Public: Can be retrieved by :c:func:`PyConfig_Get` and set by :c:func:`PyConfig_Set`. -* Read-only: Can by get by :c:func:`PyConfig_Get`, but cannot be set by +* Read-only: Can be retrieved by :c:func:`PyConfig_Get`, but cannot be set by :c:func:`PyConfig_Set`. @@ -1155,7 +1155,7 @@ PyConfig Most ``PyConfig`` methods :ref:`preinitialize Python ` if needed. In that case, the Python preinitialization configuration - (:c:type:`PyPreConfig`) in based on the :c:type:`PyConfig`. If configuration + (:c:type:`PyPreConfig`) is based on the :c:type:`PyConfig`. If configuration fields which are in common with :c:type:`PyPreConfig` are tuned, they must be set before calling a :c:type:`PyConfig` method: diff --git a/Doc/c-api/interp-lifecycle.rst b/Doc/c-api/interp-lifecycle.rst new file mode 100644 index 00000000000..f8c8fdaa05b --- /dev/null +++ b/Doc/c-api/interp-lifecycle.rst @@ -0,0 +1,956 @@ +.. highlight:: c + +.. _initialization: + +Interpreter initialization and finalization +=========================================== + +See :ref:`Python Initialization Configuration ` for details +on how to configure the interpreter prior to initialization. + +.. _pre-init-safe: + +Before Python initialization +---------------------------- + +In an application embedding Python, the :c:func:`Py_Initialize` function must +be called before using any other Python/C API functions; with the exception of +a few functions and the :ref:`global configuration variables +`. + +The following functions can be safely called before Python is initialized: + +* Functions that initialize the interpreter: + + * :c:func:`Py_Initialize` + * :c:func:`Py_InitializeEx` + * :c:func:`Py_InitializeFromConfig` + * :c:func:`Py_BytesMain` + * :c:func:`Py_Main` + * the runtime pre-initialization functions covered in :ref:`init-config` + +* Configuration functions: + + * :c:func:`PyImport_AppendInittab` + * :c:func:`PyImport_ExtendInittab` + * :c:func:`!PyInitFrozenExtensions` + * :c:func:`PyMem_SetAllocator` + * :c:func:`PyMem_SetupDebugHooks` + * :c:func:`PyObject_SetArenaAllocator` + * :c:func:`Py_SetProgramName` + * :c:func:`Py_SetPythonHome` + * the configuration functions covered in :ref:`init-config` + +* Informative functions: + + * :c:func:`Py_IsInitialized` + * :c:func:`PyMem_GetAllocator` + * :c:func:`PyObject_GetArenaAllocator` + * :c:func:`Py_GetBuildInfo` + * :c:func:`Py_GetCompiler` + * :c:func:`Py_GetCopyright` + * :c:func:`Py_GetPlatform` + * :c:func:`Py_GetVersion` + * :c:func:`Py_IsInitialized` + +* Utilities: + + * :c:func:`Py_DecodeLocale` + * the status reporting and utility functions covered in :ref:`init-config` + +* Memory allocators: + + * :c:func:`PyMem_RawMalloc` + * :c:func:`PyMem_RawRealloc` + * :c:func:`PyMem_RawCalloc` + * :c:func:`PyMem_RawFree` + +* Synchronization: + + * :c:func:`PyMutex_Lock` + * :c:func:`PyMutex_Unlock` + +.. note:: + + Despite their apparent similarity to some of the functions listed above, + the following functions **should not be called** before the interpreter has + been initialized: :c:func:`Py_EncodeLocale`, :c:func:`PyEval_InitThreads`, and + :c:func:`Py_RunMain`. + + +.. _global-conf-vars: + +Global configuration variables +------------------------------ + +Python has variables for the global configuration to control different features +and options. By default, these flags are controlled by :ref:`command line +options `. + +When a flag is set by an option, the value of the flag is the number of times +that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` +to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. + + +.. c:var:: int Py_BytesWarningFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.bytes_warning` should be used instead, see :ref:`Python + Initialization Configuration `. + + Issue a warning when comparing :class:`bytes` or :class:`bytearray` with + :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater + or equal to ``2``. + + Set by the :option:`-b` option. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_DebugFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.parser_debug` should be used instead, see :ref:`Python + Initialization Configuration `. + + Turn on parser debugging output (for expert only, depending on compilation + options). + + Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment + variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_DontWriteBytecodeFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.write_bytecode` should be used instead, see :ref:`Python + Initialization Configuration `. + + If set to non-zero, Python won't try to write ``.pyc`` files on the + import of source modules. + + Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` + environment variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_FrozenFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.pathconfig_warnings` should be used instead, see + :ref:`Python Initialization Configuration `. + + Private flag used by ``_freeze_module`` and ``frozenmain`` programs. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_HashRandomizationFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.hash_seed` and :c:member:`PyConfig.use_hash_seed` should + be used instead, see :ref:`Python Initialization Configuration + `. + + Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to + a non-empty string. + + If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment + variable to initialize the secret hash seed. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_IgnoreEnvironmentFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.use_environment` should be used instead, see + :ref:`Python Initialization Configuration `. + + Ignore all :envvar:`!PYTHON*` environment variables, e.g. + :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. + + Set by the :option:`-E` and :option:`-I` options. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_InspectFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.inspect` should be used instead, see + :ref:`Python Initialization Configuration `. + + When a script is passed as first argument or the :option:`-c` option is used, + enter interactive mode after executing the script or the command, even when + :data:`sys.stdin` does not appear to be a terminal. + + Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment + variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_InteractiveFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.interactive` should be used instead, see + :ref:`Python Initialization Configuration `. + + Set by the :option:`-i` option. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_IsolatedFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.isolated` should be used instead, see + :ref:`Python Initialization Configuration `. + + Run Python in isolated mode. In isolated mode :data:`sys.path` contains + neither the script's directory nor the user's site-packages directory. + + Set by the :option:`-I` option. + + .. versionadded:: 3.4 + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_LegacyWindowsFSEncodingFlag + + This API is kept for backward compatibility: setting + :c:member:`PyPreConfig.legacy_windows_fs_encoding` should be used instead, see + :ref:`Python Initialization Configuration `. + + If the flag is non-zero, use the ``mbcs`` encoding with ``replace`` error + handler, instead of the UTF-8 encoding with ``surrogatepass`` error handler, + for the :term:`filesystem encoding and error handler`. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment + variable is set to a non-empty string. + + See :pep:`529` for more details. + + .. availability:: Windows. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_LegacyWindowsStdioFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.legacy_windows_stdio` should be used instead, see + :ref:`Python Initialization Configuration `. + + If the flag is non-zero, use :class:`io.FileIO` instead of + :class:`!io._WindowsConsoleIO` for :mod:`sys` standard streams. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment + variable is set to a non-empty string. + + See :pep:`528` for more details. + + .. availability:: Windows. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_NoSiteFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.site_import` should be used instead, see + :ref:`Python Initialization Configuration `. + + Disable the import of the module :mod:`site` and the site-dependent + manipulations of :data:`sys.path` that it entails. Also disable these + manipulations if :mod:`site` is explicitly imported later (call + :func:`site.main` if you want them to be triggered). + + Set by the :option:`-S` option. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_NoUserSiteDirectory + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.user_site_directory` should be used instead, see + :ref:`Python Initialization Configuration `. + + Don't add the :data:`user site-packages directory ` to + :data:`sys.path`. + + Set by the :option:`-s` and :option:`-I` options, and the + :envvar:`PYTHONNOUSERSITE` environment variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_OptimizeFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.optimization_level` should be used instead, see + :ref:`Python Initialization Configuration `. + + Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment + variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_QuietFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.quiet` should be used instead, see :ref:`Python + Initialization Configuration `. + + Don't display the copyright and version messages even in interactive mode. + + Set by the :option:`-q` option. + + .. versionadded:: 3.2 + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_UnbufferedStdioFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.buffered_stdio` should be used instead, see :ref:`Python + Initialization Configuration `. + + Force the stdout and stderr streams to be unbuffered. + + Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` + environment variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_VerboseFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.verbose` should be used instead, see :ref:`Python + Initialization Configuration `. + + Print a message each time a module is initialized, showing the place + (filename or built-in module) from which it is loaded. If greater or equal + to ``2``, print a message for each file that is checked for when + searching for a module. Also provides information on module cleanup at exit. + + Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment + variable. + + .. deprecated-removed:: 3.12 3.15 + + +Initializing and finalizing the interpreter +------------------------------------------- + +.. c:function:: void Py_Initialize() + + .. index:: + single: PyEval_InitThreads() + single: modules (in module sys) + single: path (in module sys) + pair: module; builtins + pair: module; __main__ + pair: module; sys + triple: module; search; path + single: Py_FinalizeEx (C function) + + Initialize the Python interpreter. In an application embedding Python, + this should be called before using any other Python/C API functions; see + :ref:`Before Python Initialization ` for the few exceptions. + + This initializes the table of loaded modules (``sys.modules``), and creates + the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. + It also initializes the module search path (``sys.path``). It does not set + ``sys.argv``; use the :ref:`Python Initialization Configuration ` + API for that. This is a no-op when called for a second time (without calling + :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal + error if the initialization fails. + + Use :c:func:`Py_InitializeFromConfig` to customize the + :ref:`Python Initialization Configuration `. + + .. note:: + On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, + which will also affect non-Python uses of the console using the C Runtime. + + +.. c:function:: void Py_InitializeEx(int initsigs) + + This function works like :c:func:`Py_Initialize` if *initsigs* is ``1``. If + *initsigs* is ``0``, it skips initialization registration of signal handlers, + which may be useful when CPython is embedded as part of a larger application. + + Use :c:func:`Py_InitializeFromConfig` to customize the + :ref:`Python Initialization Configuration `. + + +.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) + + Initialize Python from *config* configuration, as described in + :ref:`init-from-config`. + + See the :ref:`init-config` section for details on pre-initializing the + interpreter, populating the runtime configuration structure, and querying + the returned status structure. + + +.. c:function:: int Py_IsInitialized() + + Return true (nonzero) when the Python interpreter has been initialized, false + (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until + :c:func:`Py_Initialize` is called again. + + +.. c:function:: int Py_IsFinalizing() + + Return true (non-zero) if the main Python interpreter is + :term:`shutting down `. Return false (zero) otherwise. + + .. versionadded:: 3.13 + + +.. c:function:: int Py_FinalizeEx() + + Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of + Python/C API functions, and destroy all sub-interpreters (see + :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since + the last call to :c:func:`Py_Initialize`. This is a no-op when called for a second + time (without calling :c:func:`Py_Initialize` again first). + + Since this is the reverse of :c:func:`Py_Initialize`, it should be called + in the same thread with the same interpreter active. That means + the main thread and the main interpreter. + This should never be called while :c:func:`Py_RunMain` is running. + + Normally the return value is ``0``. + If there were errors during finalization (flushing buffered data), + ``-1`` is returned. + + Note that Python will do a best effort at freeing all memory allocated by the Python + interpreter. Therefore, any C-Extension should make sure to correctly clean up all + of the previously allocated PyObjects before using them in subsequent calls to + :c:func:`Py_Initialize`. Otherwise it could introduce vulnerabilities and incorrect + behavior. + + This function is provided for a number of reasons. An embedding application + might want to restart Python without having to restart the application itself. + An application that has loaded the Python interpreter from a dynamically + loadable library (or DLL) might want to free all memory allocated by Python + before unloading the DLL. During a hunt for memory leaks in an application a + developer might want to free all memory allocated by Python before exiting from + the application. + + **Bugs and caveats:** The destruction of modules and objects in modules is done + in random order; this may cause destructors (:meth:`~object.__del__` methods) to fail + when they depend on other objects (even functions) or modules. Dynamically + loaded extension modules loaded by Python are not unloaded. Small amounts of + memory allocated by the Python interpreter may not be freed (if you find a leak, + please report it). Memory tied up in circular references between objects is not + freed. Interned strings will all be deallocated regardless of their reference count. + Some memory allocated by extension modules may not be freed. Some extensions may not + work properly if their initialization routine is called more than once; this can + happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` + more than once. :c:func:`Py_FinalizeEx` must not be called recursively from + within itself. Therefore, it must not be called by any code that may be run + as part of the interpreter shutdown process, such as :py:mod:`atexit` + handlers, object finalizers, or any code that may be run while flushing the + stdout and stderr files. + + .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx + + .. versionadded:: 3.6 + + +.. c:function:: void Py_Finalize() + + This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that + disregards the return value. + + +.. c:function:: int Py_BytesMain(int argc, char **argv) + + Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings, + allowing the calling application to delegate the text decoding step to + the CPython runtime. + + .. versionadded:: 3.8 + + +.. c:function:: int Py_Main(int argc, wchar_t **argv) + + The main program for the standard interpreter, encapsulating a full + initialization/finalization cycle, as well as additional + behaviour to implement reading configurations settings from the environment + and command line, and then executing ``__main__`` in accordance with + :ref:`using-on-cmdline`. + + This is made available for programs which wish to support the full CPython + command line interface, rather than just embedding a Python runtime in a + larger application. + + The *argc* and *argv* parameters are similar to those which are passed to a + C program's :c:func:`main` function, except that the *argv* entries are first + converted to ``wchar_t`` using :c:func:`Py_DecodeLocale`. It is also + important to note that the argument list entries may be modified to point to + strings other than those passed in (however, the contents of the strings + pointed to by the argument list are not modified). + + The return value is ``2`` if the argument list does not represent a valid + Python command line, and otherwise the same as :c:func:`Py_RunMain`. + + In terms of the CPython runtime configuration APIs documented in the + :ref:`runtime configuration ` section (and without accounting + for error handling), ``Py_Main`` is approximately equivalent to:: + + PyConfig config; + PyConfig_InitPythonConfig(&config); + PyConfig_SetArgv(&config, argc, argv); + Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + + Py_RunMain(); + + In normal usage, an embedding application will call this function + *instead* of calling :c:func:`Py_Initialize`, :c:func:`Py_InitializeEx` or + :c:func:`Py_InitializeFromConfig` directly, and all settings will be applied + as described elsewhere in this documentation. If this function is instead + called *after* a preceding runtime initialization API call, then exactly + which environmental and command line configuration settings will be updated + is version dependent (as it depends on which settings correctly support + being modified after they have already been set once when the runtime was + first initialized). + + +.. c:function:: int Py_RunMain(void) + + Executes the main module in a fully configured CPython runtime. + + Executes the command (:c:member:`PyConfig.run_command`), the script + (:c:member:`PyConfig.run_filename`) or the module + (:c:member:`PyConfig.run_module`) specified on the command line or in the + configuration. If none of these values are set, runs the interactive Python + prompt (REPL) using the ``__main__`` module's global namespace. + + If :c:member:`PyConfig.inspect` is not set (the default), the return value + will be ``0`` if the interpreter exits normally (that is, without raising + an exception), the exit status of an unhandled :exc:`SystemExit`, or ``1`` + for any other unhandled exception. + + If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option + is used), rather than returning when the interpreter exits, execution will + instead resume in an interactive Python prompt (REPL) using the ``__main__`` + module's global namespace. If the interpreter exited with an exception, it + is immediately raised in the REPL session. The function return value is + then determined by the way the *REPL session* terminates: ``0``, ``1``, or + the status of a :exc:`SystemExit`, as specified above. + + This function always finalizes the Python interpreter before it returns. + + See :ref:`Python Configuration ` for an example of a + customized Python that always runs in isolated mode using + :c:func:`Py_RunMain`. + +.. c:function:: int PyUnstable_AtExit(PyInterpreterState *interp, void (*func)(void *), void *data) + + Register an :mod:`atexit` callback for the target interpreter *interp*. + This is similar to :c:func:`Py_AtExit`, but takes an explicit interpreter and + data pointer for the callback. + + There must be an :term:`attached thread state` for *interp*. + + .. versionadded:: 3.13 + + +.. _cautions-regarding-runtime-finalization: + +Cautions regarding runtime finalization +--------------------------------------- + +In the late stage of :term:`interpreter shutdown`, after attempting to wait for +non-daemon threads to exit (though this can be interrupted by +:class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime +is marked as *finalizing*: :c:func:`Py_IsFinalizing` and +:func:`sys.is_finalizing` return true. At this point, only the *finalization +thread* that initiated finalization (typically the main thread) is allowed to +acquire the :term:`GIL`. + +If any thread, other than the finalization thread, attempts to attach a :term:`thread state` +during finalization, either explicitly or +implicitly, the thread enters **a permanently blocked state** +where it remains until the program exits. In most cases this is harmless, but this can result +in deadlock if a later stage of finalization attempts to acquire a lock owned by the +blocked thread, or otherwise waits on the blocked thread. + +Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++ +finalizations further up the call stack when such threads were forcibly exited +here in CPython 3.13 and earlier. The CPython runtime :term:`thread state` C APIs +have never had any error reporting or handling expectations at :term:`thread state` +attachment time that would've allowed for graceful exit from this situation. Changing that +would require new stable C APIs and rewriting the majority of C code in the +CPython ecosystem to use those with error handling. + + +Process-wide parameters +----------------------- + +.. c:function:: void Py_SetProgramName(const wchar_t *name) + + .. index:: + single: Py_Initialize() + single: main() + single: Py_GetPath() + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python + Initialization Configuration `. + + This function should be called before :c:func:`Py_Initialize` is called for + the first time, if it is called at all. It tells the interpreter the value + of the ``argv[0]`` argument to the :c:func:`main` function of the program + (converted to wide characters). + This is used by :c:func:`Py_GetPath` and some other functions below to find + the Python run-time libraries relative to the interpreter executable. The + default value is ``'python'``. The argument should point to a + zero-terminated wide character string in static storage whose contents will not + change for the duration of the program's execution. No code in the Python + interpreter will change the contents of this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + .. deprecated-removed:: 3.11 3.15 + + +.. c:function:: wchar_t* Py_GetProgramName() + + Return the program name set with :c:member:`PyConfig.program_name`, or the default. + The returned string points into static storage; the caller should not modify its + value. + + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + + .. deprecated-removed:: 3.13 3.15 + Use :c:func:`PyConfig_Get("executable") ` + (:data:`sys.executable`) instead. + + +.. c:function:: wchar_t* Py_GetPrefix() + + Return the *prefix* for installed platform-independent files. This is derived + through a number of complicated rules from the program name set with + :c:member:`PyConfig.program_name` and some environment variables; for example, if the + program name is ``'/usr/local/bin/python'``, the prefix is ``'/usr/local'``. The + returned string points into static storage; the caller should not modify its + value. This corresponds to the :makevar:`prefix` variable in the top-level + :file:`Makefile` and the :option:`--prefix` argument to the :program:`configure` + script at build time. The value is available to Python code as ``sys.base_prefix``. + It is only useful on Unix. See also the next function. + + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + + .. deprecated-removed:: 3.13 3.15 + Use :c:func:`PyConfig_Get("base_prefix") ` + (:data:`sys.base_prefix`) instead. Use :c:func:`PyConfig_Get("prefix") + ` (:data:`sys.prefix`) if :ref:`virtual environments + ` need to be handled. + + +.. c:function:: wchar_t* Py_GetExecPrefix() + + Return the *exec-prefix* for installed platform-*dependent* files. This is + derived through a number of complicated rules from the program name set with + :c:member:`PyConfig.program_name` and some environment variables; for example, if the + program name is ``'/usr/local/bin/python'``, the exec-prefix is + ``'/usr/local'``. The returned string points into static storage; the caller + should not modify its value. This corresponds to the :makevar:`exec_prefix` + variable in the top-level :file:`Makefile` and the ``--exec-prefix`` + argument to the :program:`configure` script at build time. The value is + available to Python code as ``sys.base_exec_prefix``. It is only useful on + Unix. + + Background: The exec-prefix differs from the prefix when platform dependent + files (such as executables and shared libraries) are installed in a different + directory tree. In a typical installation, platform dependent files may be + installed in the :file:`/usr/local/plat` subtree while platform independent may + be installed in :file:`/usr/local`. + + Generally speaking, a platform is a combination of hardware and software + families, e.g. Sparc machines running the Solaris 2.x operating system are + considered the same platform, but Intel machines running Solaris 2.x are another + platform, and Intel machines running Linux are yet another platform. Different + major revisions of the same operating system generally also form different + platforms. Non-Unix operating systems are a different story; the installation + strategies on those systems are so different that the prefix and exec-prefix are + meaningless, and set to the empty string. Note that compiled Python bytecode + files are platform independent (but not independent from the Python version by + which they were compiled!). + + System administrators will know how to configure the :program:`mount` or + :program:`automount` programs to share :file:`/usr/local` between platforms + while having :file:`/usr/local/plat` be a different filesystem for each + platform. + + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + + .. deprecated-removed:: 3.13 3.15 + Use :c:func:`PyConfig_Get("base_exec_prefix") ` + (:data:`sys.base_exec_prefix`) instead. Use + :c:func:`PyConfig_Get("exec_prefix") ` + (:data:`sys.exec_prefix`) if :ref:`virtual environments ` need + to be handled. + + +.. c:function:: wchar_t* Py_GetProgramFullPath() + + .. index:: + single: executable (in module sys) + + Return the full program name of the Python executable; this is computed as a + side-effect of deriving the default module search path from the program name + (set by :c:member:`PyConfig.program_name`). The returned string points into + static storage; the caller should not modify its value. The value is available + to Python code as ``sys.executable``. + + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + + .. deprecated-removed:: 3.13 3.15 + Use :c:func:`PyConfig_Get("executable") ` + (:data:`sys.executable`) instead. + + +.. c:function:: wchar_t* Py_GetPath() + + .. index:: + triple: module; search; path + single: path (in module sys) + + Return the default module search path; this is computed from the program name + (set by :c:member:`PyConfig.program_name`) and some environment variables. + The returned string consists of a series of directory names separated by a + platform dependent delimiter character. The delimiter character is ``':'`` + on Unix and macOS, ``';'`` on Windows. The returned string points into + static storage; the caller should not modify its value. The list + :data:`sys.path` is initialized with this value on interpreter startup; it + can be (and usually is) modified later to change the search path for loading + modules. + + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. XXX should give the exact rules + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + + .. deprecated-removed:: 3.13 3.15 + Use :c:func:`PyConfig_Get("module_search_paths") ` + (:data:`sys.path`) instead. + +.. c:function:: const char* Py_GetVersion() + + Return the version of this Python interpreter. This is a string that looks + something like :: + + "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]" + + .. index:: single: version (in module sys) + + The first word (up to the first space character) is the current Python version; + the first characters are the major and minor version separated by a + period. The returned string points into static storage; the caller should not + modify its value. The value is available to Python code as :data:`sys.version`. + + See also the :c:var:`Py_Version` constant. + + +.. c:function:: const char* Py_GetPlatform() + + .. index:: single: platform (in module sys) + + Return the platform identifier for the current platform. On Unix, this is + formed from the "official" name of the operating system, converted to lower + case, followed by the major revision number; e.g., for Solaris 2.x, which is + also known as SunOS 5.x, the value is ``'sunos5'``. On macOS, it is + ``'darwin'``. On Windows, it is ``'win'``. The returned string points into + static storage; the caller should not modify its value. The value is available + to Python code as ``sys.platform``. + + +.. c:function:: const char* Py_GetCopyright() + + Return the official copyright string for the current Python version, for example + + ``'Copyright 1991-1995 Stichting Mathematisch Centrum, Amsterdam'`` + + .. index:: single: copyright (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as ``sys.copyright``. + + +.. c:function:: const char* Py_GetCompiler() + + Return an indication of the compiler used to build the current Python version, + in square brackets, for example:: + + "[GCC 2.7.2.2]" + + .. index:: single: version (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as part of the variable + ``sys.version``. + + +.. c:function:: const char* Py_GetBuildInfo() + + Return information about the sequence number and build date and time of the + current Python interpreter instance, for example :: + + "#67, Aug 1 1997, 22:34:28" + + .. index:: single: version (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as part of the variable + ``sys.version``. + + +.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) + + .. index:: + single: main() + single: Py_FatalError() + single: argv (in module sys) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and + :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python + Initialization Configuration `. + + Set :data:`sys.argv` based on *argc* and *argv*. These parameters are + similar to those passed to the program's :c:func:`main` function with the + difference that the first entry should refer to the script file to be + executed rather than the executable hosting the Python interpreter. If there + isn't a script that will be run, the first entry in *argv* can be an empty + string. If this function fails to initialize :data:`sys.argv`, a fatal + condition is signalled using :c:func:`Py_FatalError`. + + If *updatepath* is zero, this is all the function does. If *updatepath* + is non-zero, the function also modifies :data:`sys.path` according to the + following algorithm: + + - If the name of an existing script is passed in ``argv[0]``, the absolute + path of the directory where the script is located is prepended to + :data:`sys.path`. + - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point + to an existing file name), an empty string is prepended to + :data:`sys.path`, which is the same as prepending the current working + directory (``"."``). + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration `. + + .. note:: + It is recommended that applications embedding the Python interpreter + for purposes other than executing a single script pass ``0`` as *updatepath*, + and update :data:`sys.path` themselves if desired. + See :cve:`2008-5983`. + + On versions before 3.1.3, you can achieve the same effect by manually + popping the first :data:`sys.path` element after having called + :c:func:`PySys_SetArgv`, for example using:: + + PyRun_SimpleString("import sys; sys.path.pop(0)\n"); + + .. versionadded:: 3.1.3 + + .. deprecated-removed:: 3.11 3.15 + + +.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used + instead, see :ref:`Python Initialization Configuration `. + + This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set + to ``1`` unless the :program:`python` interpreter was started with the + :option:`-I`. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration `. + + .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. + + .. deprecated-removed:: 3.11 3.15 + + +.. c:function:: void Py_SetPythonHome(const wchar_t *home) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.home` should be used instead, see :ref:`Python + Initialization Configuration `. + + Set the default "home" directory, that is, the location of the standard + Python libraries. See :envvar:`PYTHONHOME` for the meaning of the + argument string. + + The argument should point to a zero-terminated character string in static + storage whose contents will not change for the duration of the program's + execution. No code in the Python interpreter will change the contents of + this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + .. deprecated-removed:: 3.11 3.15 + + +.. c:function:: wchar_t* Py_GetPythonHome() + + Return the default "home", that is, the value set by + :c:member:`PyConfig.home`, or the value of the :envvar:`PYTHONHOME` + environment variable if it is set. + + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + + .. deprecated-removed:: 3.13 3.15 + Use :c:func:`PyConfig_Get("home") ` or the + :envvar:`PYTHONHOME` environment variable instead. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index c7520adc121..d73377fb6e6 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -116,124 +116,152 @@ defined closer to where they are useful (for example, :c:macro:`Py_RETURN_NONE`, Others of a more general utility are defined here. This is not necessarily a complete listing. +.. c:macro:: Py_CAN_START_THREADS -.. c:macro:: Py_ABS(x) - - Return the absolute value of ``x``. + If this macro is defined, then the current system is able to start threads. - If the result cannot be represented (for example, if ``x`` has - :c:macro:`!INT_MIN` value for :c:expr:`int` type), the behavior is - undefined. + Currently, all systems supported by CPython (per :pep:`11`), with the + exception of some WebAssembly platforms, support starting threads. - .. versionadded:: 3.3 + .. versionadded:: 3.13 -.. c:macro:: Py_ALWAYS_INLINE +.. c:macro:: Py_GETENV(s) - Ask the compiler to always inline a static inline function. The compiler can - ignore it and decide to not inline the function. + Like :samp:`getenv({s})`, but returns ``NULL`` if :option:`-E` was passed + on the command line (see :c:member:`PyConfig.use_environment`). - It can be used to inline performance critical static inline functions when - building Python in debug mode with function inlining disabled. For example, - MSC disables function inlining when building in debug mode. - Marking blindly a static inline function with Py_ALWAYS_INLINE can result in - worse performances (due to increased code size for example). The compiler is - usually smarter than the developer for the cost/benefit analysis. +Docstring macros +---------------- - If Python is :ref:`built in debug mode ` (if the :c:macro:`Py_DEBUG` - macro is defined), the :c:macro:`Py_ALWAYS_INLINE` macro does nothing. +.. c:macro:: PyDoc_STRVAR(name, str) - It must be specified before the function return type. Usage:: + Creates a variable with name *name* that can be used in docstrings. + If Python is built without docstrings (:option:`--without-doc-strings`), + the value will be an empty string. - static inline Py_ALWAYS_INLINE int random(void) { return 4; } + Example:: - .. versionadded:: 3.11 + PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); -.. c:macro:: Py_CHARMASK(c) + static PyMethodDef deque_methods[] = { + // ... + {"pop", (PyCFunction)deque_pop, METH_NOARGS, pop_doc}, + // ... + } - Argument must be a character or an integer in the range [-128, 127] or [0, - 255]. This macro returns ``c`` cast to an ``unsigned char``. + Expands to :samp:`PyDoc_VAR({name}) = PyDoc_STR({str})`. -.. c:macro:: Py_DEPRECATED(version) +.. c:macro:: PyDoc_STR(str) - Use this for deprecated declarations. The macro must be placed before the - symbol name. + Expands to the given input string, or an empty string + if docstrings are disabled (:option:`--without-doc-strings`). Example:: - Py_DEPRECATED(3.8) PyAPI_FUNC(int) Py_OldFunction(void); + static PyMethodDef pysqlite_row_methods[] = { + {"keys", (PyCFunction)pysqlite_row_keys, METH_NOARGS, + PyDoc_STR("Returns the keys of the row.")}, + {NULL, NULL} + }; - .. versionchanged:: 3.8 - MSVC support was added. +.. c:macro:: PyDoc_VAR(name) -.. c:macro:: Py_GETENV(s) + Declares a static character array variable with the given *name*. + Expands to :samp:`static const char {name}[]` - Like ``getenv(s)``, but returns ``NULL`` if :option:`-E` was passed on the - command line (see :c:member:`PyConfig.use_environment`). + For example:: -.. c:macro:: Py_LOCAL(type) + PyDoc_VAR(python_doc) = PyDoc_STR( + "A genus of constricting snakes in the Pythonidae family native " + "to the tropics and subtropics of the Eastern Hemisphere."); - Declare a function returning the specified *type* using a fast-calling - qualifier for functions that are local to the current file. - Semantically, this is equivalent to ``static type``. -.. c:macro:: Py_LOCAL_INLINE(type) +General utility macros +---------------------- - Equivalent to :c:macro:`Py_LOCAL` but additionally requests the function - be inlined. +The following macros are for common tasks not specific to Python. -.. c:macro:: Py_LOCAL_SYMBOL +.. c:macro:: Py_UNUSED(arg) - Macro used to declare a symbol as local to the shared library (hidden). - On supported platforms, it ensures the symbol is not exported. + Use this for unused arguments in a function definition to silence compiler + warnings. Example: ``int func(int a, int Py_UNUSED(b)) { return a; }``. - On compatible versions of GCC/Clang, it - expands to ``__attribute__((visibility("hidden")))``. + .. versionadded:: 3.4 -.. c:macro:: Py_MAX(x, y) +.. c:macro:: Py_GCC_ATTRIBUTE(name) - Return the maximum value between ``x`` and ``y``. + Use a GCC attribute *name*, hiding it from compilers that don't support GCC + attributes (such as MSVC). - .. versionadded:: 3.3 + This expands to :samp:`__attribute__(({name)})` on a GCC compiler, + and expands to nothing on compilers that don't support GCC attributes. -.. c:macro:: Py_MEMBER_SIZE(type, member) - Return the size of a structure (``type``) ``member`` in bytes. +Numeric utilities +^^^^^^^^^^^^^^^^^ - .. versionadded:: 3.6 +.. c:macro:: Py_ABS(x) -.. c:macro:: Py_MEMCPY(dest, src, n) + Return the absolute value of ``x``. - This is a :term:`soft deprecated` alias to :c:func:`!memcpy`. - Use :c:func:`!memcpy` directly instead. + The argument may be evaluated more than once. + Consequently, do not pass an expression with side-effects directly + to this macro. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. + If the result cannot be represented (for example, if ``x`` has + :c:macro:`!INT_MIN` value for :c:expr:`int` type), the behavior is + undefined. + + Corresponds roughly to :samp:`(({x}) < 0 ? -({x}) : ({x}))` + + .. versionadded:: 3.3 -.. c:macro:: Py_MIN(x, y) +.. c:macro:: Py_MAX(x, y) + Py_MIN(x, y) + + Return the larger or smaller of the arguments, respectively. - Return the minimum value between ``x`` and ``y``. + Any arguments may be evaluated more than once. + Consequently, do not pass an expression with side-effects directly + to this macro. + + :c:macro:`!Py_MAX` corresponds roughly to + :samp:`((({x}) > ({y})) ? ({x}) : ({y}))`. .. versionadded:: 3.3 -.. c:macro:: Py_NO_INLINE +.. c:macro:: Py_ARITHMETIC_RIGHT_SHIFT(type, integer, positions) - Disable inlining on a function. For example, it reduces the C stack - consumption: useful on LTO+PGO builds which heavily inline code (see - :issue:`33720`). + Similar to :samp:`{integer} >> {positions}`, but forces sign extension, + as the C standard does not define whether a right-shift of a signed + integer will perform sign extension or a zero-fill. - Usage:: + *integer* should be any signed integer type. + *positions* is the number of positions to shift to the right. - Py_NO_INLINE static int random(void) { return 4; } + Both *integer* and *positions* can be evaluated more than once; + consequently, avoid directly passing a function call or some other + operation with side-effects to this macro. Instead, store the result as a + variable and then pass it. - .. versionadded:: 3.11 + *type* is unused and only kept for backwards compatibility. Historically, + *type* was used to cast *integer*. -.. c:macro:: Py_STRINGIFY(x) + .. versionchanged:: 3.1 - Convert ``x`` to a C string. E.g. ``Py_STRINGIFY(123)`` returns - ``"123"``. + This macro is now valid for all signed integer types, not just those for + which ``unsigned type`` is legal. As a result, *type* is no longer + used. - .. versionadded:: 3.4 +.. c:macro:: Py_CHARMASK(c) + + Argument must be a character or an integer in the range [-128, 127] or [0, + 255]. This macro returns ``c`` cast to an ``unsigned char``. + + +Assertion utilities +^^^^^^^^^^^^^^^^^^^ .. c:macro:: Py_UNREACHABLE() @@ -246,8 +274,11 @@ complete listing. avoids a warning about unreachable code. For example, the macro is implemented with ``__builtin_unreachable()`` on GCC in release mode. - A use for ``Py_UNREACHABLE()`` is following a call a function that - never returns but that is not declared :c:macro:`_Py_NO_RETURN`. + In debug mode, and on unsupported compilers, the macro expands to a call to + :c:func:`Py_FatalError`. + + A use for ``Py_UNREACHABLE()`` is following a call to a function that + never returns but that is not declared ``_Noreturn``. If a code path is very unlikely code but can be reached under exceptional case, this macro must not be used. For example, under low memory condition @@ -257,18 +288,29 @@ complete listing. .. versionadded:: 3.7 -.. c:macro:: Py_UNUSED(arg) +.. c:macro:: Py_SAFE_DOWNCAST(value, larger, smaller) - Use this for unused arguments in a function definition to silence compiler - warnings. Example: ``int func(int a, int Py_UNUSED(b)) { return a; }``. + Cast *value* to type *smaller* from type *larger*, validating that no + information was lost. - .. versionadded:: 3.4 + On release builds of Python, this is roughly equivalent to + :samp:`(({smaller}) {value})` + (in C++, :samp:`static_cast<{smaller}>({value})` will be used instead). + + On debug builds (implying that :c:macro:`Py_DEBUG` is defined), this asserts + that no information was lost with the cast from *larger* to *smaller*. + + *value*, *larger*, and *smaller* may all be evaluated more than once in the + expression; consequently, do not pass an expression with side-effects + directly to this macro. .. c:macro:: Py_BUILD_ASSERT(cond) Asserts a compile-time condition *cond*, as a statement. The build will fail if the condition is false or cannot be evaluated at compile time. + Corresponds roughly to :samp:`static_assert({cond})` on C23 and above. + For example:: Py_BUILD_ASSERT(sizeof(PyTime_t) == sizeof(int64_t)); @@ -287,62 +329,127 @@ complete listing. .. versionadded:: 3.3 -.. c:macro:: PyDoc_STRVAR(name, str) - Creates a variable with name *name* that can be used in docstrings. - If Python is built without docstrings, the value will be empty. +Type size utilities +^^^^^^^^^^^^^^^^^^^ - Use :c:macro:`PyDoc_STRVAR` for docstrings to support building - Python without docstrings, as specified in :pep:`7`. +.. c:macro:: Py_ARRAY_LENGTH(array) - Example:: + Compute the length of a statically allocated C array at compile time. - PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); + The *array* argument must be a C array with a size known at compile time. + Passing an array with an unknown size, such as a heap-allocated array, + will result in a compilation error on some compilers, or otherwise produce + incorrect results. - static PyMethodDef deque_methods[] = { - // ... - {"pop", (PyCFunction)deque_pop, METH_NOARGS, pop_doc}, - // ... - } + This is roughly equivalent to:: -.. c:macro:: PyDoc_STR(str) + sizeof(array) / sizeof((array)[0]) - Creates a docstring for the given input string or an empty string - if docstrings are disabled. +.. c:macro:: Py_MEMBER_SIZE(type, member) - Use :c:macro:`PyDoc_STR` in specifying docstrings to support - building Python without docstrings, as specified in :pep:`7`. + Return the size of a structure (*type*) *member* in bytes. - Example:: + Corresponds roughly to :samp:`sizeof((({type} *)NULL)->{member})`. - static PyMethodDef pysqlite_row_methods[] = { - {"keys", (PyCFunction)pysqlite_row_keys, METH_NOARGS, - PyDoc_STR("Returns the keys of the row.")}, - {NULL, NULL} - }; + .. versionadded:: 3.6 -.. c:macro:: PyDoc_VAR(name) - Declares a static character array variable with the given name *name*. +Macro definition utilities +^^^^^^^^^^^^^^^^^^^^^^^^^^ - For example:: +.. c:macro:: Py_FORCE_EXPANSION(X) - PyDoc_VAR(python_doc) = PyDoc_STR("A genus of constricting snakes in the Pythonidae family native " - "to the tropics and subtropics of the Eastern Hemisphere."); + This is equivalent to :samp:`{X}`, which is useful for token-pasting in + macros, as macro expansions in *X* are forcefully evaluated by the + preprocessor. -.. c:macro:: Py_ARRAY_LENGTH(array) +.. c:macro:: Py_STRINGIFY(x) - Compute the length of a statically allocated C array at compile time. + Convert ``x`` to a C string. For example, ``Py_STRINGIFY(123)`` returns + ``"123"``. - The *array* argument must be a C array with a size known at compile time. - Passing an array with an unknown size, such as a heap-allocated array, - will result in a compilation error on some compilers, or otherwise produce - incorrect results. + .. versionadded:: 3.4 - This is roughly equivalent to:: - sizeof(array) / sizeof((array)[0]) +Declaration utilities +--------------------- + +The following macros can be used in declarations. +They are most useful for defining the C API itself, and have limited use +for extension authors. +Most of them expand to compiler-specific spellings of common extensions +to the C language. + +.. c:macro:: Py_ALWAYS_INLINE + + Ask the compiler to always inline a static inline function. The compiler can + ignore it and decide to not inline the function. + + Corresponds to ``always_inline`` attribute in GCC and ``__forceinline`` + in MSVC. + + It can be used to inline performance critical static inline functions when + building Python in debug mode with function inlining disabled. For example, + MSC disables function inlining when building in debug mode. + + Marking blindly a static inline function with Py_ALWAYS_INLINE can result in + worse performances (due to increased code size for example). The compiler is + usually smarter than the developer for the cost/benefit analysis. + + If Python is :ref:`built in debug mode ` (if the :c:macro:`Py_DEBUG` + macro is defined), the :c:macro:`Py_ALWAYS_INLINE` macro does nothing. + + It must be specified before the function return type. Usage:: + + static inline Py_ALWAYS_INLINE int random(void) { return 4; } + + .. versionadded:: 3.11 + +.. c:macro:: Py_NO_INLINE + + Disable inlining on a function. For example, it reduces the C stack + consumption: useful on LTO+PGO builds which heavily inline code (see + :issue:`33720`). + + Corresponds to the ``noinline`` attribute/specification on GCC and MSVC. + + Usage:: + + Py_NO_INLINE static int random(void) { return 4; } + + .. versionadded:: 3.11 + +.. c:macro:: Py_DEPRECATED(version) + + Use this to declare APIs that were deprecated in a specific CPython version. + The macro must be placed before the symbol name. + + Example:: + + Py_DEPRECATED(3.8) PyAPI_FUNC(int) Py_OldFunction(void); + + .. versionchanged:: 3.8 + MSVC support was added. + +.. c:macro:: Py_LOCAL(type) + + Declare a function returning the specified *type* using a fast-calling + qualifier for functions that are local to the current file. + Semantically, this is equivalent to :samp:`static {type}`. + +.. c:macro:: Py_LOCAL_INLINE(type) + + Equivalent to :c:macro:`Py_LOCAL` but additionally requests the function + be inlined. +.. c:macro:: Py_LOCAL_SYMBOL + + Macro used to declare a symbol as local to the shared library (hidden). + On supported platforms, it ensures the symbol is not exported. + + On compatible versions of GCC/Clang, it + expands to ``__attribute__((visibility("hidden")))``. .. c:macro:: Py_EXPORTED_SYMBOL @@ -376,6 +483,48 @@ complete listing. extension modules should not use it for their own symbols. +Outdated macros +--------------- + +The following macros have been used to features that have been standardized +in C11. + +.. c:macro:: Py_ALIGNED(num) + + Specify alignment to *num* bytes on compilers that support it. + + Consider using the C11 standard ``_Alignas`` specifier over this macro. + +.. c:macro:: Py_LL(number) + Py_ULL(number) + + Use *number* as a ``long long`` or ``unsigned long long`` integer literal, + respectively. + + Expands to *number* followed by ``LL`` or ``LLU``, respectively, but will + expand to some compiler-specific suffixes on some older compilers. + + Consider using the C99 standard suffixes ``LL`` and ``LLU`` directly. + +.. c:macro:: Py_MEMCPY(dest, src, n) + + This is a :term:`soft deprecated` alias to :c:func:`!memcpy`. + Use :c:func:`!memcpy` directly instead. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. + +.. c:macro:: Py_VA_COPY + + This is a :term:`soft deprecated` alias to the C99-standard ``va_copy`` + function. + + Historically, this would use a compiler-specific method to copy a ``va_list``. + + .. versionchanged:: 3.6 + This is now an alias to ``va_copy``. + + .. _api-objects: Objects, Types and Reference Counts diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index a3be75a2a76..0dd57eaf37e 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -208,8 +208,11 @@ The following function sets, modeled after the ANSI C standard, but specifying behavior when requesting zero bytes, are available for allocating and releasing memory from the Python heap. -The :ref:`default memory allocator ` uses the -:ref:`pymalloc memory allocator `. +In the GIL-enabled build (default build) the +:ref:`default memory allocator ` uses the +:ref:`pymalloc memory allocator `, whereas in the +:term:`free-threaded build`, the default is the +:ref:`mimalloc memory allocator ` instead. .. warning:: @@ -219,6 +222,11 @@ The :ref:`default memory allocator ` uses the The default allocator is now pymalloc instead of system :c:func:`malloc`. +.. versionchanged:: 3.13 + + In the :term:`free-threaded ` build, the default allocator + is now :ref:`mimalloc `. + .. c:function:: void* PyMem_Malloc(size_t n) Allocates *n* bytes and returns a pointer of type :c:expr:`void*` to the @@ -344,7 +352,9 @@ memory from the Python heap. the :ref:`Customize Memory Allocators ` section. The :ref:`default object allocator ` uses the -:ref:`pymalloc memory allocator `. +:ref:`pymalloc memory allocator `. In the +:term:`free-threaded ` build, the default is the +:ref:`mimalloc memory allocator ` instead. .. warning:: @@ -424,14 +434,16 @@ Default Memory Allocators Default memory allocators: -=============================== ==================== ================== ===================== ==================== -Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc -=============================== ==================== ================== ===================== ==================== -Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` -Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug -Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` -Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug -=============================== ==================== ================== ===================== ==================== +=================================== ======================= ==================== ====================== ====================== +Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc +=================================== ======================= ==================== ====================== ====================== +Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` +Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug +Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` +Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug +Free-threaded build ``"mimalloc"`` ``mimalloc`` ``mimalloc`` ``mimalloc`` +Free-threaded debug build ``"mimalloc_debug"`` ``mimalloc`` + debug ``mimalloc`` + debug ``mimalloc`` + debug +=================================== ======================= ==================== ====================== ====================== Legend: @@ -439,8 +451,7 @@ Legend: * ``malloc``: system allocators from the standard C library, C functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. * ``pymalloc``: :ref:`pymalloc memory allocator `. -* ``mimalloc``: :ref:`mimalloc memory allocator `. The pymalloc - allocator will be used if mimalloc support isn't available. +* ``mimalloc``: :ref:`mimalloc memory allocator `. * "+ debug": with :ref:`debug hooks on the Python memory allocators `. * "Debug build": :ref:`Python build in debug mode `. @@ -733,9 +744,27 @@ The mimalloc allocator .. versionadded:: 3.13 -Python supports the mimalloc allocator when the underlying platform support is available. -mimalloc "is a general purpose allocator with excellent performance characteristics. -Initially developed by Daan Leijen for the runtime systems of the Koka and Lean languages." +Python supports the `mimalloc `__ +allocator when the underlying platform support is available. +mimalloc is a general purpose allocator with excellent performance +characteristics, initially developed by Daan Leijen for the runtime systems +of the Koka and Lean languages. + +Unlike :ref:`pymalloc `, which is optimized for small objects (512 +bytes or fewer), mimalloc handles allocations of any size. + +In the :term:`free-threaded ` build, mimalloc is the default +and **required** allocator for the :c:macro:`PYMEM_DOMAIN_MEM` and +:c:macro:`PYMEM_DOMAIN_OBJ` domains. It cannot be disabled in free-threaded +builds. The free-threaded build uses per-thread mimalloc heaps, which allows +allocation and deallocation to proceed without locking in most cases. + +In the default (non-free-threaded) build, mimalloc is available but not the +default allocator. It can be selected at runtime using +:envvar:`PYTHONMALLOC`\ ``=mimalloc`` (or ``mimalloc_debug`` to include +:ref:`debug hooks `). It can be disabled at build time +using the :option:`--without-mimalloc` configure option, but this option +cannot be combined with :option:`--disable-gil`. tracemalloc C API ================= diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index eed47c8966c..49ac4de6655 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -277,6 +277,11 @@ data stored in module state. No longer called before the module state is allocated. +.. c:var:: PyTypeObject PyModuleDef_Type + + The type of ``PyModuleDef`` objects. + + Module slots ............ diff --git a/Doc/c-api/profiling.rst b/Doc/c-api/profiling.rst new file mode 100644 index 00000000000..0200f2eac6d --- /dev/null +++ b/Doc/c-api/profiling.rst @@ -0,0 +1,239 @@ +.. highlight:: c + +.. _profiling: + +Profiling and tracing +===================== + +The Python interpreter provides some low-level support for attaching profiling +and execution tracing facilities. These are used for profiling, debugging, and +coverage analysis tools. + +This C interface allows the profiling or tracing code to avoid the overhead of +calling through Python-level callable objects, making a direct C function call +instead. The essential attributes of the facility have not changed; the +interface allows trace functions to be installed per-thread, and the basic +events reported to the trace function are the same as had been reported to the +Python-level trace functions in previous versions. + + +.. c:type:: int (*Py_tracefunc)(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) + + The type of the trace function registered using :c:func:`PyEval_SetProfile` and + :c:func:`PyEval_SetTrace`. The first parameter is the object passed to the + registration function as *obj*, *frame* is the frame object to which the event + pertains, *what* is one of the constants :c:data:`PyTrace_CALL`, + :c:data:`PyTrace_EXCEPTION`, :c:data:`PyTrace_LINE`, :c:data:`PyTrace_RETURN`, + :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION`, :c:data:`PyTrace_C_RETURN`, + or :c:data:`PyTrace_OPCODE`, and *arg* depends on the value of *what*: + + +-------------------------------+----------------------------------------+ + | Value of *what* | Meaning of *arg* | + +===============================+========================================+ + | :c:data:`PyTrace_CALL` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_EXCEPTION` | Exception information as returned by | + | | :func:`sys.exc_info`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_LINE` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_RETURN` | Value being returned to the caller, | + | | or ``NULL`` if caused by an exception. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_CALL` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_EXCEPTION` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_RETURN` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + +.. c:var:: int PyTrace_CALL + + The value of the *what* parameter to a :c:type:`Py_tracefunc` function when a new + call to a function or method is being reported, or a new entry into a generator. + Note that the creation of the iterator for a generator function is not reported + as there is no control transfer to the Python bytecode in the corresponding + frame. + + +.. c:var:: int PyTrace_EXCEPTION + + The value of the *what* parameter to a :c:type:`Py_tracefunc` function when an + exception has been raised. The callback function is called with this value for + *what* when after any bytecode is processed after which the exception becomes + set within the frame being executed. The effect of this is that as exception + propagation causes the Python stack to unwind, the callback is called upon + return to each frame as the exception propagates. Only trace functions receive + these events; they are not needed by the profiler. + + +.. c:var:: int PyTrace_LINE + + The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function + (but not a profiling function) when a line-number event is being reported. + It may be disabled for a frame by setting :attr:`~frame.f_trace_lines` to + *0* on that frame. + + +.. c:var:: int PyTrace_RETURN + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a + call is about to return. + + +.. c:var:: int PyTrace_C_CALL + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function is about to be called. + + +.. c:var:: int PyTrace_C_EXCEPTION + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function has raised an exception. + + +.. c:var:: int PyTrace_C_RETURN + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function has returned. + + +.. c:var:: int PyTrace_OPCODE + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not + profiling functions) when a new opcode is about to be executed. This event is + not emitted by default: it must be explicitly requested by setting + :attr:`~frame.f_trace_opcodes` to *1* on the frame. + + +.. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) + + Set the profiler function to *func*. The *obj* parameter is passed to the + function as its first parameter, and may be any Python object, or ``NULL``. If + the profile function needs to maintain state, using a different value for *obj* + for each thread provides a convenient and thread-safe place to store it. The + profile function is called for all monitored events except :c:data:`PyTrace_LINE` + :c:data:`PyTrace_OPCODE` and :c:data:`PyTrace_EXCEPTION`. + + See also the :func:`sys.setprofile` function. + + The caller must have an :term:`attached thread state`. + + +.. c:function:: void PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *obj) + + Like :c:func:`PyEval_SetProfile` but sets the profile function in all running threads + belonging to the current interpreter instead of the setting it only on the current thread. + + The caller must have an :term:`attached thread state`. + + As :c:func:`PyEval_SetProfile`, this function ignores any exceptions raised while + setting the profile functions in all threads. + +.. versionadded:: 3.12 + + +.. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj) + + Set the tracing function to *func*. This is similar to + :c:func:`PyEval_SetProfile`, except the tracing function does receive line-number + events and per-opcode events, but does not receive any event related to C function + objects being called. Any trace function registered using :c:func:`PyEval_SetTrace` + will not receive :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION` or + :c:data:`PyTrace_C_RETURN` as a value for the *what* parameter. + + See also the :func:`sys.settrace` function. + + The caller must have an :term:`attached thread state`. + + +.. c:function:: void PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *obj) + + Like :c:func:`PyEval_SetTrace` but sets the tracing function in all running threads + belonging to the current interpreter instead of the setting it only on the current thread. + + The caller must have an :term:`attached thread state`. + + As :c:func:`PyEval_SetTrace`, this function ignores any exceptions raised while + setting the trace functions in all threads. + +.. versionadded:: 3.12 + + +Reference tracing +================= + +.. versionadded:: 3.13 + + +.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) + + The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. + The first parameter is a Python object that has been just created (when **event** + is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** + is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer + that was provided when :c:func:`PyRefTracer_SetTracer` was called. + + If a new tracing function is registered replacing the current one, a call to the + trace function will be made with the object set to **NULL** and **event** set to + :c:data:`PyRefTracer_TRACKER_REMOVED`. This will happen just before the new + function is registered. + +.. versionadded:: 3.13 + + +.. c:var:: int PyRefTracer_CREATE + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been created. + + +.. c:var:: int PyRefTracer_DESTROY + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been destroyed. + + +.. c:var:: int PyRefTracer_TRACKER_REMOVED + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when the + current tracer is about to be replaced by a new one. + + .. versionadded:: 3.14 + + +.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) + + Register a reference tracer function. The function will be called when a new + Python object has been created or when an object is going to be destroyed. If + **data** is provided it must be an opaque pointer that will be provided when + the tracer function is called. Return ``0`` on success. Set an exception and + return ``-1`` on error. + + Note that tracer functions **must not** create Python objects inside or + otherwise the call will be re-entrant. The tracer also **must not** clear + any existing exception or set an exception. A :term:`thread state` will be active + every time the tracer function is called. + + There must be an :term:`attached thread state` when calling this function. + + If another tracer function was already registered, the old function will be + called with **event** set to :c:data:`PyRefTracer_TRACKER_REMOVED` just before + the new function is registered. + +.. versionadded:: 3.13 + + +.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) + + Get the registered reference tracer function and the value of the opaque data + pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. + If no tracer was registered this function will return NULL and will set the + **data** pointer to NULL. + + There must be an :term:`attached thread state` when calling this function. + +.. versionadded:: 3.13 diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index b4e7cb1d77e..a3e164011a8 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -408,7 +408,7 @@ There are these calling conventions: These two constants are not used to indicate the calling convention but the -binding when use with methods of classes. These may not be used for functions +binding when used with methods of classes. These may not be used for functions defined for modules. At most one of these flags may be set for any given method. diff --git a/Doc/c-api/subinterpreters.rst b/Doc/c-api/subinterpreters.rst new file mode 100644 index 00000000000..44e3fc96841 --- /dev/null +++ b/Doc/c-api/subinterpreters.rst @@ -0,0 +1,470 @@ +.. highlight:: c + +.. _sub-interpreter-support: + +Multiple interpreters in a Python process +========================================= + +While in most uses, you will only embed a single Python interpreter, there +are cases where you need to create several independent interpreters in the +same process and perhaps even in the same thread. Sub-interpreters allow +you to do that. + +The "main" interpreter is the first one created when the runtime initializes. +It is usually the only Python interpreter in a process. Unlike sub-interpreters, +the main interpreter has unique process-global responsibilities like signal +handling. It is also responsible for execution during runtime initialization and +is usually the active interpreter during runtime finalization. The +:c:func:`PyInterpreterState_Main` function returns a pointer to its state. + +You can switch between sub-interpreters using the :c:func:`PyThreadState_Swap` +function. You can create and destroy them using the following functions: + + +.. c:type:: PyInterpreterConfig + + Structure containing most parameters to configure a sub-interpreter. + Its values are used only in :c:func:`Py_NewInterpreterFromConfig` and + never modified by the runtime. + + .. versionadded:: 3.12 + + Structure fields: + + .. c:member:: int use_main_obmalloc + + If this is ``0`` then the sub-interpreter will use its own + "object" allocator state. + Otherwise it will use (share) the main interpreter's. + + If this is ``0`` then + :c:member:`~PyInterpreterConfig.check_multi_interp_extensions` + must be ``1`` (non-zero). + If this is ``1`` then :c:member:`~PyInterpreterConfig.gil` + must not be :c:macro:`PyInterpreterConfig_OWN_GIL`. + + .. c:member:: int allow_fork + + If this is ``0`` then the runtime will not support forking the + process in any thread where the sub-interpreter is currently active. + Otherwise fork is unrestricted. + + Note that the :mod:`subprocess` module still works + when fork is disallowed. + + .. c:member:: int allow_exec + + If this is ``0`` then the runtime will not support replacing the + current process via exec (e.g. :func:`os.execv`) in any thread + where the sub-interpreter is currently active. + Otherwise exec is unrestricted. + + Note that the :mod:`subprocess` module still works + when exec is disallowed. + + .. c:member:: int allow_threads + + If this is ``0`` then the sub-interpreter's :mod:`threading` module + won't create threads. + Otherwise threads are allowed. + + .. c:member:: int allow_daemon_threads + + If this is ``0`` then the sub-interpreter's :mod:`threading` module + won't create daemon threads. + Otherwise daemon threads are allowed (as long as + :c:member:`~PyInterpreterConfig.allow_threads` is non-zero). + + .. c:member:: int check_multi_interp_extensions + + If this is ``0`` then all extension modules may be imported, + including legacy (single-phase init) modules, + in any thread where the sub-interpreter is currently active. + Otherwise only multi-phase init extension modules + (see :pep:`489`) may be imported. + (Also see :c:macro:`Py_mod_multiple_interpreters`.) + + This must be ``1`` (non-zero) if + :c:member:`~PyInterpreterConfig.use_main_obmalloc` is ``0``. + + .. c:member:: int gil + + This determines the operation of the GIL for the sub-interpreter. + It may be one of the following: + + .. c:namespace:: NULL + + .. c:macro:: PyInterpreterConfig_DEFAULT_GIL + + Use the default selection (:c:macro:`PyInterpreterConfig_SHARED_GIL`). + + .. c:macro:: PyInterpreterConfig_SHARED_GIL + + Use (share) the main interpreter's GIL. + + .. c:macro:: PyInterpreterConfig_OWN_GIL + + Use the sub-interpreter's own GIL. + + If this is :c:macro:`PyInterpreterConfig_OWN_GIL` then + :c:member:`PyInterpreterConfig.use_main_obmalloc` must be ``0``. + + +.. c:function:: PyStatus Py_NewInterpreterFromConfig(PyThreadState **tstate_p, const PyInterpreterConfig *config) + + .. index:: + pair: module; builtins + pair: module; __main__ + pair: module; sys + single: stdout (in module sys) + single: stderr (in module sys) + single: stdin (in module sys) + + Create a new sub-interpreter. This is an (almost) totally separate environment + for the execution of Python code. In particular, the new interpreter has + separate, independent versions of all imported modules, including the + fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. The + table of loaded modules (``sys.modules``) and the module search path + (``sys.path``) are also separate. The new environment has no ``sys.argv`` + variable. It has new standard I/O stream file objects ``sys.stdin``, + ``sys.stdout`` and ``sys.stderr`` (however these refer to the same underlying + file descriptors). + + The given *config* controls the options with which the interpreter + is initialized. + + Upon success, *tstate_p* will be set to the first :term:`thread state` + created in the new sub-interpreter. This thread state is + :term:`attached `. + Note that no actual thread is created; see the discussion of thread states + below. If creation of the new interpreter is unsuccessful, + *tstate_p* is set to ``NULL``; + no exception is set since the exception state is stored in the + :term:`attached thread state`, which might not exist. + + Like all other Python/C API functions, an :term:`attached thread state` + must be present before calling this function, but it might be detached upon + returning. On success, the returned thread state will be :term:`attached `. + If the sub-interpreter is created with its own :term:`GIL` then the + :term:`attached thread state` of the calling interpreter will be detached. + When the function returns, the new interpreter's :term:`thread state` + will be :term:`attached ` to the current thread and + the previous interpreter's :term:`attached thread state` will remain detached. + + .. versionadded:: 3.12 + + Sub-interpreters are most effective when isolated from each other, + with certain functionality restricted:: + + PyInterpreterConfig config = { + .use_main_obmalloc = 0, + .allow_fork = 0, + .allow_exec = 0, + .allow_threads = 1, + .allow_daemon_threads = 0, + .check_multi_interp_extensions = 1, + .gil = PyInterpreterConfig_OWN_GIL, + }; + PyThreadState *tstate = NULL; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + Note that the config is used only briefly and does not get modified. + During initialization the config's values are converted into various + :c:type:`PyInterpreterState` values. A read-only copy of the config + may be stored internally on the :c:type:`PyInterpreterState`. + + .. index:: + single: Py_FinalizeEx (C function) + single: Py_Initialize (C function) + + Extension modules are shared between (sub-)interpreters as follows: + + * For modules using multi-phase initialization, + e.g. :c:func:`PyModule_FromDefAndSpec`, a separate module object is + created and initialized for each interpreter. + Only C-level static and global variables are shared between these + module objects. + + * For modules using legacy + :ref:`single-phase initialization `, + e.g. :c:func:`PyModule_Create`, the first time a particular extension + is imported, it is initialized normally, and a (shallow) copy of its + module's dictionary is squirreled away. + When the same extension is imported by another (sub-)interpreter, a new + module is initialized and filled with the contents of this copy; the + extension's ``init`` function is not called. + Objects in the module's dictionary thus end up shared across + (sub-)interpreters, which might cause unwanted behavior (see + `Bugs and caveats`_ below). + + Note that this is different from what happens when an extension is + imported after the interpreter has been completely re-initialized by + calling :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that + case, the extension's ``initmodule`` function *is* called again. + As with multi-phase initialization, this means that only C-level static + and global variables are shared between these modules. + + .. index:: single: close (in module os) + + +.. c:function:: PyThreadState* Py_NewInterpreter(void) + + .. index:: + pair: module; builtins + pair: module; __main__ + pair: module; sys + single: stdout (in module sys) + single: stderr (in module sys) + single: stdin (in module sys) + + Create a new sub-interpreter. This is essentially just a wrapper + around :c:func:`Py_NewInterpreterFromConfig` with a config that + preserves the existing behavior. The result is an unisolated + sub-interpreter that shares the main interpreter's GIL, allows + fork/exec, allows daemon threads, and allows single-phase init + modules. + + +.. c:function:: void Py_EndInterpreter(PyThreadState *tstate) + + .. index:: single: Py_FinalizeEx (C function) + + Destroy the (sub-)interpreter represented by the given :term:`thread state`. + The given thread state must be :term:`attached `. + When the call returns, there will be no :term:`attached thread state`. + All thread states associated with this interpreter are destroyed. + + :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that + haven't been explicitly destroyed at that point. + + +.. _per-interpreter-gil: + +A per-interpreter GIL +--------------------- + +.. versionadded:: 3.12 + +Using :c:func:`Py_NewInterpreterFromConfig` you can create +a sub-interpreter that is completely isolated from other interpreters, +including having its own GIL. The most important benefit of this +isolation is that such an interpreter can execute Python code without +being blocked by other interpreters or blocking any others. Thus a +single Python process can truly take advantage of multiple CPU cores +when running Python code. The isolation also encourages a different +approach to concurrency than that of just using threads. +(See :pep:`554` and :pep:`684`.) + +Using an isolated interpreter requires vigilance in preserving that +isolation. That especially means not sharing any objects or mutable +state without guarantees about thread-safety. Even objects that are +otherwise immutable (e.g. ``None``, ``(1, 5)``) can't normally be shared +because of the refcount. One simple but less-efficient approach around +this is to use a global lock around all use of some state (or object). +Alternately, effectively immutable objects (like integers or strings) +can be made safe in spite of their refcounts by making them :term:`immortal`. +In fact, this has been done for the builtin singletons, small integers, +and a number of other builtin objects. + +If you preserve isolation then you will have access to proper multi-core +computing without the complications that come with free-threading. +Failure to preserve isolation will expose you to the full consequences +of free-threading, including races and hard-to-debug crashes. + +Aside from that, one of the main challenges of using multiple isolated +interpreters is how to communicate between them safely (not break +isolation) and efficiently. The runtime and stdlib do not provide +any standard approach to this yet. A future stdlib module would help +mitigate the effort of preserving isolation and expose effective tools +for communicating (and sharing) data between interpreters. + + +Bugs and caveats +---------------- + +Because sub-interpreters (and the main interpreter) are part of the same +process, the insulation between them isn't perfect --- for example, using +low-level file operations like :func:`os.close` they can +(accidentally or maliciously) affect each other's open files. Because of the +way extensions are shared between (sub-)interpreters, some extensions may not +work properly; this is especially likely when using single-phase initialization +or (static) global variables. +It is possible to insert objects created in one sub-interpreter into +a namespace of another (sub-)interpreter; this should be avoided if possible. + +Special care should be taken to avoid sharing user-defined functions, +methods, instances or classes between sub-interpreters, since import +operations executed by such objects may affect the wrong (sub-)interpreter's +dictionary of loaded modules. It is equally important to avoid sharing +objects from which the above are reachable. + +Also note that combining this functionality with ``PyGILState_*`` APIs +is delicate, because these APIs assume a bijection between Python thread states +and OS-level threads, an assumption broken by the presence of sub-interpreters. +It is highly recommended that you don't switch sub-interpreters between a pair +of matching :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` calls. +Furthermore, extensions (such as :mod:`ctypes`) using these APIs to allow calling +of Python code from non-Python created threads will probably be broken when using +sub-interpreters. + + +High-level APIs +--------------- + +.. c:type:: PyInterpreterState + + This data structure represents the state shared by a number of cooperating + threads. Threads belonging to the same interpreter share their module + administration and a few other internal items. There are no public members in + this structure. + + Threads belonging to different interpreters initially share nothing, except + process state like available memory, open file descriptors and such. The global + interpreter lock is also shared by all threads, regardless of to which + interpreter they belong. + + .. versionchanged:: 3.12 + + :pep:`684` introduced the possibility + of a :ref:`per-interpreter GIL `. + See :c:func:`Py_NewInterpreterFromConfig`. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Get(void) + + Get the current interpreter. + + Issue a fatal error if there is no :term:`attached thread state`. + It cannot return NULL. + + .. versionadded:: 3.9 + + +.. c:function:: int64_t PyInterpreterState_GetID(PyInterpreterState *interp) + + Return the interpreter's unique ID. If there was any error in doing + so then ``-1`` is returned and an error is set. + + The caller must have an :term:`attached thread state`. + + .. versionadded:: 3.7 + + +.. c:function:: PyObject* PyInterpreterState_GetDict(PyInterpreterState *interp) + + Return a dictionary in which interpreter-specific data may be stored. + If this function returns ``NULL`` then no exception has been raised and + the caller should assume no interpreter-specific dict is available. + + This is not a replacement for :c:func:`PyModule_GetState()`, which + extensions should use to store interpreter-specific state information. + + The returned dictionary is borrowed from the interpreter and is valid until + interpreter shutdown. + + .. versionadded:: 3.8 + + +.. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) + + Type of a frame evaluation function. + + The *throwflag* parameter is used by the ``throw()`` method of generators: + if non-zero, handle the current exception. + + .. versionchanged:: 3.9 + The function now takes a *tstate* parameter. + + .. versionchanged:: 3.11 + The *frame* parameter changed from ``PyFrameObject*`` to ``_PyInterpreterFrame*``. + + +.. c:function:: _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) + + Get the frame evaluation function. + + See the :pep:`523` "Adding a frame evaluation API to CPython". + + .. versionadded:: 3.9 + + +.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) + + Set the frame evaluation function. + + See the :pep:`523` "Adding a frame evaluation API to CPython". + + .. versionadded:: 3.9 + + +Low-level APIs +-------------- + +All of the following functions must be called after :c:func:`Py_Initialize`. + +.. versionchanged:: 3.7 + :c:func:`Py_Initialize()` now initializes the :term:`GIL` + and sets an :term:`attached thread state`. + + +.. c:function:: PyInterpreterState* PyInterpreterState_New() + + Create a new interpreter state object. An :term:`attached thread state` is not needed, + but may optionally exist if it is necessary to serialize calls to this + function. + + .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New + + +.. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) + + Reset all information in an interpreter state object. There must be + an :term:`attached thread state` for the interpreter. + + .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear + + +.. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) + + Destroy an interpreter state object. There **should not** be an + :term:`attached thread state` for the target interpreter. The interpreter + state must have been reset with a previous call to :c:func:`PyInterpreterState_Clear`. + + +.. _advanced-debugging: + +Advanced debugger support +------------------------- + +These functions are only intended to be used by advanced debugging tools. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Head() + + Return the interpreter state object at the head of the list of all such objects. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Main() + + Return the main interpreter state object. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Next(PyInterpreterState *interp) + + Return the next interpreter state object after *interp* from the list of all + such objects. + + +.. c:function:: PyThreadState * PyInterpreterState_ThreadHead(PyInterpreterState *interp) + + Return the pointer to the first :c:type:`PyThreadState` object in the list of + threads associated with the interpreter *interp*. + + +.. c:function:: PyThreadState* PyThreadState_Next(PyThreadState *tstate) + + Return the next thread state object after *tstate* from the list of all such + objects belonging to the same :c:type:`PyInterpreterState` object. diff --git a/Doc/c-api/synchronization.rst b/Doc/c-api/synchronization.rst new file mode 100644 index 00000000000..954ac55e538 --- /dev/null +++ b/Doc/c-api/synchronization.rst @@ -0,0 +1,301 @@ +.. highlight:: c + +.. _synchronization: + +Synchronization primitives +========================== + +The C-API provides a basic mutual exclusion lock. + +.. c:type:: PyMutex + + A mutual exclusion lock. The :c:type:`!PyMutex` should be initialized to + zero to represent the unlocked state. For example:: + + PyMutex mutex = {0}; + + Instances of :c:type:`!PyMutex` should not be copied or moved. Both the + contents and address of a :c:type:`!PyMutex` are meaningful, and it must + remain at a fixed, writable location in memory. + + .. note:: + + A :c:type:`!PyMutex` currently occupies one byte, but the size should be + considered unstable. The size may change in future Python releases + without a deprecation period. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Lock(PyMutex *m) + + Lock mutex *m*. If another thread has already locked it, the calling + thread will block until the mutex is unlocked. While blocked, the thread + will temporarily detach the :term:`thread state ` if one exists. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Unlock(PyMutex *m) + + Unlock mutex *m*. The mutex must be locked --- otherwise, the function will + issue a fatal error. + + .. versionadded:: 3.13 + +.. c:function:: int PyMutex_IsLocked(PyMutex *m) + + Returns non-zero if the mutex *m* is currently locked, zero otherwise. + + .. note:: + + This function is intended for use in assertions and debugging only and + should not be used to make concurrency control decisions, as the lock + state may change immediately after the check. + + .. versionadded:: 3.14 + +.. _python-critical-section-api: + +Python critical section API +--------------------------- + +The critical section API provides a deadlock avoidance layer on top of +per-object locks for :term:`free-threaded ` CPython. They are +intended to replace reliance on the :term:`global interpreter lock`, and are +no-ops in versions of Python with the global interpreter lock. + +Critical sections are intended to be used for custom types implemented +in C-API extensions. They should generally not be used with built-in types like +:class:`list` and :class:`dict` because their public C-APIs +already use critical sections internally, with the notable +exception of :c:func:`PyDict_Next`, which requires critical section +to be acquired externally. + +Critical sections avoid deadlocks by implicitly suspending active critical +sections, hence, they do not provide exclusive access such as provided by +traditional locks like :c:type:`PyMutex`. When a critical section is started, +the per-object lock for the object is acquired. If the code executed inside the +critical section calls C-API functions then it can suspend the critical section thereby +releasing the per-object lock, so other threads can acquire the per-object lock +for the same object. + +Variants that accept :c:type:`PyMutex` pointers rather than Python objects are also +available. Use these variants to start a critical section in a situation where +there is no :c:type:`PyObject` -- for example, when working with a C type that +does not extend or wrap :c:type:`PyObject` but still needs to call into the C +API in a manner that might lead to deadlocks. + +The functions and structs used by the macros are exposed for cases +where C macros are not available. They should only be used as in the +given macro expansions. Note that the sizes and contents of the structures may +change in future Python versions. + +.. note:: + + Operations that need to lock two objects at once must use + :c:macro:`Py_BEGIN_CRITICAL_SECTION2`. You *cannot* use nested critical + sections to lock more than one object at once, because the inner critical + section may suspend the outer critical sections. This API does not provide + a way to lock more than two objects at once. + +Example usage:: + + static PyObject * + set_field(MyObject *self, PyObject *value) + { + Py_BEGIN_CRITICAL_SECTION(self); + Py_SETREF(self->field, Py_XNewRef(value)); + Py_END_CRITICAL_SECTION(); + Py_RETURN_NONE; + } + +In the above example, :c:macro:`Py_SETREF` calls :c:macro:`Py_DECREF`, which +can call arbitrary code through an object's deallocation function. The critical +section API avoids potential deadlocks due to reentrancy and lock ordering +by allowing the runtime to temporarily suspend the critical section if the +code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION(op) + + Acquires the per-object lock for the object *op* and begins a + critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_Begin(&_py_cs, (PyObject*)(op)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION_MUTEX(m) + + Locks the mutex *m* and begins a critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_BeginMutex(&_py_cs, m) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION`, there is no cast for + the argument of the macro - it must be a :c:type:`PyMutex` pointer. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + +.. c:macro:: Py_END_CRITICAL_SECTION() + + Ends the critical section and releases the per-object lock. + + In the free-threaded build, this macro expands to:: + + PyCriticalSection_End(&_py_cs); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2(a, b) + + Acquires the per-object locks for the objects *a* and *b* and begins a + critical section. The locks are acquired in a consistent order (lowest + address first) to avoid lock ordering deadlocks. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection2_Begin(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) + + Locks the mutexes *m1* and *m2* and begins a critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION2`, there is no cast for + the arguments of the macro - they must be :c:type:`PyMutex` pointers. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + +.. c:macro:: Py_END_CRITICAL_SECTION2() + + Ends the critical section and releases the per-object locks. + + In the free-threaded build, this macro expands to:: + + PyCriticalSection2_End(&_py_cs2); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 + + +Legacy locking APIs +------------------- + +These APIs are obsolete since Python 3.13 with the introduction of +:c:type:`PyMutex`. + + +.. c:type:: PyThread_type_lock + + A pointer to a mutual exclusion lock. + + +.. c:type:: PyLockStatus + + The result of acquiring a lock with a timeout. + + .. c:namespace:: NULL + + .. c:enumerator:: PY_LOCK_FAILURE + + Failed to acquire the lock. + + .. c:enumerator:: PY_LOCK_ACQUIRED + + The lock was successfully acquired. + + .. c:enumerator:: PY_LOCK_INTR + + The lock was interrupted by a signal. + + +.. c:function:: PyThread_type_lock PyThread_allocate_lock(void) + + Allocate a new lock. + + On success, this function returns a lock; on failure, this + function returns ``0`` without an exception set. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: void PyThread_free_lock(PyThread_type_lock lock) + + Destroy *lock*. The lock should not be held by any thread when calling + this. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: PyLockStatus PyThread_acquire_lock_timed(PyThread_type_lock lock, long long microseconds, int intr_flag) + + Acquire *lock* with a timeout. + + This will wait for *microseconds* microseconds to acquire the lock. If the + timeout expires, this function returns :c:enumerator:`PY_LOCK_FAILURE`. + If *microseconds* is ``-1``, this will wait indefinitely until the lock has + been released. + + If *intr_flag* is ``1``, acquiring the lock may be interrupted by a signal, + in which case this function returns :c:enumerator:`PY_LOCK_INTR`. Upon + interruption, it's generally expected that the caller makes a call to + :c:func:`Py_MakePendingCalls` to propagate an exception to Python code. + + If the lock is successfully acquired, this function returns + :c:enumerator:`PY_LOCK_ACQUIRED`. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: int PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) + + Acquire *lock*. + + If *waitflag* is ``1`` and another thread currently holds the lock, this + function will wait until the lock can be acquired and will always return + ``1``. + + If *waitflag* is ``0`` and another thread holds the lock, this function will + not wait and instead return ``0``. If the lock is not held by any other + thread, then this function will acquire it and return ``1``. + + Unlike :c:func:`PyThread_acquire_lock_timed`, acquiring the lock cannot be + interrupted by a signal. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: int PyThread_release_lock(PyThread_type_lock lock) + + Release *lock*. If *lock* is not held, then this function issues a + fatal error. + + The caller does not need to hold an :term:`attached thread state`. diff --git a/Doc/c-api/threads.rst b/Doc/c-api/threads.rst new file mode 100644 index 00000000000..badbdee564d --- /dev/null +++ b/Doc/c-api/threads.rst @@ -0,0 +1,867 @@ +.. highlight:: c + +.. _threads: + +Thread states and the global interpreter lock +============================================= + +.. index:: + single: global interpreter lock + single: interpreter lock + single: lock, interpreter + +Unless on a :term:`free-threaded build` of :term:`CPython`, +the Python interpreter is generally not thread-safe. In order to support +multi-threaded Python programs, there's a global lock, called the :term:`global +interpreter lock` or :term:`GIL`, that must be held by a thread before +accessing Python objects. Without the lock, even the simplest operations +could cause problems in a multi-threaded program: for example, when +two threads simultaneously increment the reference count of the same object, the +reference count could end up being incremented only once instead of twice. + +As such, only a thread that holds the GIL may operate on Python objects or +invoke Python's C API. + +.. index:: single: setswitchinterval (in module sys) + +In order to emulate concurrency, the interpreter regularly tries to switch +threads between bytecode instructions (see :func:`sys.setswitchinterval`). +This is why locks are also necessary for thread-safety in pure-Python code. + +Additionally, the global interpreter lock is released around blocking I/O +operations, such as reading or writing to a file. From the C API, this is done +by :ref:`detaching the thread state `. + + +.. index:: + single: PyThreadState (C type) + +The Python interpreter keeps some thread-local information inside +a data structure called :c:type:`PyThreadState`, known as a :term:`thread state`. +Each thread has a thread-local pointer to a :c:type:`PyThreadState`; a thread state +referenced by this pointer is considered to be :term:`attached `. + +A thread can only have one :term:`attached thread state` at a time. An attached +thread state is typically analogous with holding the GIL, except on +free-threaded builds. On builds with the GIL enabled, attaching a thread state +will block until the GIL can be acquired. However, even on builds with the GIL +disabled, it is still required to have an attached thread state, as the interpreter +needs to keep track of which threads may access Python objects. + +.. note:: + + Even on the free-threaded build, attaching a thread state may block, as the + GIL can be re-enabled or threads might be temporarily suspended (such as during + a garbage collection). + +Generally, there will always be an attached thread state when using Python's +C API, including during embedding and when implementing methods, so it's uncommon +to need to set up a thread state on your own. Only in some specific cases, such +as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block or in a fresh thread, will the +thread not have an attached thread state. +If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns ``NULL``. + +If it turns out that you do need to create a thread state, call :c:func:`PyThreadState_New` +followed by :c:func:`PyThreadState_Swap`, or use the dangerous +:c:func:`PyGILState_Ensure` function. + + +.. _detaching-thread-state: + +Detaching the thread state from extension code +---------------------------------------------- + +Most extension code manipulating the :term:`thread state` has the following simple +structure:: + + Save the thread state in a local variable. + ... Do some blocking I/O operation ... + Restore the thread state from the local variable. + +This is so common that a pair of macros exists to simplify it:: + + Py_BEGIN_ALLOW_THREADS + ... Do some blocking I/O operation ... + Py_END_ALLOW_THREADS + +.. index:: + single: Py_BEGIN_ALLOW_THREADS (C macro) + single: Py_END_ALLOW_THREADS (C macro) + +The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a +hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the +block. + +The block above expands to the following code:: + + PyThreadState *_save; + + _save = PyEval_SaveThread(); + ... Do some blocking I/O operation ... + PyEval_RestoreThread(_save); + +.. index:: + single: PyEval_RestoreThread (C function) + single: PyEval_SaveThread (C function) + +Here is how these functions work: + +The attached thread state implies that the GIL is held for the interpreter. +To detach it, :c:func:`PyEval_SaveThread` is called and the result is stored +in a local variable. + +By detaching the thread state, the GIL is released, which allows other threads +to attach to the interpreter and execute while the current thread performs +blocking I/O. When the I/O operation is complete, the old thread state is +reattached by calling :c:func:`PyEval_RestoreThread`, which will wait until +the GIL can be acquired. + +.. note:: + Performing blocking I/O is the most common use case for detaching + the thread state, but it is also useful to call it over long-running + native code that doesn't need access to Python objects or Python's C API. + For example, the standard :mod:`zlib` and :mod:`hashlib` modules detach the + :term:`thread state ` when compressing or hashing + data. + +On a :term:`free-threaded build`, the :term:`GIL` is usually out of the question, +but **detaching the thread state is still required**, because the interpreter +periodically needs to block all threads to get a consistent view of Python objects +without the risk of race conditions. +For example, CPython currently suspends all threads for a short period of time +while running the garbage collector. + +.. warning:: + + Detaching the thread state can lead to unexpected behavior during interpreter + finalization. See :ref:`cautions-regarding-runtime-finalization` for more + details. + + +APIs +^^^^ + +The following macros are normally used without a trailing semicolon; look for +example usage in the Python source distribution. + +.. note:: + + These macros are still necessary on the :term:`free-threaded build` to prevent + deadlocks. + +.. c:macro:: Py_BEGIN_ALLOW_THREADS + + This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. + Note that it contains an opening brace; it must be matched with a following + :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this + macro. + + +.. c:macro:: Py_END_ALLOW_THREADS + + This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains + a closing brace; it must be matched with an earlier + :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of + this macro. + + +.. c:macro:: Py_BLOCK_THREADS + + This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to + :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. + + +.. c:macro:: Py_UNBLOCK_THREADS + + This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to + :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable + declaration. + + +Non-Python created threads +-------------------------- + +When threads are created using the dedicated Python APIs (such as the +:mod:`threading` module), a thread state is automatically associated with them, +However, when a thread is created from native code (for example, by a +third-party library with its own thread management), it doesn't hold an +attached thread state. + +If you need to call Python code from these threads (often this will be part +of a callback API provided by the aforementioned third-party library), +you must first register these threads with the interpreter by +creating a new thread state and attaching it. + +The most robust way to do this is through :c:func:`PyThreadState_New` followed +by :c:func:`PyThreadState_Swap`. + +.. note:: + ``PyThreadState_New`` requires an argument pointing to the desired + interpreter; such a pointer can be acquired via a call to + :c:func:`PyInterpreterState_Get` from the code where the thread was + created. + +For example:: + + /* The return value of PyInterpreterState_Get() from the + function that created this thread. */ + PyInterpreterState *interp = thread_data->interp; + + /* Create a new thread state for the interpreter. It does not start out + attached. */ + PyThreadState *tstate = PyThreadState_New(interp); + + /* Attach the thread state, which will acquire the GIL. */ + PyThreadState_Swap(tstate); + + /* Perform Python actions here. */ + result = CallSomeFunction(); + /* evaluate result or handle exception */ + + /* Destroy the thread state. No Python API allowed beyond this point. */ + PyThreadState_Clear(tstate); + PyThreadState_DeleteCurrent(); + +.. warning:: + + If the interpreter finalized before ``PyThreadState_Swap`` was called, then + ``interp`` will be a dangling pointer! + +.. _gilstate: + +Legacy API +---------- + +Another common pattern to call Python code from a non-Python thread is to use +:c:func:`PyGILState_Ensure` followed by a call to :c:func:`PyGILState_Release`. + +These functions do not work well when multiple interpreters exist in the Python +process. If no Python interpreter has ever been used in the current thread (which +is common for threads created outside Python), ``PyGILState_Ensure`` will create +and attach a thread state for the "main" interpreter (the first interpreter in +the Python process). + +Additionally, these functions have thread-safety issues during interpreter +finalization. Using ``PyGILState_Ensure`` during finalization will likely +crash the process. + +Usage of these functions look like such:: + + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + + /* Perform Python actions here. */ + result = CallSomeFunction(); + /* evaluate result or handle exception */ + + /* Release the thread. No Python API allowed beyond this point. */ + PyGILState_Release(gstate); + + +.. _fork-and-threads: + +Cautions about fork() +--------------------- + +Another important thing to note about threads is their behaviour in the face +of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a +process forks only the thread that issued the fork will exist. This has a +concrete impact both on how locks must be handled and on all stored state +in CPython's runtime. + +The fact that only the "current" thread remains +means any locks held by other threads will never be released. Python solves +this for :func:`os.fork` by acquiring the locks it uses internally before +the fork, and releasing them afterwards. In addition, it resets any +:ref:`lock-objects` in the child. When extending or embedding Python, there +is no way to inform Python of additional (non-Python) locks that need to be +acquired before or reset after a fork. OS facilities such as +:c:func:`!pthread_atfork` would need to be used to accomplish the same thing. +Additionally, when extending or embedding Python, calling :c:func:`fork` +directly rather than through :func:`os.fork` (and returning to or calling +into Python) may result in a deadlock by one of Python's internal locks +being held by a thread that is defunct after the fork. +:c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not +always able to. + +The fact that all other threads go away also means that CPython's +runtime state there must be cleaned up properly, which :func:`os.fork` +does. This means finalizing all other :c:type:`PyThreadState` objects +belonging to the current interpreter and all other +:c:type:`PyInterpreterState` objects. Due to this and the special +nature of the :ref:`"main" interpreter `, +:c:func:`fork` should only be called in that interpreter's "main" +thread, where the CPython global runtime was originally initialized. +The only exception is if :c:func:`exec` will be called immediately +after. + + +High-level APIs +--------------- + +These are the most commonly used types and functions when writing multi-threaded +C extensions. + + +.. c:type:: PyThreadState + + This data structure represents the state of a single thread. The only public + data member is: + + .. c:member:: PyInterpreterState *interp + + This thread's interpreter state. + + +.. c:function:: void PyEval_InitThreads() + + .. index:: + single: PyEval_AcquireThread() + single: PyEval_ReleaseThread() + single: PyEval_SaveThread() + single: PyEval_RestoreThread() + + Deprecated function which does nothing. + + In Python 3.6 and older, this function created the GIL if it didn't exist. + + .. versionchanged:: 3.9 + The function now does nothing. + + .. versionchanged:: 3.7 + This function is now called by :c:func:`Py_Initialize()`, so you don't + have to call it yourself anymore. + + .. versionchanged:: 3.2 + This function cannot be called before :c:func:`Py_Initialize()` anymore. + + .. deprecated:: 3.9 + + .. index:: pair: module; _thread + + +.. c:function:: PyThreadState* PyEval_SaveThread() + + Detach the :term:`attached thread state` and return it. + The thread will have no :term:`thread state` upon returning. + + +.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) + + Set the :term:`attached thread state` to *tstate*. + The passed :term:`thread state` **should not** be :term:`attached `, + otherwise deadlock ensues. *tstate* will be attached upon returning. + + .. note:: + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + +.. c:function:: PyThreadState* PyThreadState_Get() + + Return the :term:`attached thread state`. If the thread has no attached + thread state, (such as when inside of :c:macro:`Py_BEGIN_ALLOW_THREADS` + block), then this issues a fatal error (so that the caller needn't check + for ``NULL``). + + See also :c:func:`PyThreadState_GetUnchecked`. + +.. c:function:: PyThreadState* PyThreadState_GetUnchecked() + + Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a + fatal error if it is NULL. The caller is responsible to check if the result + is NULL. + + .. versionadded:: 3.13 + In Python 3.5 to 3.12, the function was private and known as + ``_PyThreadState_UncheckedGet()``. + + +.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) + + Set the :term:`attached thread state` to *tstate*, and return the + :term:`thread state` that was attached prior to calling. + + This function is safe to call without an :term:`attached thread state`; it + will simply return ``NULL`` indicating that there was no prior thread state. + + .. seealso:: + :c:func:`PyEval_ReleaseThread` + + .. note:: + Similar to :c:func:`PyGILState_Ensure`, this function will hang the + thread if the runtime is finalizing. + + +GIL-state APIs +-------------- + +The following functions use thread-local storage, and are not compatible +with sub-interpreters: + +.. c:type:: PyGILState_STATE + + The type of the value returned by :c:func:`PyGILState_Ensure` and passed to + :c:func:`PyGILState_Release`. + + .. c:enumerator:: PyGILState_LOCKED + + The GIL was already held when :c:func:`PyGILState_Ensure` was called. + + .. c:enumerator:: PyGILState_UNLOCKED + + The GIL was not held when :c:func:`PyGILState_Ensure` was called. + +.. c:function:: PyGILState_STATE PyGILState_Ensure() + + Ensure that the current thread is ready to call the Python C API regardless + of the current state of Python, or of the :term:`attached thread state`. This may + be called as many times as desired by a thread as long as each call is + matched with a call to :c:func:`PyGILState_Release`. In general, other + thread-related APIs may be used between :c:func:`PyGILState_Ensure` and + :c:func:`PyGILState_Release` calls as long as the thread state is restored to + its previous state before the Release(). For example, normal usage of the + :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is + acceptable. + + The return value is an opaque "handle" to the :term:`attached thread state` when + :c:func:`PyGILState_Ensure` was called, and must be passed to + :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even + though recursive calls are allowed, these handles *cannot* be shared - each + unique call to :c:func:`PyGILState_Ensure` must save the handle for its call + to :c:func:`PyGILState_Release`. + + When the function returns, there will be an :term:`attached thread state` + and the thread will be able to call arbitrary Python code. Failure is a fatal error. + + .. warning:: + Calling this function when the runtime is finalizing is unsafe. Doing + so will either hang the thread until the program ends, or fully crash + the interpreter in rare cases. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + +.. c:function:: void PyGILState_Release(PyGILState_STATE) + + Release any resources previously acquired. After this call, Python's state will + be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call + (but generally this state will be unknown to the caller, hence the use of the + GILState API). + + Every call to :c:func:`PyGILState_Ensure` must be matched by a call to + :c:func:`PyGILState_Release` on the same thread. + +.. c:function:: PyThreadState* PyGILState_GetThisThreadState() + + Get the :term:`attached thread state` for this thread. May return ``NULL`` if no + GILState API has been used on the current thread. Note that the main thread + always has such a thread-state, even if no auto-thread-state call has been + made on the main thread. This is mainly a helper/diagnostic function. + + .. note:: + This function may return non-``NULL`` even when the :term:`thread state` + is detached. + Prefer :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` + for most cases. + + .. seealso:: :c:func:`PyThreadState_Get` + +.. c:function:: int PyGILState_Check() + + Return ``1`` if the current thread is holding the :term:`GIL` and ``0`` otherwise. + This function can be called from any thread at any time. + Only if it has had its :term:`thread state ` initialized + via :c:func:`PyGILState_Ensure` will it return ``1``. + This is mainly a helper/diagnostic function. It can be useful + for example in callback contexts or memory allocation functions when + knowing that the :term:`GIL` is locked can allow the caller to perform sensitive + actions or otherwise behave differently. + + .. note:: + If the current Python process has ever created a subinterpreter, this + function will *always* return ``1``. Prefer :c:func:`PyThreadState_GetUnchecked` + for most cases. + + .. versionadded:: 3.4 + + +Low-level APIs +-------------- + +.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) + + Create a new thread state object belonging to the given interpreter object. + An :term:`attached thread state` is not needed. + +.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) + + Reset all information in a :term:`thread state` object. *tstate* + must be :term:`attached ` + + .. versionchanged:: 3.9 + This function now calls the :c:member:`!PyThreadState.on_delete` callback. + Previously, that happened in :c:func:`PyThreadState_Delete`. + + .. versionchanged:: 3.13 + The :c:member:`!PyThreadState.on_delete` callback was removed. + + +.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) + + Destroy a :term:`thread state` object. *tstate* should not + be :term:`attached ` to any thread. + *tstate* must have been reset with a previous call to + :c:func:`PyThreadState_Clear`. + + +.. c:function:: void PyThreadState_DeleteCurrent(void) + + Detach the :term:`attached thread state` (which must have been reset + with a previous call to :c:func:`PyThreadState_Clear`) and then destroy it. + + No :term:`thread state` will be :term:`attached ` upon + returning. + +.. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) + + Get the current frame of the Python thread state *tstate*. + + Return a :term:`strong reference`. Return ``NULL`` if no frame is currently + executing. + + See also :c:func:`PyEval_GetFrame`. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) + + Get the unique :term:`thread state` identifier of the Python thread state *tstate*. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: PyInterpreterState* PyThreadState_GetInterpreter(PyThreadState *tstate) + + Get the interpreter of the Python thread state *tstate*. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) + + Suspend tracing and profiling in the Python thread state *tstate*. + + Resume them using the :c:func:`PyThreadState_LeaveTracing` function. + + .. versionadded:: 3.11 + + +.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) + + Resume tracing and profiling in the Python thread state *tstate* suspended + by the :c:func:`PyThreadState_EnterTracing` function. + + See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` + functions. + + .. versionadded:: 3.11 + + +.. c:function:: int PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, void *stack_start_addr, size_t stack_size) + + Set the stack protection start address and stack protection size + of a Python thread state. + + On success, return ``0``. + On failure, set an exception and return ``-1``. + + CPython implements :ref:`recursion control ` for C code by raising + :py:exc:`RecursionError` when it notices that the machine execution stack is close + to overflow. See for example the :c:func:`Py_EnterRecursiveCall` function. + For this, it needs to know the location of the current thread's stack, which it + normally gets from the operating system. + When the stack is changed, for example using context switching techniques like the + Boost library's ``boost::context``, you must call + :c:func:`~PyUnstable_ThreadState_SetStackProtection` to inform CPython of the change. + + Call :c:func:`~PyUnstable_ThreadState_SetStackProtection` either before + or after changing the stack. + Do not call any other Python C API between the call and the stack + change. + + See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. + + .. versionadded:: 3.15 + + +.. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) + + Reset the stack protection start address and stack protection size + of a Python thread state to the operating system defaults. + + See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. + + .. versionadded:: 3.15 + + +.. c:function:: PyObject* PyThreadState_GetDict() + + Return a dictionary in which extensions can store thread-specific state + information. Each extension should use a unique key to use to store state in + the dictionary. It is okay to call this function when no :term:`thread state` + is :term:`attached `. If this function returns + ``NULL``, no exception has been raised and the caller should assume no + thread state is attached. + + +.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) + + :term:`Attach ` *tstate* to the current thread, + which must not be ``NULL`` or already :term:`attached `. + + The calling thread must not already have an :term:`attached thread state`. + + .. note:: + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.8 + Updated to be consistent with :c:func:`PyEval_RestoreThread`, + :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, + and terminate the current thread if called while the interpreter is finalizing. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + + :c:func:`PyEval_RestoreThread` is a higher-level function which is always + available (even when threads have not been initialized). + + +.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) + + Detach the :term:`attached thread state`. + The *tstate* argument, which must not be ``NULL``, is only used to check + that it represents the :term:`attached thread state` --- if it isn't, a fatal error is + reported. + + :c:func:`PyEval_SaveThread` is a higher-level function which is always + available (even when threads have not been initialized). + + +Asynchronous notifications +========================== + +A mechanism is provided to make asynchronous notifications to the main +interpreter thread. These notifications take the form of a function +pointer and a void pointer argument. + + +.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) + + Schedule a function to be called from the main interpreter thread. On + success, ``0`` is returned and *func* is queued for being called in the + main thread. On failure, ``-1`` is returned without setting any exception. + + When successfully queued, *func* will be *eventually* called from the + main interpreter thread with the argument *arg*. It will be called + asynchronously with respect to normally running Python code, but with + both these conditions met: + + * on a :term:`bytecode` boundary; + * with the main thread holding an :term:`attached thread state` + (*func* can therefore use the full C API). + + *func* must return ``0`` on success, or ``-1`` on failure with an exception + set. *func* won't be interrupted to perform another asynchronous + notification recursively, but it can still be interrupted to switch + threads if the :term:`thread state ` is detached. + + This function doesn't need an :term:`attached thread state`. However, to call this + function in a subinterpreter, the caller must have an :term:`attached thread state`. + Otherwise, the function *func* can be scheduled to be called from the wrong interpreter. + + .. warning:: + This is a low-level function, only useful for very special cases. + There is no guarantee that *func* will be called as quick as + possible. If the main thread is busy executing a system call, + *func* won't be called before the system call returns. This + function is generally **not** suitable for calling Python code from + arbitrary C threads. Instead, use the :ref:`PyGILState API`. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.9 + If this function is called in a subinterpreter, the function *func* is + now scheduled to be called from the subinterpreter, rather than being + called from the main interpreter. Each subinterpreter now has its own + list of scheduled calls. + + .. versionchanged:: 3.12 + This function now always schedules *func* to be run in the main + interpreter. + + +.. c:function:: int Py_MakePendingCalls(void) + + Execute all pending calls. This is usually executed automatically by the + interpreter. + + This function returns ``0`` on success, and returns ``-1`` with an exception + set on failure. + + If this is not called in the main thread of the main + interpreter, this function does nothing and returns ``0``. + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.12 + This function only runs pending calls in the main interpreter. + + +.. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) + + Asynchronously raise an exception in a thread. The *id* argument is the thread + id of the target thread; *exc* is the exception object to be raised. This + function does not steal any references to *exc*. To prevent naive misuse, you + must write your own C extension to call this. Must be called with an :term:`attached thread state`. + Returns the number of thread states modified; this is normally one, but will be + zero if the thread id isn't found. If *exc* is ``NULL``, the pending + exception (if any) for the thread is cleared. This raises no exceptions. + + .. versionchanged:: 3.7 + The type of the *id* parameter changed from :c:expr:`long` to + :c:expr:`unsigned long`. + + +Operating system thread APIs +============================ + +.. c:macro:: PYTHREAD_INVALID_THREAD_ID + + Sentinel value for an invalid thread ID. + + This is currently equivalent to ``(unsigned long)-1``. + + +.. c:function:: unsigned long PyThread_start_new_thread(void (*func)(void *), void *arg) + + Start function *func* in a new thread with argument *arg*. + The resulting thread is not intended to be joined. + + *func* must not be ``NULL``, but *arg* may be ``NULL``. + + On success, this function returns the identifier of the new thread; on failure, + this returns :c:macro:`PYTHREAD_INVALID_THREAD_ID`. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: unsigned long PyThread_get_thread_ident(void) + + Return the identifier of the current thread, which will never be zero. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_ident` + + +.. c:function:: PyObject *PyThread_GetInfo(void) + + Get general information about the current thread in the form of a + :ref:`struct sequence ` object. This information is + accessible as :py:attr:`sys.thread_info` in Python. + + On success, this returns a new :term:`strong reference` to the thread + information; on failure, this returns ``NULL`` with an exception set. + + The caller must hold an :term:`attached thread state`. + + +.. c:macro:: PY_HAVE_THREAD_NATIVE_ID + + This macro is defined when the system supports native thread IDs. + + +.. c:function:: unsigned long PyThread_get_thread_native_id(void) + + Get the native identifier of the current thread as it was assigned by the operating + system's kernel, which will never be less than zero. + + This function is only available when :c:macro:`PY_HAVE_THREAD_NATIVE_ID` is + defined. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_native_id` + + +.. c:function:: void PyThread_exit_thread(void) + + Terminate the current thread. This function is generally considered unsafe + and should be avoided. It is kept solely for backwards compatibility. + + This function is only safe to call if all functions in the full call + stack are written to safely allow it. + + .. warning:: + + If the current system uses POSIX threads (also known as "pthreads"), + this calls :manpage:`pthread_exit(3)`, which attempts to unwind the stack + and call C++ destructors on some libc implementations. However, if a + ``noexcept`` function is reached, it may terminate the process. + Other systems, such as macOS, do unwinding. + + On Windows, this function calls ``_endthreadex()``, which kills the thread + without calling C++ destructors. + + In any case, there is a risk of corruption on the thread's stack. + + .. deprecated:: 3.14 + + +.. c:function:: void PyThread_init_thread(void) + + Initialize ``PyThread*`` APIs. Python executes this function automatically, + so there's little need to call it from an extension module. + + +.. c:function:: int PyThread_set_stacksize(size_t size) + + Set the stack size of the current thread to *size* bytes. + + This function returns ``0`` on success, ``-1`` if *size* is invalid, or + ``-2`` if the system does not support changing the stack size. This function + does not set exceptions. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: size_t PyThread_get_stacksize(void) + + Return the stack size of the current thread in bytes, or ``0`` if the system's + default stack size is in use. + + The caller does not need to hold an :term:`attached thread state`. diff --git a/Doc/c-api/tls.rst b/Doc/c-api/tls.rst new file mode 100644 index 00000000000..93ac5557141 --- /dev/null +++ b/Doc/c-api/tls.rst @@ -0,0 +1,155 @@ +.. highlight:: c + +.. _thread-local-storage: + +Thread-local storage support +============================ + +The Python interpreter provides low-level support for thread-local storage +(TLS) which wraps the underlying native TLS implementation to support the +Python-level thread-local storage API (:class:`threading.local`). The +CPython C level APIs are similar to those offered by pthreads and Windows: +use a thread key and functions to associate a :c:expr:`void*` value per +thread. + +A :term:`thread state` does *not* need to be :term:`attached ` +when calling these functions; they supply their own locking. + +Note that :file:`Python.h` does not include the declaration of the TLS APIs, +you need to include :file:`pythread.h` to use thread-local storage. + +.. note:: + None of these API functions handle memory management on behalf of the + :c:expr:`void*` values. You need to allocate and deallocate them yourself. + If the :c:expr:`void*` values happen to be :c:expr:`PyObject*`, these + functions don't do refcount operations on them either. + +.. _thread-specific-storage-api: + +Thread-specific storage API +--------------------------- + +The thread-specific storage (TSS) API was introduced to supersede the use of the existing TLS API within the +CPython interpreter. This API uses a new type :c:type:`Py_tss_t` instead of +:c:expr:`int` to represent thread keys. + +.. versionadded:: 3.7 + +.. seealso:: "A New C-API for Thread-Local Storage in CPython" (:pep:`539`) + + +.. c:type:: Py_tss_t + + This data structure represents the state of a thread key, the definition of + which may depend on the underlying TLS implementation, and it has an + internal field representing the key's initialization state. There are no + public members in this structure. + + When :ref:`Py_LIMITED_API ` is not defined, static allocation of + this type by :c:macro:`Py_tss_NEEDS_INIT` is allowed. + + +.. c:macro:: Py_tss_NEEDS_INIT + + This macro expands to the initializer for :c:type:`Py_tss_t` variables. + Note that this macro won't be defined with :ref:`Py_LIMITED_API `. + + +Dynamic allocation +------------------ + +Dynamic allocation of the :c:type:`Py_tss_t`, required in extension modules +built with :ref:`Py_LIMITED_API `, where static allocation of this type +is not possible due to its implementation being opaque at build time. + + +.. c:function:: Py_tss_t* PyThread_tss_alloc() + + Return a value which is the same state as a value initialized with + :c:macro:`Py_tss_NEEDS_INIT`, or ``NULL`` in the case of dynamic allocation + failure. + + +.. c:function:: void PyThread_tss_free(Py_tss_t *key) + + Free the given *key* allocated by :c:func:`PyThread_tss_alloc`, after + first calling :c:func:`PyThread_tss_delete` to ensure any associated + thread locals have been unassigned. This is a no-op if the *key* + argument is ``NULL``. + + .. note:: + A freed key becomes a dangling pointer. You should reset the key to + ``NULL``. + + +Methods +------- + +The parameter *key* of these functions must not be ``NULL``. Moreover, the +behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are +undefined if the given :c:type:`Py_tss_t` has not been initialized by +:c:func:`PyThread_tss_create`. + + +.. c:function:: int PyThread_tss_is_created(Py_tss_t *key) + + Return a non-zero value if the given :c:type:`Py_tss_t` has been initialized + by :c:func:`PyThread_tss_create`. + + +.. c:function:: int PyThread_tss_create(Py_tss_t *key) + + Return a zero value on successful initialization of a TSS key. The behavior + is undefined if the value pointed to by the *key* argument is not + initialized by :c:macro:`Py_tss_NEEDS_INIT`. This function can be called + repeatedly on the same key -- calling it on an already initialized key is a + no-op and immediately returns success. + + +.. c:function:: void PyThread_tss_delete(Py_tss_t *key) + + Destroy a TSS key to forget the values associated with the key across all + threads, and change the key's initialization state to uninitialized. A + destroyed key is able to be initialized again by + :c:func:`PyThread_tss_create`. This function can be called repeatedly on + the same key -- calling it on an already destroyed key is a no-op. + + +.. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) + + Return a zero value to indicate successfully associating a :c:expr:`void*` + value with a TSS key in the current thread. Each thread has a distinct + mapping of the key to a :c:expr:`void*` value. + + +.. c:function:: void* PyThread_tss_get(Py_tss_t *key) + + Return the :c:expr:`void*` value associated with a TSS key in the current + thread. This returns ``NULL`` if no value is associated with the key in the + current thread. + + +.. _thread-local-storage-api: + +Legacy APIs +----------- + +.. deprecated:: 3.7 + This API is superseded by the + :ref:`thread-specific storage (TSS) API `. + +.. note:: + This version of the API does not support platforms where the native TLS key + is defined in a way that cannot be safely cast to ``int``. On such platforms, + :c:func:`PyThread_create_key` will return immediately with a failure status, + and the other TLS functions will all be no-ops on such platforms. + +Due to the compatibility problem noted above, this version of the API should not +be used in new code. + +.. c:function:: int PyThread_create_key() +.. c:function:: void PyThread_delete_key(int key) +.. c:function:: int PyThread_set_key_value(int key, void *value) +.. c:function:: void* PyThread_get_key_value(int key) +.. c:function:: void PyThread_delete_key_value(int key) +.. c:function:: void PyThread_ReInitTLS() diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 2f2060d0582..7fe810f585f 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -274,6 +274,10 @@ Type Objects Return the module object associated with the given type when the type was created using :c:func:`PyType_FromModuleAndSpec`. + The returned reference is :term:`borrowed ` from *type*, + and will be valid as long as you hold a reference to *type*. + Do not release it with :c:func:`Py_DECREF` or similar. + If no module is associated with the given type, sets :py:class:`TypeError` and returns ``NULL``. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 5129e9ee6f0..a0db4c31065 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1484,6 +1484,52 @@ and :c:data:`PyType_Type` effectively act as defaults.) It will be removed in a future version of CPython + .. c:macro:: Py_TPFLAGS_HAVE_VERSION_TAG + + This is a :term:`soft deprecated` macro that does nothing. + Historically, this would indicate that the + :c:member:`~PyTypeObject.tp_version_tag` field was available and + initialized. + + + .. c:macro:: Py_TPFLAGS_INLINE_VALUES + + This bit indicates that instances of this type will have an "inline values" + array (containing the object's attributes) placed directly after the end + of the object. + + This requires that :c:macro:`Py_TPFLAGS_HAVE_GC` is set. + + **Inheritance:** + + This flag is not inherited. + + .. versionadded:: 3.13 + + + .. c:macro:: Py_TPFLAGS_IS_ABSTRACT + + This bit indicates that this is an abstract type and therefore cannot + be instantiated. + + **Inheritance:** + + This flag is not inherited. + + .. seealso:: + :mod:`abc` + + + .. c:macro:: Py_TPFLAGS_HAVE_STACKLESS_EXTENSION + + Internal. Do not set or unset this flag. + Historically, this was a reserved flag for use in Stackless Python. + + .. warning:: + This flag is present in header files, but is not be used. + This may be removed in a future version of CPython. + + .. c:member:: const char* PyTypeObject.tp_doc .. corresponding-type-slot:: Py_tp_doc diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 7eb9f0b54ab..6256bf7a145 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -191,7 +191,7 @@ the same library that the Python runtime is using. objects *globals* and *locals* with the compiler flags specified by *flags*. *globals* must be a dictionary; *locals* can be any object that implements the mapping protocol. The parameter *start* specifies - the start symbol and must one of the :ref:`available start symbols `. + the start symbol and must be one of the :ref:`available start symbols `. Returns the result of executing the code as a Python object, or ``NULL`` if an exception was raised. diff --git a/Doc/conf.py b/Doc/conf.py index a4275835059..a6819d4af26 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -42,6 +42,7 @@ # Skip if downstream redistributors haven't installed them _OPTIONAL_EXTENSIONS = ( + 'linklint.ext', 'notfound.extension', 'sphinxext.opengraph', ) @@ -553,6 +554,7 @@ # mapping unique short aliases to a base URL and a prefix. # https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html extlinks = { + "oss-fuzz": ("https://issues.oss-fuzz.com/issues/%s", "#%s"), "pypi": ("https://pypi.org/project/%s/", "%s"), "source": (SOURCE_URI, "%s"), } diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 1cc1b44a5b8..48b800fdf9a 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -2415,6 +2415,9 @@ PyType_GetFlags:PyTypeObject*:type:0: PyType_GetName:PyObject*::+1: PyType_GetName:PyTypeObject*:type:0: +PyType_GetModule:PyObject*::0: +PyType_GetModule:PyTypeObject*:type:0: + PyType_GetModuleByDef:PyObject*::0: PyType_GetModuleByDef:PyTypeObject*:type:0: PyType_GetModuleByDef:PyModuleDef*:def:: diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst index c80588b27b6..600510cf7f0 100644 --- a/Doc/deprecations/pending-removal-in-3.15.rst +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -64,7 +64,7 @@ Pending removal in Python 3.15 * :func:`~threading.RLock` will take no arguments in Python 3.15. Passing any arguments has been deprecated since Python 3.14, - as the Python version does not permit any arguments, + as the Python version does not permit any arguments, but the C version allows any number of positional or keyword arguments, ignoring every argument. diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index 43e06da6c28..d9a2d50378e 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -35,7 +35,6 @@ although there is currently no date scheduled for their removal. * Support for ``__complex__()`` method returning a strict subclass of :class:`complex`: these methods will be required to return an instance of :class:`complex`. - * Delegation of ``int()`` to ``__trunc__()`` method. * Passing a complex number as the *real* or *imag* argument in the :func:`complex` constructor is now deprecated; it should only be passed as a single positional argument. diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 138a5ca7a75..ff34bb5d71c 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -8,11 +8,11 @@ Programming FAQ .. contents:: -General Questions +General questions ================= -Is there a source code level debugger with breakpoints, single-stepping, etc.? ------------------------------------------------------------------------------- +Is there a source code-level debugger with breakpoints and single-stepping? +--------------------------------------------------------------------------- Yes. @@ -25,8 +25,7 @@ Reference Manual `. You can also write your own debugger by using the code for pdb as an example. The IDLE interactive development environment, which is part of the standard -Python distribution (normally available as -`Tools/scripts/idle3 `_), +Python distribution (normally available as :mod:`idlelib`), includes a graphical debugger. PythonWin is a Python IDE that includes a GUI debugger based on pdb. The @@ -48,7 +47,6 @@ There are a number of commercial Python IDEs that include graphical debuggers. They include: * `Wing IDE `_ -* `Komodo IDE `_ * `PyCharm `_ @@ -57,13 +55,15 @@ Are there tools to help find bugs or perform static analysis? Yes. -`Pylint `_ and -`Pyflakes `_ do basic checking that will +`Ruff `__, +`Pylint `__ and +`Pyflakes `__ do basic checking that will help you catch bugs sooner. -Static type checkers such as `Mypy `_, -`Pyre `_, and -`Pytype `_ can check type hints in Python +Static type checkers such as `mypy `__, +`ty `__, +`Pyrefly `__, and +`pytype `__ can check type hints in Python source code. @@ -79,7 +79,7 @@ set of modules required by a program and bind these modules together with a Python binary to produce a single executable. One is to use the freeze tool, which is included in the Python source tree as -`Tools/freeze `_. +:source:`Tools/freeze`. It converts Python byte code to C arrays; with a C compiler you can embed all your modules into a new program, which is then linked with the standard Python modules. @@ -103,6 +103,7 @@ executables: * `py2app `_ (macOS only) * `py2exe `_ (Windows only) + Are there coding standards or a style guide for Python programs? ---------------------------------------------------------------- @@ -110,7 +111,7 @@ Yes. The coding style required for standard library modules is documented as :pep:`8`. -Core Language +Core language ============= .. _faq-unboundlocalerror: @@ -143,7 +144,7 @@ results in an :exc:`!UnboundLocalError`: >>> foo() Traceback (most recent call last): ... - UnboundLocalError: local variable 'x' referenced before assignment + UnboundLocalError: cannot access local variable 'x' where it is not associated with a value This is because when you make an assignment to a variable in a scope, that variable becomes local to that scope and shadows any similarly named variable @@ -208,7 +209,7 @@ Why do lambdas defined in a loop with different values all return the same resul ---------------------------------------------------------------------------------- Assume you use a for loop to define a few different lambdas (or even plain -functions), e.g.:: +functions), for example:: >>> squares = [] >>> for x in range(5): @@ -227,7 +228,7 @@ they all return ``16``:: This happens because ``x`` is not local to the lambdas, but is defined in the outer scope, and it is accessed when the lambda is called --- not when it is defined. At the end of the loop, the value of ``x`` is ``4``, so all the -functions now return ``4**2``, i.e. ``16``. You can also verify this by +functions now return ``4**2``, that is ``16``. You can also verify this by changing the value of ``x`` and see how the results of the lambdas change:: >>> x = 8 @@ -298,9 +299,9 @@ using multiple imports per line uses less screen space. It's good practice if you import modules in the following order: -1. standard library modules -- e.g. :mod:`sys`, :mod:`os`, :mod:`argparse`, :mod:`re` +1. standard library modules -- such as :mod:`sys`, :mod:`os`, :mod:`argparse`, :mod:`re` 2. third-party library modules (anything installed in Python's site-packages - directory) -- e.g. :mod:`!dateutil`, :mod:`!requests`, :mod:`!PIL.Image` + directory) -- such as :pypi:`dateutil`, :pypi:`requests`, :pypi:`tzdata` 3. locally developed modules It is sometimes necessary to move imports to a function or class to avoid @@ -494,11 +495,11 @@ new objects). In other words: -* If we have a mutable object (:class:`list`, :class:`dict`, :class:`set`, - etc.), we can use some specific operations to mutate it and all the variables +* If we have a mutable object (such as :class:`list`, :class:`dict`, :class:`set`), + we can use some specific operations to mutate it and all the variables that refer to it will see the change. -* If we have an immutable object (:class:`str`, :class:`int`, :class:`tuple`, - etc.), all the variables that refer to it will always see the same value, +* If we have an immutable object (such as :class:`str`, :class:`int`, :class:`tuple`), + all the variables that refer to it will always see the same value, but operations that transform that value into a new value always return a new object. @@ -511,7 +512,7 @@ How do I write a function with output parameters (call by reference)? Remember that arguments are passed by assignment in Python. Since assignment just creates references to objects, there's no alias between an argument name in -the caller and callee, and so no call-by-reference per se. You can achieve the +the caller and callee, and consequently no call-by-reference. You can achieve the desired effect in a number of ways. 1) By returning a tuple of the results:: @@ -714,8 +715,8 @@ not:: "a" in ("b", "a") -The same is true of the various assignment operators (``=``, ``+=`` etc). They -are not truly operators but syntactic delimiters in assignment statements. +The same is true of the various assignment operators (``=``, ``+=``, and so on). +They are not truly operators but syntactic delimiters in assignment statements. Is there an equivalent of C's "?:" ternary operator? @@ -868,9 +869,9 @@ with either a space or parentheses. How do I convert a string to a number? -------------------------------------- -For integers, use the built-in :func:`int` type constructor, e.g. ``int('144') +For integers, use the built-in :func:`int` type constructor, for example, ``int('144') == 144``. Similarly, :func:`float` converts to a floating-point number, -e.g. ``float('144') == 144.0``. +for example, ``float('144') == 144.0``. By default, these interpret the number as decimal, so that ``int('0144') == 144`` holds true, and ``int('0x144')`` raises :exc:`ValueError`. ``int(string, @@ -887,18 +888,18 @@ unwanted side effects. For example, someone could pass directory. :func:`eval` also has the effect of interpreting numbers as Python expressions, -so that e.g. ``eval('09')`` gives a syntax error because Python does not allow +so that, for example, ``eval('09')`` gives a syntax error because Python does not allow leading '0' in a decimal number (except '0'). How do I convert a number to a string? -------------------------------------- -To convert, e.g., the number ``144`` to the string ``'144'``, use the built-in type +For example, to convert the number ``144`` to the string ``'144'``, use the built-in type constructor :func:`str`. If you want a hexadecimal or octal representation, use the built-in functions :func:`hex` or :func:`oct`. For fancy formatting, see -the :ref:`f-strings` and :ref:`formatstrings` sections, -e.g. ``"{:04d}".format(144)`` yields +the :ref:`f-strings` and :ref:`formatstrings` sections. +For example, ``"{:04d}".format(144)`` yields ``'0144'`` and ``"{:.3f}".format(1.0/3.0)`` yields ``'0.333'``. @@ -908,7 +909,7 @@ How do I modify a string in place? You can't, because strings are immutable. In most situations, you should simply construct a new string from the various parts you want to assemble it from. However, if you need an object with the ability to modify in-place -unicode data, try using an :class:`io.StringIO` object or the :mod:`array` +Unicode data, try using an :class:`io.StringIO` object or the :mod:`array` module:: >>> import io @@ -1066,13 +1067,14 @@ the raw string:: Also see the specification in the :ref:`language reference `. + Performance =========== My program is too slow. How do I speed it up? --------------------------------------------- -That's a tough one, in general. First, here are a list of things to +That's a tough one, in general. First, here is a list of things to remember before diving further: * Performance characteristics vary across Python implementations. This FAQ @@ -1125,6 +1127,7 @@ yourself. The wiki page devoted to `performance tips `_. + .. _efficient_string_concatenation: What is the most efficient way to concatenate many strings together? @@ -1143,7 +1146,7 @@ them into a list and call :meth:`str.join` at the end:: chunks.append(s) result = ''.join(chunks) -(another reasonably efficient idiom is to use :class:`io.StringIO`) +(Another reasonably efficient idiom is to use :class:`io.StringIO`.) To accumulate many :class:`bytes` objects, the recommended idiom is to extend a :class:`bytearray` object using in-place concatenation (the ``+=`` operator):: @@ -1153,7 +1156,7 @@ a :class:`bytearray` object using in-place concatenation (the ``+=`` operator):: result += b -Sequences (Tuples/Lists) +Sequences (tuples/lists) ======================== How do I convert between tuples and lists? @@ -1217,8 +1220,8 @@ list, deleting duplicates as you go:: else: last = mylist[i] -If all elements of the list may be used as set keys (i.e. they are all -:term:`hashable`) this is often faster :: +If all elements of the list may be used as set keys (that is, they are all +:term:`hashable`) this is often faster:: mylist = list(set(mylist)) @@ -1254,7 +1257,7 @@ difference is that a Python list can contain objects of many different types. The ``array`` module also provides methods for creating arrays of fixed types with compact representations, but they are slower to index than lists. Also note that `NumPy `_ -and other third party packages define array-like structures with +and other third-party packages define array-like structures with various characteristics as well. To get Lisp-style linked lists, you can emulate *cons cells* using tuples:: @@ -1324,7 +1327,7 @@ Or, you can use an extension that provides a matrix datatype; `NumPy How do I apply a method or function to a sequence of objects? ------------------------------------------------------------- -To call a method or function and accumulate the return values is a list, +To call a method or function and accumulate the return values in a list, a :term:`list comprehension` is an elegant solution:: result = [obj.method() for obj in mylist] @@ -1340,6 +1343,7 @@ a plain :keyword:`for` loop will suffice:: for obj in mylist: function(obj) + .. _faq-augmented-assignment-tuple-error: Why does a_tuple[i] += ['item'] raise an exception when the addition works? @@ -1444,7 +1448,7 @@ How can I sort one list by values from another list? ---------------------------------------------------- Merge them into an iterator of tuples, sort the resulting list, and then pick -out the element you want. :: +out the element you want. >>> list1 = ["what", "I'm", "sorting", "by"] >>> list2 = ["something", "else", "to", "sort"] @@ -1504,14 +1508,15 @@ How do I check if an object is an instance of a given class or of a subclass of Use the built-in function :func:`isinstance(obj, cls) `. You can check if an object is an instance of any of a number of classes by providing a tuple instead of a -single class, e.g. ``isinstance(obj, (class1, class2, ...))``, and can also -check whether an object is one of Python's built-in types, e.g. +single class, for example, ``isinstance(obj, (class1, class2, ...))``, and can also +check whether an object is one of Python's built-in types, for example, ``isinstance(obj, str)`` or ``isinstance(obj, (int, float, complex))``. Note that :func:`isinstance` also checks for virtual inheritance from an :term:`abstract base class`. So, the test will return ``True`` for a registered class even if hasn't directly or indirectly inherited from it. To -test for "true inheritance", scan the :term:`MRO` of the class: +test for "true inheritance", scan the :term:`method resolution order` (MRO) of +the class: .. testcode:: @@ -1574,7 +1579,7 @@ call it:: What is delegation? ------------------- -Delegation is an object oriented technique (also called a design pattern). +Delegation is an object-oriented technique (also called a design pattern). Let's say you have an object ``x`` and want to change the behaviour of just one of its methods. You can create a new class that provides a new implementation of the method you're interested in changing and delegates all other methods to @@ -1645,7 +1650,7 @@ How can I organize my code to make it easier to change the base class? You could assign the base class to an alias and derive from the alias. Then all you have to change is the value assigned to the alias. Incidentally, this trick -is also handy if you want to decide dynamically (e.g. depending on availability +is also handy if you want to decide dynamically (such as depending on availability of resources) which base class to use. Example:: class Base: @@ -1710,9 +1715,9 @@ How can I overload constructors (or methods) in Python? This answer actually applies to all methods, but the question usually comes up first in the context of constructors. -In C++ you'd write +In C++ you'd write: -.. code-block:: c +.. code-block:: c++ class C { C() { cout << "No arguments\n"; } @@ -1731,7 +1736,7 @@ default arguments. For example:: This is not entirely equivalent, but close enough in practice. -You could also try a variable-length argument list, e.g. :: +You could also try a variable-length argument list, for example:: def __init__(self, *args): ... @@ -1774,6 +1779,7 @@ to use private variable names at all. The :ref:`private name mangling specifications ` for details and special cases. + My class defines __del__ but it is not called when I delete the object. ----------------------------------------------------------------------- @@ -1783,7 +1789,7 @@ The :keyword:`del` statement does not necessarily call :meth:`~object.__del__` - decrements the object's reference count, and if this reaches zero :meth:`!__del__` is called. -If your data structures contain circular links (e.g. a tree where each child has +If your data structures contain circular links (for example, a tree where each child has a parent reference and each parent has a list of children) the reference counts will never go back to zero. Once in a while Python runs an algorithm to detect such cycles, but the garbage collector might run some time after the last @@ -1852,6 +1858,8 @@ to the object: 13891296 +.. _faq-identity-with-is: + When can I rely on identity tests with the *is* operator? --------------------------------------------------------- @@ -1883,9 +1891,9 @@ are preferred. In particular, identity tests should not be used to check constants such as :class:`int` and :class:`str` which aren't guaranteed to be singletons:: - >>> a = 1000 - >>> b = 500 - >>> c = b + 500 + >>> a = 10_000_000 + >>> b = 5_000_000 + >>> c = b + 5_000_000 >>> a is c False @@ -1954,9 +1962,9 @@ parent class: .. testcode:: - from datetime import date + import datetime as dt - class FirstOfMonthDate(date): + class FirstOfMonthDate(dt.date): "Always choose the first day of the month" def __new__(cls, year, month, day): return super().__new__(cls, year, month, 1) @@ -1999,7 +2007,7 @@ The two principal tools for caching methods are former stores results at the instance level and the latter at the class level. -The *cached_property* approach only works with methods that do not take +The ``cached_property`` approach only works with methods that do not take any arguments. It does not create a reference to the instance. The cached method result will be kept only as long as the instance is alive. @@ -2008,7 +2016,7 @@ method result will be released right away. The disadvantage is that if instances accumulate, so too will the accumulated method results. They can grow without bound. -The *lru_cache* approach works with methods that have :term:`hashable` +The ``lru_cache`` approach works with methods that have :term:`hashable` arguments. It creates a reference to the instance unless special efforts are made to pass in weak references. @@ -2042,11 +2050,11 @@ This example shows the various techniques:: # Depends on the station_id, date, and units. The above example assumes that the *station_id* never changes. If the -relevant instance attributes are mutable, the *cached_property* approach +relevant instance attributes are mutable, the ``cached_property`` approach can't be made to work because it cannot detect changes to the attributes. -To make the *lru_cache* approach work when the *station_id* is mutable, +To make the ``lru_cache`` approach work when the *station_id* is mutable, the class needs to define the :meth:`~object.__eq__` and :meth:`~object.__hash__` methods so that the cache can detect relevant attribute updates:: @@ -2092,10 +2100,10 @@ one user but run as another, such as if you are testing with a web server. Unless the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable is set, creation of a .pyc file is automatic if you're importing a module and Python -has the ability (permissions, free space, etc...) to create a ``__pycache__`` +has the ability (permissions, free space, and so on) to create a ``__pycache__`` subdirectory and write the compiled module to that subdirectory. -Running Python on a top level script is not considered an import and no +Running Python on a top-level script is not considered an import and no ``.pyc`` will be created. For example, if you have a top-level module ``foo.py`` that imports another module ``xyz.py``, when you run ``foo`` (by typing ``python foo.py`` as a shell command), a ``.pyc`` will be created for @@ -2114,7 +2122,7 @@ the ``compile()`` function in that module interactively:: This will write the ``.pyc`` to a ``__pycache__`` subdirectory in the same location as ``foo.py`` (or you can override that with the optional parameter -``cfile``). +*cfile*). You can also automatically compile all files in a directory or directories using the :mod:`compileall` module. You can do it from the shell prompt by running @@ -2219,7 +2227,7 @@ changed module, do this:: importlib.reload(modname) Warning: this technique is not 100% fool-proof. In particular, modules -containing statements like :: +containing statements like:: from modname import some_objects diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 7c41f5bc27b..6151143a97b 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -786,6 +786,19 @@ Glossary An object that both finds and loads a module; both a :term:`finder` and :term:`loader` object. + index + A numeric value that represents the position of an element in + a :term:`sequence`. + + In Python, indexing starts at zero. + For example, ``things[0]`` names the *first* element of ``things``; + ``things[1]`` names the second one. + + In some contexts, Python allows negative indexes for counting from the + end of a sequence, and indexing using :term:`slices `. + + See also :term:`subscript`. + interactive Python has an interactive interpreter which means you can enter statements and expressions at the interpreter prompt, immediately @@ -863,6 +876,9 @@ Glossary CPython does not guarantee :term:`thread-safe` behavior of iterator operations. + key + A value that identifies an entry in a :term:`mapping`. + See also :term:`subscript`. key function A key function or collation function is a callable that returns a value @@ -935,6 +951,16 @@ Glossary to locks exist such as queues, producer/consumer patterns, and thread-local state. See also :term:`deadlock`, and :term:`reentrant`. + lock-free + An operation that does not acquire any :term:`lock` and uses atomic CPU + instructions to ensure correctness. Lock-free operations can execute + concurrently without blocking each other and cannot be blocked by + operations that hold locks. In :term:`free-threaded ` + Python, built-in types like :class:`dict` and :class:`list` provide + lock-free read operations, which means other threads may observe + intermediate states during multi-step modifications even when those + modifications hold the :term:`per-object lock`. + loader An object that loads a module. It must define the :meth:`!exec_module` and :meth:`!create_module` methods @@ -1201,6 +1227,16 @@ Glossary `, the :class:`inspect.Parameter` class, the :ref:`function` section, and :pep:`362`. + per-object lock + A :term:`lock` associated with an individual object instance rather than + a global lock shared across all objects. In :term:`free-threaded + ` Python, built-in types like :class:`dict` and + :class:`list` use per-object locks to allow concurrent operations on + different objects while serializing operations on the same object. + Operations that hold the per-object lock prevent other locking operations + on the same object from proceeding, but do not block :term:`lock-free` + operations. + path entry A single location on the :term:`import path` which the :term:`path based finder` consults to find modules for importing. @@ -1323,7 +1359,7 @@ Glossary 'email.mime.text' race condition - A condition of a program where the its behavior + A condition of a program where the behavior depends on the relative timing or ordering of events, particularly in multi-threaded programs. Race conditions can lead to :term:`non-deterministic` behavior and bugs that are difficult to @@ -1417,10 +1453,11 @@ Glossary chosen based on the type of a single argument. slice - An object usually containing a portion of a :term:`sequence`. A slice is - created using the subscript notation, ``[]`` with colons between numbers - when several are given, such as in ``variable_name[1:3:5]``. The bracket - (subscript) notation uses :class:`slice` objects internally. + An object of type :class:`slice`, used to describe a portion of + a :term:`sequence`. + A slice object is created when using the :ref:`slicing ` form + of :ref:`subscript notation `, with colons inside square + brackets, such as in ``variable_name[1:3:5]``. soft deprecated A soft deprecated API should not be used in new code, @@ -1478,6 +1515,14 @@ Glossary See also :term:`borrowed reference`. + subscript + The expression in square brackets of a + :ref:`subscription expression `, for example, + the ``3`` in ``items[3]``. + Usually used to select an element of a container. + Also called a :term:`key` when subscripting a :term:`mapping`, + or an :term:`index` when subscripting a :term:`sequence`. + synchronization primitive A basic building block for coordinating (synchronizing) the execution of multiple threads to ensure :term:`thread-safe` access to shared resources. diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index b7225ff1c2c..454e2f4930e 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -28,7 +28,7 @@ When to use logging ^^^^^^^^^^^^^^^^^^^ You can access logging functionality by creating a logger via ``logger = -getLogger(__name__)``, and then calling the logger's :meth:`~Logger.debug`, +logging.getLogger(__name__)``, and then calling the logger's :meth:`~Logger.debug`, :meth:`~Logger.info`, :meth:`~Logger.warning`, :meth:`~Logger.error` and :meth:`~Logger.critical` methods. To determine when to use logging, and to see which logger methods to use when, see the table below. It states, for each of a diff --git a/Doc/improve-page-nojs.rst b/Doc/improve-page-nojs.rst new file mode 100644 index 00000000000..91b3a88b95d --- /dev/null +++ b/Doc/improve-page-nojs.rst @@ -0,0 +1,29 @@ +:orphan: + +**************************** +Improve a documentation page +**************************** + +.. This is the no-javascript version of this page. The one most people + will see (with JavaScript enabled) is improve-page.rst. If you edit + this page, please also edit that one, and vice versa. + +.. only:: html and not epub + +We are always interested to hear ideas about improvements to the documentation. + +.. only:: translation + + If the bug or suggested improvement concerns the translation of this + documentation, open an issue or edit the page in + `translation's repository `_ instead. + +You have a few ways to ask questions or suggest changes: + +- You can start a discussion about the page on the Python discussion forum. + This link will start a topic in the Documentation category: + `New Documentation topic `_. + +- You can open an issue on the Python GitHub issue tracker. This link will + create a new issue with the "docs" label: + `New docs issue `_. diff --git a/Doc/improve-page.rst b/Doc/improve-page.rst new file mode 100644 index 00000000000..dc89fcb22fb --- /dev/null +++ b/Doc/improve-page.rst @@ -0,0 +1,65 @@ +:orphan: + +**************************** +Improve a documentation page +**************************** + +.. This is the JavaScript-enabled version of this page. Another version + (for those with JavaScript disabled) is improve-page-nojs.rst. If you + edit this page, please also edit that one, and vice versa. + +.. only:: html and not epub + + .. raw:: html + + + +We are always interested to hear ideas about improvements to the documentation. + +You were reading "PAGETITLE" at ``_. The source for that page is on +`GitHub `_. + +.. only:: translation + + If the bug or suggested improvement concerns the translation of this + documentation, open an issue or edit the page in + `translation's repository `_ instead. + +You have a few ways to ask questions or suggest changes: + +- You can start a discussion about the page on the Python discussion forum. + This link will start a pre-populated topic: + `Question about page "PAGETITLE" `_. + +- You can open an issue on the Python GitHub issue tracker. This link will + create a new pre-populated issue: + `Docs: problem with page "PAGETITLE" `_. + +- You can `edit the page on GitHub `_ + to open a pull request and begin the contribution process. diff --git a/Doc/includes/tzinfo_examples.py b/Doc/includes/tzinfo_examples.py index 1fa6e615e46..762b1b62fc8 100644 --- a/Doc/includes/tzinfo_examples.py +++ b/Doc/includes/tzinfo_examples.py @@ -1,68 +1,70 @@ -from datetime import tzinfo, timedelta, datetime - -ZERO = timedelta(0) -HOUR = timedelta(hours=1) -SECOND = timedelta(seconds=1) +import datetime as dt # A class capturing the platform's idea of local time. # (May result in wrong values on historical times in # timezones where UTC offset and/or the DST rules had # changed in the past.) -import time as _time +import time + +ZERO = dt.timedelta(0) +HOUR = dt.timedelta(hours=1) +SECOND = dt.timedelta(seconds=1) -STDOFFSET = timedelta(seconds = -_time.timezone) -if _time.daylight: - DSTOFFSET = timedelta(seconds = -_time.altzone) +STDOFFSET = dt.timedelta(seconds=-time.timezone) +if time.daylight: + DSTOFFSET = dt.timedelta(seconds=-time.altzone) else: DSTOFFSET = STDOFFSET DSTDIFF = DSTOFFSET - STDOFFSET -class LocalTimezone(tzinfo): - def fromutc(self, dt): - assert dt.tzinfo is self - stamp = (dt - datetime(1970, 1, 1, tzinfo=self)) // SECOND - args = _time.localtime(stamp)[:6] +class LocalTimezone(dt.tzinfo): + + def fromutc(self, when): + assert when.tzinfo is self + stamp = (when - dt.datetime(1970, 1, 1, tzinfo=self)) // SECOND + args = time.localtime(stamp)[:6] dst_diff = DSTDIFF // SECOND # Detect fold - fold = (args == _time.localtime(stamp - dst_diff)) - return datetime(*args, microsecond=dt.microsecond, - tzinfo=self, fold=fold) + fold = (args == time.localtime(stamp - dst_diff)) + return dt.datetime(*args, microsecond=when.microsecond, + tzinfo=self, fold=fold) - def utcoffset(self, dt): - if self._isdst(dt): + def utcoffset(self, when): + if self._isdst(when): return DSTOFFSET else: return STDOFFSET - def dst(self, dt): - if self._isdst(dt): + def dst(self, when): + if self._isdst(when): return DSTDIFF else: return ZERO - def tzname(self, dt): - return _time.tzname[self._isdst(dt)] + def tzname(self, when): + return time.tzname[self._isdst(when)] - def _isdst(self, dt): - tt = (dt.year, dt.month, dt.day, - dt.hour, dt.minute, dt.second, - dt.weekday(), 0, 0) - stamp = _time.mktime(tt) - tt = _time.localtime(stamp) + def _isdst(self, when): + tt = (when.year, when.month, when.day, + when.hour, when.minute, when.second, + when.weekday(), 0, 0) + stamp = time.mktime(tt) + tt = time.localtime(stamp) return tt.tm_isdst > 0 + Local = LocalTimezone() # A complete implementation of current DST rules for major US time zones. -def first_sunday_on_or_after(dt): - days_to_go = 6 - dt.weekday() +def first_sunday_on_or_after(when): + days_to_go = 6 - when.weekday() if days_to_go: - dt += timedelta(days_to_go) - return dt + when += dt.timedelta(days_to_go) + return when # US DST Rules @@ -75,21 +77,22 @@ def first_sunday_on_or_after(dt): # # In the US, since 2007, DST starts at 2am (standard time) on the second # Sunday in March, which is the first Sunday on or after Mar 8. -DSTSTART_2007 = datetime(1, 3, 8, 2) +DSTSTART_2007 = dt.datetime(1, 3, 8, 2) # and ends at 2am (DST time) on the first Sunday of Nov. -DSTEND_2007 = datetime(1, 11, 1, 2) +DSTEND_2007 = dt.datetime(1, 11, 1, 2) # From 1987 to 2006, DST used to start at 2am (standard time) on the first # Sunday in April and to end at 2am (DST time) on the last # Sunday of October, which is the first Sunday on or after Oct 25. -DSTSTART_1987_2006 = datetime(1, 4, 1, 2) -DSTEND_1987_2006 = datetime(1, 10, 25, 2) +DSTSTART_1987_2006 = dt.datetime(1, 4, 1, 2) +DSTEND_1987_2006 = dt.datetime(1, 10, 25, 2) # From 1967 to 1986, DST used to start at 2am (standard time) on the last # Sunday in April (the one on or after April 24) and to end at 2am (DST time) # on the last Sunday of October, which is the first Sunday # on or after Oct 25. -DSTSTART_1967_1986 = datetime(1, 4, 24, 2) +DSTSTART_1967_1986 = dt.datetime(1, 4, 24, 2) DSTEND_1967_1986 = DSTEND_1987_2006 + def us_dst_range(year): # Find start and end times for US DST. For years before 1967, return # start = end for no DST. @@ -100,17 +103,17 @@ def us_dst_range(year): elif 1966 < year < 1987: dststart, dstend = DSTSTART_1967_1986, DSTEND_1967_1986 else: - return (datetime(year, 1, 1), ) * 2 + return (dt.datetime(year, 1, 1), ) * 2 start = first_sunday_on_or_after(dststart.replace(year=year)) end = first_sunday_on_or_after(dstend.replace(year=year)) return start, end -class USTimeZone(tzinfo): +class USTimeZone(dt.tzinfo): def __init__(self, hours, reprname, stdname, dstname): - self.stdoffset = timedelta(hours=hours) + self.stdoffset = dt.timedelta(hours=hours) self.reprname = reprname self.stdname = stdname self.dstname = dstname @@ -118,45 +121,45 @@ def __init__(self, hours, reprname, stdname, dstname): def __repr__(self): return self.reprname - def tzname(self, dt): - if self.dst(dt): + def tzname(self, when): + if self.dst(when): return self.dstname else: return self.stdname - def utcoffset(self, dt): - return self.stdoffset + self.dst(dt) + def utcoffset(self, when): + return self.stdoffset + self.dst(when) - def dst(self, dt): - if dt is None or dt.tzinfo is None: + def dst(self, when): + if when is None or when.tzinfo is None: # An exception may be sensible here, in one or both cases. # It depends on how you want to treat them. The default # fromutc() implementation (called by the default astimezone() - # implementation) passes a datetime with dt.tzinfo is self. + # implementation) passes a datetime with when.tzinfo is self. return ZERO - assert dt.tzinfo is self - start, end = us_dst_range(dt.year) + assert when.tzinfo is self + start, end = us_dst_range(when.year) # Can't compare naive to aware objects, so strip the timezone from - # dt first. - dt = dt.replace(tzinfo=None) - if start + HOUR <= dt < end - HOUR: + # when first. + when = when.replace(tzinfo=None) + if start + HOUR <= when < end - HOUR: # DST is in effect. return HOUR - if end - HOUR <= dt < end: - # Fold (an ambiguous hour): use dt.fold to disambiguate. - return ZERO if dt.fold else HOUR - if start <= dt < start + HOUR: + if end - HOUR <= when < end: + # Fold (an ambiguous hour): use when.fold to disambiguate. + return ZERO if when.fold else HOUR + if start <= when < start + HOUR: # Gap (a non-existent hour): reverse the fold rule. - return HOUR if dt.fold else ZERO + return HOUR if when.fold else ZERO # DST is off. return ZERO - def fromutc(self, dt): - assert dt.tzinfo is self - start, end = us_dst_range(dt.year) + def fromutc(self, when): + assert when.tzinfo is self + start, end = us_dst_range(when.year) start = start.replace(tzinfo=self) end = end.replace(tzinfo=self) - std_time = dt + self.stdoffset + std_time = when + self.stdoffset dst_time = std_time + HOUR if end <= dst_time < end + HOUR: # Repeated hour diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index 3a485a43a5a..412005f3ec8 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -6,8 +6,6 @@ Installing Python Modules ************************* -:Email: distutils-sig@python.org - As a popular open source development project, Python has an active supporting community of contributors and users that also make their software available for other Python developers to use under open source license terms. diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 5d916b30112..749e4543c5b 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -15,7 +15,7 @@ before the release in which the feature becomes standard. While these future statements are given additional special meaning by the Python compiler, they are still executed like any other import statement and -the :mod:`__future__` exists and is handled by the import system the same way +the :mod:`!__future__` exists and is handled by the import system the same way any other Python module would be. This design serves three purposes: * To avoid confusing existing tools that analyze import statements and expect to @@ -23,17 +23,17 @@ any other Python module would be. This design serves three purposes: * To document when incompatible changes were introduced, and when they will be --- or were --- made mandatory. This is a form of executable documentation, and - can be inspected programmatically via importing :mod:`__future__` and examining + can be inspected programmatically via importing :mod:`!__future__` and examining its contents. * To ensure that :ref:`future statements ` run under releases prior to - Python 2.1 at least yield runtime exceptions (the import of :mod:`__future__` + Python 2.1 at least yield runtime exceptions (the import of :mod:`!__future__` will fail, because there was no module of that name prior to 2.1). Module Contents --------------- -No feature description will ever be deleted from :mod:`__future__`. Since its +No feature description will ever be deleted from :mod:`!__future__`. Since its introduction in Python 2.1 the following features have found their way into the language using this mechanism: diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index d8df91c882f..2b2bf0a9d6e 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -13,7 +13,7 @@ .. note:: - While :mod:`argparse` is the default recommended standard library module + While :mod:`!argparse` is the default recommended standard library module for implementing basic command line applications, authors with more exacting requirements for exactly how their command line applications behave may find it doesn't provide the necessary level of control. @@ -604,13 +604,11 @@ subparser names, the feature can be enabled by setting ``suggest_on_error`` to ``True``. Note that this only applies for arguments when the choices specified are strings:: - >>> parser = argparse.ArgumentParser(description='Process some integers.', - suggest_on_error=True) - >>> parser.add_argument('--action', choices=['sum', 'max']) - >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', - ... help='an integer for the accumulator') - >>> parser.parse_args(['--action', 'sumn', 1, 2, 3]) - tester.py: error: argument --action: invalid choice: 'sumn', maybe you meant 'sum'? (choose from 'sum', 'max') + >>> parser = argparse.ArgumentParser(suggest_on_error=True) + >>> parser.add_argument('--action', choices=['debug', 'dryrun']) + >>> parser.parse_args(['--action', 'debugg']) + usage: tester.py [-h] [--action {debug,dryrun}] + tester.py: error: argument --action: invalid choice: 'debugg', maybe you meant 'debug'? (choose from debug, dryrun) If you're writing code that needs to be compatible with older Python versions and want to opportunistically use ``suggest_on_error`` when it's available, you @@ -734,9 +732,9 @@ By default, :mod:`!argparse` automatically handles the internal naming and display names of arguments, simplifying the process without requiring additional configuration. As such, you do not need to specify the dest_ and metavar_ parameters. -The dest_ parameter defaults to the argument name with underscores ``_`` -replacing hyphens ``-`` . The metavar_ parameter defaults to the -upper-cased name. For example:: +For optional arguments, the dest_ parameter defaults to the argument name, with +underscores ``_`` replacing hyphens ``-``. The metavar_ parameter defaults to +the upper-cased name. For example:: >>> parser = argparse.ArgumentParser(prog='PROG') >>> parser.add_argument('--foo-bar') diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 424c0a92fb2..d2bff573588 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -15,7 +15,7 @@ -------------- -The :mod:`ast` module helps Python applications to process trees of the Python +The :mod:`!ast` module helps Python applications to process trees of the Python abstract syntax grammar. The abstract syntax itself might change with each Python release; this module helps to find out programmatically what the current grammar looks like. @@ -46,7 +46,7 @@ Node classes This is the base of all AST node classes. The actual node classes are derived from the :file:`Parser/Python.asdl` file, which is reproduced :ref:`above `. They are defined in the :mod:`!_ast` C - module and re-exported in :mod:`ast`. + module and re-exported in :mod:`!ast`. There is one class defined for each left-hand side symbol in the abstract grammar (for example, :class:`ast.stmt` or :class:`ast.expr`). In addition, @@ -2217,10 +2217,10 @@ Async and await occurrences of the same value (for example, :class:`ast.Add`). -:mod:`ast` helpers ------------------- +:mod:`!ast` helpers +------------------- -Apart from the node classes, the :mod:`ast` module defines these utility functions +Apart from the node classes, the :mod:`!ast` module defines these utility functions and classes for traversing abstract syntax trees: .. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1) @@ -2597,7 +2597,7 @@ Command-line usage .. versionadded:: 3.9 -The :mod:`ast` module can be executed as a script from the command line. +The :mod:`!ast` module can be executed as a script from the command line. It is as simple as: .. code-block:: sh diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 72f484fd1cb..bdb24b3a58c 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -297,8 +297,9 @@ clocks to track time. are called is undefined. The optional positional *args* will be passed to the callback when - it is called. If you want the callback to be called with keyword - arguments use :func:`functools.partial`. + it is called. Use :func:`functools.partial` + :ref:`to pass keyword arguments ` to + *callback*. An optional keyword-only *context* argument allows specifying a custom :class:`contextvars.Context` for the *callback* to run in. @@ -1034,8 +1035,8 @@ Watching file descriptors .. method:: loop.add_writer(fd, callback, *args) Start monitoring the *fd* file descriptor for write availability and - invoke *callback* with the specified arguments once *fd* is available for - writing. + invoke *callback* with the specified arguments *args* once *fd* is + available for writing. Any preexisting callback registered for *fd* is cancelled and replaced by *callback*. @@ -1308,7 +1309,8 @@ Unix signals .. method:: loop.add_signal_handler(signum, callback, *args) - Set *callback* as the handler for the *signum* signal. + Set *callback* as the handler for the *signum* signal, + passing *args* as positional arguments. The callback will be invoked by *loop*, along with other queued callbacks and runnable coroutines of that event loop. Unlike signal handlers @@ -1343,7 +1345,8 @@ Executing code in thread or process pools .. awaitablemethod:: loop.run_in_executor(executor, func, *args) - Arrange for *func* to be called in the specified executor. + Arrange for *func* to be called in the specified executor + passing *args* as positional arguments. The *executor* argument should be an :class:`concurrent.futures.Executor` instance. The default executor is used if *executor* is ``None``. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index f825ae92ec7..5331db04aeb 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -771,6 +771,9 @@ Timeouts An :ref:`asynchronous context manager ` for cancelling overdue coroutines. + Prefer using :func:`asyncio.timeout` or :func:`asyncio.timeout_at` + rather than instantiating :class:`!Timeout` directly. + ``when`` should be an absolute time at which the context should time out, as measured by the event loop's clock: diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 02d2f0807df..24a3492ba10 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -9,9 +9,9 @@ -------------- -The :mod:`atexit` module defines functions to register and unregister cleanup +The :mod:`!atexit` module defines functions to register and unregister cleanup functions. Functions thus registered are automatically executed upon normal -interpreter termination. :mod:`atexit` runs these functions in the *reverse* +interpreter termination. :mod:`!atexit` runs these functions in the *reverse* order in which they were registered; if you register ``A``, ``B``, and ``C``, at interpreter termination time they will be run in the order ``C``, ``B``, ``A``. @@ -64,7 +64,7 @@ a cleanup function is undefined. Remove *func* from the list of functions to be run at interpreter shutdown. :func:`unregister` silently does nothing if *func* was not previously registered. If *func* has been registered more than once, every occurrence - of that function in the :mod:`atexit` call stack will be removed. Equality + of that function in the :mod:`!atexit` call stack will be removed. Equality comparisons (``==``) are used internally during unregistration, so function references do not need to have matching identities. @@ -72,14 +72,14 @@ a cleanup function is undefined. .. seealso:: Module :mod:`readline` - Useful example of :mod:`atexit` to read and write :mod:`readline` history + Useful example of :mod:`!atexit` to read and write :mod:`readline` history files. .. _atexit-example: -:mod:`atexit` Example ---------------------- +:mod:`!atexit` Example +---------------------- The following simple example demonstrates how a module can initialize a counter from a file when it is imported and save the counter's updated value diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index 2b87f2626b2..8d357f18952 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`bdb` module handles basic debugger functions, like setting breakpoints +The :mod:`!bdb` module handles basic debugger functions, like setting breakpoints or managing execution via the debugger. The following exception is defined: @@ -18,7 +18,7 @@ The following exception is defined: Exception raised by the :class:`Bdb` class for quitting the debugger. -The :mod:`bdb` module also defines two classes: +The :mod:`!bdb` module also defines two classes: .. class:: Breakpoint(self, file, line, temporary=False, cond=None, funcname=None) diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 1bab785684b..91b43332016 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -10,10 +10,10 @@ -------------- -The :mod:`binascii` module contains a number of methods to convert between +The :mod:`!binascii` module contains a number of methods to convert between binary and various ASCII-encoded binary representations. Normally, you will not use these functions directly but use wrapper modules like -:mod:`base64` instead. The :mod:`binascii` module contains +:mod:`base64` instead. The :mod:`!binascii` module contains low-level functions written in C for greater speed that are used by the higher-level modules. @@ -28,7 +28,7 @@ higher-level modules. ASCII-only unicode strings are now accepted by the ``a2b_*`` functions. -The :mod:`binascii` module defines the following functions: +The :mod:`!binascii` module defines the following functions: .. function:: a2b_uu(string) diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index d5ec4212c1f..ecc8d69a2b3 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -16,7 +16,7 @@ having to sort the list after each insertion. For long lists of items with expensive comparison operations, this can be an improvement over linear searches or frequent resorting. -The module is called :mod:`bisect` because it uses a basic bisection +The module is called :mod:`!bisect` because it uses a basic bisection algorithm to do its work. Unlike other bisection tools that search for a specific value, the functions in this module are designed to locate an insertion point. Accordingly, the functions never call an :meth:`~object.__eq__` @@ -27,9 +27,9 @@ point between values in an array. .. note:: The functions in this module are not thread-safe. If multiple threads - concurrently use :mod:`bisect` functions on the same sequence, this + concurrently use :mod:`!bisect` functions on the same sequence, this may result in undefined behaviour. Likewise, if the provided sequence - is mutated by a different thread while a :mod:`bisect` function + is mutated by a different thread while a :mod:`!bisect` function is operating on it, the result is undefined. For example, using :py:func:`~bisect.insort_left` on the same list from multiple threads may result in the list becoming unsorted. @@ -203,9 +203,9 @@ example uses :py:func:`~bisect.bisect` to look up a letter grade for an exam sco based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is a 'B', and so on:: - >>> def grade(score, breakpoints=[60, 70, 80, 90], grades='FDCBA'): - ... i = bisect(breakpoints, score) - ... return grades[i] + >>> def grade(score) + ... i = bisect([60, 70, 80, 90], score) + ... return "FDCBA"[i] ... >>> [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]] ['F', 'A', 'C', 'C', 'B', 'A', 'A'] diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index 12650861c0f..32e223ddbdd 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -16,7 +16,7 @@ This module provides a comprehensive interface for compressing and decompressing data using the bzip2 compression algorithm. -The :mod:`bz2` module contains: +The :mod:`!bz2` module contains: * The :func:`.open` function and :class:`BZ2File` class for reading and writing compressed files. @@ -317,7 +317,7 @@ One-shot (de)compression Examples of usage ----------------- -Below are some examples of typical usage of the :mod:`bz2` module. +Below are some examples of typical usage of the :mod:`!bz2` module. Using :func:`compress` and :func:`decompress` to demonstrate round-trip compression: diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 39090e36ed9..73b3b8b4da4 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -447,7 +447,7 @@ For simple text calendars this module provides the following functions. inverse. -The :mod:`calendar` module exports the following data attributes: +The :mod:`!calendar` module exports the following data attributes: .. data:: day_name @@ -540,7 +540,7 @@ The :mod:`calendar` module exports the following data attributes: .. versionadded:: 3.12 -The :mod:`calendar` module defines the following exceptions: +The :mod:`!calendar` module defines the following exceptions: .. exception:: IllegalMonthError(month) @@ -579,7 +579,7 @@ Command-line usage .. versionadded:: 2.5 -The :mod:`calendar` module can be executed as a script from the command line +The :mod:`!calendar` module can be executed as a script from the command line to interactively print a calendar. .. code-block:: shell diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index b6d5dbee21d..f602003e49b 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -124,7 +124,7 @@ rectangular coordinates to polar coordinates and back. The modulus (absolute value) of a complex number *z* can be computed using the built-in :func:`abs` function. There is no - separate :mod:`cmath` module function for this operation. + separate :mod:`!cmath` module function for this operation. .. function:: polar(z) @@ -357,7 +357,7 @@ Note that the selection of functions is similar, but not identical, to that in module :mod:`math`. The reason for having two modules is that some users aren't interested in complex numbers, and perhaps don't even know what they are. They would rather have ``math.sqrt(-1)`` raise an exception than return a complex -number. Also note that the functions defined in :mod:`cmath` always return a +number. Also note that the functions defined in :mod:`!cmath` always return a complex number, even if the answer can be expressed as a real number (in which case the complex number has an imaginary part of zero). diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index 66544f82f6f..1757dedabbf 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -245,7 +245,7 @@ Cmd Example .. sectionauthor:: Raymond Hettinger -The :mod:`cmd` module is mainly useful for building custom shells that let a +The :mod:`!cmd` module is mainly useful for building custom shells that let a user work with a program interactively. This section presents a simple example of how to build a shell around a few of diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 24b5a9d64b2..b4a8326e9a8 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -310,7 +310,7 @@ and writing to platform dependent files: Codec Base Classes ------------------ -The :mod:`codecs` module defines a set of base classes which define the +The :mod:`!codecs` module defines a set of base classes which define the interfaces for working with codec objects, and can also be used as the basis for custom codec implementations. @@ -1544,8 +1544,8 @@ mapping. It is not supported by :meth:`str.encode` (which only produces Restoration of the ``rot13`` alias. -:mod:`encodings` --- Encodings package --------------------------------------- +:mod:`!encodings` --- Encodings package +--------------------------------------- .. module:: encodings :synopsis: Encodings package @@ -1604,8 +1604,8 @@ This module implements the following exception: Raised when a codec is invalid or incompatible. -:mod:`encodings.idna` --- Internationalized Domain Names in Applications ------------------------------------------------------------------------- +:mod:`!encodings.idna` --- Internationalized Domain Names in Applications +------------------------------------------------------------------------- .. module:: encodings.idna :synopsis: Internationalized Domain Names implementation @@ -1647,7 +1647,7 @@ When receiving host names from the wire (such as in reverse name lookup), no automatic conversion to Unicode is performed: applications wishing to present such host names to the user should decode them to Unicode. -The module :mod:`encodings.idna` also implements the nameprep procedure, which +The module :mod:`!encodings.idna` also implements the nameprep procedure, which performs certain normalizations on host names, to achieve case-insensitivity of international domain names, and to unify similar characters. The nameprep functions can be used directly if desired. @@ -1670,8 +1670,8 @@ functions can be used directly if desired. Convert a label to Unicode, as specified in :rfc:`3490`. -:mod:`encodings.mbcs` --- Windows ANSI codepage ------------------------------------------------ +:mod:`!encodings.mbcs` --- Windows ANSI codepage +------------------------------------------------ .. module:: encodings.mbcs :synopsis: Windows ANSI codepage @@ -1688,8 +1688,8 @@ This module implements the ANSI codepage (CP_ACP). Support any error handler. -:mod:`encodings.utf_8_sig` --- UTF-8 codec with BOM signature -------------------------------------------------------------- +:mod:`!encodings.utf_8_sig` --- UTF-8 codec with BOM signature +-------------------------------------------------------------- .. module:: encodings.utf_8_sig :synopsis: UTF-8 codec with BOM signature diff --git a/Doc/library/codeop.rst b/Doc/library/codeop.rst index 16f674adb4b..2e6d6598038 100644 --- a/Doc/library/codeop.rst +++ b/Doc/library/codeop.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`codeop` module provides utilities upon which the Python +The :mod:`!codeop` module provides utilities upon which the Python read-eval-print loop can be emulated, as is done in the :mod:`code` module. As a result, you probably don't want to use the module directly; if you want to include such a loop in your program you probably want to use the :mod:`code` @@ -25,7 +25,7 @@ There are two parts to this job: #. Remembering which future statements the user has entered, so subsequent input can be compiled with these in effect. -The :mod:`codeop` module provides a way of doing each of these things, and a way +The :mod:`!codeop` module provides a way of doing each of these things, and a way of doing them both. To do just the former: diff --git a/Doc/library/colorsys.rst b/Doc/library/colorsys.rst index ffebf4e40dd..2d3dc2b8b57 100644 --- a/Doc/library/colorsys.rst +++ b/Doc/library/colorsys.rst @@ -10,7 +10,7 @@ -------------- -The :mod:`colorsys` module defines bidirectional conversions of color values +The :mod:`!colorsys` module defines bidirectional conversions of color values between colors expressed in the RGB (Red Green Blue) color space used in computer monitors and three other coordinate systems: YIQ, HLS (Hue Lightness Saturation) and HSV (Hue Saturation Value). Coordinates in all of these color @@ -24,7 +24,7 @@ spaces, the coordinates are all between 0 and 1. https://poynton.ca/ColorFAQ.html and https://www.cambridgeincolour.com/tutorials/color-spaces.htm. -The :mod:`colorsys` module defines the following functions: +The :mod:`!colorsys` module defines the following functions: .. function:: rgb_to_yiq(r, g, b) diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index e4b505c3f97..a32c3828313 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -12,7 +12,7 @@ and :source:`Lib/concurrent/futures/interpreter.py` -------------- -The :mod:`concurrent.futures` module provides a high-level interface for +The :mod:`!concurrent.futures` module provides a high-level interface for asynchronously executing callables. The asynchronous execution can be performed with threads, using @@ -156,7 +156,9 @@ And:: print(f.result()) executor = ThreadPoolExecutor(max_workers=1) - executor.submit(wait_on_future) + future = executor.submit(wait_on_future) + # Note: calling future.result() would also cause a deadlock because + # the single worker thread is already waiting for wait_on_future(). .. class:: ThreadPoolExecutor(max_workers=None, thread_name_prefix='', initializer=None, initargs=()) diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index bb109a9b742..f73252a9026 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -80,7 +80,7 @@ Let's take a very basic configuration file that looks like this: The structure of INI files is described `in the following section <#supported-ini-file-structure>`_. Essentially, the file consists of sections, each of which contains keys with values. -:mod:`configparser` classes can read and write such files. Let's start by +:mod:`!configparser` classes can read and write such files. Let's start by creating the above configuration file programmatically. .. doctest:: @@ -449,7 +449,7 @@ Mapping Protocol Access .. versionadded:: 3.2 Mapping protocol access is a generic name for functionality that enables using -custom objects as if they were dictionaries. In case of :mod:`configparser`, +custom objects as if they were dictionaries. In case of :mod:`!configparser`, the mapping interface implementation is using the ``parser['section']['option']`` notation. @@ -459,7 +459,7 @@ the original parser on demand. What's even more important is that when values are changed on a section proxy, they are actually mutated in the original parser. -:mod:`configparser` objects behave as close to actual dictionaries as possible. +:mod:`!configparser` objects behave as close to actual dictionaries as possible. The mapping interface is complete and adheres to the :class:`~collections.abc.MutableMapping` ABC. However, there are a few differences that should be taken into account: @@ -507,7 +507,7 @@ Customizing Parser Behaviour ---------------------------- There are nearly as many INI format variants as there are applications using it. -:mod:`configparser` goes a long way to provide support for the largest sensible +:mod:`!configparser` goes a long way to provide support for the largest sensible set of INI styles available. The default functionality is mainly dictated by historical background and it's very likely that you will want to customize some of the features. @@ -560,7 +560,7 @@ the :meth:`!__init__` options: * *allow_no_value*, default value: ``False`` Some configuration files are known to include settings without values, but - which otherwise conform to the syntax supported by :mod:`configparser`. The + which otherwise conform to the syntax supported by :mod:`!configparser`. The *allow_no_value* parameter to the constructor can be used to indicate that such values should be accepted: @@ -615,7 +615,7 @@ the :meth:`!__init__` options: prefixes for whole line comments. .. versionchanged:: 3.2 - In previous versions of :mod:`configparser` behaviour matched + In previous versions of :mod:`!configparser` behaviour matched ``comment_prefixes=('#',';')`` and ``inline_comment_prefixes=(';',)``. Please note that config parsers don't support escaping of comment prefixes so @@ -672,7 +672,7 @@ the :meth:`!__init__` options: parsers in new applications. .. versionchanged:: 3.2 - In previous versions of :mod:`configparser` behaviour matched + In previous versions of :mod:`!configparser` behaviour matched ``strict=False``. * *empty_lines_in_values*, default value: ``True`` @@ -842,7 +842,7 @@ be overridden by subclasses or by attribute assignment. Legacy API Examples ------------------- -Mainly because of backwards compatibility concerns, :mod:`configparser` +Mainly because of backwards compatibility concerns, :mod:`!configparser` provides also a legacy API with explicit ``get``/``set`` methods. While there are valid use cases for the methods outlined below, mapping protocol access is preferred for new projects. The legacy API is at times more advanced, @@ -1378,7 +1378,7 @@ Exceptions .. exception:: Error - Base class for all other :mod:`configparser` exceptions. + Base class for all other :mod:`!configparser` exceptions. .. exception:: NoSectionError diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 176be4ff333..7e221c9cabb 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -21,9 +21,9 @@ Functions and classes provided: .. class:: AbstractContextManager An :term:`abstract base class` for classes that implement - :meth:`object.__enter__` and :meth:`object.__exit__`. A default - implementation for :meth:`object.__enter__` is provided which returns - ``self`` while :meth:`object.__exit__` is an abstract method which by default + :meth:`~object.__enter__` and :meth:`~object.__exit__`. A default + implementation for :meth:`~object.__enter__` is provided which returns + ``self`` while :meth:`~object.__exit__` is an abstract method which by default returns ``None``. See also the definition of :ref:`typecontextmanager`. .. versionadded:: 3.6 @@ -32,9 +32,9 @@ Functions and classes provided: .. class:: AbstractAsyncContextManager An :term:`abstract base class` for classes that implement - :meth:`object.__aenter__` and :meth:`object.__aexit__`. A default - implementation for :meth:`object.__aenter__` is provided which returns - ``self`` while :meth:`object.__aexit__` is an abstract method which by default + :meth:`~object.__aenter__` and :meth:`~object.__aexit__`. A default + implementation for :meth:`~object.__aenter__` is provided which returns + ``self`` while :meth:`~object.__aexit__` is an abstract method which by default returns ``None``. See also the definition of :ref:`async-context-managers`. @@ -228,7 +228,7 @@ Functions and classes provided: .. function:: nullcontext(enter_result=None) - Return a context manager that returns *enter_result* from ``__enter__``, but + Return a context manager that returns *enter_result* from :meth:`~object.__enter__`, but otherwise does nothing. It is intended to be used as a stand-in for an optional context manager, for example:: @@ -335,7 +335,7 @@ Functions and classes provided: For example, the output of :func:`help` normally is sent to *sys.stdout*. You can capture that output in a string by redirecting the output to an :class:`io.StringIO` object. The replacement stream is returned from the - ``__enter__`` method and so is available as the target of the + :meth:`~object.__enter__` method and so is available as the target of the :keyword:`with` statement:: with redirect_stdout(io.StringIO()) as f: @@ -396,7 +396,8 @@ Functions and classes provided: A base class that enables a context manager to also be used as a decorator. Context managers inheriting from ``ContextDecorator`` have to implement - ``__enter__`` and ``__exit__`` as normal. ``__exit__`` retains its optional + :meth:`~object.__enter__` and :meth:`~object.__exit__` as normal. + ``__exit__`` retains its optional exception handling even when used as a decorator. ``ContextDecorator`` is used by :func:`contextmanager`, so you get this @@ -668,7 +669,7 @@ Examples and Recipes -------------------- This section describes some examples and recipes for making effective use of -the tools provided by :mod:`contextlib`. +the tools provided by :mod:`!contextlib`. Supporting a variable number of context managers @@ -697,9 +698,9 @@ context management protocol. Catching exceptions from ``__enter__`` methods ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -It is occasionally desirable to catch exceptions from an ``__enter__`` +It is occasionally desirable to catch exceptions from an :meth:`~object.__enter__` method implementation, *without* inadvertently catching exceptions from -the :keyword:`with` statement body or the context manager's ``__exit__`` +the :keyword:`with` statement body or the context manager's :meth:`~object.__exit__` method. By using :class:`ExitStack` the steps in the context management protocol can be separated slightly in order to allow this:: diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 210ad718800..121c44a16ad 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -80,7 +80,7 @@ of lists by assigning a slice of the entire list, for example, Classes can use the same interfaces to control copying that they use to control pickling. See the description of module :mod:`pickle` for information on these -methods. In fact, the :mod:`copy` module uses the registered +methods. In fact, the :mod:`!copy` module uses the registered pickle functions from the :mod:`copyreg` module. .. index:: diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 6e3144824eb..d59936029da 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -12,7 +12,7 @@ -------------- -The :mod:`copyreg` module offers a way to define functions used while pickling +The :mod:`!copyreg` module offers a way to define functions used while pickling specific objects. The :mod:`pickle` and :mod:`copy` modules use those functions when pickling/copying those objects. The module provides configuration information about object constructors which are not classes. diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 4a033d823e6..5c086ab9422 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -25,14 +25,14 @@ similar enough that it is possible to write a single module which can efficiently manipulate such data, hiding the details of reading and writing the data from the programmer. -The :mod:`csv` module implements classes to read and write tabular data in CSV +The :mod:`!csv` module implements classes to read and write tabular data in CSV format. It allows programmers to say, "write this data in the format preferred by Excel," or "read data from this file which was generated by Excel," without knowing the precise details of the CSV format used by Excel. Programmers can also describe the CSV formats understood by other applications or define their own special-purpose CSV formats. -The :mod:`csv` module's :class:`reader` and :class:`writer` objects read and +The :mod:`!csv` module's :class:`reader` and :class:`writer` objects read and write sequences. Programmers can also read and write data in dictionary form using the :class:`DictReader` and :class:`DictWriter` classes. @@ -47,7 +47,7 @@ using the :class:`DictReader` and :class:`DictWriter` classes. Module Contents --------------- -The :mod:`csv` module defines the following functions: +The :mod:`!csv` module defines the following functions: .. index:: @@ -146,7 +146,7 @@ The :mod:`csv` module defines the following functions: given, this becomes the new limit. -The :mod:`csv` module defines the following classes: +The :mod:`!csv` module defines the following classes: .. class:: DictReader(f, fieldnames=None, restkey=None, restval=None, \ dialect='excel', *args, **kwds) @@ -314,7 +314,7 @@ An example for :class:`Sniffer` use:: .. _csv-constants: -The :mod:`csv` module defines the following constants: +The :mod:`!csv` module defines the following constants: .. data:: QUOTE_ALL @@ -375,7 +375,7 @@ The :mod:`csv` module defines the following constants: .. versionadded:: 3.12 -The :mod:`csv` module defines the following exception: +The :mod:`!csv` module defines the following exception: .. exception:: Error diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index d2f4da08327..7ec4941c444 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -10,7 +10,7 @@ -------------- -:mod:`ctypes` is a foreign function library for Python. It provides C compatible +:mod:`!ctypes` is a foreign function library for Python. It provides C compatible data types, and allows calling functions in DLLs or shared libraries. It can be used to wrap these libraries in pure Python. @@ -22,10 +22,6 @@ used to wrap these libraries in pure Python. ctypes tutorial --------------- -Note: The code samples in this tutorial use :mod:`doctest` to make sure that -they actually work. Since some code samples behave differently under Linux, -Windows, or macOS, they contain doctest directives in comments. - Note: Some code samples reference the ctypes :class:`c_int` type. On platforms where ``sizeof(long) == sizeof(int)`` it is an alias to :class:`c_long`. So, you should not be confused if :class:`c_long` is printed if you would expect @@ -36,13 +32,16 @@ So, you should not be confused if :class:`c_long` is printed if you would expect Loading dynamic link libraries ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` exports the *cdll*, and on Windows *windll* and *oledll* +:mod:`!ctypes` exports the :py:data:`~ctypes.cdll`, and on Windows +:py:data:`~ctypes.windll` and :py:data:`~ctypes.oledll` objects, for loading dynamic link libraries. -You load libraries by accessing them as attributes of these objects. *cdll* -loads libraries which export functions using the standard ``cdecl`` calling -convention, while *windll* libraries call functions using the ``stdcall`` -calling convention. *oledll* also uses the ``stdcall`` calling convention, and +You load libraries by accessing them as attributes of these objects. +:py:data:`!cdll` loads libraries which export functions using the +standard ``cdecl`` calling convention, while :py:data:`!windll` +libraries call functions using the ``stdcall`` +calling convention. +:py:data:`~oledll` also uses the ``stdcall`` calling convention, and assumes the functions return a Windows :c:type:`!HRESULT` error code. The error code is used to automatically raise an :class:`OSError` exception when the function call fails. @@ -72,11 +71,13 @@ Windows appends the usual ``.dll`` file suffix automatically. being used by Python. Where possible, use native Python functionality, or else import and use the ``msvcrt`` module. -On Linux, it is required to specify the filename *including* the extension to +Other systems require the filename *including* the extension to load a library, so attribute access can not be used to load libraries. Either the :meth:`~LibraryLoader.LoadLibrary` method of the dll loaders should be used, -or you should load the library by creating an instance of CDLL by calling -the constructor:: +or you should load the library by creating an instance of :py:class:`CDLL` +by calling the constructor. + +For example, on Linux:: >>> cdll.LoadLibrary("libc.so.6") # doctest: +LINUX @@ -85,7 +86,14 @@ the constructor:: >>> -.. XXX Add section for macOS. +On macOS:: + + >>> cdll.LoadLibrary("libc.dylib") # doctest: +MACOS + + >>> libc = CDLL("libc.dylib") # doctest: +MACOS + >>> libc # doctest: +MACOS + + .. _ctypes-accessing-functions-from-loaded-dlls: @@ -182,7 +190,7 @@ handle (passing ``None`` as single argument to call it with a ``NULL`` pointer): To find out the correct calling convention you have to look into the C header file or the documentation for the function you want to call. -On Windows, :mod:`ctypes` uses win32 structured exception handling to prevent +On Windows, :mod:`!ctypes` uses win32 structured exception handling to prevent crashes from general protection faults when functions are called with invalid argument values:: @@ -192,7 +200,7 @@ argument values:: OSError: exception: access violation reading 0x00000020 >>> -There are, however, enough ways to crash Python with :mod:`ctypes`, so you +There are, however, enough ways to crash Python with :mod:`!ctypes`, so you should be careful anyway. The :mod:`faulthandler` module can be helpful in debugging crashes (e.g. from segmentation faults produced by erroneous C library calls). @@ -205,7 +213,7 @@ as pointer to the memory block that contains their data (:c:expr:`char *` or :c:expr:`int` type, their value is masked to fit into the C type. Before we move on calling functions with other parameter types, we have to learn -more about :mod:`ctypes` data types. +more about :mod:`!ctypes` data types. .. _ctypes-fundamental-data-types: @@ -213,7 +221,7 @@ more about :mod:`ctypes` data types. Fundamental data types ^^^^^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` defines a number of primitive C compatible data types: +:mod:`!ctypes` defines a number of primitive C compatible data types: +----------------------+------------------------------------------+----------------------------+ | ctypes type | C type | Python type | @@ -397,7 +405,7 @@ from within *IDLE* or *PythonWin*:: >>> As has been mentioned before, all Python types except integers, strings, and -bytes objects have to be wrapped in their corresponding :mod:`ctypes` type, so +bytes objects have to be wrapped in their corresponding :mod:`!ctypes` type, so that they can be converted to the required C data type:: >>> printf(b"An int %d, a double %f\n", 1234, c_double(3.14)) @@ -431,10 +439,10 @@ specify :attr:`~_CFuncPtr.argtypes` for all variadic functions. Calling functions with your own custom data types ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -You can also customize :mod:`ctypes` argument conversion to allow instances of -your own classes be used as function arguments. :mod:`ctypes` looks for an +You can also customize :mod:`!ctypes` argument conversion to allow instances of +your own classes be used as function arguments. :mod:`!ctypes` looks for an :attr:`!_as_parameter_` attribute and uses this as the function argument. The -attribute must be an integer, string, bytes, a :mod:`ctypes` instance, or an +attribute must be an integer, string, bytes, a :mod:`!ctypes` instance, or an object with an :attr:`!_as_parameter_` attribute:: >>> class Bottles: @@ -490,7 +498,7 @@ the Python object passed to the function call, it should do a typecheck or whatever is needed to make sure this object is acceptable, and then return the object itself, its :attr:`!_as_parameter_` attribute, or whatever you want to pass as the C function argument in this case. Again, the result should be an -integer, string, bytes, a :mod:`ctypes` instance, or an object with an +integer, string, bytes, a :mod:`!ctypes` instance, or an object with an :attr:`!_as_parameter_` attribute. @@ -600,7 +608,7 @@ Sometimes a C api function expects a *pointer* to a data type as parameter, probably to write into the corresponding location, or if the data is too large to be passed by value. This is also known as *passing parameters by reference*. -:mod:`ctypes` exports the :func:`byref` function which is used to pass parameters +:mod:`!ctypes` exports the :func:`byref` function which is used to pass parameters by reference. The same effect can be achieved with the :func:`pointer` function, although :func:`pointer` does a lot more work since it constructs a real pointer object, so it is faster to use :func:`byref` if you don't need the pointer @@ -625,12 +633,12 @@ Structures and unions ^^^^^^^^^^^^^^^^^^^^^ Structures and unions must derive from the :class:`Structure` and :class:`Union` -base classes which are defined in the :mod:`ctypes` module. Each subclass must +base classes which are defined in the :mod:`!ctypes` module. Each subclass must define a :attr:`~Structure._fields_` attribute. :attr:`!_fields_` must be a list of *2-tuples*, containing a *field name* and a *field type*. -The field type must be a :mod:`ctypes` type like :class:`c_int`, or any other -derived :mod:`ctypes` type: structure, union, array, pointer. +The field type must be a :mod:`!ctypes` type like :class:`c_int`, or any other +derived :mod:`!ctypes` type: structure, union, array, pointer. Here is a simple example of a POINT structure, which contains two integers named *x* and *y*, and also shows how to initialize a structure in the constructor:: @@ -689,7 +697,7 @@ See :class:`CField`:: .. warning:: - :mod:`ctypes` does not support passing unions or structures with bit-fields + :mod:`!ctypes` does not support passing unions or structures with bit-fields to functions by value. While this may work on 32-bit x86, it's not guaranteed by the library to work in the general case. Unions and structures with bit-fields should always be passed to functions by pointer. @@ -707,7 +715,7 @@ structure itself by setting the class attributes :attr:`~Structure._pack_` and/or :attr:`~Structure._align_`, respectively. See the attribute documentation for details. -:mod:`ctypes` uses the native byte order for Structures and Unions. To build +:mod:`!ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the :class:`BigEndianStructure`, :class:`LittleEndianStructure`, :class:`BigEndianUnion`, and :class:`LittleEndianUnion` base classes. These @@ -796,7 +804,7 @@ Pointers ^^^^^^^^ Pointer instances are created by calling the :func:`pointer` function on a -:mod:`ctypes` type:: +:mod:`!ctypes` type:: >>> from ctypes import * >>> i = c_int(42) @@ -810,7 +818,7 @@ returns the object to which the pointer points, the ``i`` object above:: c_long(42) >>> -Note that :mod:`ctypes` does not have OOR (original object return), it constructs a +Note that :mod:`!ctypes` does not have OOR (original object return), it constructs a new, equivalent object each time you retrieve an attribute:: >>> pi.contents is i @@ -854,7 +862,7 @@ item. Behind the scenes, the :func:`pointer` function does more than simply create pointer instances, it has to create pointer *types* first. This is done with the -:func:`POINTER` function, which accepts any :mod:`ctypes` type, and returns a +:func:`POINTER` function, which accepts any :mod:`!ctypes` type, and returns a new type:: >>> PI = POINTER(c_int) @@ -876,7 +884,7 @@ Calling the pointer type without an argument creates a ``NULL`` pointer. False >>> -:mod:`ctypes` checks for ``NULL`` when dereferencing pointers (but dereferencing +:mod:`!ctypes` checks for ``NULL`` when dereferencing pointers (but dereferencing invalid non-\ ``NULL`` pointers would crash Python):: >>> null_ptr[0] @@ -961,7 +969,7 @@ To set a POINTER type field to ``NULL``, you can assign ``None``:: .. XXX list other conversions... Sometimes you have instances of incompatible types. In C, you can cast one type -into another type. :mod:`ctypes` provides a :func:`cast` function which can be +into another type. :mod:`!ctypes` provides a :func:`cast` function which can be used in the same way. The ``Bar`` structure defined above accepts ``POINTER(c_int)`` pointers or :class:`c_int` arrays for its ``values`` field, but not instances of other types:: @@ -1025,7 +1033,7 @@ work:: >>> because the new ``class cell`` is not available in the class statement itself. -In :mod:`ctypes`, we can define the ``cell`` class and set the +In :mod:`!ctypes`, we can define the ``cell`` class and set the :attr:`~Structure._fields_` attribute later, after the class statement:: >>> from ctypes import * @@ -1059,7 +1067,7 @@ other, and finally follow the pointer chain a few times:: Callback functions ^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` allows creating C callable function pointers from Python callables. +:mod:`!ctypes` allows creating C callable function pointers from Python callables. These are sometimes called *callback functions*. First, you must create a class for the callback function. The class knows the @@ -1158,7 +1166,7 @@ write:: .. note:: Make sure you keep references to :func:`CFUNCTYPE` objects as long as they - are used from C code. :mod:`ctypes` doesn't, and if you don't, they may be + are used from C code. :mod:`!ctypes` doesn't, and if you don't, they may be garbage collected, crashing your program when a callback is made. Also, note that if the callback function is called in a thread created @@ -1177,7 +1185,7 @@ Some shared libraries not only export functions, they also export variables. An example in the Python library itself is the :c:data:`Py_Version`, Python runtime version number encoded in a single constant integer. -:mod:`ctypes` can access values like this with the :meth:`~_CData.in_dll` class methods of +:mod:`!ctypes` can access values like this with the :meth:`~_CData.in_dll` class methods of the type. *pythonapi* is a predefined symbol giving access to the Python C api:: @@ -1196,7 +1204,7 @@ Quoting the docs for that value: tricks with this to provide a dynamically created collection of frozen modules. So manipulating this pointer could even prove useful. To restrict the example -size, we show only how this table can be read with :mod:`ctypes`:: +size, we show only how this table can be read with :mod:`!ctypes`:: >>> from ctypes import * >>> @@ -1242,7 +1250,7 @@ for testing. Try it out with ``import __hello__`` for example. Surprises ^^^^^^^^^ -There are some edges in :mod:`ctypes` where you might expect something other +There are some edges in :mod:`!ctypes` where you might expect something other than what actually happens. Consider the following example:: @@ -1310,7 +1318,7 @@ constructs a new Python object each time! Variable-sized data types ^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` provides some support for variable-sized arrays and structures. +:mod:`!ctypes` provides some support for variable-sized arrays and structures. The :func:`resize` function can be used to resize the memory buffer of an existing ctypes object. The function takes the object as first argument, and @@ -1344,7 +1352,7 @@ get errors accessing other elements:: IndexError: invalid index >>> -Another way to use variable-sized data types with :mod:`ctypes` is to use the +Another way to use variable-sized data types with :mod:`!ctypes` is to use the dynamic nature of Python, and (re-)define the data type after the required size is already known, on a case by case basis. @@ -1425,7 +1433,7 @@ On Windows, :func:`~ctypes.util.find_library` searches along the system search p returns the full pathname, but since there is no predefined naming scheme a call like ``find_library("c")`` will fail and return ``None``. -If wrapping a shared library with :mod:`ctypes`, it *may* be better to determine +If wrapping a shared library with :mod:`!ctypes`, it *may* be better to determine the shared library name at development time, and hardcode that into the wrapper module instead of using :func:`~ctypes.util.find_library` to locate the library at runtime. @@ -1458,14 +1466,82 @@ Loading shared libraries ^^^^^^^^^^^^^^^^^^^^^^^^ There are several ways to load shared libraries into the Python process. One -way is to instantiate one of the following classes: +way is to instantiate :py:class:`CDLL` or one of its subclasses: .. class:: CDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) - Instances of this class represent loaded shared libraries. Functions in these - libraries use the standard C calling convention, and are assumed to return - :c:expr:`int`. + Represents a loaded shared library. + + Functions in this library use the standard C calling convention, and are + assumed to return :c:expr:`int`. + The Python :term:`global interpreter lock` is released before calling any + function exported by these libraries, and reacquired afterwards. + For different function behavior, use a subclass: :py:class:`~ctypes.OleDLL`, + :py:class:`~ctypes.WinDLL`, or :py:class:`~ctypes.PyDLL`. + + If you have an existing :py:attr:`handle ` to an already + loaded shared library, it can be passed as the *handle* argument to wrap + the opened library in a new :py:class:`!CDLL` object. + In this case, *name* is only used to set the :py:attr:`~ctypes.CDLL._name` + attribute, but it may be adjusted and/or validated. + + If *handle* is ``None``, the underlying platform's :manpage:`dlopen(3)` or + :c:func:`!LoadLibrary` function is used to load the library into + the process, and to get a handle to it. + + *name* is the pathname of the shared library to open. + If *name* does not contain a path separator, the library is found + in a platform-specific way. + + On non-Windows systems, *name* can be ``None``. In this case, + :c:func:`!dlopen` is called with ``NULL``, which opens the main program + as a "library". + (Some systems do the same is *name* is empty; ``None``/``NULL`` is more + portable.) + + .. admonition:: CPython implementation detail + + Since CPython is linked to ``libc``, a ``None`` *name* is often used + to access the C standard library:: + + >>> printf = ctypes.CDLL(None).printf + >>> printf.argtypes = [ctypes.c_char_p] + >>> printf(b"hello\n") + hello + 6 + + To access the Python C API, prefer :py:data:`ctypes.pythonapi` which + works across platforms. + + The *mode* parameter can be used to specify how the library is loaded. For + details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is + ignored. On posix systems, RTLD_NOW is always added, and is not + configurable. + + The *use_errno* parameter, when set to true, enables a ctypes mechanism that + allows accessing the system :data:`errno` error number in a safe way. + :mod:`!ctypes` maintains a thread-local copy of the system's :data:`errno` + variable; if you call foreign functions created with ``use_errno=True`` then the + :data:`errno` value before the function call is swapped with the ctypes private + copy, the same happens immediately after the function call. + + The function :func:`ctypes.get_errno` returns the value of the ctypes private + copy, and the function :func:`ctypes.set_errno` changes the ctypes private copy + to a new value and returns the former value. + + The *use_last_error* parameter, when set to true, enables the same mechanism for + the Windows error code which is managed by the :func:`GetLastError` and + :func:`!SetLastError` Windows API functions; :func:`ctypes.get_last_error` and + :func:`ctypes.set_last_error` are used to request and change the ctypes private + copy of the windows error code. + + The *winmode* parameter is used on Windows to specify how the library is loaded + (since *mode* is ignored). It takes any value that is valid for the Win32 API + ``LoadLibraryEx`` flags parameter. When omitted, the default is to use the + flags that result in the most secure DLL load, which avoids issues such as DLL + hijacking. Passing the full path to the DLL is the safest way to ensure the + correct library and dependencies are loaded. On Windows creating a :class:`CDLL` instance may fail even if the DLL name exists. When a dependent DLL of the loaded DLL is not found, a @@ -1477,20 +1553,47 @@ way is to instantiate one of the following classes: DLLs and determine which one is not found using Windows debugging and tracing tools. + .. seealso:: + + `Microsoft DUMPBIN tool `_ + -- A tool to find DLL dependents. + + .. versionchanged:: 3.8 + Added *winmode* parameter. + .. versionchanged:: 3.12 The *name* parameter can now be a :term:`path-like object`. -.. seealso:: + Instances of this class have no public methods. Functions exported by the + shared library can be accessed as attributes or by index. Please note that + accessing the function through an attribute caches the result and therefore + accessing it repeatedly returns the same object each time. On the other hand, + accessing it through an index returns a new object each time:: + + >>> from ctypes import CDLL + >>> libc = CDLL("libc.so.6") # On Linux + >>> libc.time == libc.time + True + >>> libc['time'] == libc['time'] + False + + The following public attributes are available. Their name starts with an + underscore to not clash with exported function names: + + .. attribute:: _handle + + The system handle used to access the library. - `Microsoft DUMPBIN tool `_ - -- A tool to find DLL dependents. + .. attribute:: _name + The name of the library passed in the constructor. -.. class:: OleDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) +.. class:: OleDLL - Instances of this class represent loaded shared libraries, - functions in these libraries use the ``stdcall`` calling convention, and are + See :py:class:`~ctypes.CDLL`, the superclass, for common information. + + Functions in this library use the ``stdcall`` calling convention, and are assumed to return the windows specific :class:`HRESULT` code. :class:`HRESULT` values contain information specifying whether the function call failed or succeeded, together with additional error code. If the return value signals a @@ -1502,133 +1605,51 @@ way is to instantiate one of the following classes: :exc:`WindowsError` used to be raised, which is now an alias of :exc:`OSError`. - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. +.. class:: WinDLL -.. class:: WinDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) + See :py:class:`~ctypes.CDLL`, the superclass, for common information. - Instances of this class represent loaded shared libraries, - functions in these libraries use the ``stdcall`` calling convention, and are + Functions in these libraries use the ``stdcall`` calling convention, and are assumed to return :c:expr:`int` by default. .. availability:: Windows - .. versionchanged:: 3.12 +.. class:: PyDLL - The *name* parameter can now be a :term:`path-like object`. + See :py:class:`~ctypes.CDLL`, the superclass, for common information. - -The Python :term:`global interpreter lock` is released before calling any -function exported by these libraries, and reacquired afterwards. - - -.. class:: PyDLL(name, mode=DEFAULT_MODE, handle=None) - - Instances of this class behave like :class:`CDLL` instances, except that the + When functions in this library are called, the Python GIL is *not* released during the function call, and after the function execution the Python error flag is checked. If the error flag is set, a Python exception is raised. - Thus, this is only useful to call Python C api functions directly. - - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. - -All these classes can be instantiated by calling them with at least one -argument, the pathname of the shared library. If you have an existing handle to -an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platform's :c:func:`!dlopen` or -:c:func:`!LoadLibrary` function is used to load the library into -the process, and to get a handle to it. - -The *mode* parameter can be used to specify how the library is loaded. For -details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is -ignored. On posix systems, RTLD_NOW is always added, and is not -configurable. - -The *use_errno* parameter, when set to true, enables a ctypes mechanism that -allows accessing the system :data:`errno` error number in a safe way. -:mod:`ctypes` maintains a thread-local copy of the system's :data:`errno` -variable; if you call foreign functions created with ``use_errno=True`` then the -:data:`errno` value before the function call is swapped with the ctypes private -copy, the same happens immediately after the function call. - -The function :func:`ctypes.get_errno` returns the value of the ctypes private -copy, and the function :func:`ctypes.set_errno` changes the ctypes private copy -to a new value and returns the former value. - -The *use_last_error* parameter, when set to true, enables the same mechanism for -the Windows error code which is managed by the :func:`GetLastError` and -:func:`!SetLastError` Windows API functions; :func:`ctypes.get_last_error` and -:func:`ctypes.set_last_error` are used to request and change the ctypes private -copy of the windows error code. - -The *winmode* parameter is used on Windows to specify how the library is loaded -(since *mode* is ignored). It takes any value that is valid for the Win32 API -``LoadLibraryEx`` flags parameter. When omitted, the default is to use the -flags that result in the most secure DLL load, which avoids issues such as DLL -hijacking. Passing the full path to the DLL is the safest way to ensure the -correct library and dependencies are loaded. - -.. versionchanged:: 3.8 - Added *winmode* parameter. + Thus, this is only useful to call Python C API functions directly. .. data:: RTLD_GLOBAL - :noindex: Flag to use as *mode* parameter. On platforms where this flag is not available, it is defined as the integer zero. .. data:: RTLD_LOCAL - :noindex: Flag to use as *mode* parameter. On platforms where this is not available, it is the same as *RTLD_GLOBAL*. .. data:: DEFAULT_MODE - :noindex: The default mode which is used to load shared libraries. On OSX 10.3, this is *RTLD_GLOBAL*, otherwise it is the same as *RTLD_LOCAL*. -Instances of these classes have no public methods. Functions exported by the -shared library can be accessed as attributes or by index. Please note that -accessing the function through an attribute caches the result and therefore -accessing it repeatedly returns the same object each time. On the other hand, -accessing it through an index returns a new object each time:: - - >>> from ctypes import CDLL - >>> libc = CDLL("libc.so.6") # On Linux - >>> libc.time == libc.time - True - >>> libc['time'] == libc['time'] - False - -The following public attributes are available, their name starts with an -underscore to not clash with exported function names: - - -.. attribute:: PyDLL._handle - - The system handle used to access the library. - - -.. attribute:: PyDLL._name - - The name of the library passed in the constructor. Shared libraries can also be loaded by using one of the prefabricated objects, which are instances of the :class:`LibraryLoader` class, either by calling the :meth:`~LibraryLoader.LoadLibrary` method, or by retrieving the library as attribute of the loader instance. - .. class:: LibraryLoader(dlltype) Class which loads shared libraries. *dlltype* should be one of the @@ -1647,13 +1668,11 @@ attribute of the loader instance. These prefabricated library loaders are available: .. data:: cdll - :noindex: Creates :class:`CDLL` instances. .. data:: windll - :noindex: Creates :class:`WinDLL` instances. @@ -1661,7 +1680,6 @@ These prefabricated library loaders are available: .. data:: oledll - :noindex: Creates :class:`OleDLL` instances. @@ -1669,7 +1687,6 @@ These prefabricated library loaders are available: .. data:: pydll - :noindex: Creates :class:`PyDLL` instances. @@ -1678,7 +1695,6 @@ For accessing the C Python api directly, a ready-to-use Python shared library object is available: .. data:: pythonapi - :noindex: An instance of :class:`PyDLL` that exposes Python C API functions as attributes. Note that all these functions are assumed to return C @@ -1929,7 +1945,7 @@ the windows header file is this:: LPCWSTR lpCaption, UINT uType); -Here is the wrapping with :mod:`ctypes`:: +Here is the wrapping with :mod:`!ctypes`:: >>> from ctypes import c_int, WINFUNCTYPE, windll >>> from ctypes.wintypes import HWND, LPCWSTR, UINT @@ -1952,7 +1968,7 @@ function retrieves the dimensions of a specified window by copying them into HWND hWnd, LPRECT lpRect); -Here is the wrapping with :mod:`ctypes`:: +Here is the wrapping with :mod:`!ctypes`:: >>> from ctypes import POINTER, WINFUNCTYPE, windll, WinError >>> from ctypes.wintypes import BOOL, HWND, RECT @@ -1980,7 +1996,7 @@ do the error checking, and raises an exception when the api call failed:: >>> If the :attr:`~_CFuncPtr.errcheck` function returns the argument tuple it receives -unchanged, :mod:`ctypes` continues the normal processing it does on the output +unchanged, :mod:`!ctypes` continues the normal processing it does on the output parameters. If you want to return a tuple of window coordinates instead of a ``RECT`` instance, you can retrieve the fields in the function and return them instead, the normal processing will no longer take place:: @@ -2450,7 +2466,7 @@ Fundamental data types Python bytes object or string. When the ``value`` attribute is retrieved from a ctypes instance, usually - a new object is returned each time. :mod:`ctypes` does *not* implement + a new object is returned each time. :mod:`!ctypes` does *not* implement original object return, always a new object is constructed. The same is true for all other ctypes object instances. @@ -2749,7 +2765,7 @@ fields, or any other data types containing pointer type fields. Abstract base class for structures in *native* byte order. Concrete structure and union types must be created by subclassing one of these - types, and at least define a :attr:`_fields_` class variable. :mod:`ctypes` will + types, and at least define a :attr:`_fields_` class variable. :mod:`!ctypes` will create :term:`descriptor`\s which allow reading and writing the fields by direct attribute accesses. These are the @@ -2803,7 +2819,7 @@ fields, or any other data types containing pointer type fields. Setting :attr:`!_pack_` to 0 is the same as not setting it at all. Otherwise, the value must be a positive power of two. The effect is equivalent to ``#pragma pack(N)`` in C, except - :mod:`ctypes` may allow larger *n* than what the compiler accepts. + :mod:`!ctypes` may allow larger *n* than what the compiler accepts. :attr:`!_pack_` must already be defined when :attr:`_fields_` is assigned, otherwise it will have no effect. @@ -2824,7 +2840,7 @@ fields, or any other data types containing pointer type fields. The value must not be negative. The effect is equivalent to ``__attribute__((aligned(N)))`` on GCC - or ``#pragma align(N)`` on MSVC, except :mod:`ctypes` may allow + or ``#pragma align(N)`` on MSVC, except :mod:`!ctypes` may allow values that the compiler would reject. :attr:`!_align_` can only *increase* a structure's alignment @@ -2873,7 +2889,7 @@ fields, or any other data types containing pointer type fields. assigned, otherwise it will have no effect. The fields listed in this variable must be structure or union type fields. - :mod:`ctypes` will create descriptors in the structure type that allows + :mod:`!ctypes` will create descriptors in the structure type that allows accessing the nested fields directly, without the need to create the structure or union field. @@ -3017,7 +3033,7 @@ Arrays and pointers Abstract base class for arrays. The recommended way to create concrete array types is by multiplying any - :mod:`ctypes` data type with a non-negative integer. Alternatively, you can subclass + :mod:`!ctypes` data type with a non-negative integer. Alternatively, you can subclass this type and define :attr:`_length_` and :attr:`_type_` class variables. Array elements can be read and written using standard subscript and slice accesses; for slice reads, the resulting object is @@ -3043,7 +3059,7 @@ Arrays and pointers Create an array. Equivalent to ``type * length``, where *type* is a - :mod:`ctypes` data type and *length* an integer. + :mod:`!ctypes` data type and *length* an integer. This function is :term:`soft deprecated` in favor of multiplication. There are no plans to remove it. diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index cb895664ff1..4910954b778 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`curses.ascii` module supplies name constants for ASCII characters and +The :mod:`!curses.ascii` module supplies name constants for ASCII characters and functions to test membership in various ASCII character classes. The constants supplied are names for control characters as follows: diff --git a/Doc/library/curses.panel.rst b/Doc/library/curses.panel.rst index 11fd841d381..e52f588c5bc 100644 --- a/Doc/library/curses.panel.rst +++ b/Doc/library/curses.panel.rst @@ -18,7 +18,7 @@ displayed. Panels can be added, moved up or down in the stack, and removed. Functions --------- -The module :mod:`curses.panel` defines the following functions: +The module :mod:`!curses.panel` defines the following functions: .. function:: bottom_panel() diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 057d338edda..2dc638b3d40 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -4,7 +4,6 @@ .. module:: curses :synopsis: An interface to the curses library, providing portable terminal handling. - :platform: Unix .. sectionauthor:: Moshe Zadka .. sectionauthor:: Eric Raymond @@ -13,7 +12,7 @@ -------------- -The :mod:`curses` module provides an interface to the curses library, the +The :mod:`!curses` module provides an interface to the curses library, the de-facto standard for portable advanced terminal handling. While curses is most widely used in the Unix environment, versions are available @@ -25,6 +24,8 @@ Linux and the BSD variants of Unix. .. include:: ../includes/optional-module.rst +.. availability:: Unix. + .. note:: Whenever the documentation mentions a *character* it can be specified @@ -54,7 +55,7 @@ Linux and the BSD variants of Unix. Functions --------- -The module :mod:`curses` defines the following exception: +The module :mod:`!curses` defines the following exception: .. exception:: error @@ -67,7 +68,7 @@ The module :mod:`curses` defines the following exception: default to the current cursor location. Whenever *attr* is optional, it defaults to :const:`A_NORMAL`. -The module :mod:`curses` defines the following functions: +The module :mod:`!curses` defines the following functions: .. function:: assume_default_colors(fg, bg, /) @@ -581,7 +582,7 @@ The module :mod:`curses` defines the following functions: after :func:`initscr`. :func:`start_color` initializes eight basic colors (black, red, green, yellow, - blue, magenta, cyan, and white), and two global variables in the :mod:`curses` + blue, magenta, cyan, and white), and two global variables in the :mod:`!curses` module, :const:`COLORS` and :const:`COLOR_PAIRS`, containing the maximum number of colors and color-pairs the terminal can support. It also restores the colors on the terminal to the values they had when the terminal was just turned on. @@ -1021,7 +1022,7 @@ Window Objects .. method:: window.idlok(flag) - If *flag* is ``True``, :mod:`curses` will try and use hardware line + If *flag* is ``True``, :mod:`!curses` will try and use hardware line editing facilities. Otherwise, line insertion/deletion are disabled. @@ -1109,7 +1110,7 @@ Window Objects .. method:: window.keypad(flag) If *flag* is ``True``, escape sequences generated by some keys (keypad, function keys) - will be interpreted by :mod:`curses`. If *flag* is ``False``, escape sequences will be + will be interpreted by :mod:`!curses`. If *flag* is ``False``, escape sequences will be left as is in the input stream. @@ -1335,7 +1336,7 @@ Window Objects Constants --------- -The :mod:`curses` module defines the following data members: +The :mod:`!curses` module defines the following data members: .. data:: ERR @@ -1824,8 +1825,8 @@ The following table lists the predefined colors: +-------------------------+----------------------------+ -:mod:`curses.textpad` --- Text input widget for curses programs -=============================================================== +:mod:`!curses.textpad` --- Text input widget for curses programs +================================================================ .. module:: curses.textpad :synopsis: Emacs-like input editing in a curses window. @@ -1833,13 +1834,13 @@ The following table lists the predefined colors: .. sectionauthor:: Eric Raymond -The :mod:`curses.textpad` module provides a :class:`Textbox` class that handles +The :mod:`!curses.textpad` module provides a :class:`Textbox` class that handles elementary text editing in a curses window, supporting a set of keybindings resembling those of Emacs (thus, also of Netscape Navigator, BBedit 6.x, FrameMaker, and many other programs). The module also provides a rectangle-drawing function useful for framing text boxes or for other purposes. -The module :mod:`curses.textpad` defines the following function: +The module :mod:`!curses.textpad` defines the following function: .. function:: rectangle(win, uly, ulx, lry, lrx) diff --git a/Doc/library/datetime-inheritance.dot b/Doc/library/datetime-inheritance.dot new file mode 100644 index 00000000000..3c6b9b4beb7 --- /dev/null +++ b/Doc/library/datetime-inheritance.dot @@ -0,0 +1,31 @@ +// Used to generate datetime-inheritance.svg with Graphviz +// (https://graphviz.org/) for the datetime documentation. + +digraph { + comment="Generated with datetime-inheritance.dot" + graph [ + bgcolor="transparent" + fontnames="svg" + layout="dot" + ranksep=0.5 + nodesep=0.5 + splines=line + ] + node [ + fontname="Courier" + fontsize=14.0 + shape=box + style=rounded + margin="0.15,0.07" + ] + edge [ + arrowhead=none + ] + + object -> tzinfo + object -> timedelta + object -> time + object -> date + tzinfo -> timezone + date -> datetime +} diff --git a/Doc/library/datetime-inheritance.svg b/Doc/library/datetime-inheritance.svg new file mode 100644 index 00000000000..e6b1cf877a5 --- /dev/null +++ b/Doc/library/datetime-inheritance.svg @@ -0,0 +1,84 @@ + + + + + + +datetime class hierarchy + + +object + +object + + + +tzinfo + +tzinfo + + + +object->tzinfo + + + + +timedelta + +timedelta + + + +object->timedelta + + + + +time + +time + + + +object->time + + + + +date + +date + + + +object->date + + + + +timezone + +timezone + + + +tzinfo->timezone + + + + +datetime + +datetime + + + +date->datetime + + + + diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index c17ff8986ab..beef4f09f02 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -12,8 +12,6 @@ -------------- -.. XXX what order should the types be discussed in? - The :mod:`!datetime` module supplies classes for manipulating dates and times. While date and time arithmetic is supported, the focus of the implementation is @@ -38,13 +36,14 @@ on efficient attribute extraction for output formatting and manipulation. Third-party library with expanded time zone and parsing support. Package :pypi:`DateType` - Third-party library that introduces distinct static types to e.g. allow - :term:`static type checkers ` + Third-party library that introduces distinct static types to for example, + allow :term:`static type checkers ` to differentiate between naive and aware datetimes. + .. _datetime-naive-aware: -Aware and Naive Objects +Aware and naive objects ----------------------- Date and time objects may be categorized as "aware" or "naive" depending on @@ -65,7 +64,7 @@ understand and to work with, at the cost of ignoring some aspects of reality. For applications requiring aware objects, :class:`.datetime` and :class:`.time` objects have an optional time zone information attribute, :attr:`!tzinfo`, that -can be set to an instance of a subclass of the abstract :class:`tzinfo` class. +can be set to an instance of a subclass of the abstract :class:`!tzinfo` class. These :class:`tzinfo` objects capture information about the offset from UTC time, the time zone name, and whether daylight saving time is in effect. @@ -77,6 +76,7 @@ detail is up to the application. The rules for time adjustment across the world are more political than rational, change frequently, and there is no standard suitable for every application aside from UTC. + Constants --------- @@ -93,13 +93,15 @@ The :mod:`!datetime` module exports the following constants: The largest year number allowed in a :class:`date` or :class:`.datetime` object. :const:`MAXYEAR` is 9999. + .. data:: UTC Alias for the UTC time zone singleton :attr:`datetime.timezone.utc`. .. versionadded:: 3.11 -Available Types + +Available types --------------- .. class:: date @@ -142,6 +144,7 @@ Available Types time adjustment (for example, to account for time zone and/or daylight saving time). + .. class:: timezone :noindex: @@ -150,19 +153,19 @@ Available Types .. versionadded:: 3.2 + Objects of these types are immutable. -Subclass relationships:: +Subclass relationships: - object - timedelta - tzinfo - timezone - time - date - datetime +.. figure:: datetime-inheritance.svg + :class: invert-in-dark-mode + :align: center + :alt: timedelta, tzinfo, time, and date inherit from object; timezone inherits + from tzinfo; and datetime inherits from date. -Common Properties + +Common properties ^^^^^^^^^^^^^^^^^ The :class:`date`, :class:`.datetime`, :class:`.time`, and :class:`timezone` types @@ -173,7 +176,8 @@ share these common features: dictionary keys. - Objects of these types support efficient pickling via the :mod:`pickle` module. -Determining if an Object is Aware or Naive + +Determining if an object is aware or naive ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Objects of the :class:`date` type are always naive. @@ -197,10 +201,11 @@ Otherwise, ``t`` is naive. The distinction between aware and naive doesn't apply to :class:`timedelta` objects. + .. _datetime-timedelta: -:class:`timedelta` Objects --------------------------- +:class:`!timedelta` objects +--------------------------- A :class:`timedelta` object represents a duration, the difference between two :class:`.datetime` or :class:`date` instances. @@ -229,8 +234,8 @@ A :class:`timedelta` object represents a duration, the difference between two *days*, *seconds* and *microseconds* are "merged" and normalized into those three resulting attributes:: - >>> from datetime import timedelta - >>> delta = timedelta( + >>> import datetime as dt + >>> delta = dt.timedelta( ... days=50, ... seconds=27, ... microseconds=10, @@ -243,6 +248,12 @@ A :class:`timedelta` object represents a duration, the difference between two >>> delta datetime.timedelta(days=64, seconds=29156, microseconds=10) + .. tip:: + ``import datetime as dt`` instead of ``import datetime`` or + ``from datetime import datetime`` to avoid confusion between the module + and the class. See `How I Import Python’s datetime Module + `__. + If any argument is a float and there are fractional microseconds, the fractional microseconds left over from all arguments are combined and their sum is rounded to the nearest microsecond using @@ -256,8 +267,8 @@ A :class:`timedelta` object represents a duration, the difference between two Note that normalization of negative values may be surprising at first. For example:: - >>> from datetime import timedelta - >>> d = timedelta(microseconds=-1) + >>> import datetime as dt + >>> d = dt.timedelta(microseconds=-1) >>> (d.days, d.seconds, d.microseconds) (-1, 86399, 999999) @@ -296,6 +307,7 @@ Class attributes: The smallest possible difference between non-equal :class:`timedelta` objects, ``timedelta(microseconds=1)``. + Note that, because of normalization, ``timedelta.max`` is greater than ``-timedelta.min``. ``-timedelta.max`` is not representable as a :class:`timedelta` object. @@ -319,13 +331,14 @@ Instance attributes (read-only): .. doctest:: - >>> from datetime import timedelta - >>> duration = timedelta(seconds=11235813) + >>> import datetime as dt + >>> duration = dt.timedelta(seconds=11235813) >>> duration.days, duration.seconds (130, 3813) >>> duration.total_seconds() 11235813.0 + .. attribute:: timedelta.microseconds Between 0 and 999,999 inclusive. @@ -333,8 +346,6 @@ Instance attributes (read-only): Supported operations: -.. XXX this table is too wide! - +--------------------------------+-----------------------------------------------+ | Operation | Result | +================================+===============================================+ @@ -396,7 +407,6 @@ Supported operations: | | call with canonical attribute values. | +--------------------------------+-----------------------------------------------+ - Notes: (1) @@ -432,9 +442,9 @@ objects (see below). .. versionchanged:: 3.2 Floor division and true division of a :class:`timedelta` object by another - :class:`timedelta` object are now supported, as are remainder operations and + :class:`!timedelta` object are now supported, as are remainder operations and the :func:`divmod` function. True division and multiplication of a - :class:`timedelta` object by a :class:`float` object are now supported. + :class:`!timedelta` object by a :class:`float` object are now supported. :class:`timedelta` objects support equality and order comparisons. @@ -447,23 +457,24 @@ Instance methods: Return the total number of seconds contained in the duration. Equivalent to ``td / timedelta(seconds=1)``. For interval units other than seconds, use the - division form directly (e.g. ``td / timedelta(microseconds=1)``). + division form directly (for example, ``td / timedelta(microseconds=1)``). Note that for very large time intervals (greater than 270 years on most platforms) this method will lose microsecond accuracy. .. versionadded:: 3.2 -Examples of usage: :class:`timedelta` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Examples of usage: :class:`!timedelta` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ An additional example of normalization:: >>> # Components of another_year add up to exactly 365 days - >>> from datetime import timedelta - >>> year = timedelta(days=365) - >>> another_year = timedelta(weeks=40, days=84, hours=23, - ... minutes=50, seconds=600) + >>> import datetime as dt + >>> year = dt.timedelta(days=365) + >>> another_year = dt.timedelta(weeks=40, days=84, hours=23, + ... minutes=50, seconds=600) >>> year == another_year True >>> year.total_seconds() @@ -471,8 +482,8 @@ An additional example of normalization:: Examples of :class:`timedelta` arithmetic:: - >>> from datetime import timedelta - >>> year = timedelta(days=365) + >>> import datetime as dt + >>> year = dt.timedelta(days=365) >>> ten_years = 10 * year >>> ten_years datetime.timedelta(days=3650) @@ -485,10 +496,11 @@ Examples of :class:`timedelta` arithmetic:: >>> three_years, three_years.days // 365 (datetime.timedelta(days=1095), 3) + .. _datetime-date: -:class:`date` Objects ---------------------- +:class:`!date` objects +---------------------- A :class:`date` object represents a date (year, month and day) in an idealized calendar, the current Gregorian calendar indefinitely extended in both @@ -517,9 +529,10 @@ Other constructors, all class methods: This is equivalent to ``date.fromtimestamp(time.time())``. + .. classmethod:: date.fromtimestamp(timestamp) - Return the local date corresponding to the POSIX timestamp, such as is + Return the local date corresponding to the POSIX *timestamp*, such as is returned by :func:`time.time`. This may raise :exc:`OverflowError`, if the timestamp is out @@ -538,7 +551,7 @@ Other constructors, all class methods: .. classmethod:: date.fromordinal(ordinal) - Return the date corresponding to the proleptic Gregorian ordinal, where + Return the date corresponding to the proleptic Gregorian *ordinal*, where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 <= ordinal <= @@ -559,25 +572,27 @@ Other constructors, all class methods: Examples:: - >>> from datetime import date - >>> date.fromisoformat('2019-12-04') + >>> import datetime as dt + >>> dt.date.fromisoformat('2019-12-04') datetime.date(2019, 12, 4) - >>> date.fromisoformat('20191204') + >>> dt.date.fromisoformat('20191204') datetime.date(2019, 12, 4) - >>> date.fromisoformat('2021-W01-1') + >>> dt.date.fromisoformat('2021-W01-1') datetime.date(2021, 1, 4) .. versionadded:: 3.7 .. versionchanged:: 3.11 Previously, this method only supported the format ``YYYY-MM-DD``. + .. classmethod:: date.fromisocalendar(year, week, day) Return a :class:`date` corresponding to the ISO calendar date specified by - year, week and day. This is the inverse of the function :meth:`date.isocalendar`. + *year*, *week* and *day*. This is the inverse of the function :meth:`date.isocalendar`. .. versionadded:: 3.8 + .. classmethod:: date.strptime(date_string, format) Return a :class:`.date` corresponding to *date_string*, parsed according to @@ -603,9 +618,9 @@ Other constructors, all class methods: .. doctest:: - >>> from datetime import date + >>> import datetime as dt >>> date_string = "02/29" - >>> when = date.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when = dt.date.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. >>> when.strftime("%B %d") # doctest: +SKIP 'February 29' @@ -697,7 +712,7 @@ Notes: In other words, ``date1 < date2`` if and only if ``date1.toordinal() < date2.toordinal()``. - Order comparison between a :class:`!date` object that is not also a + Order comparison between a :class:`date` object that is not also a :class:`.datetime` instance and a :class:`!datetime` object raises :exc:`TypeError`. @@ -720,8 +735,8 @@ Instance methods: Example:: - >>> from datetime import date - >>> d = date(2002, 12, 31) + >>> import datetime as dt + >>> d = dt.date(2002, 12, 31) >>> d.replace(day=26) datetime.date(2002, 12, 26) @@ -779,23 +794,25 @@ Instance methods: For example, 2004 begins on a Thursday, so the first week of ISO year 2004 begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004:: - >>> from datetime import date - >>> date(2003, 12, 29).isocalendar() + >>> import datetime as dt + >>> dt.date(2003, 12, 29).isocalendar() datetime.IsoCalendarDate(year=2004, week=1, weekday=1) - >>> date(2004, 1, 4).isocalendar() + >>> dt.date(2004, 1, 4).isocalendar() datetime.IsoCalendarDate(year=2004, week=1, weekday=7) .. versionchanged:: 3.9 Result changed from a tuple to a :term:`named tuple`. + .. method:: date.isoformat() Return a string representing the date in ISO 8601 format, ``YYYY-MM-DD``:: - >>> from datetime import date - >>> date(2002, 12, 4).isoformat() + >>> import datetime as dt + >>> dt.date(2002, 12, 4).isoformat() '2002-12-04' + .. method:: date.__str__() For a date ``d``, ``str(d)`` is equivalent to ``d.isoformat()``. @@ -805,8 +822,8 @@ Instance methods: Return a string representing the date:: - >>> from datetime import date - >>> date(2002, 12, 4).ctime() + >>> import datetime as dt + >>> dt.date(2002, 12, 4).ctime() 'Wed Dec 4 00:00:00 2002' ``d.ctime()`` is equivalent to:: @@ -832,19 +849,20 @@ Instance methods: literals ` and when using :meth:`str.format`. See also :ref:`strftime-strptime-behavior` and :meth:`date.isoformat`. -Examples of Usage: :class:`date` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Examples of usage: :class:`!date` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Example of counting days to an event:: >>> import time - >>> from datetime import date - >>> today = date.today() + >>> import datetime as dt + >>> today = dt.date.today() >>> today datetime.date(2007, 12, 5) - >>> today == date.fromtimestamp(time.time()) + >>> today == dt.date.fromtimestamp(time.time()) True - >>> my_birthday = date(today.year, 6, 24) + >>> my_birthday = dt.date(today.year, 6, 24) >>> if my_birthday < today: ... my_birthday = my_birthday.replace(year=today.year + 1) ... @@ -858,8 +876,8 @@ More examples of working with :class:`date`: .. doctest:: - >>> from datetime import date - >>> d = date.fromordinal(730920) # 730920th day after 1. 1. 0001 + >>> import datetime as dt + >>> d = dt.date.fromordinal(730920) # 730920th day after 1. 1. 0001 >>> d datetime.date(2002, 3, 11) @@ -875,7 +893,7 @@ More examples of working with :class:`date`: >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month") 'The day is 11, the month is March.' - >>> # Methods for to extracting 'components' under different calendars + >>> # Methods for extracting 'components' under different calendars >>> t = d.timetuple() >>> for i in t: # doctest: +SKIP ... print(i) @@ -902,7 +920,7 @@ More examples of working with :class:`date`: .. _datetime-datetime: -:class:`.datetime` Objects +:class:`!datetime` objects -------------------------- A :class:`.datetime` object is a single object containing all the information @@ -910,7 +928,7 @@ from a :class:`date` object and a :class:`.time` object. Like a :class:`date` object, :class:`.datetime` assumes the current Gregorian calendar extended in both directions; like a :class:`.time` object, -:class:`.datetime` assumes there are exactly 3600\*24 seconds in every day. +:class:`!datetime` assumes there are exactly 3600\*24 seconds in every day. Constructor: @@ -934,6 +952,7 @@ Constructor: .. versionchanged:: 3.6 Added the *fold* parameter. + Other constructors, all class methods: .. classmethod:: datetime.today() @@ -949,6 +968,7 @@ Other constructors, all class methods: This method is functionally equivalent to :meth:`now`, but without a ``tz`` parameter. + .. classmethod:: datetime.now(tz=None) Return the current local date and time. @@ -969,6 +989,7 @@ Other constructors, all class methods: Subsequent calls to :meth:`!datetime.now` may return the same instant depending on the precision of the underlying clock. + .. classmethod:: datetime.utcnow() Return the current UTC date and time, with :attr:`.tzinfo` ``None``. @@ -1056,6 +1077,9 @@ Other constructors, all class methods: :c:func:`gmtime` function. Raise :exc:`OSError` instead of :exc:`ValueError` on :c:func:`gmtime` failure. + .. versionchanged:: 3.15 + Accepts any real number as *timestamp*, not only integer or float. + .. deprecated:: 3.12 Use :meth:`datetime.fromtimestamp` with :const:`UTC` instead. @@ -1076,7 +1100,7 @@ Other constructors, all class methods: are equal to the given :class:`.time` object's. If the *tzinfo* argument is provided, its value is used to set the :attr:`.tzinfo` attribute of the result, otherwise the :attr:`~.time.tzinfo` attribute of the *time* argument - is used. If the *date* argument is a :class:`.datetime` object, its time components + is used. If the *date* argument is a :class:`!datetime` object, its time components and :attr:`.tzinfo` attributes are ignored. For any :class:`.datetime` object ``d``, @@ -1102,24 +1126,24 @@ Other constructors, all class methods: Examples:: - >>> from datetime import datetime - >>> datetime.fromisoformat('2011-11-04') + >>> import datetime as dt + >>> dt.datetime.fromisoformat('2011-11-04') datetime.datetime(2011, 11, 4, 0, 0) - >>> datetime.fromisoformat('20111104') + >>> dt.datetime.fromisoformat('20111104') datetime.datetime(2011, 11, 4, 0, 0) - >>> datetime.fromisoformat('2011-11-04T00:05:23') + >>> dt.datetime.fromisoformat('2011-11-04T00:05:23') datetime.datetime(2011, 11, 4, 0, 5, 23) - >>> datetime.fromisoformat('2011-11-04T00:05:23Z') + >>> dt.datetime.fromisoformat('2011-11-04T00:05:23Z') datetime.datetime(2011, 11, 4, 0, 5, 23, tzinfo=datetime.timezone.utc) - >>> datetime.fromisoformat('20111104T000523') + >>> dt.datetime.fromisoformat('20111104T000523') datetime.datetime(2011, 11, 4, 0, 5, 23) - >>> datetime.fromisoformat('2011-W01-2T00:05:23.283') + >>> dt.datetime.fromisoformat('2011-W01-2T00:05:23.283') datetime.datetime(2011, 1, 4, 0, 5, 23, 283000) - >>> datetime.fromisoformat('2011-11-04 00:05:23.283') + >>> dt.datetime.fromisoformat('2011-11-04 00:05:23.283') datetime.datetime(2011, 11, 4, 0, 5, 23, 283000) - >>> datetime.fromisoformat('2011-11-04 00:05:23.283+00:00') + >>> dt.datetime.fromisoformat('2011-11-04 00:05:23.283+00:00') datetime.datetime(2011, 11, 4, 0, 5, 23, 283000, tzinfo=datetime.timezone.utc) - >>> datetime.fromisoformat('2011-11-04T00:05:23+04:00') # doctest: +NORMALIZE_WHITESPACE + >>> dt.datetime.fromisoformat('2011-11-04T00:05:23+04:00') # doctest: +NORMALIZE_WHITESPACE datetime.datetime(2011, 11, 4, 0, 5, 23, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) @@ -1132,12 +1156,13 @@ Other constructors, all class methods: .. classmethod:: datetime.fromisocalendar(year, week, day) Return a :class:`.datetime` corresponding to the ISO calendar date specified - by year, week and day. The non-date components of the datetime are populated + by *year*, *week* and *day*. The non-date components of the datetime are populated with their normal default values. This is the inverse of the function :meth:`datetime.isocalendar`. .. versionadded:: 3.8 + .. classmethod:: datetime.strptime(date_string, format) Return a :class:`.datetime` corresponding to *date_string*, parsed according to @@ -1165,9 +1190,9 @@ Other constructors, all class methods: .. doctest:: - >>> from datetime import datetime + >>> import datetime as dt >>> date_string = "02/29" - >>> when = datetime.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when = dt.datetime.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. >>> when.strftime("%B %d") # doctest: +SKIP 'February 29' @@ -1245,6 +1270,7 @@ Instance attributes (read-only): .. versionadded:: 3.6 + Supported operations: +---------------------------------------+--------------------------------+ @@ -1280,7 +1306,7 @@ Supported operations: datetime, and no time zone adjustments are done even if the input is aware. (3) - Subtraction of a :class:`.datetime` from a :class:`.datetime` is defined only if + Subtraction of a :class:`.datetime` from a :class:`!datetime` is defined only if both operands are naive, or if both are aware. If one is aware and the other is naive, :exc:`TypeError` is raised. @@ -1298,7 +1324,7 @@ Supported operations: :class:`.datetime` objects are equal if they represent the same date and time, taking into account the time zone. - Naive and aware :class:`!datetime` objects are never equal. + Naive and aware :class:`.datetime` objects are never equal. If both comparands are aware, and have the same :attr:`!tzinfo` attribute, the :attr:`!tzinfo` and :attr:`~.datetime.fold` attributes are ignored and @@ -1306,7 +1332,7 @@ Supported operations: If both comparands are aware and have different :attr:`~.datetime.tzinfo` attributes, the comparison acts as comparands were first converted to UTC datetimes except that the implementation never overflows. - :class:`!datetime` instances in a repeated interval are never equal to + :class:`.datetime` instances in a repeated interval are never equal to :class:`!datetime` instances in other time zone. (5) @@ -1335,6 +1361,7 @@ Supported operations: The default behavior can be changed by overriding the special comparison methods in subclasses. + Instance methods: .. method:: datetime.date() @@ -1490,11 +1517,13 @@ Instance methods: ``datetime.replace(tzinfo=timezone.utc)`` to make it aware, at which point you can use :meth:`.datetime.timetuple`. + .. method:: datetime.toordinal() Return the proleptic Gregorian ordinal of the date. The same as ``self.date().toordinal()``. + .. method:: datetime.timestamp() Return POSIX timestamp corresponding to the :class:`.datetime` @@ -1503,7 +1532,7 @@ Instance methods: Naive :class:`.datetime` instances are assumed to represent local time and this method relies on the platform C :c:func:`mktime` - function to perform the conversion. Since :class:`.datetime` + function to perform the conversion. Since :class:`!datetime` supports wider range of values than :c:func:`mktime` on many platforms, this method may raise :exc:`OverflowError` or :exc:`OSError` for times far in the past or far in the future. @@ -1513,12 +1542,6 @@ Instance methods: (dt - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() - .. versionadded:: 3.3 - - .. versionchanged:: 3.6 - The :meth:`timestamp` method uses the :attr:`.fold` attribute to - disambiguate the times during a repeated interval. - .. note:: There is no method to obtain the POSIX timestamp directly from a @@ -1533,6 +1556,13 @@ Instance methods: timestamp = (dt - datetime(1970, 1, 1)) / timedelta(seconds=1) + .. versionadded:: 3.3 + + .. versionchanged:: 3.6 + The :meth:`timestamp` method uses the :attr:`.fold` attribute to + disambiguate the times during a repeated interval. + + .. method:: datetime.weekday() Return the day of the week as an integer, where Monday is 0 and Sunday is 6. @@ -1568,24 +1598,24 @@ Instance methods: Examples:: - >>> from datetime import datetime, timezone - >>> datetime(2019, 5, 18, 15, 17, 8, 132263).isoformat() + >>> import datetime as dt + >>> dt.datetime(2019, 5, 18, 15, 17, 8, 132263).isoformat() '2019-05-18T15:17:08.132263' - >>> datetime(2019, 5, 18, 15, 17, tzinfo=timezone.utc).isoformat() + >>> dt.datetime(2019, 5, 18, 15, 17, tzinfo=dt.timezone.utc).isoformat() '2019-05-18T15:17:00+00:00' The optional argument *sep* (default ``'T'``) is a one-character separator, placed between the date and time portions of the result. For example:: - >>> from datetime import tzinfo, timedelta, datetime - >>> class TZ(tzinfo): + >>> import datetime as dt + >>> class TZ(dt.tzinfo): ... """A time zone with an arbitrary, constant -06:39 offset.""" - ... def utcoffset(self, dt): - ... return timedelta(hours=-6, minutes=-39) + ... def utcoffset(self, when): + ... return dt.timedelta(hours=-6, minutes=-39) ... - >>> datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ') + >>> dt.datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ') '2002-12-25 00:00:00-06:39' - >>> datetime(2009, 11, 27, microsecond=100, tzinfo=TZ()).isoformat() + >>> dt.datetime(2009, 11, 27, microsecond=100, tzinfo=TZ()).isoformat() '2009-11-27T00:00:00.000100-06:39' The optional argument *timespec* specifies the number of additional @@ -1609,11 +1639,11 @@ Instance methods: :exc:`ValueError` will be raised on an invalid *timespec* argument:: - >>> from datetime import datetime - >>> datetime.now().isoformat(timespec='minutes') # doctest: +SKIP + >>> import datetime as dt + >>> dt.datetime.now().isoformat(timespec='minutes') # doctest: +SKIP '2002-12-25T00:00' - >>> dt = datetime(2015, 1, 1, 12, 30, 59, 0) - >>> dt.isoformat(timespec='microseconds') + >>> my_datetime = dt.datetime(2015, 1, 1, 12, 30, 59, 0) + >>> my_datetime.isoformat(timespec='microseconds') '2015-01-01T12:30:59.000000' .. versionchanged:: 3.6 @@ -1630,8 +1660,8 @@ Instance methods: Return a string representing the date and time:: - >>> from datetime import datetime - >>> datetime(2002, 12, 4, 20, 30, 40).ctime() + >>> import datetime as dt + >>> dt.datetime(2002, 12, 4, 20, 30, 40).ctime() 'Wed Dec 4 20:30:40 2002' The output string will *not* include time zone information, regardless @@ -1661,34 +1691,34 @@ Instance methods: See also :ref:`strftime-strptime-behavior` and :meth:`datetime.isoformat`. -Examples of Usage: :class:`.datetime` +Examples of usage: :class:`!datetime` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Examples of working with :class:`.datetime` objects: .. doctest:: - >>> from datetime import datetime, date, time, timezone + >>> import datetime as dt >>> # Using datetime.combine() - >>> d = date(2005, 7, 14) - >>> t = time(12, 30) - >>> datetime.combine(d, t) + >>> d = dt.date(2005, 7, 14) + >>> t = dt.time(12, 30) + >>> dt.datetime.combine(d, t) datetime.datetime(2005, 7, 14, 12, 30) >>> # Using datetime.now() - >>> datetime.now() # doctest: +SKIP + >>> dt.datetime.now() # doctest: +SKIP datetime.datetime(2007, 12, 6, 16, 29, 43, 79043) # GMT +1 - >>> datetime.now(timezone.utc) # doctest: +SKIP + >>> dt.datetime.now(dt.timezone.utc) # doctest: +SKIP datetime.datetime(2007, 12, 6, 15, 29, 43, 79060, tzinfo=datetime.timezone.utc) >>> # Using datetime.strptime() - >>> dt = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M") - >>> dt + >>> my_datetime = dt.datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M") + >>> my_datetime datetime.datetime(2006, 11, 21, 16, 30) >>> # Using datetime.timetuple() to get tuple of all attributes - >>> tt = dt.timetuple() + >>> tt = my_datetime.timetuple() >>> for it in tt: # doctest: +SKIP ... print(it) ... @@ -1703,7 +1733,7 @@ Examples of working with :class:`.datetime` objects: -1 # dst - method tzinfo.dst() returned None >>> # Date in ISO format - >>> ic = dt.isocalendar() + >>> ic = my_datetime.isocalendar() >>> for it in ic: # doctest: +SKIP ... print(it) ... @@ -1712,55 +1742,55 @@ Examples of working with :class:`.datetime` objects: 2 # ISO weekday >>> # Formatting a datetime - >>> dt.strftime("%A, %d. %B %Y %I:%M%p") + >>> my_datetime.strftime("%A, %d. %B %Y %I:%M%p") 'Tuesday, 21. November 2006 04:30PM' - >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(dt, "day", "month", "time") + >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(my_datetime, "day", "month", "time") 'The day is 21, the month is November, the time is 04:30PM.' The example below defines a :class:`tzinfo` subclass capturing time zone information for Kabul, Afghanistan, which used +4 UTC until 1945 and then +4:30 UTC thereafter:: - from datetime import timedelta, datetime, tzinfo, timezone + import datetime as dt - class KabulTz(tzinfo): + class KabulTz(dt.tzinfo): # Kabul used +4 until 1945, when they moved to +4:30 - UTC_MOVE_DATE = datetime(1944, 12, 31, 20, tzinfo=timezone.utc) + UTC_MOVE_DATE = dt.datetime(1944, 12, 31, 20, tzinfo=dt.timezone.utc) - def utcoffset(self, dt): - if dt.year < 1945: - return timedelta(hours=4) - elif (1945, 1, 1, 0, 0) <= dt.timetuple()[:5] < (1945, 1, 1, 0, 30): + def utcoffset(self, when): + if when.year < 1945: + return dt.timedelta(hours=4) + elif (1945, 1, 1, 0, 0) <= when.timetuple()[:5] < (1945, 1, 1, 0, 30): # An ambiguous ("imaginary") half-hour range representing # a 'fold' in time due to the shift from +4 to +4:30. - # If dt falls in the imaginary range, use fold to decide how - # to resolve. See PEP495. - return timedelta(hours=4, minutes=(30 if dt.fold else 0)) + # If when falls in the imaginary range, use fold to decide how + # to resolve. See PEP 495. + return dt.timedelta(hours=4, minutes=(30 if when.fold else 0)) else: - return timedelta(hours=4, minutes=30) + return dt.timedelta(hours=4, minutes=30) - def fromutc(self, dt): + def fromutc(self, when): # Follow same validations as in datetime.tzinfo - if not isinstance(dt, datetime): + if not isinstance(when, dt.datetime): raise TypeError("fromutc() requires a datetime argument") - if dt.tzinfo is not self: - raise ValueError("dt.tzinfo is not self") + if when.tzinfo is not self: + raise ValueError("when.tzinfo is not self") # A custom implementation is required for fromutc as # the input to this function is a datetime with utc values # but with a tzinfo set to self. # See datetime.astimezone or fromtimestamp. - if dt.replace(tzinfo=timezone.utc) >= self.UTC_MOVE_DATE: - return dt + timedelta(hours=4, minutes=30) + if when.replace(tzinfo=dt.timezone.utc) >= self.UTC_MOVE_DATE: + return when + dt.timedelta(hours=4, minutes=30) else: - return dt + timedelta(hours=4) + return when + dt.timedelta(hours=4) - def dst(self, dt): + def dst(self, when): # Kabul does not observe daylight saving time. - return timedelta(0) + return dt.timedelta(0) - def tzname(self, dt): - if dt >= self.UTC_MOVE_DATE: + def tzname(self, when): + if when >= self.UTC_MOVE_DATE: return "+04:30" return "+04" @@ -1769,17 +1799,17 @@ Usage of ``KabulTz`` from above:: >>> tz1 = KabulTz() >>> # Datetime before the change - >>> dt1 = datetime(1900, 11, 21, 16, 30, tzinfo=tz1) + >>> dt1 = dt.datetime(1900, 11, 21, 16, 30, tzinfo=tz1) >>> print(dt1.utcoffset()) 4:00:00 >>> # Datetime after the change - >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=tz1) + >>> dt2 = dt.datetime(2006, 6, 14, 13, 0, tzinfo=tz1) >>> print(dt2.utcoffset()) 4:30:00 >>> # Convert datetime to another time zone - >>> dt3 = dt2.astimezone(timezone.utc) + >>> dt3 = dt2.astimezone(dt.timezone.utc) >>> dt3 datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc) >>> dt2 @@ -1787,9 +1817,10 @@ Usage of ``KabulTz`` from above:: >>> dt2 == dt3 True + .. _datetime-time: -:class:`.time` Objects +:class:`!time` objects ---------------------- A :class:`.time` object represents a (local) time of day, independent of any particular @@ -1810,6 +1841,7 @@ day, and subject to adjustment via a :class:`tzinfo` object. If an argument outside those ranges is given, :exc:`ValueError` is raised. All default to 0 except *tzinfo*, which defaults to ``None``. + Class attributes: @@ -1868,6 +1900,7 @@ Instance attributes (read-only): .. versionadded:: 3.6 + :class:`.time` objects support equality and order comparisons, where ``a`` is considered less than ``b`` when ``a`` precedes ``b`` in time. @@ -1890,8 +1923,8 @@ In Boolean contexts, a :class:`.time` object is always considered to be true. .. versionchanged:: 3.5 Before Python 3.5, a :class:`.time` object was considered to be false if it represented midnight in UTC. This behavior was considered obscure and - error-prone and has been removed in Python 3.5. See :issue:`13936` for full - details. + error-prone and has been removed in Python 3.5. See :issue:`13936` for more + information. Other constructors: @@ -1912,22 +1945,22 @@ Other constructors: .. doctest:: - >>> from datetime import time - >>> time.fromisoformat('04:23:01') + >>> import datetime as dt + >>> dt.time.fromisoformat('04:23:01') datetime.time(4, 23, 1) - >>> time.fromisoformat('T04:23:01') + >>> dt.time.fromisoformat('T04:23:01') datetime.time(4, 23, 1) - >>> time.fromisoformat('T042301') + >>> dt.time.fromisoformat('T042301') datetime.time(4, 23, 1) - >>> time.fromisoformat('04:23:01.000384') + >>> dt.time.fromisoformat('04:23:01.000384') datetime.time(4, 23, 1, 384) - >>> time.fromisoformat('04:23:01,000384') + >>> dt.time.fromisoformat('04:23:01,000384') datetime.time(4, 23, 1, 384) - >>> time.fromisoformat('04:23:01+04:00') + >>> dt.time.fromisoformat('04:23:01+04:00') datetime.time(4, 23, 1, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) - >>> time.fromisoformat('04:23:01Z') + >>> dt.time.fromisoformat('04:23:01Z') datetime.time(4, 23, 1, tzinfo=datetime.timezone.utc) - >>> time.fromisoformat('04:23:01+00:00') + >>> dt.time.fromisoformat('04:23:01+00:00') datetime.time(4, 23, 1, tzinfo=datetime.timezone.utc) @@ -1936,6 +1969,7 @@ Other constructors: Previously, this method only supported formats that could be emitted by :meth:`time.isoformat`. + .. classmethod:: time.strptime(date_string, format) Return a :class:`.time` corresponding to *date_string*, parsed according to @@ -1960,7 +1994,7 @@ Instance methods: Return a new :class:`.time` with the same values, but with specified parameters updated. Note that ``tzinfo=None`` can be specified to create a - naive :class:`.time` from an aware :class:`.time`, without conversion of the + naive :class:`!time` from an aware :class:`!time`, without conversion of the time data. :class:`.time` objects are also supported by generic function @@ -2001,13 +2035,13 @@ Instance methods: Example:: - >>> from datetime import time - >>> time(hour=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes') + >>> import datetime as dt + >>> dt.time(hour=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes') '12:34' - >>> dt = time(hour=12, minute=34, second=56, microsecond=0) - >>> dt.isoformat(timespec='microseconds') + >>> my_time = dt.time(hour=12, minute=34, second=56, microsecond=0) + >>> my_time.isoformat(timespec='microseconds') '12:34:56.000000' - >>> dt.isoformat(timespec='auto') + >>> my_time.isoformat(timespec='auto') '12:34:56' .. versionchanged:: 3.6 @@ -2052,29 +2086,31 @@ Instance methods: .. versionchanged:: 3.7 The DST offset is not restricted to a whole number of minutes. + .. method:: time.tzname() If :attr:`.tzinfo` is ``None``, returns ``None``, else returns ``self.tzinfo.tzname(None)``, or raises an exception if the latter doesn't return ``None`` or a string object. -Examples of Usage: :class:`.time` + +Examples of usage: :class:`!time` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Examples of working with a :class:`.time` object:: - >>> from datetime import time, tzinfo, timedelta - >>> class TZ1(tzinfo): - ... def utcoffset(self, dt): - ... return timedelta(hours=1) - ... def dst(self, dt): - ... return timedelta(0) - ... def tzname(self,dt): + >>> import datetime as dt + >>> class TZ1(dt.tzinfo): + ... def utcoffset(self, when): + ... return dt.timedelta(hours=1) + ... def dst(self, when): + ... return dt.timedelta(0) + ... def tzname(self, when): ... return "+01:00" ... def __repr__(self): ... return f"{self.__class__.__name__}()" ... - >>> t = time(12, 10, 30, tzinfo=TZ1()) + >>> t = dt.time(12, 10, 30, tzinfo=TZ1()) >>> t datetime.time(12, 10, 30, tzinfo=TZ1()) >>> t.isoformat() @@ -2091,25 +2127,25 @@ Examples of working with a :class:`.time` object:: .. _datetime-tzinfo: -:class:`tzinfo` Objects ------------------------ +:class:`!tzinfo` objects +------------------------ .. class:: tzinfo() - This is an abstract base class, meaning that this class should not be + This is an :term:`abstract base class`, meaning that this class should not be instantiated directly. Define a subclass of :class:`tzinfo` to capture information about a particular time zone. An instance of (a concrete subclass of) :class:`tzinfo` can be passed to the constructors for :class:`.datetime` and :class:`.time` objects. The latter objects - view their attributes as being in local time, and the :class:`tzinfo` object + view their attributes as being in local time, and the :class:`!tzinfo` object supports methods revealing offset of local time from UTC, the name of the time zone, and DST offset, all relative to a date or time object passed to them. You need to derive a concrete subclass, and (at least) supply implementations of the standard :class:`tzinfo` methods needed by the :class:`.datetime` methods you use. The :mod:`!datetime` module provides - :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can + :class:`timezone`, a simple concrete subclass of :class:`!tzinfo` which can represent time zones with fixed offset from UTC such as UTC itself or North American EST and EDT. @@ -2172,31 +2208,35 @@ Examples of working with a :class:`.time` object:: ``tz.utcoffset(dt) - tz.dst(dt)`` must return the same result for every :class:`.datetime` *dt* with ``dt.tzinfo == - tz``. For sane :class:`tzinfo` subclasses, this expression yields the time + tz``. For sane :class:`!tzinfo` subclasses, this expression yields the time zone's "standard offset", which should not depend on the date or the time, but only on geographic location. The implementation of :meth:`datetime.astimezone` relies on this, but cannot detect violations; it's the programmer's - responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee + responsibility to ensure it. If a :class:`!tzinfo` subclass cannot guarantee this, it may be able to override the default implementation of :meth:`tzinfo.fromutc` to work correctly with :meth:`~.datetime.astimezone` regardless. Most implementations of :meth:`dst` will probably look like one of these two:: - def dst(self, dt): + import datetime as dt + + def dst(self, when): # a fixed-offset class: doesn't account for DST - return timedelta(0) + return dt.timedelta(0) or:: - def dst(self, dt): + import datetime as dt + + def dst(self, when): # Code to set dston and dstoff to the time zone's DST - # transition times based on the input dt.year, and expressed + # transition times based on the input when.year, and expressed # in standard local time. - if dston <= dt.replace(tzinfo=None) < dstoff: - return timedelta(hours=1) + if dston <= when.replace(tzinfo=None) < dstoff: + return dt.timedelta(hours=1) else: - return timedelta(0) + return dt.timedelta(0) The default implementation of :meth:`dst` raises :exc:`NotImplementedError`. @@ -2213,17 +2253,17 @@ Examples of working with a :class:`.time` object:: valid replies. Return ``None`` if a string name isn't known. Note that this is a method rather than a fixed string primarily because some :class:`tzinfo` subclasses will wish to return different names depending on the specific value - of *dt* passed, especially if the :class:`tzinfo` class is accounting for + of *dt* passed, especially if the :class:`!tzinfo` class is accounting for daylight time. The default implementation of :meth:`tzname` raises :exc:`NotImplementedError`. These methods are called by a :class:`.datetime` or :class:`.time` object, in -response to their methods of the same names. A :class:`.datetime` object passes -itself as the argument, and a :class:`.time` object passes ``None`` as the +response to their methods of the same names. A :class:`!datetime` object passes +itself as the argument, and a :class:`!time` object passes ``None`` as the argument. A :class:`tzinfo` subclass's methods should therefore be prepared to -accept a *dt* argument of ``None``, or of class :class:`.datetime`. +accept a *dt* argument of ``None``, or of class :class:`!datetime`. When ``None`` is passed, it's up to the class designer to decide the best response. For example, returning ``None`` is appropriate if the class wishes to @@ -2231,10 +2271,10 @@ say that time objects don't participate in the :class:`tzinfo` protocols. It may be more useful for ``utcoffset(None)`` to return the standard UTC offset, as there is no other convention for discovering the standard offset. -When a :class:`.datetime` object is passed in response to a :class:`.datetime` +When a :class:`.datetime` object is passed in response to a :class:`!datetime` method, ``dt.tzinfo`` is the same object as *self*. :class:`tzinfo` methods can -rely on this, unless user code calls :class:`tzinfo` methods directly. The -intent is that the :class:`tzinfo` methods interpret *dt* as being in local +rely on this, unless user code calls :class:`!tzinfo` methods directly. The +intent is that the :class:`!tzinfo` methods interpret *dt* as being in local time, and not need worry about objects in other time zones. There is one more :class:`tzinfo` method that a subclass may wish to override: @@ -2262,20 +2302,22 @@ There is one more :class:`tzinfo` method that a subclass may wish to override: Skipping code for error cases, the default :meth:`fromutc` implementation acts like:: - def fromutc(self, dt): - # raise ValueError error if dt.tzinfo is not self - dtoff = dt.utcoffset() - dtdst = dt.dst() + import datetime as dt + + def fromutc(self, when): + # raise ValueError error if when.tzinfo is not self + dtoff = when.utcoffset() + dtdst = when.dst() # raise ValueError if dtoff is None or dtdst is None delta = dtoff - dtdst # this is self's standard offset if delta: - dt += delta # convert to standard local time - dtdst = dt.dst() + when += delta # convert to standard local time + dtdst = when.dst() # raise ValueError if dtdst is None if dtdst: - return dt + dtdst + return when + dtdst else: - return dt + return when In the following :download:`tzinfo_examples.py <../includes/tzinfo_examples.py>` file there are some examples of @@ -2302,9 +2344,9 @@ When DST starts (the "start" line), the local wall clock leaps from 1:59 to ``astimezone(Eastern)`` won't deliver a result with ``hour == 2`` on the day DST begins. For example, at the Spring forward transition of 2016, we get:: - >>> from datetime import datetime, timezone + >>> import datetime as dt >>> from tzinfo_examples import HOUR, Eastern - >>> u0 = datetime(2016, 3, 13, 5, tzinfo=timezone.utc) + >>> u0 = dt.datetime(2016, 3, 13, 5, tzinfo=dt.timezone.utc) >>> for i in range(4): ... u = u0 + i*HOUR ... t = u.astimezone(Eastern) @@ -2327,7 +2369,9 @@ form 5:MM and 6:MM both map to 1:MM when converted to Eastern, but earlier times have the :attr:`~.datetime.fold` attribute set to 0 and the later times have it set to 1. For example, at the Fall back transition of 2016, we get:: - >>> u0 = datetime(2016, 11, 6, 4, tzinfo=timezone.utc) + >>> import datetime as dt + >>> from tzinfo_examples import HOUR, Eastern + >>> u0 = dt.datetime(2016, 11, 6, 4, tzinfo=dt.timezone.utc) >>> for i in range(4): ... u = u0 + i*HOUR ... t = u.astimezone(Eastern) @@ -2344,7 +2388,7 @@ Note that the :class:`.datetime` instances that differ only by the value of the Applications that can't bear wall-time ambiguities should explicitly check the value of the :attr:`~.datetime.fold` attribute or avoid using hybrid :class:`tzinfo` subclasses; there are no ambiguities when using :class:`timezone`, -or any other fixed-offset :class:`tzinfo` subclass (such as a class representing +or any other fixed-offset :class:`!tzinfo` subclass (such as a class representing only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: @@ -2367,8 +2411,8 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. _datetime-timezone: -:class:`timezone` Objects -------------------------- +:class:`!timezone` objects +-------------------------- The :class:`timezone` class is a subclass of :class:`tzinfo`, each instance of which represents a time zone defined by a fixed offset from @@ -2406,6 +2450,7 @@ where historical changes have been made to civil time. .. versionchanged:: 3.7 The UTC offset is not restricted to a whole number of minutes. + .. method:: timezone.tzname(dt) Return the fixed value specified when the :class:`timezone` instance @@ -2426,11 +2471,13 @@ where historical changes have been made to civil time. Always returns ``None``. + .. method:: timezone.fromutc(dt) Return ``dt + offset``. The *dt* argument must be an aware :class:`.datetime` instance, with ``tzinfo`` set to ``self``. + Class attributes: .. attribute:: timezone.utc @@ -2443,8 +2490,8 @@ Class attributes: .. _strftime-strptime-behavior: -:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Behavior --------------------------------------------------------------------- +:meth:`!strftime` and :meth:`!strptime` behavior +------------------------------------------------ :class:`date`, :class:`.datetime`, and :class:`.time` objects all support a ``strftime(format)`` method, to create a string representing the time under the @@ -2470,13 +2517,14 @@ versus :meth:`~.datetime.strptime`: .. _format-codes: -:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Format Codes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:meth:`!strftime` and :meth:`!strptime` format codes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ These methods accept format codes that can be used to parse and format dates:: - >>> datetime.strptime('31/01/22 23:59:59.999999', - ... '%d/%m/%y %H:%M:%S.%f') + >>> import datetime as dt + >>> dt.datetime.strptime('31/01/22 23:59:59.999999', + ... '%d/%m/%y %H:%M:%S.%f') datetime.datetime(2022, 1, 31, 23, 59, 59, 999999) >>> _.strftime('%a %d %b %Y, %I:%M%p') 'Mon 31 Jan 2022, 11:59PM' @@ -2627,7 +2675,8 @@ differences between platforms in handling of unsupported format specifiers. .. versionadded:: 3.12 ``%:z`` was added. -Technical Detail + +Technical detail ^^^^^^^^^^^^^^^^ Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's @@ -2638,13 +2687,18 @@ For the :meth:`.datetime.strptime` and :meth:`.date.strptime` class methods, the default value is ``1900-01-01T00:00:00.000``: any components not specified in the format string will be pulled from the default value. +.. note:: + Format strings without separators can be ambiguous for parsing. For + example, with ``%Y%m%d``, the string ``2026111`` may be parsed either as + ``2026-11-01`` or as ``2026-01-11``. + Use separators to ensure the input is parsed as intended. + .. note:: When used to parse partial dates lacking a year, :meth:`.datetime.strptime` and :meth:`.date.strptime` will raise when encountering February 29 because the default year of 1900 is *not* a leap year. Always add a default leap year to partial date strings before parsing. - .. testsetup:: # doctest seems to turn the warning into an error which makes it @@ -2662,13 +2716,13 @@ in the format string will be pulled from the default value. .. doctest:: - >>> from datetime import datetime + >>> import datetime as dt >>> value = "2/29" - >>> datetime.strptime(value, "%m/%d") + >>> dt.datetime.strptime(value, "%m/%d") Traceback (most recent call last): ... ValueError: day 29 must be in range 1..28 for month 2 in year 1900 - >>> datetime.strptime(f"1904 {value}", "%Y %m/%d") + >>> dt.datetime.strptime(f"1904 {value}", "%Y %m/%d") datetime.datetime(1904, 2, 29, 0, 0) Using ``datetime.strptime(date_string, format)`` is equivalent to:: @@ -2805,7 +2859,7 @@ Notes: .. doctest:: >>> month_day = "02/29" - >>> datetime.strptime(f"{month_day};1984", "%m/%d;%Y") # No leap year bug. + >>> dt.datetime.strptime(f"{month_day};1984", "%m/%d;%Y") # No leap year bug. datetime.datetime(1984, 2, 29, 0, 0) .. deprecated-removed:: 3.13 3.15 @@ -2816,7 +2870,7 @@ Notes: .. rubric:: Footnotes -.. [#] If, that is, we ignore the effects of Relativity +.. [#] If, that is, we ignore the effects of relativity. .. [#] This matches the definition of the "proleptic Gregorian" calendar in Dershowitz and Reingold's book *Calendrical Calculations*, diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 628232cb631..c736ee0d705 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -8,7 +8,7 @@ -------------- -:mod:`dbm` is a generic interface to variants of the DBM database: +:mod:`!dbm` is a generic interface to variants of the DBM database: * :mod:`dbm.sqlite3` * :mod:`dbm.gnu` @@ -101,7 +101,7 @@ will automatically close them when done. .. versionchanged:: 3.2 :meth:`!get` and :meth:`!setdefault` methods are now available for all - :mod:`dbm` backends. + :mod:`!dbm` backends. .. versionchanged:: 3.4 Added native support for the context management protocol to the objects @@ -112,7 +112,7 @@ will automatically close them when done. instead of :exc:`KeyError`. .. versionchanged:: 3.13 - :meth:`!clear` methods are now available for all :mod:`dbm` backends. + :meth:`!clear` methods are now available for all :mod:`!dbm` backends. The following example records some hostnames and a corresponding title, and @@ -151,11 +151,10 @@ then prints out the contents of the database:: The individual submodules are described in the following sections. -:mod:`dbm.sqlite3` --- SQLite backend for dbm ---------------------------------------------- +:mod:`!dbm.sqlite3` --- SQLite backend for dbm +---------------------------------------------- .. module:: dbm.sqlite3 - :platform: All :synopsis: SQLite backend for dbm .. versionadded:: 3.13 @@ -165,8 +164,8 @@ The individual submodules are described in the following sections. -------------- This module uses the standard library :mod:`sqlite3` module to provide an -SQLite backend for the :mod:`dbm` module. -The files created by :mod:`dbm.sqlite3` can thus be opened by :mod:`sqlite3`, +SQLite backend for the :mod:`!dbm` module. +The files created by :mod:`!dbm.sqlite3` can thus be opened by :mod:`sqlite3`, or any other SQLite browser, including the SQLite CLI. .. include:: ../includes/wasm-notavail.rst @@ -202,31 +201,32 @@ or any other SQLite browser, including the SQLite CLI. Close the SQLite database. -:mod:`dbm.gnu` --- GNU database manager ---------------------------------------- +:mod:`!dbm.gnu` --- GNU database manager +---------------------------------------- .. module:: dbm.gnu - :platform: Unix :synopsis: GNU database manager **Source code:** :source:`Lib/dbm/gnu.py` -------------- -The :mod:`dbm.gnu` module provides an interface to the :abbr:`GDBM (GNU dbm)` +The :mod:`!dbm.gnu` module provides an interface to the :abbr:`GDBM (GNU dbm)` library, similar to the :mod:`dbm.ndbm` module, but with additional functionality like crash tolerance. .. note:: - The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + The file formats created by :mod:`!dbm.gnu` and :mod:`dbm.ndbm` are incompatible and can not be used interchangeably. .. include:: ../includes/wasm-mobile-notavail.rst +.. availability:: Unix. + .. exception:: error - Raised on :mod:`dbm.gnu`-specific errors, such as I/O errors. :exc:`KeyError` is + Raised on :mod:`!dbm.gnu`-specific errors, such as I/O errors. :exc:`KeyError` is raised for general mapping errors like specifying an incorrect key. @@ -321,25 +321,24 @@ functionality like crash tolerance. unwritten data to be written to the disk. -:mod:`dbm.ndbm` --- New Database Manager ----------------------------------------- +:mod:`!dbm.ndbm` --- New Database Manager +----------------------------------------- .. module:: dbm.ndbm - :platform: Unix :synopsis: The New Database Manager **Source code:** :source:`Lib/dbm/ndbm.py` -------------- -The :mod:`dbm.ndbm` module provides an interface to the +The :mod:`!dbm.ndbm` module provides an interface to the :abbr:`NDBM (New Database Manager)` library. This module can be used with the "classic" NDBM interface or the :abbr:`GDBM (GNU dbm)` compatibility interface. .. note:: - The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + The file formats created by :mod:`dbm.gnu` and :mod:`!dbm.ndbm` are incompatible and can not be used interchangeably. .. warning:: @@ -351,9 +350,11 @@ This module can be used with the "classic" NDBM interface or the .. include:: ../includes/wasm-mobile-notavail.rst +.. availability:: Unix. + .. exception:: error - Raised on :mod:`dbm.ndbm`-specific errors, such as I/O errors. :exc:`KeyError` is raised + Raised on :mod:`!dbm.ndbm`-specific errors, such as I/O errors. :exc:`KeyError` is raised for general mapping errors like specifying an incorrect key. @@ -403,8 +404,8 @@ This module can be used with the "classic" NDBM interface or the Close the NDBM database. -:mod:`dbm.dumb` --- Portable DBM implementation ------------------------------------------------ +:mod:`!dbm.dumb` --- Portable DBM implementation +------------------------------------------------ .. module:: dbm.dumb :synopsis: Portable implementation of the simple DBM interface. @@ -415,23 +416,23 @@ This module can be used with the "classic" NDBM interface or the .. note:: - The :mod:`dbm.dumb` module is intended as a last resort fallback for the - :mod:`dbm` module when a more robust module is not available. The :mod:`dbm.dumb` + The :mod:`!dbm.dumb` module is intended as a last resort fallback for the + :mod:`!dbm` module when a more robust module is not available. The :mod:`!dbm.dumb` module is not written for speed and is not nearly as heavily used as the other database modules. -------------- -The :mod:`dbm.dumb` module provides a persistent :class:`dict`-like +The :mod:`!dbm.dumb` module provides a persistent :class:`dict`-like interface which is written entirely in Python. -Unlike other :mod:`dbm` backends, such as :mod:`dbm.gnu`, no +Unlike other :mod:`!dbm` backends, such as :mod:`dbm.gnu`, no external library is required. The :mod:`!dbm.dumb` module defines the following: .. exception:: error - Raised on :mod:`dbm.dumb`-specific errors, such as I/O errors. :exc:`KeyError` is + Raised on :mod:`!dbm.dumb`-specific errors, such as I/O errors. :exc:`KeyError` is raised for general mapping errors like specifying an incorrect key. diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 43f2e14f92e..d4c089f5a14 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -30,7 +30,7 @@ -------------- -The :mod:`decimal` module provides support for fast correctly rounded +The :mod:`!decimal` module provides support for fast correctly rounded decimal floating-point arithmetic. It offers several advantages over the :class:`float` datatype: @@ -289,7 +289,7 @@ For more advanced work, it may be useful to create alternate contexts using the :meth:`Context` constructor. To make an alternate active, use the :func:`setcontext` function. -In accordance with the standard, the :mod:`decimal` module provides two ready to +In accordance with the standard, the :mod:`!decimal` module provides two ready to use standard contexts, :const:`BasicContext` and :const:`ExtendedContext`. The former is especially useful for debugging because many of the traps are enabled: @@ -1838,7 +1838,7 @@ properties of addition: >>> u * (v+w) Decimal('0.0060000') -The :mod:`decimal` module makes it possible to restore the identities by +The :mod:`!decimal` module makes it possible to restore the identities by expanding the precision sufficiently to avoid loss of significance: .. doctest:: newcontext @@ -1860,7 +1860,7 @@ expanding the precision sufficiently to avoid loss of significance: Special values ^^^^^^^^^^^^^^ -The number system for the :mod:`decimal` module provides special values +The number system for the :mod:`!decimal` module provides special values including ``NaN``, ``sNaN``, ``-Infinity``, ``Infinity``, and two zeros, ``+0`` and ``-0``. diff --git a/Doc/library/dialog.rst b/Doc/library/dialog.rst index e0693e8eb6e..5d522556235 100644 --- a/Doc/library/dialog.rst +++ b/Doc/library/dialog.rst @@ -1,18 +1,17 @@ Tkinter Dialogs =============== -:mod:`tkinter.simpledialog` --- Standard Tkinter input dialogs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:mod:`!tkinter.simpledialog` --- Standard Tkinter input dialogs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: tkinter.simpledialog - :platform: Tk :synopsis: Simple dialog windows **Source code:** :source:`Lib/tkinter/simpledialog.py` -------------- -The :mod:`tkinter.simpledialog` module contains convenience classes and +The :mod:`!tkinter.simpledialog` module contains convenience classes and functions for creating simple modal dialogs to get a value from the user. @@ -39,18 +38,17 @@ functions for creating simple modal dialogs to get a value from the user. -:mod:`tkinter.filedialog` --- File selection dialogs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:mod:`!tkinter.filedialog` --- File selection dialogs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: tkinter.filedialog - :platform: Tk :synopsis: Dialog classes for file selection **Source code:** :source:`Lib/tkinter/filedialog.py` -------------- -The :mod:`tkinter.filedialog` module provides classes and factory functions for +The :mod:`!tkinter.filedialog` module provides classes and factory functions for creating file/directory selection windows. Native Load/Save Dialogs @@ -204,18 +202,17 @@ These do not emulate the native look-and-feel of the platform. directory. Confirmation is required if an already existing file is selected. -:mod:`tkinter.commondialog` --- Dialog window templates -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:mod:`!tkinter.commondialog` --- Dialog window templates +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: tkinter.commondialog - :platform: Tk :synopsis: Tkinter base class for dialogs **Source code:** :source:`Lib/tkinter/commondialog.py` -------------- -The :mod:`tkinter.commondialog` module provides the :class:`Dialog` class that +The :mod:`!tkinter.commondialog` module provides the :class:`Dialog` class that is the base class for dialogs defined in other supporting modules. .. class:: Dialog(master=None, **options) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 61556fd5be3..d6b728b0848 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -14,7 +14,7 @@ -------------- -The :mod:`dis` module supports the analysis of CPython :term:`bytecode` by +The :mod:`!dis` module supports the analysis of CPython :term:`bytecode` by disassembling it. The CPython bytecode which this module takes as an input is defined in the file :file:`Include/opcode.h` and used by the compiler and the interpreter. @@ -38,7 +38,7 @@ interpreter. Some instructions are accompanied by one or more inline cache entries, which take the form of :opcode:`CACHE` instructions. These instructions are hidden by default, but can be shown by passing ``show_caches=True`` to - any :mod:`dis` utility. Furthermore, the interpreter now adapts the + any :mod:`!dis` utility. Furthermore, the interpreter now adapts the bytecode to specialize it for different runtime conditions. The adaptive bytecode can be shown by passing ``adaptive=True``. @@ -87,7 +87,7 @@ the following command can be used to display the disassembly of Command-line interface ---------------------- -The :mod:`dis` module can be invoked as a script from the command line: +The :mod:`!dis` module can be invoked as a script from the command line: .. code-block:: sh @@ -223,7 +223,7 @@ Example: Analysis functions ------------------ -The :mod:`dis` module also defines the following analysis functions that convert +The :mod:`!dis` module also defines the following analysis functions that convert the input directly to the desired output. They can be useful if only a single operation is being performed, so the intermediate analysis object isn't useful: @@ -1827,7 +1827,7 @@ iterations of the loop. ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument. .. versionchanged:: 3.12 - Pseudo instructions were added to the :mod:`dis` module, and for them + Pseudo instructions were added to the :mod:`!dis` module, and for them it is not true that comparison with ``HAVE_ARGUMENT`` indicates whether they use their arg. diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 61463d6adcd..a303afe60b7 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -13,7 +13,7 @@ -------------- -The :mod:`doctest` module searches for pieces of text that look like interactive +The :mod:`!doctest` module searches for pieces of text that look like interactive Python sessions, and then executes those sessions to verify that they work exactly as shown. There are several common ways to use doctest: @@ -85,7 +85,7 @@ Here's a complete but small example module:: import doctest doctest.testmod() -If you run :file:`example.py` directly from the command line, :mod:`doctest` +If you run :file:`example.py` directly from the command line, :mod:`!doctest` works its magic: .. code-block:: shell-session @@ -94,7 +94,7 @@ works its magic: $ There's no output! That's normal, and it means all the examples worked. Pass -``-v`` to the script, and :mod:`doctest` prints a detailed log of what +``-v`` to the script, and :mod:`!doctest` prints a detailed log of what it's trying, and prints a summary at the end: .. code-block:: shell-session @@ -130,7 +130,7 @@ And so on, eventually ending with: Test passed. $ -That's all you need to know to start making productive use of :mod:`doctest`! +That's all you need to know to start making productive use of :mod:`!doctest`! Jump in. The following sections provide full details. Note that there are many examples of doctests in the standard Python test suite and libraries. Especially useful examples can be found in the standard test file @@ -252,7 +252,7 @@ For more information on :func:`testfile`, see section :ref:`doctest-basic-api`. Command-line Usage ------------------ -The :mod:`doctest` module can be invoked as a script from the command line: +The :mod:`!doctest` module can be invoked as a script from the command line: .. code-block:: bash @@ -450,7 +450,7 @@ The fine print: What's the Execution Context? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -By default, each time :mod:`doctest` finds a docstring to test, it uses a +By default, each time :mod:`!doctest` finds a docstring to test, it uses a *shallow copy* of :mod:`!M`'s globals, so that running tests doesn't change the module's real globals, and so that one test in :mod:`!M` can't leave behind crumbs that accidentally allow another test to work. This means examples can @@ -730,7 +730,7 @@ The second group of options controls how test failures are reported: There is also a way to register new option flag names, though this isn't -useful unless you intend to extend :mod:`doctest` internals via subclassing: +useful unless you intend to extend :mod:`!doctest` internals via subclassing: .. function:: register_optionflag(name) @@ -833,7 +833,7 @@ disabling an option via ``-`` in a directive can be useful. Warnings ^^^^^^^^ -:mod:`doctest` is serious about requiring exact matches in expected output. If +:mod:`!doctest` is serious about requiring exact matches in expected output. If even a single character doesn't match, the test fails. This will probably surprise you a few times, as you learn exactly what Python does and doesn't guarantee about output. For example, when printing a set, Python doesn't @@ -1035,7 +1035,7 @@ Unittest API ------------ As your collection of doctest'ed modules grows, you'll want a way to run all -their doctests systematically. :mod:`doctest` provides two functions that can +their doctests systematically. :mod:`!doctest` provides two functions that can be used to create :mod:`unittest` test suites from modules and text files containing doctests. To integrate with :mod:`unittest` test discovery, include a :ref:`load_tests ` function in your test module:: @@ -1168,7 +1168,7 @@ of :class:`!DocTestCase`. So both ways of creating a :class:`unittest.TestSuite` run instances of :class:`!DocTestCase`. This is important for a subtle reason: when you run -:mod:`doctest` functions yourself, you can control the :mod:`!doctest` options in +:mod:`!doctest` functions yourself, you can control the :mod:`!doctest` options in use directly, by passing option flags to :mod:`!doctest` functions. However, if you're writing a :mod:`unittest` framework, :mod:`!unittest` ultimately controls when and how tests get run. The framework author typically wants to control @@ -1176,13 +1176,13 @@ when and how tests get run. The framework author typically wants to control options), but there's no way to pass options through :mod:`!unittest` to :mod:`!doctest` test runners. -For this reason, :mod:`doctest` also supports a notion of :mod:`!doctest` +For this reason, :mod:`!doctest` also supports a notion of :mod:`!doctest` reporting flags specific to :mod:`unittest` support, via this function: .. function:: set_unittest_reportflags(flags) - Set the :mod:`doctest` reporting flags to use. + Set the :mod:`!doctest` reporting flags to use. Argument *flags* takes the :ref:`bitwise OR ` of option flags. See section :ref:`doctest-options`. Only "reporting flags" can be used. @@ -1899,7 +1899,7 @@ There are two exceptions that may be raised by :class:`DebugRunner` instances: Soapbox ------- -As mentioned in the introduction, :mod:`doctest` has grown to have three primary +As mentioned in the introduction, :mod:`!doctest` has grown to have three primary uses: #. Checking examples in docstrings. @@ -1917,7 +1917,7 @@ this that needs to be learned---it may not be natural at first. Examples should add genuine value to the documentation. A good example can often be worth many words. If done with care, the examples will be invaluable for your users, and will pay back the time it takes to collect them many times over as the years go -by and things change. I'm still amazed at how often one of my :mod:`doctest` +by and things change. I'm still amazed at how often one of my :mod:`!doctest` examples stops working after a "harmless" change. Doctest also makes an excellent tool for regression testing, especially if you diff --git a/Doc/library/email.charset.rst b/Doc/library/email.charset.rst index 6875af2be49..76a57031862 100644 --- a/Doc/library/email.charset.rst +++ b/Doc/library/email.charset.rst @@ -19,7 +19,7 @@ registry and several convenience methods for manipulating this registry. Instances of :class:`Charset` are used in several other modules within the :mod:`email` package. -Import this class from the :mod:`email.charset` module. +Import this class from the :mod:`!email.charset` module. .. class:: Charset(input_charset=DEFAULT_CHARSET) @@ -164,7 +164,7 @@ Import this class from the :mod:`email.charset` module. This method allows you to compare two :class:`Charset` instances for inequality. -The :mod:`email.charset` module also provides the following functions for adding +The :mod:`!email.charset` module also provides the following functions for adding new entries to the global character set, alias, and codec registries: diff --git a/Doc/library/email.encoders.rst b/Doc/library/email.encoders.rst index 9c8c8c9234e..1a9a1cad3a6 100644 --- a/Doc/library/email.encoders.rst +++ b/Doc/library/email.encoders.rst @@ -25,7 +25,7 @@ is especially true for :mimetype:`image/\*` and :mimetype:`text/\*` type message containing binary data. The :mod:`email` package provides some convenient encoders in its -:mod:`~email.encoders` module. These encoders are actually used by the +:mod:`!encoders` module. These encoders are actually used by the :class:`~email.mime.audio.MIMEAudio` and :class:`~email.mime.image.MIMEImage` class constructors to provide default encodings. All encoder functions take exactly one argument, the message object to encode. They usually extract the diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 689e7397cbc..2f7c9140cfc 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -8,7 +8,7 @@ -------------- -The following exception classes are defined in the :mod:`email.errors` module: +The following exception classes are defined in the :mod:`!email.errors` module: .. exception:: MessageError() diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index a3132d02687..6f4f813a0f8 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -232,7 +232,7 @@ a formatted string representation of a message object. For more detail, see :mod:`email.message`. -The :mod:`email.generator` module also provides a derived class, +The :mod:`!email.generator` module also provides a derived class, :class:`DecodedGenerator`, which is like the :class:`Generator` base class, except that non-\ :mimetype:`text` parts are not serialized, but are instead represented in the output stream by a string derived from a template filled diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst index f49885b8785..e7e21d036e0 100644 --- a/Doc/library/email.header.rst +++ b/Doc/library/email.header.rst @@ -28,13 +28,13 @@ transferred using only 7-bit ASCII characters, so a slew of RFCs have been written describing how to encode email containing non-ASCII characters into :rfc:`2822`\ -compliant format. These RFCs include :rfc:`2045`, :rfc:`2046`, :rfc:`2047`, and :rfc:`2231`. The :mod:`email` package supports these standards -in its :mod:`email.header` and :mod:`email.charset` modules. +in its :mod:`!email.header` and :mod:`email.charset` modules. If you want to include non-ASCII characters in your email headers, say in the :mailheader:`Subject` or :mailheader:`To` fields, you should use the :class:`Header` class and assign the field in the :class:`~email.message.Message` object to an instance of :class:`Header` instead of using a string for the header -value. Import the :class:`Header` class from the :mod:`email.header` module. +value. Import the :class:`Header` class from the :mod:`!email.header` module. For example:: >>> from email.message import Message @@ -170,7 +170,7 @@ Here is the :class:`Header` class description: This method allows you to compare two :class:`Header` instances for inequality. -The :mod:`email.header` module also provides the following convenient functions. +The :mod:`!email.header` module also provides the following convenient functions. .. function:: decode_header(header) diff --git a/Doc/library/email.iterators.rst b/Doc/library/email.iterators.rst index 090981d84b4..ed300cdb30f 100644 --- a/Doc/library/email.iterators.rst +++ b/Doc/library/email.iterators.rst @@ -10,7 +10,7 @@ Iterating over a message object tree is fairly easy with the :meth:`Message.walk ` method. The -:mod:`email.iterators` module provides some useful higher level iterations over +:mod:`!email.iterators` module provides some useful higher level iterations over message object trees. diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index 0aa8e632c2c..f6908d2e6e9 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -14,7 +14,7 @@ .. versionadded:: 3.6 [1]_ The central class in the :mod:`email` package is the :class:`EmailMessage` -class, imported from the :mod:`email.message` module. It is the base class for +class, imported from the :mod:`!email.message` module. It is the base class for the :mod:`email` object model. :class:`EmailMessage` provides the core functionality for setting and querying header fields, for accessing message bodies, and for creating or modifying structured messages. diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index 6a70714dc3e..6a67bf7c8e5 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -125,10 +125,10 @@ Here is the API for the :class:`BytesFeedParser`: Parser API ^^^^^^^^^^ -The :class:`BytesParser` class, imported from the :mod:`email.parser` module, +The :class:`BytesParser` class, imported from the :mod:`!email.parser` module, provides an API that can be used to parse a message when the complete contents of the message are available in a :term:`bytes-like object` or file. The -:mod:`email.parser` module also provides :class:`Parser` for parsing strings, +:mod:`!email.parser` module also provides :class:`Parser` for parsing strings, and header-only parsers, :class:`BytesHeaderParser` and :class:`HeaderParser`, which can be used if you're only interested in the headers of the message. :class:`BytesHeaderParser` and :class:`HeaderParser` @@ -155,7 +155,7 @@ message body, instead setting the payload to the raw body. Read all the data from the binary file-like object *fp*, parse the resulting bytes, and return the message object. *fp* must support - both the :meth:`~io.IOBase.readline` and the :meth:`~io.IOBase.read` + both the :meth:`~io.IOBase.readline` and the :meth:`~io.BufferedIOBase.read` methods. The bytes contained in *fp* must be formatted as a block of :rfc:`5322` diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 1ff3e2c3f8d..fef064114ec 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -602,7 +602,7 @@ The header objects and their attributes are described in This concrete :class:`Policy` is the backward compatibility policy. It replicates the behavior of the email package in Python 3.2. The - :mod:`~email.policy` module also defines an instance of this class, + :mod:`!policy` module also defines an instance of this class, :const:`compat32`, that is used as the default policy. Thus the default behavior of the email package is to maintain compatibility with Python 3.2. diff --git a/Doc/library/email.rst b/Doc/library/email.rst index 66c42e4a500..03ac1783be0 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -12,10 +12,10 @@ -------------- -The :mod:`email` package is a library for managing email messages. It is +The :mod:`!email` package is a library for managing email messages. It is specifically *not* designed to do any sending of email messages to SMTP (:rfc:`2821`), NNTP, or other servers; those are functions of modules such as -:mod:`smtplib`. The :mod:`email` package attempts to be as +:mod:`smtplib`. The :mod:`!email` package attempts to be as RFC-compliant as possible, supporting :rfc:`5322` and :rfc:`6532`, as well as such MIME-related RFCs as :rfc:`2045`, :rfc:`2046`, :rfc:`2047`, :rfc:`2183`, and :rfc:`2231`. @@ -68,7 +68,7 @@ high level structure in question, and not the details of how those structures are represented. Since MIME content types are used widely in modern internet software (not just email), this will be a familiar concept to many programmers. -The following sections describe the functionality of the :mod:`email` package. +The following sections describe the functionality of the :mod:`!email` package. We start with the :mod:`~email.message` object model, which is the primary interface an application will use, and follow that with the :mod:`~email.parser` and :mod:`~email.generator` components. Then we cover the @@ -102,7 +102,7 @@ compatibility reasons. :class:`~email.message.EmailMessage`/:class:`~email.policy.EmailPolicy` API. -Contents of the :mod:`email` package documentation: +Contents of the :mod:`!email` package documentation: .. toctree:: diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index 611549604fd..e0d2c19a3b0 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -8,7 +8,7 @@ -------------- -There are a couple of useful utilities provided in the :mod:`email.utils` +There are a couple of useful utilities provided in the :mod:`!email.utils` module: .. function:: localtime(dt=None) diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index 32b92c01570..e0d77229b11 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`ensurepip` package provides support for bootstrapping the ``pip`` +The :mod:`!ensurepip` package provides support for bootstrapping the ``pip`` installer into an existing Python installation or virtual environment. This bootstrapping approach reflects the fact that ``pip`` is an independent project with its own release cycle, and the latest available stable version @@ -99,7 +99,7 @@ Providing both of the script selection options will trigger an exception. Module API ---------- -:mod:`ensurepip` exposes two functions for programmatic use: +:mod:`!ensurepip` exposes two functions for programmatic use: .. function:: version() diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index b39164e5475..ec7cbfb52b6 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -1053,7 +1053,7 @@ Utilities and Decorators >>> enum.bin(~10) # ~10 is -11 '0b1 0101' - .. versionadded:: 3.10 + .. versionadded:: 3.11 --------------- diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 3f1ec95005c..27d5caa43f8 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -221,7 +221,7 @@ The following exceptions are the exceptions that are usually raised. .. exception:: EOFError Raised when the :func:`input` function hits an end-of-file condition (EOF) - without reading any data. (Note: the :meth:`!io.IOBase.read` and + without reading any data. (Note: the :meth:`io.TextIOBase.read` and :meth:`io.IOBase.readline` methods return an empty string when they hit EOF.) diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index c8ce86cc7af..4a08af4f90d 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -2,7 +2,6 @@ ========================================================== .. module:: fcntl - :platform: Unix :synopsis: The fcntl() and ioctl() system calls. .. sectionauthor:: Jaap Vermeulen @@ -53,7 +52,7 @@ descriptor. the latter setting ``FD_CLOEXEC`` flag in addition. .. versionchanged:: 3.12 - On Linux >= 4.5, the :mod:`fcntl` module exposes the ``FICLONE`` and + On Linux >= 4.5, the :mod:`!fcntl` module exposes the ``FICLONE`` and ``FICLONERANGE`` constants, which allow to share some data of one file with another file by reflinking on some filesystems (e.g., btrfs, OCFS2, and XFS). This behavior is commonly referred to as "copy-on-write". @@ -91,7 +90,7 @@ The module defines the following functions: Perform the operation *cmd* on file descriptor *fd* (file objects providing a :meth:`~io.IOBase.fileno` method are accepted as well). The values used for *cmd* are operating system dependent, and are available as constants - in the :mod:`fcntl` module, using the same names as used in the relevant C + in the :mod:`!fcntl` module, using the same names as used in the relevant C header files. The argument *arg* can either be an integer value, a :term:`bytes-like object`, or a string. The type and size of *arg* must match the type and size of diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst index abd1b8c826d..e87a7869685 100644 --- a/Doc/library/filecmp.rst +++ b/Doc/library/filecmp.rst @@ -10,11 +10,11 @@ -------------- -The :mod:`filecmp` module defines functions to compare files and directories, +The :mod:`!filecmp` module defines functions to compare files and directories, with various optional time/correctness trade-offs. For comparing files, see also the :mod:`difflib` module. -The :mod:`filecmp` module defines the following functions: +The :mod:`!filecmp` module defines the following functions: .. function:: cmp(f1, f2, shallow=True) diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index d6d1c7a461c..575e90942d4 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`fractions` module provides support for rational number arithmetic. +The :mod:`!fractions` module provides support for rational number arithmetic. A Fraction instance can be constructed from a pair of rational numbers, from diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index bb153220672..9cbb387f40f 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -23,7 +23,7 @@ The default encoding is UTF-8, following :rfc:`2640`. .. include:: ../includes/wasm-notavail.rst -Here's a sample session using the :mod:`ftplib` module:: +Here's a sample session using the :mod:`!ftplib` module:: >>> from ftplib import FTP >>> ftp = FTP('ftp.us.debian.org') # connect to host, default port diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 893e4f60e01..96b88b38039 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -597,17 +597,18 @@ are always available. They are listed here in alphabetical order. .. warning:: This function executes arbitrary code. Calling it with - user-supplied input may lead to security vulnerabilities. + untrusted user-supplied input will lead to security vulnerabilities. The *source* argument is parsed and evaluated as a Python expression (technically speaking, a condition list) using the *globals* and *locals* mappings as global and local namespace. If the *globals* dictionary is present and does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is - inserted under that key before *source* is parsed. That way you can - control what builtins are available to the executed code by inserting your - own ``__builtins__`` dictionary into *globals* before passing it to - :func:`eval`. If the *locals* mapping is omitted it defaults to the + inserted under that key before *source* is parsed. + Overriding ``__builtins__`` can be used to restrict or change the available + names, but this is **not** a security mechanism: the executed code can + still access all builtins. + If the *locals* mapping is omitted it defaults to the *globals* dictionary. If both mappings are omitted, the source is executed with the *globals* and *locals* in the environment where :func:`eval` is called. Note, *eval()* will only have access to the @@ -658,7 +659,7 @@ are always available. They are listed here in alphabetical order. .. warning:: This function executes arbitrary code. Calling it with - user-supplied input may lead to security vulnerabilities. + untrusted user-supplied input will lead to security vulnerabilities. This function supports dynamic execution of Python code. *source* must be either a string or a code object. If it is a string, the string is parsed as @@ -689,9 +690,10 @@ are always available. They are listed here in alphabetical order. If the *globals* dictionary does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module - :mod:`builtins` is inserted under that key. That way you can control what - builtins are available to the executed code by inserting your own - ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. + :mod:`builtins` is inserted under that key. + Overriding ``__builtins__`` can be used to restrict or change the available + names, but this is **not** a security mechanism: the executed code can + still access all builtins. The *closure* argument specifies a closure--a tuple of cellvars. It's only valid when the *object* is a code object containing @@ -1843,19 +1845,19 @@ are always available. They are listed here in alphabetical order. ``range(start, stop, step)``. The *start* and *step* arguments default to ``None``. - Slice objects have read-only data attributes :attr:`!start`, - :attr:`!stop`, and :attr:`!step` which merely return the argument - values (or their default). They have no other explicit functionality; - however, they are used by NumPy and other third-party packages. + Slice objects are also generated when :ref:`slicing syntax ` + is used. For example: ``a[start:stop:step]`` or ``a[start:stop, i]``. + + See :func:`itertools.islice` for an alternate version that returns an + :term:`iterator`. .. attribute:: slice.start - .. attribute:: slice.stop - .. attribute:: slice.step + slice.stop + slice.step - Slice objects are also generated when extended indexing syntax is used. For - example: ``a[start:stop:step]`` or ``a[start:stop, i]``. See - :func:`itertools.islice` for an alternate version that returns an - :term:`iterator`. + These read-only attributes are set to the argument values + (or their default). They have no other explicit functionality; + however, they are used by NumPy and other third-party packages. .. versionchanged:: 3.12 Slice objects are now :term:`hashable` (provided :attr:`~slice.start`, diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 01db54bbb86..4babc246ea9 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -20,11 +20,11 @@ -------------- -The :mod:`functools` module is for higher-order functions: functions that act on +The :mod:`!functools` module is for higher-order functions: functions that act on or return other functions. In general, any callable object can be treated as a function for the purposes of this module. -The :mod:`functools` module defines the following functions: +The :mod:`!functools` module defines the following functions: .. decorator:: cache(user_function) @@ -190,7 +190,7 @@ The :mod:`functools` module defines the following functions: Note, type specificity applies only to the function's immediate arguments rather than their contents. The scalar arguments, ``Decimal(42)`` and - ``Fraction(42)`` are be treated as distinct calls with distinct results. + ``Fraction(42)`` are treated as distinct calls with distinct results. In contrast, the tuple arguments ``('answer', Decimal(42))`` and ``('answer', Fraction(42))`` are treated as equivalent. diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 35c5c7350cf..63dc8bce46d 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -20,7 +20,7 @@ can be disabled by calling ``gc.disable()``. To debug a leaking program call ``gc.DEBUG_SAVEALL``, causing garbage-collected objects to be saved in gc.garbage for inspection. -The :mod:`gc` module provides the following functions: +The :mod:`!gc` module provides the following functions: .. function:: enable() diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index a0c0c6dee2d..37ffbe1be55 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -14,7 +14,7 @@ .. include:: ../includes/wasm-notavail.rst -The :mod:`getpass` module provides two functions: +The :mod:`!getpass` module provides two functions: .. function:: getpass(prompt='Password: ', stream=None, *, echo_char=None) diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index d0de83907eb..ddd0188e661 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`gettext` module provides internationalization (I18N) and localization +The :mod:`!gettext` module provides internationalization (I18N) and localization (L10N) services for your Python modules and applications. It supports both the GNU :program:`gettext` message catalog API and a higher level, class-based API that may be more appropriate for Python files. The interface described below allows you @@ -25,7 +25,7 @@ Some hints on localizing your Python modules and applications are also given. GNU :program:`gettext` API -------------------------- -The :mod:`gettext` module defines the following API, which is very similar to +The :mod:`!gettext` module defines the following API, which is very similar to the GNU :program:`gettext` API. If you use this API you will affect the translation of your entire application globally. Often this is what you want if your application is monolingual, with the choice of language dependent on the @@ -37,7 +37,7 @@ class-based API instead. .. function:: bindtextdomain(domain, localedir=None) Bind the *domain* to the locale directory *localedir*. More concretely, - :mod:`gettext` will look for binary :file:`.mo` files for the given domain using + :mod:`!gettext` will look for binary :file:`.mo` files for the given domain using the path (on Unix): :file:`{localedir}/{language}/LC_MESSAGES/{domain}.mo`, where *language* is searched for in the environment variables :envvar:`LANGUAGE`, :envvar:`LC_ALL`, :envvar:`LC_MESSAGES`, and :envvar:`LANG` respectively. @@ -114,7 +114,7 @@ Here's an example of typical usage for this API:: Class-based API --------------- -The class-based API of the :mod:`gettext` module gives you more flexibility and +The class-based API of the :mod:`!gettext` module gives you more flexibility and greater convenience than the GNU :program:`gettext` API. It is the recommended way of localizing your Python applications and modules. :mod:`!gettext` defines a :class:`GNUTranslations` class which implements the parsing of GNU :file:`.mo` format @@ -393,7 +393,7 @@ The Catalog constructor .. index:: single: GNOME -GNOME uses a version of the :mod:`gettext` module by James Henstridge, but this +GNOME uses a version of the :mod:`!gettext` module by James Henstridge, but this version has a slightly different API. Its documented usage was:: import gettext @@ -425,7 +425,7 @@ take the following steps: #. create language-specific translations of the message catalogs -#. use the :mod:`gettext` module so that message strings are properly translated +#. use the :mod:`!gettext` module so that message strings are properly translated In order to prepare your code for I18N, you need to look at all the strings in your files. Any string that needs to be translated should be marked by wrapping @@ -473,10 +473,10 @@ supported natural language. They send back the completed language-specific versions as a :file:`.po` file that's compiled into a machine-readable :file:`.mo` binary catalog file using the :program:`msgfmt` program. The :file:`.mo` files are used by the -:mod:`gettext` module for the actual translation processing at +:mod:`!gettext` module for the actual translation processing at run-time. -How you use the :mod:`gettext` module in your code depends on whether you are +How you use the :mod:`!gettext` module in your code depends on whether you are internationalizing a single module or your entire application. The next two sections will discuss each case. diff --git a/Doc/library/graphlib.rst b/Doc/library/graphlib.rst index 053d5f8231b..21f4d1fb938 100644 --- a/Doc/library/graphlib.rst +++ b/Doc/library/graphlib.rst @@ -204,7 +204,7 @@ Exceptions ---------- -The :mod:`graphlib` module defines the following exception classes: +The :mod:`!graphlib` module defines the following exception classes: .. exception:: CycleError diff --git a/Doc/library/grp.rst b/Doc/library/grp.rst index d1c7f22a209..f436970e791 100644 --- a/Doc/library/grp.rst +++ b/Doc/library/grp.rst @@ -2,7 +2,6 @@ ================================== .. module:: grp - :platform: Unix :synopsis: The group database (getgrnam() and friends). -------------- diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index 76de92b3069..a3c98d2f42b 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -15,7 +15,7 @@ like the GNU programs :program:`gzip` and :program:`gunzip` would. The data compression is provided by the :mod:`zlib` module. -The :mod:`gzip` module provides the :class:`GzipFile` class, as well as the +The :mod:`!gzip` module provides the :class:`GzipFile` class, as well as the :func:`.open`, :func:`compress` and :func:`decompress` convenience functions. The :class:`GzipFile` class reads and writes :program:`gzip`\ -format files, automatically compressing or decompressing the data so that it looks like an @@ -272,10 +272,10 @@ Example of how to GZIP compress a binary string:: Command-line interface ---------------------- -The :mod:`gzip` module provides a simple command line interface to compress or +The :mod:`!gzip` module provides a simple command line interface to compress or decompress files. -Once executed the :mod:`gzip` module keeps the input file(s). +Once executed the :mod:`!gzip` module keeps the input file(s). .. versionchanged:: 3.8 diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 855a6efc9f7..84039ad2818 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -61,7 +61,7 @@ if you are using a rare "FIPS compliant" build of Python. These correspond to :data:`algorithms_guaranteed`. Additional algorithms may also be available if your Python distribution's -:mod:`hashlib` was linked against a build of OpenSSL that provides others. +:mod:`!hashlib` was linked against a build of OpenSSL that provides others. Others *are not guaranteed available* on all installations and will only be accessible by name via :func:`new`. See :data:`algorithms_available`. @@ -390,7 +390,7 @@ BLAKE2 supports **keyed mode** (a faster and simpler replacement for HMAC_), **salted hashing**, **personalization**, and **tree hashing**. Hash objects from this module follow the API of standard library's -:mod:`hashlib` objects. +:mod:`!hashlib` objects. Creating hash objects diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index fcb0069b760..90daaf28f8d 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`http.cookiejar` module defines classes for automatic handling of HTTP +The :mod:`!http.cookiejar` module defines classes for automatic handling of HTTP cookies. It is useful for accessing websites that require small pieces of data -- :dfn:`cookies` -- to be set on the client machine by an HTTP response from a web server, and then returned to the server in later HTTP requests. @@ -21,7 +21,7 @@ Both the regular Netscape cookie protocol and the protocol defined by :rfc:`2109` cookies are parsed as Netscape cookies and subsequently treated either as Netscape or RFC 2965 cookies according to the 'policy' in effect. Note that the great majority of cookies on the internet are Netscape cookies. -:mod:`http.cookiejar` attempts to follow the de-facto Netscape cookie protocol (which +:mod:`!http.cookiejar` attempts to follow the de-facto Netscape cookie protocol (which differs substantially from that set out in the original Netscape specification), including taking note of the ``max-age`` and ``port`` cookie-attributes introduced with RFC 2965. @@ -109,7 +109,7 @@ The following classes are provided: .. class:: Cookie() This class represents Netscape, :rfc:`2109` and :rfc:`2965` cookies. It is not - expected that users of :mod:`http.cookiejar` construct their own :class:`Cookie` + expected that users of :mod:`!http.cookiejar` construct their own :class:`Cookie` instances. Instead, if necessary, call :meth:`make_cookies` on a :class:`CookieJar` instance. @@ -121,13 +121,13 @@ The following classes are provided: Module :mod:`http.cookies` HTTP cookie classes, principally useful for server-side code. The - :mod:`http.cookiejar` and :mod:`http.cookies` modules do not depend on each + :mod:`!http.cookiejar` and :mod:`http.cookies` modules do not depend on each other. https://curl.se/rfc/cookie_spec.html The specification of the original Netscape cookie protocol. Though this is still the dominant protocol, the 'Netscape cookie protocol' implemented by all - the major browsers (and :mod:`http.cookiejar`) only bears a passing resemblance to + the major browsers (and :mod:`!http.cookiejar`) only bears a passing resemblance to the one sketched out in ``cookie_spec.html``. :rfc:`2109` - HTTP State Management Mechanism @@ -617,7 +617,7 @@ standard cookie-attributes specified in the various cookie standards. The correspondence is not one-to-one, because there are complicated rules for assigning default values, because the ``max-age`` and ``expires`` cookie-attributes contain equivalent information, and because :rfc:`2109` cookies -may be 'downgraded' by :mod:`http.cookiejar` from version 1 to version 0 (Netscape) +may be 'downgraded' by :mod:`!http.cookiejar` from version 1 to version 0 (Netscape) cookies. Assignment to these attributes should not be necessary other than in rare @@ -629,7 +629,7 @@ internal consistency, so you should know what you're doing if you do that. Integer or :const:`None`. Netscape cookies have :attr:`version` 0. :rfc:`2965` and :rfc:`2109` cookies have a ``version`` cookie-attribute of 1. However, note that - :mod:`http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which + :mod:`!http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which case :attr:`version` is 0. @@ -692,7 +692,7 @@ internal consistency, so you should know what you're doing if you do that. ``True`` if this cookie was received as an :rfc:`2109` cookie (ie. the cookie arrived in a :mailheader:`Set-Cookie` header, and the value of the Version cookie-attribute in that header was 1). This attribute is provided because - :mod:`http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in + :mod:`!http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which case :attr:`version` is 0. @@ -744,7 +744,7 @@ The :class:`Cookie` class also defines the following method: Examples -------- -The first example shows the most common usage of :mod:`http.cookiejar`:: +The first example shows the most common usage of :mod:`!http.cookiejar`:: import http.cookiejar, urllib.request cj = http.cookiejar.CookieJar() diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index 0257f74706f..1b2d17975a5 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`http.cookies` module defines classes for abstracting the concept of +The :mod:`!http.cookies` module defines classes for abstracting the concept of cookies, an HTTP state management mechanism. It supports both simple string-only cookies, and provides an abstraction for having any serializable data-type as cookie value. @@ -65,7 +65,7 @@ in a cookie name (as :attr:`~Morsel.key`). Module :mod:`http.cookiejar` HTTP cookie handling for web *clients*. The :mod:`http.cookiejar` and - :mod:`http.cookies` modules do not depend on each other. + :mod:`!http.cookies` modules do not depend on each other. :rfc:`2109` - HTTP State Management Mechanism This is the state management specification implemented by this module. @@ -264,7 +264,7 @@ Morsel Objects Example ------- -The following example demonstrates how to use the :mod:`http.cookies` module. +The following example demonstrates how to use the :mod:`!http.cookies` module. .. doctest:: :options: +NORMALIZE_WHITESPACE diff --git a/Doc/library/http.rst b/Doc/library/http.rst index b0bdfc65e45..43a801416e2 100644 --- a/Doc/library/http.rst +++ b/Doc/library/http.rst @@ -12,7 +12,7 @@ -------------- -:mod:`http` is a package that collects several modules for working with the +:mod:`!http` is a package that collects several modules for working with the HyperText Transfer Protocol: * :mod:`http.client` is a low-level HTTP protocol client; for high-level URL @@ -22,7 +22,7 @@ HyperText Transfer Protocol: * :mod:`http.cookiejar` provides persistence of cookies -The :mod:`http` module also defines the following enums that help you work with http related code: +The :mod:`!http` module also defines the following enums that help you work with http related code: .. class:: HTTPStatus diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 41026e2b303..b47da97d3f2 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -19,7 +19,7 @@ This module defines classes for implementing HTTP servers. .. warning:: - :mod:`http.server` is not recommended for production. It only implements + :mod:`!http.server` is not recommended for production. It only implements :ref:`basic security checks `. .. include:: ../includes/wasm-notavail.rst @@ -512,7 +512,7 @@ such as using different index file names by overriding the class attribute Command-line interface ---------------------- -:mod:`http.server` can also be invoked directly using the :option:`-m` +:mod:`!http.server` can also be invoked directly using the :option:`-m` switch of the interpreter. The following example illustrates how to serve files relative to the current directory:: @@ -572,7 +572,7 @@ The following options are accepted: .. deprecated-removed:: 3.13 3.15 - :mod:`http.server` command line ``--cgi`` support is being removed + :mod:`!http.server` command line ``--cgi`` support is being removed because :class:`CGIHTTPRequestHandler` is being removed. .. warning:: diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 3eee2b9e607..166455bae02 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -29,7 +29,7 @@ note that the ``STATUS`` command is not supported in IMAP4. .. include:: ../includes/wasm-notavail.rst -Three classes are provided by the :mod:`imaplib` module, :class:`IMAP4` is the +Three classes are provided by the :mod:`!imaplib` module, :class:`IMAP4` is the base class: diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index bd937811f41..2e65eff83f5 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -17,7 +17,7 @@ Introduction ------------ -The purpose of the :mod:`importlib` package is three-fold. +The purpose of the :mod:`!importlib` package is three-fold. One is to provide the implementation of the :keyword:`import` statement (and thus, by extension, the @@ -211,8 +211,8 @@ Functions in unexpected behavior. It's recommended to use the :class:`threading.Lock` or other synchronization primitives for thread-safe module reloading. -:mod:`importlib.abc` -- Abstract base classes related to import ---------------------------------------------------------------- +:mod:`!importlib.abc` -- Abstract base classes related to import +---------------------------------------------------------------- .. module:: importlib.abc :synopsis: Abstract base classes related to import @@ -222,7 +222,7 @@ Functions -------------- -The :mod:`importlib.abc` module contains all of the core abstract base classes +The :mod:`!importlib.abc` module contains all of the core abstract base classes used by :keyword:`import`. Some subclasses of the core abstract base classes are also provided to help in implementing the core ABCs. @@ -637,8 +637,8 @@ ABC hierarchy:: itself does not end in ``__init__``. -:mod:`importlib.machinery` -- Importers and path hooks ------------------------------------------------------- +:mod:`!importlib.machinery` -- Importers and path hooks +------------------------------------------------------- .. module:: importlib.machinery :synopsis: Importers and path hooks @@ -1136,8 +1136,8 @@ find and load modules. Path to the ``.fwork`` file for the extension module. -:mod:`importlib.util` -- Utility code for importers ---------------------------------------------------- +:mod:`!importlib.util` -- Utility code for importers +---------------------------------------------------- .. module:: importlib.util :synopsis: Utility code for importers diff --git a/Doc/library/index.rst b/Doc/library/index.rst index 163e1679c65..8fc77be520d 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -43,6 +43,7 @@ the `Python Package Index `_. constants.rst stdtypes.rst exceptions.rst + threadsafety.rst text.rst binary.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 7bfacc78988..5e3c5c1f155 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -16,7 +16,7 @@ -------------- -The :mod:`inspect` module provides several useful functions to help get +The :mod:`!inspect` module provides several useful functions to help get information about live objects such as modules, classes, methods, functions, tracebacks, frame objects, and code objects. For example, it can help you examine the contents of a class, retrieve the source code of a method, extract @@ -1755,7 +1755,7 @@ which is a bitmap of the following flags: The flags are specific to CPython, and may not be defined in other Python implementations. Furthermore, the flags are an implementation detail, and can be removed or deprecated in future Python releases. - It's recommended to use public APIs from the :mod:`inspect` module + It's recommended to use public APIs from the :mod:`!inspect` module for any introspection needs. @@ -1797,7 +1797,7 @@ Buffer flags Command-line interface ---------------------- -The :mod:`inspect` module also provides a basic introspection capability +The :mod:`!inspect` module also provides a basic introspection capability from the command line. .. program:: inspect diff --git a/Doc/library/io.rst b/Doc/library/io.rst index de5cab5aee6..55ec6ee5b31 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -24,7 +24,7 @@ Overview .. index:: single: file object; io module -The :mod:`io` module provides Python's main facilities for dealing with various +The :mod:`!io` module provides Python's main facilities for dealing with various types of I/O. There are three main types of I/O: *text I/O*, *binary I/O* and *raw I/O*. These are generic categories, and various backing stores can be used for each of them. A concrete object belonging to any of these @@ -292,7 +292,7 @@ interface to a buffered raw stream (:class:`BufferedIOBase`). Finally, Argument names are not part of the specification, and only the arguments of :func:`open` are intended to be used as keyword arguments. -The following table summarizes the ABCs provided by the :mod:`io` module: +The following table summarizes the ABCs provided by the :mod:`!io` module: .. tabularcolumns:: |l|l|L|L| @@ -587,7 +587,7 @@ I/O Base Classes When the underlying raw stream is non-blocking, implementations may either raise :exc:`BlockingIOError` or return ``None`` if no data is - available. :mod:`io` implementations return ``None``. + available. :mod:`!io` implementations return ``None``. .. method:: read1(size=-1, /) @@ -600,7 +600,7 @@ I/O Base Classes When the underlying raw stream is non-blocking, implementations may either raise :exc:`BlockingIOError` or return ``None`` if no data is - available. :mod:`io` implementations return ``None``. + available. :mod:`!io` implementations return ``None``. .. method:: readinto(b, /) diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 9e887d8e657..c546d913cbe 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -10,7 +10,7 @@ -------------- -:mod:`ipaddress` provides the capabilities to create, manipulate and +:mod:`!ipaddress` provides the capabilities to create, manipulate and operate on IPv4 and IPv6 addresses and networks. The functions and classes in this module make it straightforward to handle @@ -34,7 +34,7 @@ This is the full module API reference—for an overview and introduction, see Convenience factory functions ----------------------------- -The :mod:`ipaddress` module provides factory functions to conveniently create +The :mod:`!ipaddress` module provides factory functions to conveniently create IP addresses, networks and interfaces: .. function:: ip_address(address) @@ -1027,7 +1027,7 @@ The module also provides the following module level functions: IPv4Address('192.0.2.0') <= IPv4Network('192.0.2.0/24') doesn't make sense. There are some times however, where you may wish to - have :mod:`ipaddress` sort these anyway. If you need to do this, you can use + have :mod:`!ipaddress` sort these anyway. If you need to do this, you can use this function as the *key* argument to :func:`sorted`. *obj* is either a network or address object. diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 08dacb505f7..53f4b31e17d 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -30,18 +30,7 @@ For instance, SML provides a tabulation tool: ``tabulate(f)`` which produces a sequence ``f(0), f(1), ...``. The same effect can be achieved in Python by combining :func:`map` and :func:`count` to form ``map(f, count())``. - -**Infinite iterators:** - -================== ================= ================================================= ========================================= -Iterator Arguments Results Example -================== ================= ================================================= ========================================= -:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) → 10 11 12 13 14 ...`` -:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') → A B C D A B C D ...`` -:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) → 10 10 10`` -================== ================= ================================================= ========================================= - -**Iterators terminating on the shortest input sequence:** +**General iterators:** ============================ ============================ ================================================= ============================================================= Iterator Arguments Results Example @@ -51,11 +40,14 @@ Iterator Arguments Results :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') → A B C D E F`` :func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) → A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` +:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) → 10 11 12 13 14 ...`` +:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') → A B C D A B C D ...`` :func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8`` :func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8`` :func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) ``groupby(['A','B','DEF'], len) → (1, A B) (3, DEF)`` :func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) → C D E F G`` :func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') → AB BC CD DE EF FG`` +:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) → 10 10 10`` :func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000`` :func:`takewhile` predicate, seq seq[0], seq[1], until predicate fails ``takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4`` :func:`tee` it, n it1, it2, ... itn splits one iterator into n ``tee('ABC', 2) → A B C, A B C`` @@ -845,7 +837,8 @@ and :term:`generators ` which incur interpreter overhead. from contextlib import suppress from functools import reduce from math import comb, isqrt, prod, sumprod - from operator import getitem, is_not, itemgetter, mul, neg + from operator import getitem, is_not, itemgetter, mul, neg, truediv + # ==== Basic one liners ==== @@ -858,9 +851,10 @@ and :term:`generators ` which incur interpreter overhead. # prepend(1, [2, 3, 4]) → 1 2 3 4 return chain([value], iterable) - def tabulate(function, start=0): - "Return function(0), function(1), ..." - return map(function, count(start)) + def running_mean(iterable): + "Yield the average of all values seen so far." + # running_mean([8.5, 9.5, 7.5, 6.5]) -> 8.5 9.0 8.5 8.0 + return map(truediv, accumulate(iterable), count(1)) def repeatfunc(function, times=None, *args): "Repeat calls to a function with specified arguments." @@ -913,6 +907,7 @@ and :term:`generators ` which incur interpreter overhead. # all_equal('4٤௪౪໔', key=int) → True return len(take(2, groupby(iterable, key))) <= 1 + # ==== Data pipelines ==== def unique_justseen(iterable, key=None): @@ -1021,6 +1016,7 @@ and :term:`generators ` which incur interpreter overhead. while True: yield function() + # ==== Mathematical operations ==== def multinomial(*counts): @@ -1040,6 +1036,7 @@ and :term:`generators ` which incur interpreter overhead. # sum_of_squares([10, 20, 30]) → 1400 return sumprod(*tee(iterable)) + # ==== Matrix operations ==== def reshape(matrix, columns): @@ -1058,6 +1055,7 @@ and :term:`generators ` which incur interpreter overhead. n = len(m2[0]) return batched(starmap(sumprod, product(m1, transpose(m2))), n) + # ==== Polynomial arithmetic ==== def convolve(signal, kernel): @@ -1114,6 +1112,7 @@ and :term:`generators ` which incur interpreter overhead. powers = reversed(range(1, n)) return list(map(mul, coefficients, powers)) + # ==== Number theory ==== def sieve(n): @@ -1230,8 +1229,8 @@ and :term:`generators ` which incur interpreter overhead. [(0, 'a'), (1, 'b'), (2, 'c')] - >>> list(islice(tabulate(lambda x: 2*x), 4)) - [0, 2, 4, 6] + >>> list(running_mean([8.5, 9.5, 7.5, 6.5])) + [8.5, 9.0, 8.5, 8.0] >>> for _ in loops(5): @@ -1798,6 +1797,10 @@ and :term:`generators ` which incur interpreter overhead. # Old recipes and their tests which are guaranteed to continue to work. + def tabulate(function, start=0): + "Return function(0), function(1), ..." + return map(function, count(start)) + def old_sumprod_recipe(vec1, vec2): "Compute a sum of products." return sum(starmap(operator.mul, zip(vec1, vec2, strict=True))) @@ -1877,6 +1880,10 @@ and :term:`generators ` which incur interpreter overhead. .. doctest:: :hide: + >>> list(islice(tabulate(lambda x: 2*x), 4)) + [0, 2, 4, 6] + + >>> dotproduct([1,2,3], [4,5,6]) 32 diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 8b4217c210d..57aad5ba9d1 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -121,7 +121,7 @@ Extending :class:`JSONEncoder`:: ['[2.0', ', 1.0', ']'] -Using :mod:`json` from the shell to validate and pretty-print: +Using :mod:`!json` from the shell to validate and pretty-print: .. code-block:: shell-session @@ -747,8 +747,8 @@ Command-line interface -------------- -The :mod:`json` module can be invoked as a script via ``python -m json`` -to validate and pretty-print JSON objects. The :mod:`json.tool` submodule +The :mod:`!json` module can be invoked as a script via ``python -m json`` +to validate and pretty-print JSON objects. The :mod:`!json.tool` submodule implements this interface. If the optional ``infile`` and ``outfile`` arguments are not @@ -769,7 +769,7 @@ specified, :data:`sys.stdin` and :data:`sys.stdout` will be used respectively: alphabetically by key. .. versionchanged:: 3.14 - The :mod:`json` module may now be directly executed as + The :mod:`!json` module may now be directly executed as ``python -m json``. For backwards compatibility, invoking the CLI as ``python -m json.tool`` remains supported. diff --git a/Doc/library/linecache.rst b/Doc/library/linecache.rst index 07305a2a39b..0a5373ec976 100644 --- a/Doc/library/linecache.rst +++ b/Doc/library/linecache.rst @@ -10,7 +10,7 @@ -------------- -The :mod:`linecache` module allows one to get any line from a Python source file, while +The :mod:`!linecache` module allows one to get any line from a Python source file, while attempting to optimize internally, using a cache, the common case where many lines are read from a single file. This is used by the :mod:`traceback` module to retrieve source lines for inclusion in the formatted traceback. @@ -19,7 +19,7 @@ The :func:`tokenize.open` function is used to open files. This function uses :func:`tokenize.detect_encoding` to get the encoding of the file; in the absence of an encoding token, the file encoding defaults to UTF-8. -The :mod:`linecache` module defines the following functions: +The :mod:`!linecache` module defines the following functions: .. function:: getline(filename, lineno, module_globals=None) diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 0f49f30b9d7..81ac46eea87 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -11,17 +11,17 @@ -------------- -The :mod:`locale` module opens access to the POSIX locale database and +The :mod:`!locale` module opens access to the POSIX locale database and functionality. The POSIX locale mechanism allows programmers to deal with certain cultural issues in an application, without requiring the programmer to know all the specifics of each country where the software is executed. .. index:: pair: module; _locale -The :mod:`locale` module is implemented on top of the :mod:`!_locale` module, +The :mod:`!locale` module is implemented on top of the :mod:`!_locale` module, which in turn uses an ANSI C locale implementation if available. -The :mod:`locale` module defines the following exception and functions: +The :mod:`!locale` module defines the following exception and functions: .. exception:: Error @@ -535,7 +535,7 @@ The :mod:`locale` module defines the following exception and functions: .. data:: LC_COLLATE Locale category for sorting strings. The functions :func:`strcoll` and - :func:`strxfrm` of the :mod:`locale` module are affected. + :func:`strxfrm` of the :mod:`!locale` module are affected. .. data:: LC_TIME @@ -564,7 +564,7 @@ The :mod:`locale` module defines the following exception and functions: .. data:: LC_NUMERIC Locale category for formatting numbers. The functions :func:`format_string`, - :func:`atoi`, :func:`atof` and :func:`.str` of the :mod:`locale` module are + :func:`atoi`, :func:`atof` and :func:`.str` of the :mod:`!locale` module are affected by that category. All other numeric formatting operations are not affected. @@ -688,7 +688,7 @@ the current locale is. But since the return value can only be used portably to restore it, that is not very useful (except perhaps to find out whether or not the locale is ``C``). -When Python code uses the :mod:`locale` module to change the locale, this also +When Python code uses the :mod:`!locale` module to change the locale, this also affects the embedding application. If the embedding application doesn't want this to happen, it should remove the :mod:`!_locale` extension module (which does all the work) from the table of built-in modules in the :file:`config.c` file, diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 96cca3073fe..6709062dfca 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -28,7 +28,7 @@ Configuration functions ^^^^^^^^^^^^^^^^^^^^^^^ The following functions configure the logging module. They are located in the -:mod:`logging.config` module. Their use is optional --- you can configure the +:mod:`!logging.config` module. Their use is optional --- you can configure the logging module using these functions or by making calls to the main API (defined in :mod:`logging` itself) and defining handlers which are declared either in :mod:`logging` or :mod:`logging.handlers`. @@ -55,7 +55,7 @@ in :mod:`logging` itself) and defining handlers which are declared either in Parsing is performed by the :class:`DictConfigurator` class, whose constructor is passed the dictionary used for configuration, and - has a :meth:`configure` method. The :mod:`logging.config` module + has a :meth:`configure` method. The :mod:`!logging.config` module has a callable attribute :attr:`dictConfigClass` which is initially set to :class:`DictConfigurator`. You can replace the value of :attr:`dictConfigClass` with a diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index c9cfbdb4126..d128f64aae7 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -160,7 +160,7 @@ WatchedFileHandler .. currentmodule:: logging.handlers -The :class:`WatchedFileHandler` class, located in the :mod:`logging.handlers` +The :class:`WatchedFileHandler` class, located in the :mod:`!logging.handlers` module, is a :class:`FileHandler` which watches the file it is logging to. If the file changes, it is closed and reopened using the file name. @@ -213,7 +213,7 @@ for this value. BaseRotatingHandler ^^^^^^^^^^^^^^^^^^^ -The :class:`BaseRotatingHandler` class, located in the :mod:`logging.handlers` +The :class:`BaseRotatingHandler` class, located in the :mod:`!logging.handlers` module, is the base class for the rotating file handlers, :class:`RotatingFileHandler` and :class:`TimedRotatingFileHandler`. You should not need to instantiate this class, but it has attributes and methods you may @@ -307,7 +307,7 @@ For an example, see :ref:`cookbook-rotator-namer`. RotatingFileHandler ^^^^^^^^^^^^^^^^^^^ -The :class:`RotatingFileHandler` class, located in the :mod:`logging.handlers` +The :class:`RotatingFileHandler` class, located in the :mod:`!logging.handlers` module, supports rotation of disk log files. @@ -362,7 +362,7 @@ TimedRotatingFileHandler ^^^^^^^^^^^^^^^^^^^^^^^^ The :class:`TimedRotatingFileHandler` class, located in the -:mod:`logging.handlers` module, supports rotation of disk log files at certain +:mod:`!logging.handlers` module, supports rotation of disk log files at certain timed intervals. @@ -475,7 +475,7 @@ timed intervals. SocketHandler ^^^^^^^^^^^^^ -The :class:`SocketHandler` class, located in the :mod:`logging.handlers` module, +The :class:`SocketHandler` class, located in the :mod:`!logging.handlers` module, sends logging output to a network socket. The base class uses a TCP socket. @@ -571,7 +571,7 @@ sends logging output to a network socket. The base class uses a TCP socket. DatagramHandler ^^^^^^^^^^^^^^^ -The :class:`DatagramHandler` class, located in the :mod:`logging.handlers` +The :class:`DatagramHandler` class, located in the :mod:`!logging.handlers` module, inherits from :class:`SocketHandler` to support sending logging messages over UDP sockets. @@ -618,7 +618,7 @@ over UDP sockets. SysLogHandler ^^^^^^^^^^^^^ -The :class:`SysLogHandler` class, located in the :mod:`logging.handlers` module, +The :class:`SysLogHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a remote or local Unix syslog. @@ -797,7 +797,7 @@ supports sending logging messages to a remote or local Unix syslog. NTEventLogHandler ^^^^^^^^^^^^^^^^^ -The :class:`NTEventLogHandler` class, located in the :mod:`logging.handlers` +The :class:`NTEventLogHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a local Windows NT, Windows 2000 or Windows XP event log. Before you can use it, you need Mark Hammond's Win32 extensions for Python installed. @@ -864,7 +864,7 @@ extensions for Python installed. SMTPHandler ^^^^^^^^^^^ -The :class:`SMTPHandler` class, located in the :mod:`logging.handlers` module, +The :class:`SMTPHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to an email address via SMTP. @@ -905,7 +905,7 @@ supports sending logging messages to an email address via SMTP. MemoryHandler ^^^^^^^^^^^^^ -The :class:`MemoryHandler` class, located in the :mod:`logging.handlers` module, +The :class:`MemoryHandler` class, located in the :mod:`!logging.handlers` module, supports buffering of logging records in memory, periodically flushing them to a :dfn:`target` handler. Flushing occurs whenever the buffer is full, or when an event of a certain severity or greater is seen. @@ -985,7 +985,7 @@ should, then :meth:`flush` is expected to do the flushing. HTTPHandler ^^^^^^^^^^^ -The :class:`HTTPHandler` class, located in the :mod:`logging.handlers` module, +The :class:`HTTPHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a web server, using either ``GET`` or ``POST`` semantics. @@ -1037,7 +1037,7 @@ QueueHandler .. versionadded:: 3.2 -The :class:`QueueHandler` class, located in the :mod:`logging.handlers` module, +The :class:`QueueHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a queue, such as those implemented in the :mod:`queue` or :mod:`multiprocessing` modules. @@ -1130,7 +1130,7 @@ QueueListener .. versionadded:: 3.2 -The :class:`QueueListener` class, located in the :mod:`logging.handlers` +The :class:`QueueListener` class, located in the :mod:`!logging.handlers` module, supports receiving logging messages from a queue, such as those implemented in the :mod:`queue` or :mod:`multiprocessing` modules. The messages are received from a queue in an internal thread and passed, on diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 9d93ac110c9..6e38d45504c 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1534,7 +1534,7 @@ Module-Level Attributes Integration with the warnings module ------------------------------------ -The :func:`captureWarnings` function can be used to integrate :mod:`logging` +The :func:`captureWarnings` function can be used to integrate :mod:`!logging` with the :mod:`warnings` module. .. function:: captureWarnings(capture) @@ -1565,7 +1565,7 @@ with the :mod:`warnings` module. library. `Original Python logging package `_ - This is the original source for the :mod:`logging` package. The version of the + This is the original source for the :mod:`!logging` package. The version of the package available from this site is suitable for use with Python 1.5.2, 2.1.x - and 2.2.x, which do not include the :mod:`logging` package in the standard + and 2.2.x, which do not include the :mod:`!logging` package in the standard library. diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index 62e289573c0..ed135bf02cb 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -1025,7 +1025,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: remove_flag(flag) Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than + remove more than one flag at a time, *flag* may be a string of more than one character. If "info" contains experimental information rather than flags, the current "info" is not modified. @@ -1190,7 +1190,7 @@ When a :class:`!MaildirMessage` instance is created based upon a .. method:: remove_flag(flag) Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than + remove more than one flag at a time, *flag* may be a string of more than one character. When an :class:`!mboxMessage` instance is created based upon a @@ -1562,7 +1562,7 @@ When a :class:`!BabylMessage` instance is created based upon an .. method:: remove_flag(flag) Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than + remove more than one flag at a time, *flag* may be a string of more than one character. When an :class:`!MMDFMessage` instance is created based upon a @@ -1641,7 +1641,7 @@ The following exception classes are defined in the :mod:`!mailbox` module: .. exception:: Error() - The based class for all other module-specific exceptions. + The base class for all other module-specific exceptions. .. exception:: NoSuchMailboxError() @@ -1661,7 +1661,7 @@ The following exception classes are defined in the :mod:`!mailbox` module: Raised when some mailbox-related condition beyond the control of the program causes it to be unable to proceed, such as when failing to acquire a lock that - another program already holds a lock, or when a uniquely generated file name + another program already holds, or when a uniquely generated file name already exists. diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index e8e9071a5c9..ed182ea24e8 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -20,7 +20,7 @@ rarely does). [#]_ This is not a general "persistence" module. For general persistence and transfer of Python objects through RPC calls, see the modules :mod:`pickle` and -:mod:`shelve`. The :mod:`marshal` module exists mainly to support reading and +:mod:`shelve`. The :mod:`!marshal` module exists mainly to support reading and writing the "pseudo-compiled" code for Python modules of :file:`.pyc` files. Therefore, the Python maintainers reserve the right to modify the marshal format in backward incompatible ways should the need arise. @@ -34,7 +34,7 @@ supports a substantially wider range of objects than marshal. .. warning:: - The :mod:`marshal` module is not intended to be secure against erroneous or + The :mod:`!marshal` module is not intended to be secure against erroneous or maliciously constructed data. Never unmarshal data received from an untrusted or unauthenticated source. diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 8dad20bc83b..c14682f7fe5 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -816,7 +816,7 @@ Constants .. impl-detail:: - The :mod:`math` module consists mostly of thin wrappers around the platform C + The :mod:`!math` module consists mostly of thin wrappers around the platform C math library functions. Behavior in exceptional cases follows Annex F of the C99 standard where appropriate. The current implementation will raise :exc:`ValueError` for invalid operations like ``sqrt(-1.0)`` or ``log(0.0)`` diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 13511b16a0e..f489b60af3c 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -12,7 +12,7 @@ -------------- -The :mod:`mimetypes` module converts between a filename or URL and the MIME type +The :mod:`!mimetypes` module converts between a filename or URL and the MIME type associated with the filename extension. Conversions are provided from filename to MIME type and from MIME type to filename extension; encodings are not supported for the latter conversion. @@ -196,7 +196,7 @@ MimeTypes objects The :class:`MimeTypes` class may be useful for applications which may want more than one MIME-type database; it provides an interface similar to the one of the -:mod:`mimetypes` module. +:mod:`!mimetypes` module. .. class:: MimeTypes(filenames=(), strict=True) diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index a2c5e375d2c..79069c136c2 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -2,7 +2,6 @@ =========================================================== .. module:: msvcrt - :platform: Windows :synopsis: Miscellaneous useful routines from the MS VC++ runtime. .. sectionauthor:: Fred L. Drake, Jr. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 88813c6f1a4..d581b73cf0b 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -13,16 +13,16 @@ Introduction ------------ -:mod:`multiprocessing` is a package that supports spawning processes using an -API similar to the :mod:`threading` module. The :mod:`multiprocessing` package +:mod:`!multiprocessing` is a package that supports spawning processes using an +API similar to the :mod:`threading` module. The :mod:`!multiprocessing` package offers both local and remote concurrency, effectively side-stepping the :term:`Global Interpreter Lock ` by using subprocesses instead of threads. Due -to this, the :mod:`multiprocessing` module allows the programmer to fully +to this, the :mod:`!multiprocessing` module allows the programmer to fully leverage multiple processors on a given machine. It runs on both POSIX and Windows. -The :mod:`multiprocessing` module also introduces the +The :mod:`!multiprocessing` module also introduces the :class:`~multiprocessing.pool.Pool` object which offers a convenient means of parallelizing the execution of a function across multiple input values, distributing the input data across processes (data parallelism). The following @@ -43,7 +43,7 @@ will print to standard output :: [1, 4, 9] -The :mod:`multiprocessing` module also introduces APIs which do not have +The :mod:`!multiprocessing` module also introduces APIs which do not have analogs in the :mod:`threading` module, like the ability to :meth:`terminate `, :meth:`interrupt ` or :meth:`kill ` a running process. @@ -61,7 +61,7 @@ analogs in the :mod:`threading` module, like the ability to :meth:`terminate The :class:`Process` class ^^^^^^^^^^^^^^^^^^^^^^^^^^ -In :mod:`multiprocessing`, processes are spawned by creating a :class:`Process` +In :mod:`!multiprocessing`, processes are spawned by creating a :class:`Process` object and then calling its :meth:`~Process.start` method. :class:`Process` follows the API of :class:`threading.Thread`. A trivial example of a multiprocess program is :: @@ -111,7 +111,7 @@ could lead to an :exc:`AttributeError` in the child process trying to locate the Contexts and start methods ^^^^^^^^^^^^^^^^^^^^^^^^^^ -Depending on the platform, :mod:`multiprocessing` supports three ways +Depending on the platform, :mod:`!multiprocessing` supports three ways to start a process. These *start methods* are .. _multiprocessing-start-method-spawn: @@ -240,7 +240,7 @@ processes for a different context. In particular, locks created using the *fork* context cannot be passed to processes started using the *spawn* or *forkserver* start methods. -Libraries using :mod:`multiprocessing` or +Libraries using :mod:`!multiprocessing` or :class:`~concurrent.futures.ProcessPoolExecutor` should be designed to allow their users to provide their own multiprocessing context. Using a specific context of your own within a library can lead to incompatibilities with the @@ -258,7 +258,7 @@ requires a specific start method. Exchanging objects between processes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`multiprocessing` supports two types of communication channel between +:mod:`!multiprocessing` supports two types of communication channel between processes: **Queues** @@ -279,7 +279,7 @@ processes: p.join() Queues are thread and process safe. - Any object put into a :mod:`~multiprocessing` queue will be serialized. + Any object put into a :mod:`!multiprocessing` queue will be serialized. **Pipes** @@ -313,7 +313,7 @@ processes: Synchronization between processes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`multiprocessing` contains equivalents of all the synchronization +:mod:`!multiprocessing` contains equivalents of all the synchronization primitives from :mod:`threading`. For instance one can use a lock to ensure that only one process prints to standard output at a time:: @@ -344,7 +344,7 @@ avoid using shared state as far as possible. This is particularly true when using multiple processes. However, if you really do need to use some shared data then -:mod:`multiprocessing` provides a couple of ways of doing so. +:mod:`!multiprocessing` provides a couple of ways of doing so. **Shared memory** @@ -518,7 +518,7 @@ process which created it. Reference --------- -The :mod:`multiprocessing` package mostly replicates the API of the +The :mod:`!multiprocessing` package mostly replicates the API of the :mod:`threading` module. .. _global-start-method: @@ -704,7 +704,7 @@ or creating these objects. The process's authentication key (a byte string). - When :mod:`multiprocessing` is initialized the main process is assigned a + When :mod:`!multiprocessing` is initialized the main process is assigned a random string using :func:`os.urandom`. When a :class:`Process` object is created, it will inherit the @@ -805,7 +805,7 @@ or creating these objects. .. exception:: ProcessError - The base class of all :mod:`multiprocessing` exceptions. + The base class of all :mod:`!multiprocessing` exceptions. .. exception:: BufferTooShort @@ -845,7 +845,7 @@ If you use :class:`JoinableQueue` then you **must** call semaphore used to count the number of unfinished tasks may eventually overflow, raising an exception. -One difference from other Python queue implementations, is that :mod:`multiprocessing` +One difference from other Python queue implementations, is that :mod:`!multiprocessing` queues serializes all objects that are put into them using :mod:`pickle`. The object returned by the get method is a re-created object that does not share memory with the original object. @@ -855,9 +855,9 @@ Note that one can also create a shared queue by using a manager object -- see .. note:: - :mod:`multiprocessing` uses the usual :exc:`queue.Empty` and + :mod:`!multiprocessing` uses the usual :exc:`queue.Empty` and :exc:`queue.Full` exceptions to signal a timeout. They are not available in - the :mod:`multiprocessing` namespace so you need to import them from + the :mod:`!multiprocessing` namespace so you need to import them from :mod:`queue`. .. note:: @@ -1152,7 +1152,7 @@ Miscellaneous .. function:: freeze_support() - Add support for when a program which uses :mod:`multiprocessing` has been + Add support for when a program which uses :mod:`!multiprocessing` has been frozen to produce an executable. (Has been tested with **py2exe**, **PyInstaller** and **cx_Freeze**.) @@ -1188,7 +1188,7 @@ Miscellaneous .. function:: get_context(method=None) Return a context object which has the same attributes as the - :mod:`multiprocessing` module. + :mod:`!multiprocessing` module. If *method* is ``None`` then the default context is returned. Note that if the global start method has not been set, this will set it to the system default @@ -1222,7 +1222,7 @@ Miscellaneous Set the path of the Python interpreter to use when starting a child process. (By default :data:`sys.executable` is used). Embedders will probably need to - do some thing like :: + do something like :: set_executable(os.path.join(sys.exec_prefix, 'pythonw.exe')) @@ -1269,7 +1269,7 @@ Miscellaneous .. note:: - :mod:`multiprocessing` contains no analogues of + :mod:`!multiprocessing` contains no analogues of :func:`threading.active_count`, :func:`threading.enumerate`, :func:`threading.settrace`, :func:`threading.setprofile`, :class:`threading.Timer`, or :class:`threading.local`. @@ -1463,7 +1463,7 @@ object -- see :ref:`multiprocessing-managers`. A condition variable: an alias for :class:`threading.Condition`. If *lock* is specified then it should be a :class:`Lock` or :class:`RLock` - object from :mod:`multiprocessing`. + object from :mod:`!multiprocessing`. Instantiating this class may set the global start method. See :ref:`global-start-method` for more details. @@ -1715,13 +1715,13 @@ inherited by child processes. attributes which allow one to use it to store and retrieve strings. -The :mod:`multiprocessing.sharedctypes` module -"""""""""""""""""""""""""""""""""""""""""""""" +The :mod:`!multiprocessing.sharedctypes` module +""""""""""""""""""""""""""""""""""""""""""""""" .. module:: multiprocessing.sharedctypes :synopsis: Allocate ctypes objects from shared memory. -The :mod:`multiprocessing.sharedctypes` module provides functions for allocating +The :mod:`!multiprocessing.sharedctypes` module provides functions for allocating :mod:`ctypes` objects from shared memory which can be inherited by child processes. @@ -2321,7 +2321,7 @@ demonstrates a level of control over the synchronization. .. note:: - The proxy types in :mod:`multiprocessing` do nothing to support comparisons + The proxy types in :mod:`!multiprocessing` do nothing to support comparisons by value. So, for instance, we have: .. doctest:: @@ -2463,7 +2463,7 @@ with the :class:`Pool` class. duration of the Pool's work queue. A frequent pattern found in other systems (such as Apache, mod_wsgi, etc) to free resources held by workers is to allow a worker within a pool to complete only a set - amount of work before being exiting, being cleaned up and a new + amount of work before exiting, being cleaned up and a new process spawned to replace the old one. The *maxtasksperchild* argument to the :class:`Pool` exposes this ability to the end user. @@ -2648,7 +2648,7 @@ Usually message passing between processes is done using queues or by using :class:`~Connection` objects returned by :func:`~multiprocessing.Pipe`. -However, the :mod:`multiprocessing.connection` module allows some extra +However, the :mod:`!multiprocessing.connection` module allows some extra flexibility. It basically gives a high level message oriented API for dealing with sockets or Windows named pipes. It also has support for *digest authentication* using the :mod:`hmac` module, and for polling @@ -2917,7 +2917,7 @@ handler type) for messages from different processes to get mixed up. .. currentmodule:: multiprocessing .. function:: get_logger() - Returns the logger used by :mod:`multiprocessing`. If necessary, a new one + Returns the logger used by :mod:`!multiprocessing`. If necessary, a new one will be created. When first created the logger has level :const:`logging.NOTSET` and no @@ -2955,18 +2955,18 @@ Below is an example session with logging turned on:: For a full table of logging levels, see the :mod:`logging` module. -The :mod:`multiprocessing.dummy` module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The :mod:`!multiprocessing.dummy` module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: multiprocessing.dummy :synopsis: Dumb wrapper around threading. -:mod:`multiprocessing.dummy` replicates the API of :mod:`multiprocessing` but is +:mod:`!multiprocessing.dummy` replicates the API of :mod:`!multiprocessing` but is no more than a wrapper around the :mod:`threading` module. .. currentmodule:: multiprocessing.pool -In particular, the ``Pool`` function provided by :mod:`multiprocessing.dummy` +In particular, the ``Pool`` function provided by :mod:`!multiprocessing.dummy` returns an instance of :class:`ThreadPool`, which is a subclass of :class:`Pool` that supports all the same method calls but uses a pool of worker threads rather than worker processes. @@ -3011,7 +3011,7 @@ Programming guidelines ---------------------- There are certain guidelines and idioms which should be adhered to when using -:mod:`multiprocessing`. +:mod:`!multiprocessing`. All start methods @@ -3052,7 +3052,7 @@ Joining zombie processes Better to inherit than pickle/unpickle When using the *spawn* or *forkserver* start methods many types - from :mod:`multiprocessing` need to be picklable so that child + from :mod:`!multiprocessing` need to be picklable so that child processes can use them. However, one should generally avoid sending shared objects to other processes using pipes or queues. Instead you should arrange the program so that a process which @@ -3142,7 +3142,7 @@ Explicitly pass resources to child processes Beware of replacing :data:`sys.stdin` with a "file like object" - :mod:`multiprocessing` originally unconditionally called:: + :mod:`!multiprocessing` originally unconditionally called:: os.close(sys.stdin.fileno()) diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index e8e71068dd9..c715e977cca 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -15,7 +15,7 @@ -------------- -The :mod:`operator` module exports a set of efficient functions corresponding to +The :mod:`!operator` module exports a set of efficient functions corresponding to the intrinsic operators of Python. For example, ``operator.add(x, y)`` is equivalent to the expression ``x+y``. Many function names are those used for special methods, without the double underscores. For backward compatibility, @@ -275,7 +275,7 @@ The following operation works with callables: .. versionadded:: 3.11 -The :mod:`operator` module also defines tools for generalized attribute and item +The :mod:`!operator` module also defines tools for generalized attribute and item lookups. These are useful for making fast field extractors as arguments for :func:`map`, :func:`sorted`, :meth:`itertools.groupby`, or other functions that expect a function argument. @@ -390,7 +390,7 @@ Mapping Operators to Functions ------------------------------ This table shows how abstract operations correspond to operator symbols in the -Python syntax and the functions in the :mod:`operator` module. +Python syntax and the functions in the :mod:`!operator` module. +-----------------------+-------------------------+---------------------------------------+ | Operation | Syntax | Function | diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index ff327cf9162..51827e1f8da 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -20,7 +20,7 @@ The standard library includes three argument parsing libraries: * :mod:`getopt`: a module that closely mirrors the procedural C ``getopt`` API. Included in the standard library since before the initial Python 1.0 release. -* :mod:`optparse`: a declarative replacement for ``getopt`` that +* :mod:`!optparse`: a declarative replacement for ``getopt`` that provides equivalent functionality without requiring each application to implement its own procedural option parsing logic. Included in the standard library since the Python 2.3 release. @@ -37,10 +37,10 @@ the highest level of baseline functionality with the least application level cod However, it also serves a niche use case as a tool for prototyping and testing command line argument handling in ``getopt``-based C applications. -:mod:`optparse` should be considered as an alternative to :mod:`argparse` in the +:mod:`!optparse` should be considered as an alternative to :mod:`argparse` in the following cases: -* an application is already using :mod:`optparse` and doesn't want to risk the +* an application is already using :mod:`!optparse` and doesn't want to risk the subtle behavioural changes that may arise when migrating to :mod:`argparse` * the application requires additional control over the way options and positional parameters are interleaved on the command line (including @@ -55,7 +55,7 @@ following cases: behavior which ``argparse`` does not support, but which can be implemented in terms of the lower level interface offered by ``optparse`` -These considerations also mean that :mod:`optparse` is likely to provide a +These considerations also mean that :mod:`!optparse` is likely to provide a better foundation for library authors writing third party command line argument processing libraries. @@ -126,15 +126,15 @@ application use case. Introduction ------------ -:mod:`optparse` is a more convenient, flexible, and powerful library for parsing +:mod:`!optparse` is a more convenient, flexible, and powerful library for parsing command-line options than the minimalist :mod:`getopt` module. -:mod:`optparse` uses a more declarative style of command-line parsing: +:mod:`!optparse` uses a more declarative style of command-line parsing: you create an instance of :class:`OptionParser`, populate it with options, and parse the command line. -:mod:`optparse` allows users to specify options in the conventional +:mod:`!optparse` allows users to specify options in the conventional GNU/POSIX syntax, and additionally generates usage and help messages for you. -Here's an example of using :mod:`optparse` in a simple script:: +Here's an example of using :mod:`!optparse` in a simple script:: from optparse import OptionParser ... @@ -152,11 +152,11 @@ on the command-line, for example:: --file=outfile -q -As it parses the command line, :mod:`optparse` sets attributes of the +As it parses the command line, :mod:`!optparse` sets attributes of the ``options`` object returned by :meth:`~OptionParser.parse_args` based on user-supplied command-line values. When :meth:`~OptionParser.parse_args` returns from parsing this command line, ``options.filename`` will be ``"outfile"`` and ``options.verbose`` will be -``False``. :mod:`optparse` supports both long and short options, allows short +``False``. :mod:`!optparse` supports both long and short options, allows short options to be merged together, and allows options to be associated with their arguments in a variety of ways. Thus, the following command lines are all equivalent to the above example:: @@ -171,7 +171,7 @@ Additionally, users can run one of the following :: -h --help -and :mod:`optparse` will print out a brief summary of your script's options: +and :mod:`!optparse` will print out a brief summary of your script's options: .. code-block:: text @@ -191,7 +191,7 @@ where the value of *yourscript* is determined at runtime (normally from Background ---------- -:mod:`optparse` was explicitly designed to encourage the creation of programs +:mod:`!optparse` was explicitly designed to encourage the creation of programs with straightforward command-line interfaces that follow the conventions established by the :c:func:`!getopt` family of functions available to C developers. To that end, it supports only the most common command-line syntax and semantics @@ -223,7 +223,7 @@ option options to be merged into a single argument, e.g. ``-x -F`` is equivalent to ``-xF``. The GNU project introduced ``--`` followed by a series of hyphen-separated words, e.g. ``--file`` or ``--dry-run``. These are the - only two option syntaxes provided by :mod:`optparse`. + only two option syntaxes provided by :mod:`!optparse`. Some other option syntaxes that the world has seen include: @@ -240,7 +240,7 @@ option * a slash followed by a letter, or a few letters, or a word, e.g. ``/f``, ``/file`` - These option syntaxes are not supported by :mod:`optparse`, and they never + These option syntaxes are not supported by :mod:`!optparse`, and they never will be. This is deliberate: the first three are non-standard on any environment, and the last only makes sense if you're exclusively targeting Windows or certain legacy platforms (e.g. VMS, MS-DOS). @@ -248,7 +248,7 @@ option option argument an argument that follows an option, is closely associated with that option, and is consumed from the argument list when that option is. With - :mod:`optparse`, option arguments may either be in a separate argument from + :mod:`!optparse`, option arguments may either be in a separate argument from their option: .. code-block:: text @@ -268,7 +268,7 @@ option argument will take an argument if they see it, and won't if they don't. This is somewhat controversial, because it makes parsing ambiguous: if ``-a`` takes an optional argument and ``-b`` is another option entirely, how do we - interpret ``-ab``? Because of this ambiguity, :mod:`optparse` does not + interpret ``-ab``? Because of this ambiguity, :mod:`!optparse` does not support this feature. positional argument @@ -278,7 +278,7 @@ positional argument required option an option that must be supplied on the command-line; note that the phrase - "required option" is self-contradictory in English. :mod:`optparse` doesn't + "required option" is self-contradictory in English. :mod:`!optparse` doesn't prevent you from implementing required options, but doesn't give you much help at it either. @@ -357,9 +357,9 @@ too many options can overwhelm users and make your code much harder to maintain. Tutorial -------- -While :mod:`optparse` is quite flexible and powerful, it's also straightforward +While :mod:`!optparse` is quite flexible and powerful, it's also straightforward to use in most cases. This section covers the code patterns that are common to -any :mod:`optparse`\ -based program. +any :mod:`!optparse`\ -based program. First, you need to import the OptionParser class; then, early in the main program, create an OptionParser instance:: @@ -374,7 +374,7 @@ Then you can start defining options. The basic syntax is:: attr=value, ...) Each option has one or more option strings, such as ``-f`` or ``--file``, -and several option attributes that tell :mod:`optparse` what to expect and what +and several option attributes that tell :mod:`!optparse` what to expect and what to do when it encounters that option on the command line. Typically, each option will have one short option string and one long option @@ -389,10 +389,10 @@ string overall. The option strings passed to :meth:`OptionParser.add_option` are effectively labels for the option defined by that call. For brevity, we will frequently refer to -*encountering an option* on the command line; in reality, :mod:`optparse` +*encountering an option* on the command line; in reality, :mod:`!optparse` encounters *option strings* and looks up options from them. -Once all of your options are defined, instruct :mod:`optparse` to parse your +Once all of your options are defined, instruct :mod:`!optparse` to parse your program's command line:: (options, args) = parser.parse_args() @@ -420,14 +420,14 @@ most fundamental. Understanding option actions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Actions tell :mod:`optparse` what to do when it encounters an option on the -command line. There is a fixed set of actions hard-coded into :mod:`optparse`; +Actions tell :mod:`!optparse` what to do when it encounters an option on the +command line. There is a fixed set of actions hard-coded into :mod:`!optparse`; adding new actions is an advanced topic covered in section -:ref:`optparse-extending-optparse`. Most actions tell :mod:`optparse` to store +:ref:`optparse-extending-optparse`. Most actions tell :mod:`!optparse` to store a value in some variable---for example, take a string from the command line and store it in an attribute of ``options``. -If you don't specify an option action, :mod:`optparse` defaults to ``store``. +If you don't specify an option action, :mod:`!optparse` defaults to ``store``. .. _optparse-store-action: @@ -435,7 +435,7 @@ If you don't specify an option action, :mod:`optparse` defaults to ``store``. The store action ^^^^^^^^^^^^^^^^ -The most common option action is ``store``, which tells :mod:`optparse` to take +The most common option action is ``store``, which tells :mod:`!optparse` to take the next argument (or the remainder of the current argument), ensure that it is of the correct type, and store it to your chosen destination. @@ -444,16 +444,16 @@ For example:: parser.add_option("-f", "--file", action="store", type="string", dest="filename") -Now let's make up a fake command line and ask :mod:`optparse` to parse it:: +Now let's make up a fake command line and ask :mod:`!optparse` to parse it:: args = ["-f", "foo.txt"] (options, args) = parser.parse_args(args) -When :mod:`optparse` sees the option string ``-f``, it consumes the next +When :mod:`!optparse` sees the option string ``-f``, it consumes the next argument, ``foo.txt``, and stores it in ``options.filename``. So, after this call to :meth:`~OptionParser.parse_args`, ``options.filename`` is ``"foo.txt"``. -Some other option types supported by :mod:`optparse` are ``int`` and ``float``. +Some other option types supported by :mod:`!optparse` are ``int`` and ``float``. Here's an option that expects an integer argument:: parser.add_option("-n", type="int", dest="num") @@ -470,19 +470,19 @@ right up against the option: since ``-n42`` (one argument) is equivalent to will print ``42``. -If you don't specify a type, :mod:`optparse` assumes ``string``. Combined with +If you don't specify a type, :mod:`!optparse` assumes ``string``. Combined with the fact that the default action is ``store``, that means our first example can be a lot shorter:: parser.add_option("-f", "--file", dest="filename") -If you don't supply a destination, :mod:`optparse` figures out a sensible +If you don't supply a destination, :mod:`!optparse` figures out a sensible default from the option strings: if the first long option string is ``--foo-bar``, then the default destination is ``foo_bar``. If there are no -long option strings, :mod:`optparse` looks at the first short option string: the +long option strings, :mod:`!optparse` looks at the first short option string: the default destination for ``-f`` is ``f``. -:mod:`optparse` also includes the built-in ``complex`` type. Adding +:mod:`!optparse` also includes the built-in ``complex`` type. Adding types is covered in section :ref:`optparse-extending-optparse`. @@ -492,7 +492,7 @@ Handling boolean (flag) options ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Flag options---set a variable to true or false when a particular option is -seen---are quite common. :mod:`optparse` supports them with two separate actions, +seen---are quite common. :mod:`!optparse` supports them with two separate actions, ``store_true`` and ``store_false``. For example, you might have a ``verbose`` flag that is turned on with ``-v`` and off with ``-q``:: @@ -503,7 +503,7 @@ Here we have two different options with the same destination, which is perfectly OK. (It just means you have to be a bit careful when setting default values---see below.) -When :mod:`optparse` encounters ``-v`` on the command line, it sets +When :mod:`!optparse` encounters ``-v`` on the command line, it sets ``options.verbose`` to ``True``; when it encounters ``-q``, ``options.verbose`` is set to ``False``. @@ -513,7 +513,7 @@ When :mod:`optparse` encounters ``-v`` on the command line, it sets Other actions ^^^^^^^^^^^^^ -Some other actions supported by :mod:`optparse` are: +Some other actions supported by :mod:`!optparse` are: ``"store_const"`` store a constant value, pre-set via :attr:`Option.const` @@ -539,11 +539,11 @@ Default values All of the above examples involve setting some variable (the "destination") when certain command-line options are seen. What happens if those options are never seen? Since we didn't supply any defaults, they are all set to ``None``. This -is usually fine, but sometimes you want more control. :mod:`optparse` lets you +is usually fine, but sometimes you want more control. :mod:`!optparse` lets you supply a default value for each destination, which is assigned before the command line is parsed. -First, consider the verbose/quiet example. If we want :mod:`optparse` to set +First, consider the verbose/quiet example. If we want :mod:`!optparse` to set ``verbose`` to ``True`` unless ``-q`` is seen, then we can do this:: parser.add_option("-v", action="store_true", dest="verbose", default=True) @@ -582,7 +582,7 @@ values, not both. Generating help ^^^^^^^^^^^^^^^ -:mod:`optparse`'s ability to generate help and usage text automatically is +:mod:`!optparse`'s ability to generate help and usage text automatically is useful for creating user-friendly command-line interfaces. All you have to do is supply a :attr:`~Option.help` value for each option, and optionally a short usage message for your whole program. Here's an OptionParser populated with @@ -603,7 +603,7 @@ user-friendly (documented) options:: help="interaction mode: novice, intermediate, " "or expert [default: %default]") -If :mod:`optparse` encounters either ``-h`` or ``--help`` on the +If :mod:`!optparse` encounters either ``-h`` or ``--help`` on the command-line, or if you just call :meth:`parser.print_help`, it prints the following to standard output: @@ -620,26 +620,26 @@ following to standard output: -m MODE, --mode=MODE interaction mode: novice, intermediate, or expert [default: intermediate] -(If the help output is triggered by a help option, :mod:`optparse` exits after +(If the help output is triggered by a help option, :mod:`!optparse` exits after printing the help text.) -There's a lot going on here to help :mod:`optparse` generate the best possible +There's a lot going on here to help :mod:`!optparse` generate the best possible help message: * the script defines its own usage message:: usage = "usage: %prog [options] arg1 arg2" - :mod:`optparse` expands ``%prog`` in the usage string to the name of the + :mod:`!optparse` expands ``%prog`` in the usage string to the name of the current program, i.e. ``os.path.basename(sys.argv[0])``. The expanded string is then printed before the detailed option help. - If you don't supply a usage string, :mod:`optparse` uses a bland but sensible + If you don't supply a usage string, :mod:`!optparse` uses a bland but sensible default: ``"Usage: %prog [options]"``, which is fine if your script doesn't take any positional arguments. * every option defines a help string, and doesn't worry about - line-wrapping---\ :mod:`optparse` takes care of wrapping lines and making + line-wrapping---\ :mod:`!optparse` takes care of wrapping lines and making the help output look good. * options that take a value indicate this fact in their automatically generated @@ -649,7 +649,7 @@ help message: Here, "MODE" is called the meta-variable: it stands for the argument that the user is expected to supply to ``-m``/``--mode``. By default, - :mod:`optparse` converts the destination variable name to uppercase and uses + :mod:`!optparse` converts the destination variable name to uppercase and uses that for the meta-variable. Sometimes, that's not what you want---for example, the ``--filename`` option explicitly sets ``metavar="FILE"``, resulting in this automatically generated option description:: @@ -663,7 +663,7 @@ help message: way to make your help text a lot clearer and more useful for end users. * options that have a default value can include ``%default`` in the help - string---\ :mod:`optparse` will replace it with :func:`str` of the option's + string---\ :mod:`!optparse` will replace it with :func:`str` of the option's default value. If an option has no default value (or the default value is ``None``), ``%default`` expands to ``none``. @@ -779,14 +779,14 @@ option groups is: Printing a version string ^^^^^^^^^^^^^^^^^^^^^^^^^ -Similar to the brief usage string, :mod:`optparse` can also print a version +Similar to the brief usage string, :mod:`!optparse` can also print a version string for your program. You have to supply the string as the ``version`` argument to OptionParser:: parser = OptionParser(usage="%prog [-f] [-q]", version="%prog 1.0") ``%prog`` is expanded just like it is in ``usage``. Apart from that, -``version`` can contain anything you like. When you supply it, :mod:`optparse` +``version`` can contain anything you like. When you supply it, :mod:`!optparse` automatically adds a ``--version`` option to your parser. If it encounters this option on the command line, it expands your ``version`` string (by replacing ``%prog``), prints it to stdout, and exits. @@ -815,10 +815,10 @@ The following two methods can be used to print and get the ``version`` string: .. _optparse-how-optparse-handles-errors: -How :mod:`optparse` handles errors -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +How :mod:`!optparse` handles errors +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are two broad classes of errors that :mod:`optparse` has to worry about: +There are two broad classes of errors that :mod:`!optparse` has to worry about: programmer errors and user errors. Programmer errors are usually erroneous calls to :func:`OptionParser.add_option`, e.g. invalid option strings, unknown option attributes, missing option attributes, etc. These are dealt with in the @@ -826,7 +826,7 @@ usual way: raise an exception (either :exc:`optparse.OptionError` or :exc:`TypeError`) and let the program crash. Handling user errors is much more important, since they are guaranteed to happen -no matter how stable your code is. :mod:`optparse` can automatically detect +no matter how stable your code is. :mod:`!optparse` can automatically detect some user errors, such as bad option arguments (passing ``-n 4x`` where ``-n`` takes an integer argument), missing arguments (``-n`` at the end of the command line, where ``-n`` takes an argument of any type). Also, @@ -838,7 +838,7 @@ condition:: if options.a and options.b: parser.error("options -a and -b are mutually exclusive") -In either case, :mod:`optparse` handles the error the same way: it prints the +In either case, :mod:`!optparse` handles the error the same way: it prints the program's usage message and an error message to standard error and exits with error status 2. @@ -861,11 +861,11 @@ Or, where the user fails to pass a value at all: foo: error: -n option requires an argument -:mod:`optparse`\ -generated error messages take care always to mention the +:mod:`!optparse`\ -generated error messages take care always to mention the option involved in the error; be sure to do the same when calling :func:`OptionParser.error` from your application code. -If :mod:`optparse`'s default error-handling behaviour does not suit your needs, +If :mod:`!optparse`'s default error-handling behaviour does not suit your needs, you'll need to subclass OptionParser and override its :meth:`~OptionParser.exit` and/or :meth:`~OptionParser.error` methods. @@ -875,7 +875,7 @@ and/or :meth:`~OptionParser.error` methods. Putting it all together ^^^^^^^^^^^^^^^^^^^^^^^ -Here's what :mod:`optparse`\ -based scripts usually look like:: +Here's what :mod:`!optparse`\ -based scripts usually look like:: from optparse import OptionParser ... @@ -911,7 +911,7 @@ Reference Guide Creating the parser ^^^^^^^^^^^^^^^^^^^ -The first step in using :mod:`optparse` is to create an OptionParser instance. +The first step in using :mod:`!optparse` is to create an OptionParser instance. .. class:: OptionParser(...) @@ -921,7 +921,7 @@ The first step in using :mod:`optparse` is to create an OptionParser instance. ``usage`` (default: ``"%prog [options]"``) The usage summary to print when your program is run incorrectly or with a - help option. When :mod:`optparse` prints the usage string, it expands + help option. When :mod:`!optparse` prints the usage string, it expands ``%prog`` to ``os.path.basename(sys.argv[0])`` (or to ``prog`` if you passed that keyword argument). To suppress a usage message, pass the special value :const:`optparse.SUPPRESS_USAGE`. @@ -938,7 +938,7 @@ The first step in using :mod:`optparse` is to create an OptionParser instance. ``version`` (default: ``None``) A version string to print when the user supplies a version option. If you - supply a true value for ``version``, :mod:`optparse` automatically adds a + supply a true value for ``version``, :mod:`!optparse` automatically adds a version option with the single option string ``--version``. The substring ``%prog`` is expanded the same as for ``usage``. @@ -949,17 +949,17 @@ The first step in using :mod:`optparse` is to create an OptionParser instance. ``description`` (default: ``None``) A paragraph of text giving a brief overview of your program. - :mod:`optparse` reformats this paragraph to fit the current terminal width + :mod:`!optparse` reformats this paragraph to fit the current terminal width and prints it when the user requests help (after ``usage``, but before the list of options). ``formatter`` (default: a new :class:`IndentedHelpFormatter`) An instance of optparse.HelpFormatter that will be used for printing help - text. :mod:`optparse` provides two concrete classes for this purpose: + text. :mod:`!optparse` provides two concrete classes for this purpose: IndentedHelpFormatter and TitledHelpFormatter. ``add_help_option`` (default: ``True``) - If true, :mod:`optparse` will add a help option (with option strings ``-h`` + If true, :mod:`!optparse` will add a help option (with option strings ``-h`` and ``--help``) to the parser. ``prog`` @@ -997,7 +997,7 @@ the OptionParser constructor, as in:: (:func:`make_option` is a factory function for creating Option instances; currently it is an alias for the Option constructor. A future version of -:mod:`optparse` may split Option into several classes, and :func:`make_option` +:mod:`!optparse` may split Option into several classes, and :func:`make_option` will pick the right class to instantiate. Do not instantiate Option directly.) @@ -1027,12 +1027,12 @@ The canonical way to create an :class:`Option` instance is with the The keyword arguments define attributes of the new Option object. The most important option attribute is :attr:`~Option.action`, and it largely determines which other attributes are relevant or required. If you pass - irrelevant option attributes, or fail to pass required ones, :mod:`optparse` + irrelevant option attributes, or fail to pass required ones, :mod:`!optparse` raises an :exc:`OptionError` exception explaining your mistake. - An option's *action* determines what :mod:`optparse` does when it encounters + An option's *action* determines what :mod:`!optparse` does when it encounters this option on the command-line. The standard option actions hard-coded into - :mod:`optparse` are: + :mod:`!optparse` are: ``"store"`` store this option's argument (default) @@ -1066,7 +1066,7 @@ The canonical way to create an :class:`Option` instance is with the attributes; see :ref:`optparse-standard-option-actions`.) As you can see, most actions involve storing or updating a value somewhere. -:mod:`optparse` always creates a special object for this, conventionally called +:mod:`!optparse` always creates a special object for this, conventionally called ``options``, which is an instance of :class:`optparse.Values`. .. class:: Values @@ -1084,7 +1084,7 @@ For example, when you call :: parser.parse_args() -one of the first things :mod:`optparse` does is create the ``options`` object:: +one of the first things :mod:`!optparse` does is create the ``options`` object:: options = Values() @@ -1099,7 +1099,7 @@ and the command-line being parsed includes any of the following:: --file=foo --file foo -then :mod:`optparse`, on seeing this option, will do the equivalent of :: +then :mod:`!optparse`, on seeing this option, will do the equivalent of :: options.filename = "foo" @@ -1124,13 +1124,13 @@ Option attributes The following option attributes may be passed as keyword arguments to :meth:`OptionParser.add_option`. If you pass an option attribute that is not relevant to a particular option, or fail to pass a required option attribute, -:mod:`optparse` raises :exc:`OptionError`. +:mod:`!optparse` raises :exc:`OptionError`. .. attribute:: Option.action (default: ``"store"``) - Determines :mod:`optparse`'s behaviour when this option is seen on the + Determines :mod:`!optparse`'s behaviour when this option is seen on the command line; the available options are documented :ref:`here `. @@ -1147,8 +1147,8 @@ relevant to a particular option, or fail to pass a required option attribute, (default: derived from option strings) If the option's action implies writing or modifying a value somewhere, this - tells :mod:`optparse` where to write it: :attr:`~Option.dest` names an - attribute of the ``options`` object that :mod:`optparse` builds as it parses + tells :mod:`!optparse` where to write it: :attr:`~Option.dest` names an + attribute of the ``options`` object that :mod:`!optparse` builds as it parses the command line. .. attribute:: Option.default @@ -1161,7 +1161,7 @@ relevant to a particular option, or fail to pass a required option attribute, (default: 1) How many arguments of type :attr:`~Option.type` should be consumed when this - option is seen. If > 1, :mod:`optparse` will store a tuple of values to + option is seen. If > 1, :mod:`!optparse` will store a tuple of values to :attr:`~Option.dest`. .. attribute:: Option.const @@ -1207,7 +1207,7 @@ Standard option actions The various option actions all have slightly different requirements and effects. Most actions have several relevant option attributes which you may specify to -guide :mod:`optparse`'s behaviour; a few have required attributes, which you +guide :mod:`!optparse`'s behaviour; a few have required attributes, which you must specify for any option using that action. * ``"store"`` [relevant: :attr:`~Option.type`, :attr:`~Option.dest`, @@ -1225,9 +1225,9 @@ must specify for any option using that action. If :attr:`~Option.type` is not supplied, it defaults to ``"string"``. - If :attr:`~Option.dest` is not supplied, :mod:`optparse` derives a destination + If :attr:`~Option.dest` is not supplied, :mod:`!optparse` derives a destination from the first long option string (e.g., ``--foo-bar`` implies - ``foo_bar``). If there are no long option strings, :mod:`optparse` derives a + ``foo_bar``). If there are no long option strings, :mod:`!optparse` derives a destination from the first short option string (e.g., ``-f`` implies ``f``). Example:: @@ -1239,7 +1239,7 @@ must specify for any option using that action. -f foo.txt -p 1 -3.5 4 -fbar.txt - :mod:`optparse` will set :: + :mod:`!optparse` will set :: options.f = "foo.txt" options.point = (1.0, -3.5, 4.0) @@ -1259,7 +1259,7 @@ must specify for any option using that action. parser.add_option("--noisy", action="store_const", const=2, dest="verbose") - If ``--noisy`` is seen, :mod:`optparse` will set :: + If ``--noisy`` is seen, :mod:`!optparse` will set :: options.verbose = 2 @@ -1282,7 +1282,7 @@ must specify for any option using that action. The option must be followed by an argument, which is appended to the list in :attr:`~Option.dest`. If no default value for :attr:`~Option.dest` is - supplied, an empty list is automatically created when :mod:`optparse` first + supplied, an empty list is automatically created when :mod:`!optparse` first encounters this option on the command-line. If :attr:`~Option.nargs` > 1, multiple arguments are consumed, and a tuple of length :attr:`~Option.nargs` is appended to :attr:`~Option.dest`. @@ -1294,7 +1294,7 @@ must specify for any option using that action. parser.add_option("-t", "--tracks", action="append", type="int") - If ``-t3`` is seen on the command-line, :mod:`optparse` does the equivalent + If ``-t3`` is seen on the command-line, :mod:`!optparse` does the equivalent of:: options.tracks = [] @@ -1333,7 +1333,7 @@ must specify for any option using that action. parser.add_option("-v", action="count", dest="verbosity") - The first time ``-v`` is seen on the command line, :mod:`optparse` does the + The first time ``-v`` is seen on the command line, :mod:`!optparse` does the equivalent of:: options.verbosity = 0 @@ -1364,7 +1364,7 @@ must specify for any option using that action. listed in the help message. To omit an option entirely, use the special value :const:`optparse.SUPPRESS_HELP`. - :mod:`optparse` automatically adds a :attr:`~Option.help` option to all + :mod:`!optparse` automatically adds a :attr:`~Option.help` option to all OptionParsers, so you do not normally need to create one. Example:: @@ -1382,7 +1382,7 @@ must specify for any option using that action. help="Input file to read data from") parser.add_option("--secret", help=SUPPRESS_HELP) - If :mod:`optparse` sees either ``-h`` or ``--help`` on the command line, + If :mod:`!optparse` sees either ``-h`` or ``--help`` on the command line, it will print something like the following help message to stdout (assuming ``sys.argv[0]`` is ``"foo.py"``): @@ -1395,7 +1395,7 @@ must specify for any option using that action. -v Be moderately verbose --file=FILENAME Input file to read data from - After printing the help message, :mod:`optparse` terminates your process with + After printing the help message, :mod:`!optparse` terminates your process with ``sys.exit(0)``. * ``"version"`` @@ -1405,7 +1405,7 @@ must specify for any option using that action. ``print_version()`` method of OptionParser. Generally only relevant if the ``version`` argument is supplied to the OptionParser constructor. As with :attr:`~Option.help` options, you will rarely create ``version`` options, - since :mod:`optparse` automatically adds them when needed. + since :mod:`!optparse` automatically adds them when needed. .. _optparse-standard-option-types: @@ -1413,7 +1413,7 @@ must specify for any option using that action. Standard option types ^^^^^^^^^^^^^^^^^^^^^ -:mod:`optparse` has five built-in option types: ``"string"``, ``"int"``, +:mod:`!optparse` has five built-in option types: ``"string"``, ``"int"``, ``"choice"``, ``"float"`` and ``"complex"``. If you need to add new option types, see section :ref:`optparse-extending-optparse`. @@ -1432,7 +1432,7 @@ Integer arguments (type ``"int"``) are parsed as follows: The conversion is done by calling :func:`int` with the appropriate base (2, 8, -10, or 16). If this fails, so will :mod:`optparse`, although with a more useful +10, or 16). If this fails, so will :mod:`!optparse`, although with a more useful error message. ``"float"`` and ``"complex"`` option arguments are converted directly with @@ -1471,7 +1471,7 @@ The whole point of creating and populating an OptionParser is to call its ``options`` the same object that was passed in as *values*, or the ``optparse.Values`` - instance created by :mod:`optparse` + instance created by :mod:`!optparse` ``args`` the leftover positional arguments after all options have been processed @@ -1499,7 +1499,7 @@ provides several methods to help you out: .. method:: OptionParser.disable_interspersed_args() Set parsing to stop on the first non-option. For example, if ``-a`` and - ``-b`` are both simple options that take no arguments, :mod:`optparse` + ``-b`` are both simple options that take no arguments, :mod:`!optparse` normally accepts this syntax:: prog -a arg1 -b arg2 @@ -1554,7 +1554,7 @@ strings:: (This is particularly true if you've defined your own OptionParser subclass with some standard options.) -Every time you add an option, :mod:`optparse` checks for conflicts with existing +Every time you add an option, :mod:`!optparse` checks for conflicts with existing options. If it finds any, it invokes the current conflict-handling mechanism. You can set the conflict-handling mechanism either in the constructor:: @@ -1581,7 +1581,7 @@ intelligently and add conflicting options to it:: parser.add_option("-n", "--dry-run", ..., help="do no harm") parser.add_option("-n", "--noisy", ..., help="be noisy") -At this point, :mod:`optparse` detects that a previously added option is already +At this point, :mod:`!optparse` detects that a previously added option is already using the ``-n`` option string. Since ``conflict_handler`` is ``"resolve"``, it resolves the situation by removing ``-n`` from the earlier option's list of option strings. Now ``--dry-run`` is the only way for the user to activate @@ -1594,14 +1594,14 @@ that option. If the user asks for help, the help message will reflect that:: It's possible to whittle away the option strings for a previously added option until there are none left, and the user has no way of invoking that option from -the command-line. In that case, :mod:`optparse` removes that option completely, +the command-line. In that case, :mod:`!optparse` removes that option completely, so it doesn't show up in help text or anywhere else. Carrying on with our existing OptionParser:: parser.add_option("--dry-run", ..., help="new dry-run option") At this point, the original ``-n``/``--dry-run`` option is no longer -accessible, so :mod:`optparse` removes it, leaving this help text:: +accessible, so :mod:`!optparse` removes it, leaving this help text:: Options: ... @@ -1676,9 +1676,9 @@ OptionParser supports several other public methods: Option Callbacks ---------------- -When :mod:`optparse`'s built-in actions and types aren't quite enough for your -needs, you have two choices: extend :mod:`optparse` or define a callback option. -Extending :mod:`optparse` is more general, but overkill for a lot of simple +When :mod:`!optparse`'s built-in actions and types aren't quite enough for your +needs, you have two choices: extend :mod:`!optparse` or define a callback option. +Extending :mod:`!optparse` is more general, but overkill for a lot of simple cases. Quite often a simple callback is all you need. There are two steps to defining a callback option: @@ -1702,14 +1702,14 @@ only option attribute you must specify is ``callback``, the function to call:: ``callback`` is a function (or other callable object), so you must have already defined ``my_callback()`` when you create this callback option. In this simple -case, :mod:`optparse` doesn't even know if ``-c`` takes any arguments, +case, :mod:`!optparse` doesn't even know if ``-c`` takes any arguments, which usually means that the option takes no arguments---the mere presence of ``-c`` on the command-line is all it needs to know. In some circumstances, though, you might want your callback to consume an arbitrary number of command-line arguments. This is where writing callbacks gets tricky; it's covered later in this section. -:mod:`optparse` always passes four particular arguments to your callback, and it +:mod:`!optparse` always passes four particular arguments to your callback, and it will only pass additional arguments if you specify them via :attr:`~Option.callback_args` and :attr:`~Option.callback_kwargs`. Thus, the minimal callback function signature is:: @@ -1723,12 +1723,12 @@ callback option: :attr:`~Option.type` has its usual meaning: as with the ``"store"`` or ``"append"`` actions, it - instructs :mod:`optparse` to consume one argument and convert it to + instructs :mod:`!optparse` to consume one argument and convert it to :attr:`~Option.type`. Rather than storing the converted value(s) anywhere, - though, :mod:`optparse` passes it to your callback function. + though, :mod:`!optparse` passes it to your callback function. :attr:`~Option.nargs` - also has its usual meaning: if it is supplied and > 1, :mod:`optparse` will + also has its usual meaning: if it is supplied and > 1, :mod:`!optparse` will consume :attr:`~Option.nargs` arguments, each of which must be convertible to :attr:`~Option.type`. It then passes a tuple of converted values to your callback. @@ -1762,7 +1762,7 @@ where ``"--foobar"``.) ``value`` - is the argument to this option seen on the command-line. :mod:`optparse` will + is the argument to this option seen on the command-line. :mod:`!optparse` will only expect an argument if :attr:`~Option.type` is set; the type of ``value`` will be the type implied by the option's type. If :attr:`~Option.type` for this option is ``None`` (no argument expected), then ``value`` will be ``None``. If :attr:`~Option.nargs` @@ -1787,7 +1787,7 @@ where ``parser.values`` the object where option values are by default stored (an instance of optparse.OptionValues). This lets callbacks use the same mechanism as the - rest of :mod:`optparse` for storing option values; you don't need to mess + rest of :mod:`!optparse` for storing option values; you don't need to mess around with globals or closures. You can also access or modify the value(s) of any options already encountered on the command-line. @@ -1806,7 +1806,7 @@ Raising errors in a callback ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The callback function should raise :exc:`OptionValueError` if there are any -problems with the option or its argument(s). :mod:`optparse` catches this and +problems with the option or its argument(s). :mod:`!optparse` catches this and terminates the program, printing the error message you supply to stderr. Your message should be clear, concise, accurate, and mention the option at fault. Otherwise, the user will have a hard time figuring out what they did wrong. @@ -1906,7 +1906,7 @@ Here's an example that just emulates the standard ``"store"`` action:: action="callback", callback=store_value, type="int", nargs=3, dest="foo") -Note that :mod:`optparse` takes care of consuming 3 arguments and converting +Note that :mod:`!optparse` takes care of consuming 3 arguments and converting them to integers for you; all you have to do is store them. (Or whatever; obviously you don't need a callback for this example.) @@ -1917,9 +1917,9 @@ Callback example 6: variable arguments ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Things get hairy when you want an option to take a variable number of arguments. -For this case, you must write a callback, as :mod:`optparse` doesn't provide any +For this case, you must write a callback, as :mod:`!optparse` doesn't provide any built-in capabilities for it. And you have to deal with certain intricacies of -conventional Unix command-line parsing that :mod:`optparse` normally handles for +conventional Unix command-line parsing that :mod:`!optparse` normally handles for you. In particular, callbacks should implement the conventional rules for bare ``--`` and ``-`` arguments: @@ -1934,7 +1934,7 @@ you. In particular, callbacks should implement the conventional rules for bare If you want an option that takes a variable number of arguments, there are several subtle, tricky issues to worry about. The exact implementation you choose will be based on which trade-offs you're willing to make for your -application (which is why :mod:`optparse` doesn't support this sort of thing +application (which is why :mod:`!optparse` doesn't support this sort of thing directly). Nevertheless, here's a stab at a callback for an option with variable @@ -1970,10 +1970,10 @@ arguments:: .. _optparse-extending-optparse: -Extending :mod:`optparse` -------------------------- +Extending :mod:`!optparse` +-------------------------- -Since the two major controlling factors in how :mod:`optparse` interprets +Since the two major controlling factors in how :mod:`!optparse` interprets command-line options are the action and type of each option, the most likely direction of extension is to add new actions and new types. @@ -1983,9 +1983,9 @@ direction of extension is to add new actions and new types. Adding new types ^^^^^^^^^^^^^^^^ -To add new types, you need to define your own subclass of :mod:`optparse`'s +To add new types, you need to define your own subclass of :mod:`!optparse`'s :class:`Option` class. This class has a couple of attributes that define -:mod:`optparse`'s types: :attr:`~Option.TYPES` and :attr:`~Option.TYPE_CHECKER`. +:mod:`!optparse`'s types: :attr:`~Option.TYPES` and :attr:`~Option.TYPE_CHECKER`. .. attribute:: Option.TYPES @@ -2015,7 +2015,7 @@ To add new types, you need to define your own subclass of :mod:`optparse`'s Here's a silly example that demonstrates adding a ``"complex"`` option type to parse Python-style complex numbers on the command line. (This is even sillier -than it used to be, because :mod:`optparse` 1.3 added built-in support for +than it used to be, because :mod:`!optparse` 1.3 added built-in support for complex numbers, but never mind.) First, the necessary imports:: @@ -2041,12 +2041,12 @@ Finally, the Option subclass:: TYPE_CHECKER["complex"] = check_complex (If we didn't make a :func:`copy` of :attr:`Option.TYPE_CHECKER`, we would end -up modifying the :attr:`~Option.TYPE_CHECKER` attribute of :mod:`optparse`'s +up modifying the :attr:`~Option.TYPE_CHECKER` attribute of :mod:`!optparse`'s Option class. This being Python, nothing stops you from doing that except good manners and common sense.) That's it! Now you can write a script that uses the new option type just like -any other :mod:`optparse`\ -based script, except you have to instruct your +any other :mod:`!optparse`\ -based script, except you have to instruct your OptionParser to use MyOption instead of Option:: parser = OptionParser(option_class=MyOption) @@ -2066,10 +2066,10 @@ Adding new actions ^^^^^^^^^^^^^^^^^^ Adding new actions is a bit trickier, because you have to understand that -:mod:`optparse` has a couple of classifications for actions: +:mod:`!optparse` has a couple of classifications for actions: "store" actions - actions that result in :mod:`optparse` storing a value to an attribute of the + actions that result in :mod:`!optparse` storing a value to an attribute of the current OptionValues instance; these options require a :attr:`~Option.dest` attribute to be supplied to the Option constructor. @@ -2101,7 +2101,7 @@ of the following class attributes of Option (all are lists of strings): .. attribute:: Option.ALWAYS_TYPED_ACTIONS Actions that always take a type (i.e. whose options always take a value) are - additionally listed here. The only effect of this is that :mod:`optparse` + additionally listed here. The only effect of this is that :mod:`!optparse` assigns the default type, ``"string"``, to options with no explicit type whose action is listed in :attr:`ALWAYS_TYPED_ACTIONS`. @@ -2144,12 +2144,12 @@ Features of note: somewhere, so it goes in both :attr:`~Option.STORE_ACTIONS` and :attr:`~Option.TYPED_ACTIONS`. -* to ensure that :mod:`optparse` assigns the default type of ``"string"`` to +* to ensure that :mod:`!optparse` assigns the default type of ``"string"`` to ``"extend"`` actions, we put the ``"extend"`` action in :attr:`~Option.ALWAYS_TYPED_ACTIONS` as well. * :meth:`MyOption.take_action` implements just this one new action, and passes - control back to :meth:`Option.take_action` for the standard :mod:`optparse` + control back to :meth:`Option.take_action` for the standard :mod:`!optparse` actions. * ``values`` is an instance of the optparse_parser.Values class, which provides diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 0f805de0fde..48e4734cad9 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -36,7 +36,7 @@ the :mod:`glob` module.) Since different operating systems have different path name conventions, there are several versions of this module in the standard library. The - :mod:`os.path` module is always the path module suitable for the operating + :mod:`!os.path` module is always the path module suitable for the operating system Python is running on, and therefore usable for local paths. However, you can also import and use the individual modules if you want to manipulate a path that is *always* in one of the different formats. They all have the diff --git a/Doc/library/os.rst b/Doc/library/os.rst index b1112656749..721f18e6af7 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -25,7 +25,7 @@ Notes on the availability of these functions: with the POSIX interface). * Extensions peculiar to a particular operating system are also available - through the :mod:`os` module, but using them is of course a threat to + through the :mod:`!os` module, but using them is of course a threat to portability. * All functions accepting path or file names accept both bytes and string @@ -34,7 +34,7 @@ Notes on the availability of these functions: * On VxWorks, os.popen, os.fork, os.execv and os.spawn*p* are not supported. -* On WebAssembly platforms, Android and iOS, large parts of the :mod:`os` module are +* On WebAssembly platforms, Android and iOS, large parts of the :mod:`!os` module are not available or behave differently. APIs related to processes (e.g. :func:`~os.fork`, :func:`~os.execve`) and resources (e.g. :func:`~os.nice`) are not available. Others like :func:`~os.getuid` and :func:`~os.getpid` are @@ -188,7 +188,7 @@ process and user. of your home directory (on some platforms), and is equivalent to ``getenv("HOME")`` in C. - This mapping is captured the first time the :mod:`os` module is imported, + This mapping is captured the first time the :mod:`!os` module is imported, typically during Python startup as part of processing :file:`site.py`. Changes to the environment made after this time are not reflected in :data:`os.environ`, except for changes made by modifying :data:`os.environ` directly. @@ -257,7 +257,7 @@ process and user. .. warning:: This function is not thread-safe. Calling it while the environment is - being modified in an other thread is an undefined behavior. Reading from + being modified in another thread is an undefined behavior. Reading from :data:`os.environ` or :data:`os.environb`, or calling :func:`os.getenv` while reloading, may return an empty result. @@ -1282,7 +1282,7 @@ as internal buffering of data. For a description of the flag and mode values, see the C run-time documentation; flag constants (like :const:`O_RDONLY` and :const:`O_WRONLY`) are defined in - the :mod:`os` module. In particular, on Windows adding + the :mod:`!os` module. In particular, on Windows adding :const:`O_BINARY` is needed to open files in binary mode. This function can support :ref:`paths relative to directory descriptors @@ -1297,8 +1297,8 @@ as internal buffering of data. This function is intended for low-level I/O. For normal usage, use the built-in function :func:`open`, which returns a :term:`file object` with - :meth:`~file.read` and :meth:`~file.write` methods (and many more). To - wrap a file descriptor in a file object, use :func:`fdopen`. + :meth:`~io.BufferedIOBase.read` and :meth:`~io.BufferedIOBase.write` methods. + To wrap a file descriptor in a file object, use :func:`fdopen`. .. versionchanged:: 3.3 Added the *dir_fd* parameter. @@ -1652,7 +1652,7 @@ or `the MSDN `_ on Windo descriptor as returned by :func:`os.open` or :func:`pipe`. To read a "file object" returned by the built-in function :func:`open` or by :func:`popen` or :func:`fdopen`, or :data:`sys.stdin`, use its - :meth:`~file.read` or :meth:`~file.readline` methods. + :meth:`~io.TextIOBase.read` or :meth:`~io.IOBase.readline` methods. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -1887,7 +1887,7 @@ or `the MSDN `_ on Windo descriptor as returned by :func:`os.open` or :func:`pipe`. To write a "file object" returned by the built-in function :func:`open` or by :func:`popen` or :func:`fdopen`, or :data:`sys.stdout` or :data:`sys.stderr`, use its - :meth:`~file.write` method. + :meth:`~io.TextIOBase.write` method. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -2006,7 +2006,7 @@ features: .. _path_fd: * **specifying a file descriptor:** - Normally the *path* argument provided to functions in the :mod:`os` module + Normally the *path* argument provided to functions in the :mod:`!os` module must be a string specifying a file path. However, some functions now alternatively accept an open file descriptor for their *path* argument. The function will then operate on the file referred to by the descriptor. @@ -3417,7 +3417,7 @@ features: .. data:: supports_dir_fd - A :class:`set` object indicating which functions in the :mod:`os` + A :class:`set` object indicating which functions in the :mod:`!os` module accept an open file descriptor for their *dir_fd* parameter. Different platforms provide different features, and the underlying functionality Python uses to implement the *dir_fd* parameter is not @@ -3462,7 +3462,7 @@ features: .. data:: supports_fd A :class:`set` object indicating which functions in the - :mod:`os` module permit specifying their *path* parameter as an open file + :mod:`!os` module permit specifying their *path* parameter as an open file descriptor on the local platform. Different platforms provide different features, and the underlying functionality Python uses to accept open file descriptors as *path* arguments is not available on all platforms Python @@ -3481,7 +3481,7 @@ features: .. data:: supports_follow_symlinks - A :class:`set` object indicating which functions in the :mod:`os` module + A :class:`set` object indicating which functions in the :mod:`!os` module accept ``False`` for their *follow_symlinks* parameter on the local platform. Different platforms provide different features, and the underlying functionality Python uses to implement *follow_symlinks* is not available @@ -3989,7 +3989,7 @@ Naturally, they are all only available on Linux. except it includes any time that the system is suspended. The file descriptor's behaviour can be modified by specifying a *flags* value. - Any of the following variables may used, combined using bitwise OR + Any of the following variables may be used, combined using bitwise OR (the ``|`` operator): - :const:`TFD_NONBLOCK` @@ -4021,7 +4021,7 @@ Naturally, they are all only available on Linux. *fd* must be a valid timer file descriptor. The timer's behaviour can be modified by specifying a *flags* value. - Any of the following variables may used, combined using bitwise OR + Any of the following variables may be used, combined using bitwise OR (the ``|`` operator): - :const:`TFD_TIMER_ABSTIME` @@ -4090,7 +4090,7 @@ Naturally, they are all only available on Linux. Return a two-item tuple of floats (``next_expiration``, ``interval``). - ``next_expiration`` denotes the relative time until next the timer next fires, + ``next_expiration`` denotes the relative time until the timer next fires, regardless of if the :const:`TFD_TIMER_ABSTIME` flag is set. ``interval`` denotes the timer's interval. @@ -4326,7 +4326,7 @@ to be ignored. The current process is replaced immediately. Open file objects and descriptors are not flushed, so if there may be data buffered on these open files, you should flush them using - :func:`sys.stdout.flush` or :func:`os.fsync` before calling an + :func:`~io.IOBase.flush` or :func:`os.fsync` before calling an :func:`exec\* ` function. The "l" and "v" variants of the :func:`exec\* ` functions differ in how diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index b8876c4ca14..9484103e5ce 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -486,6 +486,10 @@ Pure paths provide the following methods and properties: >>> PurePosixPath('my/library').stem 'library' + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. + .. method:: PurePath.as_posix() @@ -1897,7 +1901,7 @@ Below is a table mapping various :mod:`os` functions to their corresponding :class:`PurePath`/:class:`Path` equivalent. ===================================== ============================================== -:mod:`os` and :mod:`os.path` :mod:`pathlib` +:mod:`os` and :mod:`os.path` :mod:`!pathlib` ===================================== ============================================== :func:`os.path.dirname` :attr:`PurePath.parent` :func:`os.path.basename` :attr:`PurePath.name` @@ -1956,7 +1960,7 @@ Protocols :synopsis: pathlib types for static type checking -The :mod:`pathlib.types` module provides types for static type checking. +The :mod:`!pathlib.types` module provides types for static type checking. .. versionadded:: 3.14 diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 0bbdc425352..e7dd2e5524a 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -1,7 +1,7 @@ .. _debugger: -:mod:`pdb` --- The Python Debugger -================================== +:mod:`!pdb` --- The Python Debugger +=================================== .. module:: pdb :synopsis: The Python debugger for interactive interpreters. @@ -12,7 +12,7 @@ -------------- -The module :mod:`pdb` defines an interactive source code debugger for Python +The module :mod:`!pdb` defines an interactive source code debugger for Python programs. It supports setting (conditional) breakpoints and single stepping at the source line level, inspection of stack frames, source code listing, and evaluation of arbitrary Python code in the context of any stack frame. It also @@ -82,7 +82,7 @@ Command-line interface .. program:: pdb -You can also invoke :mod:`pdb` from the command line to debug other scripts. For +You can also invoke :mod:`!pdb` from the command line to debug other scripts. For example:: python -m pdb [-c command] (-m module | -p pid | pyfile) [args ...] diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 7b0d979d61a..02b79a9f3a7 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -19,7 +19,7 @@ -------------- -The :mod:`pickle` module implements binary protocols for serializing and +The :mod:`!pickle` module implements binary protocols for serializing and de-serializing a Python object structure. *"Pickling"* is the process whereby a Python object hierarchy is converted into a byte stream, and *"unpickling"* is the inverse operation, whereby a byte stream @@ -50,14 +50,14 @@ Comparison with ``marshal`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Python has a more primitive serialization module called :mod:`marshal`, but in -general :mod:`pickle` should always be the preferred way to serialize Python +general :mod:`!pickle` should always be the preferred way to serialize Python objects. :mod:`marshal` exists primarily to support Python's :file:`.pyc` files. -The :mod:`pickle` module differs from :mod:`marshal` in several significant ways: +The :mod:`!pickle` module differs from :mod:`marshal` in several significant ways: * :mod:`marshal` cannot be used to serialize user-defined classes and their - instances. :mod:`pickle` can save and restore class instances transparently, + instances. :mod:`!pickle` can save and restore class instances transparently, however the class definition must be importable and live in the same module as when the object was stored. @@ -65,7 +65,7 @@ The :mod:`pickle` module differs from :mod:`marshal` in several significant ways across Python versions. Because its primary job in life is to support :file:`.pyc` files, the Python implementers reserve the right to change the serialization format in non-backwards compatible ways should the need arise. - The :mod:`pickle` serialization format is guaranteed to be backwards compatible + The :mod:`!pickle` serialization format is guaranteed to be backwards compatible across Python releases provided a compatible pickle protocol is chosen and pickling and unpickling code deals with Python 2 to Python 3 type differences if your data is crossing that unique breaking change language boundary. @@ -110,17 +110,17 @@ Data stream format .. index:: single: External Data Representation -The data format used by :mod:`pickle` is Python-specific. This has the +The data format used by :mod:`!pickle` is Python-specific. This has the advantage that there are no restrictions imposed by external standards such as JSON (which can't represent pointer sharing); however it means that non-Python programs may not be able to reconstruct pickled Python objects. -By default, the :mod:`pickle` data format uses a relatively compact binary +By default, the :mod:`!pickle` data format uses a relatively compact binary representation. If you need optimal size characteristics, you can efficiently :doc:`compress ` pickled data. The module :mod:`pickletools` contains tools for analyzing data streams -generated by :mod:`pickle`. :mod:`pickletools` source code has extensive +generated by :mod:`!pickle`. :mod:`pickletools` source code has extensive comments about opcodes used by pickle protocols. There are currently 6 different protocols which can be used for pickling. @@ -154,9 +154,9 @@ to read the pickle produced. .. note:: Serialization is a more primitive notion than persistence; although - :mod:`pickle` reads and writes file objects, it does not handle the issue of + :mod:`!pickle` reads and writes file objects, it does not handle the issue of naming persistent objects, nor the (even more complicated) issue of concurrent - access to persistent objects. The :mod:`pickle` module can transform a complex + access to persistent objects. The :mod:`!pickle` module can transform a complex object into a byte stream and it can transform the byte stream into an object with the same internal structure. Perhaps the most obvious thing to do with these byte streams is to write them onto a file, but it is also conceivable to @@ -173,7 +173,7 @@ Similarly, to de-serialize a data stream, you call the :func:`loads` function. However, if you want more control over serialization and de-serialization, you can create a :class:`Pickler` or an :class:`Unpickler` object, respectively. -The :mod:`pickle` module provides the following constants: +The :mod:`!pickle` module provides the following constants: .. data:: HIGHEST_PROTOCOL @@ -204,7 +204,7 @@ The :mod:`pickle` module provides the following constants: The default protocol is 5. -The :mod:`pickle` module provides the following functions to make the pickling +The :mod:`!pickle` module provides the following functions to make the pickling process more convenient: .. function:: dump(obj, file, protocol=None, *, fix_imports=True, buffer_callback=None) @@ -262,7 +262,7 @@ process more convenient: The *buffers* argument was added. -The :mod:`pickle` module defines three exceptions: +The :mod:`!pickle` module defines three exceptions: .. exception:: PickleError @@ -287,7 +287,7 @@ The :mod:`pickle` module defines three exceptions: IndexError. -The :mod:`pickle` module exports three classes, :class:`Pickler`, +The :mod:`!pickle` module exports three classes, :class:`Pickler`, :class:`Unpickler` and :class:`PickleBuffer`: .. class:: Pickler(file, protocol=None, *, fix_imports=True, buffer_callback=None) @@ -519,7 +519,7 @@ The following types can be pickled: * classes accessible from the top level of a module; -* instances of such classes whose the result of calling :meth:`~object.__getstate__` +* instances of such classes for which the result of calling :meth:`~object.__getstate__` is picklable (see section :ref:`pickle-inst` for details). Attempts to pickle unpicklable objects will raise the :exc:`PicklingError` @@ -588,7 +588,7 @@ methods: .. method:: object.__getnewargs_ex__() - In protocols 2 and newer, classes that implements the + In protocols 2 and newer, classes that implement the :meth:`__getnewargs_ex__` method can dictate the values passed to the :meth:`__new__` method upon unpickling. The method must return a pair ``(args, kwargs)`` where *args* is a tuple of positional arguments @@ -760,13 +760,13 @@ Persistence of External Objects single: persistent_id (pickle protocol) single: persistent_load (pickle protocol) -For the benefit of object persistence, the :mod:`pickle` module supports the +For the benefit of object persistence, the :mod:`!pickle` module supports the notion of a reference to an object outside the pickled data stream. Such objects are referenced by a persistent ID, which should be either a string of alphanumeric characters (for protocol 0) [#]_ or just an arbitrary object (for any newer protocol). -The resolution of such persistent IDs is not defined by the :mod:`pickle` +The resolution of such persistent IDs is not defined by the :mod:`!pickle` module; it will delegate this resolution to the user-defined methods on the pickler and unpickler, :meth:`~Pickler.persistent_id` and :meth:`~Unpickler.persistent_load` respectively. @@ -960,10 +960,10 @@ Out-of-band Buffers .. versionadded:: 3.8 -In some contexts, the :mod:`pickle` module is used to transfer massive amounts +In some contexts, the :mod:`!pickle` module is used to transfer massive amounts of data. Therefore, it can be important to minimize the number of memory copies, to preserve performance and resource consumption. However, normal -operation of the :mod:`pickle` module, as it transforms a graph-like structure +operation of the :mod:`!pickle` module, as it transforms a graph-like structure of objects into a sequential stream of bytes, intrinsically involves copying data to and from the pickle stream. @@ -982,8 +982,8 @@ for any large data. A :class:`PickleBuffer` object *signals* that the underlying buffer is eligible for out-of-band data transfer. Those objects remain compatible -with normal usage of the :mod:`pickle` module. However, consumers can also -opt-in to tell :mod:`pickle` that they will handle those buffers by +with normal usage of the :mod:`!pickle` module. However, consumers can also +opt-in to tell :mod:`!pickle` that they will handle those buffers by themselves. Consumer API @@ -1159,7 +1159,7 @@ Performance Recent versions of the pickle protocol (from protocol 2 and upwards) feature efficient binary encodings for several common features and built-in types. -Also, the :mod:`pickle` module has a transparent optimizer written in C. +Also, the :mod:`!pickle` module has a transparent optimizer written in C. .. _pickle-example: @@ -1202,7 +1202,7 @@ The following example reads the resulting pickled data. :: Command-line interface ---------------------- -The :mod:`pickle` module can be invoked as a script from the command line, +The :mod:`!pickle` module can be invoked as a script from the command line, it will display contents of the pickle files. However, when the pickle file that you want to examine comes from an untrusted source, ``-m pickletools`` is a safer option because it does not execute pickle bytecode, see @@ -1230,7 +1230,7 @@ The following option is accepted: Tools for working with and analyzing pickled data. Module :mod:`shelve` - Indexed databases of objects; uses :mod:`pickle`. + Indexed databases of objects; uses :mod:`!pickle`. Module :mod:`copy` Shallow and deep object copying. diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index 30fc2962e0b..7a771ea3ab9 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -15,7 +15,7 @@ This module contains various constants relating to the intimate details of the few useful functions for analyzing pickled data. The contents of this module are useful for Python core developers who are working on the :mod:`pickle`; ordinary users of the :mod:`pickle` module probably won't find the -:mod:`pickletools` module relevant. +:mod:`!pickletools` module relevant. .. _pickletools-cli: diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 256baa53ed5..ff254caa5fb 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -375,7 +375,7 @@ Android platform Command-line usage ------------------ -:mod:`platform` can also be invoked directly using the :option:`-m` +:mod:`!platform` can also be invoked directly using the :option:`-m` switch of the interpreter:: python -m platform [--terse] [--nonaliased] [{nonaliased,terse} ...] diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index 23f20b00e6d..51ae480338d 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -30,7 +30,7 @@ mailserver supports IMAP, you would be better off using the .. include:: ../includes/wasm-notavail.rst -The :mod:`poplib` module provides two classes: +The :mod:`!poplib` module provides two classes: .. class:: POP3(host, port=POP3_PORT[, timeout]) @@ -86,7 +86,7 @@ The :mod:`poplib` module provides two classes: .. versionchanged:: 3.12 The deprecated *keyfile* and *certfile* parameters have been removed. -One exception is defined as an attribute of the :mod:`poplib` module: +One exception is defined as an attribute of the :mod:`!poplib` module: .. exception:: error_proto diff --git a/Doc/library/posix.rst b/Doc/library/posix.rst index 14ab3e91e8a..1f1ca03dfd6 100644 --- a/Doc/library/posix.rst +++ b/Doc/library/posix.rst @@ -2,7 +2,6 @@ ==================================================== .. module:: posix - :platform: Unix :synopsis: The most common POSIX system calls (normally used via module os). -------------- @@ -17,10 +16,10 @@ interface). **Do not import this module directly.** Instead, import the module :mod:`os`, which provides a *portable* version of this interface. On Unix, the :mod:`os` -module provides a superset of the :mod:`posix` interface. On non-Unix operating -systems the :mod:`posix` module is not available, but a subset is always +module provides a superset of the :mod:`!posix` interface. On non-Unix operating +systems the :mod:`!posix` module is not available, but a subset is always available through the :mod:`os` interface. Once :mod:`os` is imported, there is -*no* performance penalty in using it instead of :mod:`posix`. In addition, +*no* performance penalty in using it instead of :mod:`!posix`. In addition, :mod:`os` provides some additional functionality, such as automatically calling :func:`~os.putenv` when an entry in ``os.environ`` is changed. @@ -67,7 +66,7 @@ Notable Module Contents ----------------------- In addition to many functions described in the :mod:`os` module documentation, -:mod:`posix` defines the following data item: +:mod:`!posix` defines the following data item: .. data:: environ @@ -91,4 +90,4 @@ In addition to many functions described in the :mod:`os` module documentation, which updates the environment on modification. Note also that updating :data:`os.environ` will render this dictionary obsolete. Use of the :mod:`os` module version of this is recommended over direct access to the - :mod:`posix` module. + :mod:`!posix` module. diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index f5189245079..be942949d3e 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`pprint` module provides a capability to "pretty-print" arbitrary +The :mod:`!pprint` module provides a capability to "pretty-print" arbitrary Python data structures in a form which can be used as input to the interpreter. If the formatted structures include objects which are not fundamental Python types, the representation may not be loadable. This may be the case if objects diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 89433af5d87..57f997792b8 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -217,14 +217,14 @@ Invoked as a script, the :mod:`pstats` module is a statistics browser for reading and examining profile dumps. It has a simple line-oriented interface (implemented using :mod:`cmd`) and interactive help. -:mod:`profile` and :mod:`cProfile` Module Reference -======================================================= +:mod:`profile` and :mod:`!cProfile` Module Reference +==================================================== .. module:: cProfile .. module:: profile :synopsis: Python source profiler. -Both the :mod:`profile` and :mod:`cProfile` modules provide the following +Both the :mod:`profile` and :mod:`!cProfile` modules provide the following functions: .. function:: run(command, filename=None, sort=-1) @@ -278,7 +278,7 @@ functions: print(s.getvalue()) The :class:`Profile` class can also be used as a context manager (supported - only in :mod:`cProfile` module. see :ref:`typecontextmanager`):: + only in :mod:`!cProfile` module. see :ref:`typecontextmanager`):: import cProfile @@ -292,11 +292,11 @@ functions: .. method:: enable() - Start collecting profiling data. Only in :mod:`cProfile`. + Start collecting profiling data. Only in :mod:`!cProfile`. .. method:: disable() - Stop collecting profiling data. Only in :mod:`cProfile`. + Stop collecting profiling data. Only in :mod:`!cProfile`. .. method:: create_stats() diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index 1a44bb13a84..9d6036d92c4 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -2,7 +2,6 @@ ========================================= .. module:: pty - :platform: Unix :synopsis: Pseudo-Terminal Handling for Unix. .. moduleauthor:: Steen Lumholt @@ -12,7 +11,7 @@ -------------- -The :mod:`pty` module defines operations for handling the pseudo-terminal +The :mod:`!pty` module defines operations for handling the pseudo-terminal concept: starting another process and being able to write to and read from its controlling terminal programmatically. @@ -22,7 +21,7 @@ Pseudo-terminal handling is highly platform dependent. This code is mainly tested on Linux, FreeBSD, and macOS (it is supposed to work on other POSIX platforms but it's not been thoroughly tested). -The :mod:`pty` module defines the following functions: +The :mod:`!pty` module defines the following functions: .. function:: fork() diff --git a/Doc/library/pwd.rst b/Doc/library/pwd.rst index e1ff3291213..7691fed2c7c 100644 --- a/Doc/library/pwd.rst +++ b/Doc/library/pwd.rst @@ -2,7 +2,6 @@ ===================================== .. module:: pwd - :platform: Unix :synopsis: The password database (getpwnam() and friends). -------------- diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 75aa739d100..1cff16b6c1b 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -13,7 +13,7 @@ -------------- -The :mod:`py_compile` module provides a function to generate a byte-code file +The :mod:`!py_compile` module provides a function to generate a byte-code file from a source file, and another function used when the module source file is invoked as a script. diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index 5efb11d89dd..40f93646af2 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -10,7 +10,7 @@ -------------- -The :mod:`pyclbr` module provides limited information about the +The :mod:`!pyclbr` module provides limited information about the functions, classes, and methods defined in a Python-coded module. The information is sufficient to implement a module browser. The information is extracted from the Python source code rather than by diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index b2dd92f7ba2..c448627e2bd 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -24,7 +24,7 @@ .. index:: single: Expat -The :mod:`xml.parsers.expat` module is a Python interface to the Expat +The :mod:`!xml.parsers.expat` module is a Python interface to the Expat non-validating XML parser. The module provides a single extension type, :class:`xmlparser`, that represents the current state of an XML parser. After an :class:`xmlparser` object has been created, various attributes of the object @@ -55,7 +55,7 @@ This module provides one exception and one type object: The type of the return values from the :func:`ParserCreate` function. -The :mod:`xml.parsers.expat` module contains two functions: +The :mod:`!xml.parsers.expat` module contains two functions: .. function:: ErrorString(errno) @@ -409,7 +409,7 @@ otherwise stated. ...``). The *doctypeName* is provided exactly as presented. The *systemId* and *publicId* parameters give the system and public identifiers if specified, or ``None`` if omitted. *has_internal_subset* will be true if the document - contains and internal document declaration subset. This requires Expat version + contains an internal document declaration subset. This requires Expat version 1.2 or newer. @@ -683,7 +683,7 @@ values: the type, the quantifier, the name, and a tuple of children. Children are simply additional content model descriptions. The values of the first two fields are constants defined in the -:mod:`xml.parsers.expat.model` module. These constants can be collected in two +:mod:`!xml.parsers.expat.model` module. These constants can be collected in two groups: the model type group and the quantifier group. The constants in the model type group are: @@ -757,7 +757,7 @@ Expat error constants .. module:: xml.parsers.expat.errors -The following constants are provided in the :mod:`xml.parsers.expat.errors` +The following constants are provided in the :mod:`!xml.parsers.expat.errors` module. These constants are useful in interpreting some of the attributes of the :exc:`ExpatError` exception objects raised when an error has occurred. Since for backwards compatibility reasons, the constants' value is the error @@ -904,7 +904,7 @@ The ``errors`` module has the following attributes: An operation was requested that requires DTD support to be compiled in, but Expat was configured without DTD support. This should never be reported by a - standard build of the :mod:`xml.parsers.expat` module. + standard build of the :mod:`!xml.parsers.expat` module. .. data:: XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 1b75582f0cf..5ac72ef7604 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`queue` module implements multi-producer, multi-consumer queues. +The :mod:`!queue` module implements multi-producer, multi-consumer queues. It is especially useful in threaded programming when information must be exchanged safely between multiple threads. The :class:`Queue` class in this module implements all the required locking semantics. @@ -30,7 +30,7 @@ In addition, the module implements a "simple" specific implementation provides additional guarantees in exchange for the smaller functionality. -The :mod:`queue` module defines the following classes and exceptions: +The :mod:`!queue` module defines the following classes and exceptions: .. class:: Queue(maxsize=0) diff --git a/Doc/library/random.rst b/Doc/library/random.rst index fdc14cd04e2..9327e5a23d6 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -37,7 +37,7 @@ Class :class:`Random` can also be subclassed if you want to use a different basic generator of your own devising: see the documentation on that class for more details. -The :mod:`random` module also provides the :class:`SystemRandom` class which +The :mod:`!random` module also provides the :class:`SystemRandom` class which uses the system function :func:`os.urandom` to generate random numbers from sources provided by the operating system. @@ -410,7 +410,7 @@ Alternative Generator .. class:: Random([seed]) Class that implements the default pseudo-random number generator used by the - :mod:`random` module. + :mod:`!random` module. .. versionchanged:: 3.11 Formerly the *seed* could be any hashable object. Now it is limited to: diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 75ebbf11c8e..6ee52c6927d 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -49,7 +49,7 @@ fine-tuning parameters. .. seealso:: The third-party :pypi:`regex` module, - which has an API compatible with the standard library :mod:`re` module, + which has an API compatible with the standard library :mod:`!re` module, but offers additional functionality and a more thorough Unicode support. @@ -1407,10 +1407,10 @@ when there is no match, you can test whether there was a match with a simple result is a single string; if there are multiple arguments, the result is a tuple with one item per argument. Without arguments, *group1* defaults to zero (the whole match is returned). If a *groupN* argument is zero, the corresponding - return value is the entire matching string; if it is in the inclusive range - [1..99], it is the string matching the corresponding parenthesized group. If a - group number is negative or larger than the number of groups defined in the - pattern, an :exc:`IndexError` exception is raised. If a group is contained in a + return value is the entire matching string; if it is a positive integer, it is + the string matching the corresponding parenthesized group. If a group number is + negative or larger than the number of groups defined in the pattern, an + :exc:`IndexError` exception is raised. If a group is contained in a part of the pattern that did not match, the corresponding result is ``None``. If a group is contained in a part of the pattern that matched multiple times, the last match is returned. :: diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 02e0d100791..be36f7b1b6a 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -2,14 +2,13 @@ =========================================== .. module:: readline - :platform: Unix :synopsis: GNU readline support for Python. .. sectionauthor:: Skip Montanaro -------------- -The :mod:`readline` module defines a number of functions to facilitate +The :mod:`!readline` module defines a number of functions to facilitate completion and reading/writing of history files from the Python interpreter. This module can be used directly, or via the :mod:`rlcompleter` module, which supports completion of Python identifiers at the interactive prompt. Settings @@ -28,11 +27,13 @@ Readline library in general. .. include:: ../includes/optional-module.rst +.. availability:: Unix. + .. note:: The underlying Readline library API may be implemented by the ``editline`` (``libedit``) library instead of GNU readline. - On macOS the :mod:`readline` module detects which library is being used + On macOS the :mod:`!readline` module detects which library is being used at run time. The configuration file for ``editline`` is different from that @@ -255,7 +256,7 @@ The following functions relate to implementing a custom word completion function. This is typically operated by the Tab key, and can suggest and automatically complete a word being typed. By default, Readline is set up to be used by :mod:`rlcompleter` to complete Python identifiers for -the interactive interpreter. If the :mod:`readline` module is to be used +the interactive interpreter. If the :mod:`!readline` module is to be used with a custom completer, a different set of word delimiters should be set. @@ -324,7 +325,7 @@ with a custom completer, a different set of word delimiters should be set. Example ------- -The following example demonstrates how to use the :mod:`readline` module's +The following example demonstrates how to use the :mod:`!readline` module's history reading and writing functions to automatically load and save a history file named :file:`.python_history` from the user's home directory. The code below would normally be executed automatically during interactive sessions diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index 512f0852dd5..357686da00c 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -2,7 +2,6 @@ =============================================== .. module:: resource - :platform: Unix :synopsis: An interface to provide resource usage information on the current process. .. moduleauthor:: Jeremy Hylton @@ -303,9 +302,9 @@ These functions are used to retrieve resource usage information: print(getrusage(RUSAGE_SELF)) The fields of the return value each describe how a particular system resource - has been used, e.g. amount of time spent running is user mode or number of times + has been used, e.g. amount of time spent running in user mode or number of times the process was swapped out of main memory. Some values are dependent on the - clock tick internal, e.g. the amount of memory the process is using. + clock tick interval, e.g. the amount of memory the process is using. For backward compatibility, the return value is also accessible as a tuple of 16 elements. diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index b07ec6e93f8..2e8081f3bf0 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -10,7 +10,7 @@ -------------- -The :mod:`runpy` module is used to locate and run Python modules without +The :mod:`!runpy` module is used to locate and run Python modules without importing them first. Its main use is to implement the :option:`-m` command line switch that allows scripts to be located using the Python module namespace rather than the filesystem. @@ -20,11 +20,11 @@ current process, and any side effects (such as cached imports of other modules) will remain in place after the functions have returned. Furthermore, any functions and classes defined by the executed code are not -guaranteed to work correctly after a :mod:`runpy` function has returned. +guaranteed to work correctly after a :mod:`!runpy` function has returned. If that limitation is not acceptable for a given use case, :mod:`importlib` is likely to be a more suitable choice than this module. -The :mod:`runpy` module provides two functions: +The :mod:`!runpy` module provides two functions: .. function:: run_module(mod_name, init_globals=None, run_name=None, alter_sys=False) diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 517dbe8c321..5560478ce15 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -12,7 +12,7 @@ -------------- -The :mod:`sched` module defines a class which implements a general purpose event +The :mod:`!sched` module defines a class which implements a general purpose event scheduler: .. class:: scheduler(timefunc=time.monotonic, delayfunc=time.sleep) diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index 75dafc54d40..e266849918a 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -17,11 +17,11 @@ ------------- -The :mod:`secrets` module is used for generating cryptographically strong +The :mod:`!secrets` module is used for generating cryptographically strong random numbers suitable for managing data such as passwords, account authentication, security tokens, and related secrets. -In particular, :mod:`secrets` should be used in preference to the +In particular, :mod:`!secrets` should be used in preference to the default pseudo-random number generator in the :mod:`random` module, which is designed for modelling and simulation, not security or cryptography. @@ -33,7 +33,7 @@ is designed for modelling and simulation, not security or cryptography. Random numbers -------------- -The :mod:`secrets` module provides access to the most secure source of +The :mod:`!secrets` module provides access to the most secure source of randomness that your operating system provides. .. class:: SystemRandom @@ -58,43 +58,48 @@ randomness that your operating system provides. Generating tokens ----------------- -The :mod:`secrets` module provides functions for generating secure +The :mod:`!secrets` module provides functions for generating secure tokens, suitable for applications such as password resets, hard-to-guess URLs, and similar. -.. function:: token_bytes([nbytes=None]) +.. function:: token_bytes(nbytes=None) Return a random byte string containing *nbytes* number of bytes. - If *nbytes* is ``None`` or not supplied, a reasonable default is - used. + + If *nbytes* is not specified or ``None``, :const:`DEFAULT_ENTROPY` + is used instead. .. doctest:: - >>> token_bytes(16) #doctest:+SKIP + >>> token_bytes(16) # doctest: +SKIP b'\xebr\x17D*t\xae\xd4\xe3S\xb6\xe2\xebP1\x8b' -.. function:: token_hex([nbytes=None]) +.. function:: token_hex(nbytes=None) Return a random text string, in hexadecimal. The string has *nbytes* - random bytes, each byte converted to two hex digits. If *nbytes* is - ``None`` or not supplied, a reasonable default is used. + random bytes, each byte converted to two hex digits. + + If *nbytes* is not specified or ``None``, :const:`DEFAULT_ENTROPY` + is used instead. .. doctest:: - >>> token_hex(16) #doctest:+SKIP + >>> token_hex(16) # doctest: +SKIP 'f9bf78b9a18ce6d46a0cd2b0b86df9da' -.. function:: token_urlsafe([nbytes=None]) +.. function:: token_urlsafe(nbytes=None) Return a random URL-safe text string, containing *nbytes* random bytes. The text is Base64 encoded, so on average each byte results - in approximately 1.3 characters. If *nbytes* is ``None`` or not - supplied, a reasonable default is used. + in approximately 1.3 characters. + + If *nbytes* is not specified or ``None``, :const:`DEFAULT_ENTROPY` + is used instead. .. doctest:: - >>> token_urlsafe(16) #doctest:+SKIP + >>> token_urlsafe(16) # doctest: +SKIP 'Drmhze6EPcv0fN_81Bj-nA' @@ -107,7 +112,7 @@ tokens need to have sufficient randomness. Unfortunately, what is considered sufficient will necessarily increase as computers get more powerful and able to make more guesses in a shorter period. As of 2015, it is believed that 32 bytes (256 bits) of randomness is sufficient for -the typical use-case expected for the :mod:`secrets` module. +the typical use-case expected for the :mod:`!secrets` module. For those who want to manage their own token length, you can explicitly specify how much randomness is used for tokens by giving an :class:`int` @@ -115,11 +120,13 @@ argument to the various ``token_*`` functions. That argument is taken as the number of bytes of randomness to use. Otherwise, if no argument is provided, or if the argument is ``None``, -the ``token_*`` functions will use a reasonable default instead. +the ``token_*`` functions use :const:`DEFAULT_ENTROPY` instead. -.. note:: +.. data:: DEFAULT_ENTROPY + + Default number of bytes of randomness used by the ``token_*`` functions. - That default is subject to change at any time, including during + The exact value is subject to change at any time, including during maintenance releases. @@ -139,7 +146,7 @@ Other functions Recipes and best practices -------------------------- -This section shows recipes and best practices for using :mod:`secrets` +This section shows recipes and best practices for using :mod:`!secrets` to manage a basic level of security. Generate an eight-character alphanumeric password: diff --git a/Doc/library/select.rst b/Doc/library/select.rst index 0f0c76060df..330b0a1c55a 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -18,7 +18,7 @@ it was last read. .. note:: The :mod:`selectors` module allows high-level and efficient I/O - multiplexing, built upon the :mod:`select` module primitives. Users are + multiplexing, built upon the :mod:`!select` module primitives. Users are encouraged to use the :mod:`selectors` module instead, unless they want precise control over the OS-level primitives used. @@ -171,7 +171,7 @@ The module defines the following: The minimum number of bytes which can be written without blocking to a pipe when the pipe has been reported as ready for writing by :func:`~select.select`, :func:`!poll` or another interface in this module. This doesn't apply - to other kind of file-like objects such as sockets. + to other kinds of file-like objects such as sockets. This value is guaranteed by POSIX to be at least 512. @@ -223,7 +223,7 @@ object. implement :meth:`!fileno`, so they can also be used as the argument. *eventmask* is an optional bitmask describing the type of events you want to - check for. The constants are the same that with :c:func:`!poll` + check for. The constants are the same as with :c:func:`!poll` object. The default value is a combination of the constants :const:`POLLIN`, :const:`POLLPRI`, and :const:`POLLOUT`. @@ -238,7 +238,7 @@ object. .. method:: devpoll.modify(fd[, eventmask]) This method does an :meth:`unregister` followed by a - :meth:`register`. It is (a bit) more efficient that doing the same + :meth:`register`. It is (a bit) more efficient than doing the same explicitly. @@ -561,9 +561,9 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +---------------------------+---------------------------------------------+ | :const:`KQ_EV_DELETE` | Removes an event from the queue | +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ENABLE` | Permitscontrol() to returns the event | + | :const:`KQ_EV_ENABLE` | Permits control() to return the event | +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DISABLE` | Disablesevent | + | :const:`KQ_EV_DISABLE` | Disables event | +---------------------------+---------------------------------------------+ | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence | +---------------------------+---------------------------------------------+ diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst index ee556f1f3ce..2d523a9d2ea 100644 --- a/Doc/library/selectors.rst +++ b/Doc/library/selectors.rst @@ -54,7 +54,7 @@ Classes hierarchy:: In the following, *events* is a bitwise mask indicating which I/O events should -be waited for on a given file object. It can be a combination of the modules +be waited for on a given file object. It can be a combination of the module's constants below: +-----------------------+-----------------------------------------------+ diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 6e74a59b82b..a2c9065ae78 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -61,11 +61,11 @@ lots of shared sub-objects. The keys are ordinary strings. .. warning:: - Because the :mod:`shelve` module is backed by :mod:`pickle`, it is insecure + Because the :mod:`!shelve` module is backed by :mod:`pickle`, it is insecure to load a shelf from an untrusted source. Like with pickle, loading a shelf can execute arbitrary code. -Shelf objects support most of methods and operations supported by dictionaries +Shelf objects support most of the methods and operations supported by dictionaries (except copying, constructors and operators ``|`` and ``|=``). This eases the transition from dictionary based scripts to those requiring persistent storage. @@ -106,7 +106,7 @@ Restrictions database should be fairly small, and in rare cases key collisions may cause the database to refuse updates. -* The :mod:`shelve` module does not support *concurrent* read/write access to +* The :mod:`!shelve` module does not support *concurrent* read/write access to shelved objects. (Multiple simultaneous read accesses are safe.) When a program has a shelf open for writing, no other program should have it open for reading or writing. Unix file locking can be used to solve this, but this @@ -219,5 +219,5 @@ object):: Generic interface to ``dbm``-style databases. Module :mod:`pickle` - Object serialization used by :mod:`shelve`. + Object serialization used by :mod:`!shelve`. diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index a96f0864dc1..0653bf2f418 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -18,7 +18,7 @@ simple syntaxes resembling that of the Unix shell. This will often be useful for writing minilanguages, (for example, in run control files for Python applications) or for parsing quoted strings. -The :mod:`shlex` module defines the following functions: +The :mod:`!shlex` module defines the following functions: .. function:: split(s, comments=False, posix=True) @@ -98,7 +98,7 @@ The :mod:`shlex` module defines the following functions: .. versionadded:: 3.3 -The :mod:`shlex` module defines the following class: +The :mod:`!shlex` module defines the following class: .. class:: shlex(instream=None, infile=None, posix=False, punctuation_chars=False) @@ -214,7 +214,7 @@ A :class:`~shlex.shlex` instance has the following methods: with the name of the current source file and the ``%d`` with the current input line number (the optional arguments can be used to override these). - This convenience is provided to encourage :mod:`shlex` users to generate error + This convenience is provided to encourage :mod:`!shlex` users to generate error messages in the standard, parseable format understood by Emacs and other Unix tools. @@ -343,7 +343,7 @@ variables which either control lexical analysis or can be used for debugging: Parsing Rules ------------- -When operating in non-POSIX mode, :class:`~shlex.shlex` will try to obey to the +When operating in non-POSIX mode, :class:`~shlex.shlex` will try to obey the following rules. * Quote characters are not recognized within words (``Do"Not"Separate`` is @@ -366,7 +366,7 @@ following rules. * It's not possible to parse empty strings, even if quoted. -When operating in POSIX mode, :class:`~shlex.shlex` will try to obey to the +When operating in POSIX mode, :class:`~shlex.shlex` will try to obey the following parsing rules. * Quotes are stripped out, and do not separate words (``"Do"Not"Separate"`` is @@ -382,7 +382,7 @@ following parsing rules. * Enclosing characters in quotes which are part of :attr:`~shlex.escapedquotes` (e.g. ``'"'``) preserves the literal value of all characters within the quotes, with the exception of the characters - mentioned in :attr:`~shlex.escape`. The escape characters retain its + mentioned in :attr:`~shlex.escape`. The escape characters retain their special meaning only when followed by the quote in use, or the escape character itself. Otherwise the escape character will be considered a normal character. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 3a4631e7c65..8564a5b72d9 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -15,7 +15,7 @@ -------------- -The :mod:`shutil` module offers a number of high-level operations on files and +The :mod:`!shutil` module offers a number of high-level operations on files and collections of files. In particular, functions are provided which support file copying and removal. For operations on individual files, see also the :mod:`os` module. @@ -543,7 +543,7 @@ instead of 64 KiB) and a :func:`memoryview`-based variant of :func:`shutil.copyfileobj` is used. If the fast-copy operation fails and no data was written in the destination -file then shutil will silently fallback on using less efficient +file then shutil will silently fall back to less efficient :func:`copyfileobj` function internally. .. versionchanged:: 3.8 @@ -674,7 +674,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Return a list of supported formats for archiving. Each element of the returned sequence is a tuple ``(name, description)``. - By default :mod:`shutil` provides these formats: + By default :mod:`!shutil` provides these formats: - *zip*: ZIP file (if the :mod:`zlib` module is available). - *tar*: Uncompressed tar file. Uses POSIX.1-2001 pax format for new archives. @@ -791,7 +791,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Each element of the returned sequence is a tuple ``(name, extensions, description)``. - By default :mod:`shutil` provides these formats: + By default :mod:`!shutil` provides these formats: - *zip*: ZIP file (unpacking compressed files works only if the corresponding module is available). diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 1a2a555f5c0..65750638042 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -36,7 +36,7 @@ Execution of Python signal handlers A Python signal handler does not get executed inside the low-level (C) signal handler. Instead, the low-level signal handler sets a flag which tells the :term:`virtual machine` to execute the corresponding Python signal handler -at a later point(for example at the next :term:`bytecode` instruction). +at a later point (for example, at the next :term:`bytecode` instruction). This has consequences: * It makes little sense to catch synchronous errors like :const:`SIGFPE` or @@ -68,6 +68,11 @@ the synchronization primitives from the :mod:`threading` module instead. Besides, only the main thread of the main interpreter is allowed to set a new signal handler. +.. warning:: + + Synchronization primitives such as :class:`threading.Lock` should not be used + within signal handlers. Doing so can lead to unexpected deadlocks. + Module contents --------------- @@ -92,13 +97,13 @@ The signal module defines three enums: .. class:: Handlers - :class:`enum.IntEnum` collection the constants :const:`SIG_DFL` and :const:`SIG_IGN`. + :class:`enum.IntEnum` collection of the constants :const:`SIG_DFL` and :const:`SIG_IGN`. .. versionadded:: 3.5 .. class:: Sigmasks - :class:`enum.IntEnum` collection the constants :const:`SIG_BLOCK`, :const:`SIG_UNBLOCK` and :const:`SIG_SETMASK`. + :class:`enum.IntEnum` collection of the constants :const:`SIG_BLOCK`, :const:`SIG_UNBLOCK` and :const:`SIG_SETMASK`. .. availability:: Unix. @@ -108,7 +113,7 @@ The signal module defines three enums: .. versionadded:: 3.5 -The variables defined in the :mod:`signal` module are: +The variables defined in the :mod:`!signal` module are: .. data:: SIG_DFL @@ -350,7 +355,7 @@ The variables defined in the :mod:`signal` module are: .. versionadded:: 3.3 -The :mod:`signal` module defines one exception: +The :mod:`!signal` module defines one exception: .. exception:: ItimerError @@ -364,7 +369,7 @@ The :mod:`signal` module defines one exception: alias of :exc:`OSError`. -The :mod:`signal` module defines the following functions: +The :mod:`!signal` module defines the following functions: .. function:: alarm(time) diff --git a/Doc/library/site.rst b/Doc/library/site.rst index ca2ac3b0098..4686c9fc92c 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -51,11 +51,11 @@ added path for configuration files. .. versionchanged:: 3.14 - :mod:`site` is no longer responsible for updating :data:`sys.prefix` and + :mod:`!site` is no longer responsible for updating :data:`sys.prefix` and :data:`sys.exec_prefix` on :ref:`sys-path-init-virtual-environments`. This is now done during the :ref:`path initialization `. As a result, under :ref:`sys-path-init-virtual-environments`, :data:`sys.prefix` and - :data:`sys.exec_prefix` no longer depend on the :mod:`site` initialization, + :data:`sys.exec_prefix` no longer depend on the :mod:`!site` initialization, and are therefore unaffected by :option:`-S`. .. _site-virtual-environments-configuration: @@ -130,38 +130,38 @@ directory precedes the :file:`foo` directory because :file:`bar.pth` comes alphabetically before :file:`foo.pth`; and :file:`spam` is omitted because it is not mentioned in either path configuration file. -:mod:`sitecustomize` --------------------- +:mod:`!sitecustomize` +--------------------- .. module:: sitecustomize After these path manipulations, an attempt is made to import a module named -:mod:`sitecustomize`, which can perform arbitrary site-specific customizations. +:mod:`!sitecustomize`, which can perform arbitrary site-specific customizations. It is typically created by a system administrator in the site-packages directory. If this import fails with an :exc:`ImportError` or its subclass exception, and the exception's :attr:`~ImportError.name` -attribute equals to ``'sitecustomize'``, +attribute equals ``'sitecustomize'``, it is silently ignored. If Python is started without output streams available, as with :file:`pythonw.exe` on Windows (which is used by default to start IDLE), -attempted output from :mod:`sitecustomize` is ignored. Any other exception +attempted output from :mod:`!sitecustomize` is ignored. Any other exception causes a silent and perhaps mysterious failure of the process. -:mod:`usercustomize` --------------------- +:mod:`!usercustomize` +--------------------- .. module:: usercustomize -After this, an attempt is made to import a module named :mod:`usercustomize`, +After this, an attempt is made to import a module named :mod:`!usercustomize`, which can perform arbitrary user-specific customizations, if :data:`~site.ENABLE_USER_SITE` is true. This file is intended to be created in the user site-packages directory (see below), which is part of ``sys.path`` unless disabled by :option:`-s`. If this import fails with an :exc:`ImportError` or its subclass exception, and the exception's :attr:`~ImportError.name` -attribute equals to ``'usercustomize'``, it is silently ignored. +attribute equals ``'usercustomize'``, it is silently ignored. Note that for some non-Unix systems, ``sys.prefix`` and ``sys.exec_prefix`` are empty, and the path manipulations are skipped; however the import of -:mod:`sitecustomize` and :mod:`usercustomize` is still attempted. +:mod:`sitecustomize` and :mod:`!usercustomize` is still attempted. .. currentmodule:: site @@ -173,7 +173,7 @@ Readline configuration On systems that support :mod:`readline`, this module will also import and configure the :mod:`rlcompleter` module, if Python is started in :ref:`interactive mode ` and without the :option:`-S` option. -The default behavior is enable tab-completion and to use +The default behavior is to enable tab completion and to use :file:`~/.python_history` as the history save file. To disable it, delete (or override) the :data:`sys.__interactivehook__` attribute in your :mod:`sitecustomize` or :mod:`usercustomize` module or your @@ -275,7 +275,7 @@ Command-line interface .. program:: site -The :mod:`site` module also provides a way to get the user directories from the +The :mod:`!site` module also provides a way to get the user directories from the command line: .. code-block:: shell-session diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index 10525c90aa9..583bc3afcad 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -14,7 +14,7 @@ -------------- -The :mod:`smtplib` module defines an SMTP client session object that can be used +The :mod:`!smtplib` module defines an SMTP client session object that can be used to send mail to any internet machine with an SMTP or ESMTP listener daemon. For details of SMTP and ESMTP operation, consult :rfc:`821` (Simple Mail Transfer Protocol) and :rfc:`1869` (SMTP Service Extensions). @@ -336,7 +336,7 @@ An :class:`SMTP` instance has the following methods: :exc:`SMTPException` No suitable authentication method was found. - Each of the authentication methods supported by :mod:`smtplib` are tried in + Each of the authentication methods supported by :mod:`!smtplib` are tried in turn if they are advertised as supported by the server. See :meth:`auth` for a list of supported authentication methods. *initial_response_ok* is passed through to :meth:`auth`. @@ -388,7 +388,7 @@ An :class:`SMTP` instance has the following methods: call the :meth:`login` method, which will try each of the above mechanisms in turn, in the order listed. ``auth`` is exposed to facilitate the implementation of authentication methods not (or not yet) supported - directly by :mod:`smtplib`. + directly by :mod:`!smtplib`. .. versionadded:: 3.5 diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 782ea7d8fda..56a4f8efe71 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -83,7 +83,7 @@ created. Socket addresses are represented as follows: - For :const:`AF_INET6` address family, a four-tuple ``(host, port, flowinfo, scope_id)`` is used, where *flowinfo* and *scope_id* represent the ``sin6_flowinfo`` and ``sin6_scope_id`` members in :const:`struct sockaddr_in6` in C. For - :mod:`socket` module methods, *flowinfo* and *scope_id* can be omitted just for + :mod:`!socket` module methods, *flowinfo* and *scope_id* can be omitted just for backward compatibility. Note, however, omission of *scope_id* can cause problems in manipulating scoped IPv6 addresses. @@ -118,10 +118,10 @@ created. Socket addresses are represented as follows: ``'can0'``. The network interface name ``''`` can be used to receive packets from all network interfaces of this family. - - :const:`CAN_ISOTP` protocol require a tuple ``(interface, rx_addr, tx_addr)`` + - :const:`CAN_ISOTP` protocol requires a tuple ``(interface, rx_addr, tx_addr)`` where both additional parameters are unsigned long integer that represent a CAN identifier (standard or extended). - - :const:`CAN_J1939` protocol require a tuple ``(interface, name, pgn, addr)`` + - :const:`CAN_J1939` protocol requires a tuple ``(interface, name, pgn, addr)`` where additional parameters are 64-bit unsigned integer representing the ECU name, a 32-bit unsigned integer representing the Parameter Group Number (PGN), and an 8-bit integer representing the address. @@ -302,7 +302,7 @@ generalization of this based on timeouts is supported through Module contents --------------- -The module :mod:`socket` exports the following elements. +The module :mod:`!socket` exports the following elements. Exceptions @@ -1028,13 +1028,13 @@ The following functions all create :ref:`socket objects `. Other functions ''''''''''''''' -The :mod:`socket` module also offers various network-related services: +The :mod:`!socket` module also offers various network-related services: .. function:: close(fd) Close a socket file descriptor. This is like :func:`os.close`, but for - sockets. On some platforms (most noticeable Windows) :func:`os.close` + sockets. On some platforms (most notably Windows) :func:`os.close` does not work for socket file descriptors. .. versionadded:: 3.7 @@ -1590,7 +1590,7 @@ to sockets. address family --- see above.) If the connection is interrupted by a signal, the method waits until the - connection completes, or raise a :exc:`TimeoutError` on timeout, if the + connection completes, or raises a :exc:`TimeoutError` on timeout, if the signal handler doesn't raise an exception and the socket is blocking or has a timeout. For non-blocking sockets, the method raises an :exc:`InterruptedError` exception if the connection is interrupted by a @@ -2094,11 +2094,11 @@ to sockets. Set the value of the given socket option (see the Unix manual page :manpage:`setsockopt(2)`). The needed symbolic constants are defined in this module (:ref:`!SO_\* etc. `). The value can be an integer, - ``None`` or a :term:`bytes-like object` representing a buffer. In the later + ``None`` or a :term:`bytes-like object` representing a buffer. In the latter case it is up to the caller to ensure that the bytestring contains the proper bits (see the optional built-in module :mod:`struct` for a way to encode C structures as bytestrings). When *value* is set to ``None``, - *optlen* argument is required. It's equivalent to call :c:func:`setsockopt` C + *optlen* argument is required. It's equivalent to calling :c:func:`setsockopt` C function with ``optval=NULL`` and ``optlen=optlen``. .. versionchanged:: 3.5 @@ -2412,7 +2412,7 @@ lead to this error:: This is because the previous execution has left the socket in a ``TIME_WAIT`` state, and can't be immediately reused. -There is a :mod:`socket` flag to set, in order to prevent this, +There is a :mod:`!socket` flag to set, in order to prevent this, :const:`socket.SO_REUSEADDR`:: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 7545b5fb0a5..e1d47b69b75 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`socketserver` module simplifies the task of writing network servers. +The :mod:`!socketserver` module simplifies the task of writing network servers. .. include:: ../includes/wasm-notavail.rst diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index cbc3f32d9b9..b708a50ed25 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -291,7 +291,7 @@ Module functions Set it to any combination (using ``|``, bitwise or) of :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES` to enable this. - Column names takes precedence over declared types if both flags are set. + Column names take precedence over declared types if both flags are set. By default (``0``), type detection is disabled. :param isolation_level: diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 93cdf6d85f0..aecc133abec 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -130,7 +130,7 @@ purposes. cafile=None, capath=None, cadata=None) Return a new :class:`SSLContext` object with default settings for - the given *purpose*. The settings are chosen by the :mod:`ssl` module, + the given *purpose*. The settings are chosen by the :mod:`!ssl` module, and usually represent a higher security level than when calling the :class:`SSLContext` constructor directly. @@ -1456,7 +1456,7 @@ to speed up repeated connections from the same clients. TLS 1.3. .. seealso:: - :func:`create_default_context` lets the :mod:`ssl` module choose + :func:`create_default_context` lets the :mod:`!ssl` module choose security settings for a given purpose. .. versionchanged:: 3.6 diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index 8434b2e8c75..bcb1dc269b6 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`stat` module defines constants and functions for interpreting the +The :mod:`!stat` module defines constants and functions for interpreting the results of :func:`os.stat`, :func:`os.fstat` and :func:`os.lstat` (if they exist). For complete details about the :c:func:`stat`, :c:func:`!fstat` and :c:func:`!lstat` calls, consult the documentation for your system. @@ -19,7 +19,7 @@ exist). For complete details about the :c:func:`stat`, :c:func:`!fstat` and .. versionchanged:: 3.4 The stat module is backed by a C implementation. -The :mod:`stat` module defines the following functions to test for specific file +The :mod:`!stat` module defines the following functions to test for specific file types: diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 7b983f25c46..ac856152e41 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -265,9 +265,17 @@ The constructors :func:`int`, :func:`float`, and pair: operator; % (percent) pair: operator; ** +.. _stdtypes-mixed-arithmetic: + Python fully supports mixed arithmetic: when a binary arithmetic operator has -operands of different numeric types, the operand with the "narrower" type is -widened to that of the other, where integer is narrower than floating point. +operands of different built-in numeric types, the operand with the "narrower" +type is widened to that of the other: + +* If both arguments are complex numbers, no conversion is performed; +* if either argument is a complex or a floating-point number, the other is + converted to a floating-point number; +* otherwise, both must be integers and no conversion is necessary. + Arithmetic with complex and real operands is defined by the usual mathematical formula, for example:: @@ -1441,108 +1449,10 @@ application). list appear empty for the duration, and raises :exc:`ValueError` if it can detect that the list has been mutated during a sort. -.. admonition:: Thread safety - - Reading a single element from a :class:`list` is - :term:`atomic `: - - .. code-block:: - :class: green - - lst[i] # list.__getitem__ - - The following methods traverse the list and use :term:`atomic ` - reads of each item to perform their function. That means that they may - return results affected by concurrent modifications: - - .. code-block:: - :class: maybe - - item in lst - lst.index(item) - lst.count(item) - - All of the above methods/operations are also lock-free. They do not block - concurrent modifications. Other operations that hold a lock will not block - these from observing intermediate states. - - All other operations from here on block using the per-object lock. - - Writing a single item via ``lst[i] = x`` is safe to call from multiple - threads and will not corrupt the list. - - The following operations return new objects and appear - :term:`atomic ` to other threads: - - .. code-block:: - :class: good - - lst1 + lst2 # concatenates two lists into a new list - x * lst # repeats lst x times into a new list - lst.copy() # returns a shallow copy of the list - - Methods that only operate on a single elements with no shifting required are - :term:`atomic `: - - .. code-block:: - :class: good - - lst.append(x) # append to the end of the list, no shifting required - lst.pop() # pop element from the end of the list, no shifting required - - The :meth:`~list.clear` method is also :term:`atomic `. - Other threads cannot observe elements being removed. - - The :meth:`~list.sort` method is not :term:`atomic `. - Other threads cannot observe intermediate states during sorting, but the - list appears empty for the duration of the sort. - - The following operations may allow lock-free operations to observe - intermediate states since they modify multiple elements in place: - - .. code-block:: - :class: maybe - - lst.insert(idx, item) # shifts elements - lst.pop(idx) # idx not at the end of the list, shifts elements - lst *= x # copies elements in place - - The :meth:`~list.remove` method may allow concurrent modifications since - element comparison may execute arbitrary Python code (via - :meth:`~object.__eq__`). - - :meth:`~list.extend` is safe to call from multiple threads. However, its - guarantees depend on the iterable passed to it. If it is a :class:`list`, a - :class:`tuple`, a :class:`set`, a :class:`frozenset`, a :class:`dict` or a - :ref:`dictionary view object ` (but not their subclasses), the - ``extend`` operation is safe from concurrent modifications to the iterable. - Otherwise, an iterator is created which can be concurrently modified by - another thread. The same applies to inplace concatenation of a list with - other iterables when using ``lst += iterable``. - - Similarly, assigning to a list slice with ``lst[i:j] = iterable`` is safe - to call from multiple threads, but ``iterable`` is only locked when it is - also a :class:`list` (but not its subclasses). - - Operations that involve multiple accesses, as well as iteration, are never - atomic. For example: - - .. code-block:: - :class: bad - - # NOT atomic: read-modify-write - lst[i] = lst[i] + 1 - - # NOT atomic: check-then-act - if lst: - item = lst.pop() - - # NOT thread-safe: iteration while modifying - for item in lst: - process(item) # another thread may modify lst +.. seealso:: - Consider external synchronization when sharing :class:`list` instances - across threads. See :ref:`freethreading-python-howto` for more information. + For detailed information on thread-safety guarantees for :class:`list` + objects, see :ref:`thread-safety-list`. .. _typesseq-tuple: @@ -2180,7 +2090,18 @@ expression support in the :mod:`re` module). Return ``True`` if all characters in the string are alphanumeric and there is at least one character, ``False`` otherwise. A character ``c`` is alphanumeric if one of the following returns ``True``: ``c.isalpha()``, ``c.isdecimal()``, - ``c.isdigit()``, or ``c.isnumeric()``. + ``c.isdigit()``, or ``c.isnumeric()``. For example: + + .. doctest:: + + >>> 'abc123'.isalnum() + True + >>> 'abc123!@#'.isalnum() + False + >>> ''.isalnum() + False + >>> ' '.isalnum() + False .. method:: str.isalpha() @@ -2326,17 +2247,34 @@ expression support in the :mod:`re` module). >>> '\t'.isprintable(), '\n'.isprintable() (False, False) + See also :meth:`isspace`. + .. method:: str.isspace() Return ``True`` if there are only whitespace characters in the string and there is at least one character, ``False`` otherwise. + For example: + + .. doctest:: + + >>> ''.isspace() + False + >>> ' '.isspace() + True + >>> '\t\n'.isspace() # TAB and BREAK LINE + True + >>> '\u3000'.isspace() # IDEOGRAPHIC SPACE + True + A character is *whitespace* if in the Unicode character database (see :mod:`unicodedata`), either its general category is ``Zs`` ("Separator, space"), or its bidirectional class is one of ``WS``, ``B``, or ``S``. + See also :meth:`isprintable`. + .. method:: str.istitle() @@ -2472,6 +2410,19 @@ expression support in the :mod:`re` module). after the separator. If the separator is not found, return a 3-tuple containing the string itself, followed by two empty strings. + For example: + + .. doctest:: + + >>> 'Monty Python'.partition(' ') + ('Monty', ' ', 'Python') + >>> "Monty Python's Flying Circus".partition(' ') + ('Monty', ' ', "Python's Flying Circus") + >>> 'Monty Python'.partition('-') + ('Monty Python', '', '') + + See also :meth:`rpartition`. + .. method:: str.removeprefix(prefix, /) @@ -2570,6 +2521,19 @@ expression support in the :mod:`re` module). done using the specified *fillchar* (default is an ASCII space). The original string is returned if *width* is less than or equal to ``len(s)``. + For example: + + .. doctest:: + + >>> 'Python'.rjust(10) + ' Python' + >>> 'Python'.rjust(10, '.') + '....Python' + >>> 'Monty Python'.rjust(10, '.') + 'Monty Python' + + See also :meth:`ljust` and :meth:`zfill`. + .. method:: str.rpartition(sep, /) @@ -2606,14 +2570,17 @@ expression support in the :mod:`re` module). Return a copy of the string with trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted or ``None``, the *chars* argument defaults to removing whitespace. The *chars* - argument is not a suffix; rather, all combinations of its values are stripped:: + argument is not a suffix; rather, all combinations of its values are stripped. + For example: + + .. doctest:: >>> ' spacious '.rstrip() ' spacious' >>> 'mississippi'.rstrip('ipz') 'mississ' - See :meth:`str.removesuffix` for a method that will remove a single suffix + See :meth:`removesuffix` for a method that will remove a single suffix string rather than all of a set of characters. For example:: >>> 'Monty Python'.rstrip(' Python') @@ -2621,6 +2588,9 @@ expression support in the :mod:`re` module). >>> 'Monty Python'.removesuffix(' Python') 'Monty' + See also :meth:`strip`. + + .. method:: str.split(sep=None, maxsplit=-1) Return a list of the words in the string, using *sep* as the delimiter @@ -2751,6 +2721,19 @@ expression support in the :mod:`re` module). test string beginning at that position. With optional *end*, stop comparing string at that position. + For example: + + .. doctest:: + + >>> 'Python'.startswith('Py') + True + >>> 'a tuple of prefixes'.startswith(('at', 'a')) + True + >>> 'Python is amazing'.startswith('is', 7) + True + + See also :meth:`endswith` and :meth:`removeprefix`. + .. method:: str.strip(chars=None, /) @@ -2758,7 +2741,11 @@ expression support in the :mod:`re` module). The *chars* argument is a string specifying the set of characters to be removed. If omitted or ``None``, the *chars* argument defaults to removing whitespace. The *chars* argument is not a prefix or suffix; rather, all combinations of its - values are stripped:: + values are stripped. + + For example: + + .. doctest:: >>> ' spacious '.strip() 'spacious' @@ -2769,12 +2756,17 @@ expression support in the :mod:`re` module). from the string. Characters are removed from the leading end until reaching a string character that is not contained in the set of characters in *chars*. A similar action takes place on the trailing end. - For example:: + + For example: + + .. doctest:: >>> comment_string = '#....... Section 3.2.1 Issue #32 .......' >>> comment_string.strip('.#! ') 'Section 3.2.1 Issue #32' + See also :meth:`rstrip`. + .. method:: str.swapcase() @@ -2858,13 +2850,17 @@ expression support in the :mod:`re` module). than before. The original string is returned if *width* is less than or equal to ``len(s)``. - For example:: + For example: + + .. doctest:: >>> "42".zfill(5) '00042' >>> "-42".zfill(5) '-0042' + See also :meth:`rjust`. + .. index:: single: ! formatted string literal @@ -3194,6 +3190,10 @@ The conversion types are: | | character in the result. | | +------------+-----------------------------------------------------+-------+ +For floating-point formats, the result should be correctly rounded to a given +precision ``p`` of digits after the decimal point. The rounding mode matches +that of the :func:`round` builtin. + Notes: (1) @@ -5506,6 +5506,12 @@ can be used interchangeably to index the same dictionary entry. of a :class:`dict`. +.. seealso:: + + For detailed information on thread-safety guarantees for :class:`dict` + objects, see :ref:`thread-safety-dict`. + + .. _dict-views: Dictionary view objects diff --git a/Doc/library/string.rst b/Doc/library/string.rst index e3ad018d1d0..8096d90317d 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -87,7 +87,7 @@ Custom String Formatting The built-in string class provides the ability to do complex variable substitutions and value formatting via the :meth:`~str.format` method described in -:pep:`3101`. The :class:`Formatter` class in the :mod:`string` module allows +:pep:`3101`. The :class:`Formatter` class in the :mod:`!string` module allows you to create and customize your own string formatting behaviors using the same implementation as the built-in :meth:`~str.format` method. @@ -840,7 +840,7 @@ Template strings support ``$``-based substitutions, using the following rules: Any other appearance of ``$`` in the string will result in a :exc:`ValueError` being raised. -The :mod:`string` module provides a :class:`Template` class that implements +The :mod:`!string` module provides a :class:`Template` class that implements these rules. The methods of :class:`Template` are: diff --git a/Doc/library/stringprep.rst b/Doc/library/stringprep.rst index 37d5adf0fa9..b9caa2aa830 100644 --- a/Doc/library/stringprep.rst +++ b/Doc/library/stringprep.rst @@ -26,14 +26,14 @@ define which tables it uses, and what other optional parts of the ``stringprep`` procedure are part of the profile. One example of a ``stringprep`` profile is ``nameprep``, which is used for internationalized domain names. -The module :mod:`stringprep` only exposes the tables from :rfc:`3454`. As these +The module :mod:`!stringprep` only exposes the tables from :rfc:`3454`. As these tables would be very large to represent as dictionaries or lists, the module uses the Unicode character database internally. The module source code itself was generated using the ``mkstringprep.py`` utility. As a result, these tables are exposed as functions, not as data structures. There are two kinds of tables in the RFC: sets and mappings. For a set, -:mod:`stringprep` provides the "characteristic function", i.e. a function that +:mod:`!stringprep` provides the "characteristic function", i.e. a function that returns ``True`` if the parameter is part of the set. For mappings, it provides the mapping function: given the key, it returns the associated value. Below is a list of all functions available in the module. diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 17fc479fd0c..c08df534128 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -36,7 +36,7 @@ and the C layer. responsible for defining byte ordering and padding between elements. See :ref:`struct-alignment` for details. -Several :mod:`struct` functions (and methods of :class:`Struct`) take a *buffer* +Several :mod:`!struct` functions (and methods of :class:`Struct`) take a *buffer* argument. This refers to objects that implement the :ref:`bufferobjects` and provide either a readable or read-writable buffer. The most common types used for that purpose are :class:`bytes` and :class:`bytearray`, but many other types @@ -479,7 +479,7 @@ at the end, assuming the platform's longs are aligned on 4-byte boundaries:: Applications ------------ -Two main applications for the :mod:`struct` module exist, data +Two main applications for the :mod:`!struct` module exist, data interchange between Python and C code within an application or another application compiled using the same compiler (:ref:`native formats`), and data interchange between applications using agreed upon data layout @@ -571,7 +571,7 @@ below were executed on a 32-bit machine:: Classes ------- -The :mod:`struct` module also defines the following type: +The :mod:`!struct` module also defines the following type: .. class:: Struct(format) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index b8dfcc31077..a0eae14aecd 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -11,14 +11,14 @@ -------------- -The :mod:`subprocess` module allows you to spawn new processes, connect to their +The :mod:`!subprocess` module allows you to spawn new processes, connect to their input/output/error pipes, and obtain their return codes. This module intends to replace several older modules and functions:: os.system os.spawn* -Information about how the :mod:`subprocess` module can be used to replace these +Information about how the :mod:`!subprocess` module can be used to replace these modules and functions can be found in the following sections. .. seealso:: @@ -27,8 +27,8 @@ modules and functions can be found in the following sections. .. include:: ../includes/wasm-mobile-notavail.rst -Using the :mod:`subprocess` Module ----------------------------------- +Using the :mod:`!subprocess` Module +----------------------------------- The recommended approach to invoking subprocesses is to use the :func:`run` function for all use cases it can handle. For more advanced use cases, the @@ -1041,7 +1041,7 @@ on Windows. Windows Constants ^^^^^^^^^^^^^^^^^ -The :mod:`subprocess` module exposes the following constants. +The :mod:`!subprocess` module exposes the following constants. .. data:: STD_INPUT_HANDLE @@ -1330,8 +1330,8 @@ calls these functions. .. _subprocess-replacements: -Replacing Older Functions with the :mod:`subprocess` Module ------------------------------------------------------------ +Replacing Older Functions with the :mod:`!subprocess` Module +------------------------------------------------------------ In this section, "a becomes b" means that b can be used as a replacement for a. @@ -1347,7 +1347,7 @@ In this section, "a becomes b" means that b can be used as a replacement for a. :attr:`~CalledProcessError.output` attribute of the raised exception. In the following examples, we assume that the relevant functions have already -been imported from the :mod:`subprocess` module. +been imported from the :mod:`!subprocess` module. Replacing :program:`/bin/sh` shell command substitution @@ -1407,7 +1407,7 @@ Notes: * The :func:`os.system` function ignores SIGINT and SIGQUIT signals while the command is running, but the caller must do this separately when - using the :mod:`subprocess` module. + using the :mod:`!subprocess` module. A more realistic example would look like this:: @@ -1591,7 +1591,7 @@ runtime): Disable use of ``posix_spawn()`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -On Linux, :mod:`subprocess` defaults to using the ``vfork()`` system call +On Linux, :mod:`!subprocess` defaults to using the ``vfork()`` system call internally when it is safe to do so rather than ``fork()``. This greatly improves performance. diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 54e19af4bd6..2f7bba2537d 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -14,7 +14,7 @@ Symbol tables are generated by the compiler from AST just before bytecode is generated. The symbol table is responsible for calculating the scope of every -identifier in the code. :mod:`symtable` provides an interface to examine these +identifier in the code. :mod:`!symtable` provides an interface to examine these tables. @@ -355,7 +355,7 @@ Command-Line Usage .. versionadded:: 3.13 -The :mod:`symtable` module can be executed as a script from the command line. +The :mod:`!symtable` module can be executed as a script from the command line. .. code-block:: sh diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 303655fb128..4a460479e4a 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -10,7 +10,7 @@ .. note:: - :mod:`sys.monitoring` is a namespace within the :mod:`sys` module, + :mod:`!sys.monitoring` is a namespace within the :mod:`sys` module, not an independent module, so there is no need to ``import sys.monitoring``, simply ``import sys`` and then use ``sys.monitoring``. @@ -20,7 +20,7 @@ This namespace provides access to the functions and constants necessary to activate and control event monitoring. As programs execute, events occur that might be of interest to tools that -monitor execution. The :mod:`sys.monitoring` namespace provides means to +monitor execution. The :mod:`!sys.monitoring` namespace provides means to receive callbacks when events of interest occur. The monitoring API consists of three components: diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 136229b4c81..3420f88e36f 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -515,7 +515,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only in the range 0--127, and produce undefined results otherwise. Some systems have a convention for assigning specific meanings to specific exit codes, but these are generally underdeveloped; Unix programs generally use 2 for command - line syntax errors and 1 for all other kind of errors. If another type of + line syntax errors and 1 for all other kinds of errors. If another type of object is passed, ``None`` is equivalent to passing zero, and any other object is printed to :data:`stderr` and results in an exit code of 1. In particular, ``sys.exit("some error message")`` is a quick way to exit a @@ -2230,7 +2230,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only The version number used to form registry keys on Windows platforms. This is stored as string resource 1000 in the Python DLL. The value is normally the - major and minor versions of the running Python interpreter. It is provided in the :mod:`sys` + major and minor versions of the running Python interpreter. It is provided in the :mod:`!sys` module for informational purposes; modifying this value has no effect on the registry keys used by Python. diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 532facb45f8..138af76a66f 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -16,7 +16,7 @@ -------------- -The :mod:`sysconfig` module provides access to Python's configuration +The :mod:`!sysconfig` module provides access to Python's configuration information like the list of installation paths and the configuration variables relevant for the current platform. @@ -28,7 +28,7 @@ A Python distribution contains a :file:`Makefile` and a :file:`pyconfig.h` header file that are necessary to build both the Python binary itself and third-party C extensions compiled using ``setuptools``. -:mod:`sysconfig` puts all variables found in these files in a dictionary that +:mod:`!sysconfig` puts all variables found in these files in a dictionary that can be accessed using :func:`get_config_vars` or :func:`get_config_var`. Notice that on Windows, it's a much smaller set. @@ -68,7 +68,7 @@ Installation paths ------------------ Python uses an installation scheme that differs depending on the platform and on -the installation options. These schemes are stored in :mod:`sysconfig` under +the installation options. These schemes are stored in :mod:`!sysconfig` under unique identifiers based on the value returned by :const:`os.name`. The schemes are used by package installers to determine where to copy files to. @@ -258,12 +258,12 @@ Path Installation directory Installation path functions --------------------------- -:mod:`sysconfig` provides some functions to determine these installation paths. +:mod:`!sysconfig` provides some functions to determine these installation paths. .. function:: get_scheme_names() Return a tuple containing all schemes currently supported in - :mod:`sysconfig`. + :mod:`!sysconfig`. .. function:: get_default_scheme() @@ -285,7 +285,7 @@ Installation path functions *key* must be either ``"prefix"``, ``"home"``, or ``"user"``. The return value is a scheme name listed in :func:`get_scheme_names`. It - can be passed to :mod:`sysconfig` functions that take a *scheme* argument, + can be passed to :mod:`!sysconfig` functions that take a *scheme* argument, such as :func:`get_paths`. .. versionadded:: 3.10 @@ -313,7 +313,7 @@ Installation path functions .. function:: get_path_names() Return a tuple containing all path names currently supported in - :mod:`sysconfig`. + :mod:`!sysconfig`. .. function:: get_path(name, [scheme, [vars, [expand]]]) @@ -323,7 +323,7 @@ Installation path functions *name* has to be a value from the list returned by :func:`get_path_names`. - :mod:`sysconfig` stores installation paths corresponding to each path name, + :mod:`!sysconfig` stores installation paths corresponding to each path name, for each platform, with variables to be expanded. For instance the *stdlib* path for the *nt* scheme is: ``{base}/Lib``. @@ -431,7 +431,7 @@ Other functions Command-line usage ------------------ -You can use :mod:`sysconfig` as a script with Python's *-m* option: +You can use :mod:`!sysconfig` as a script with Python's *-m* option: .. code-block:: shell-session diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index 548898a37bc..b6bf0124095 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -2,7 +2,6 @@ =============================================== .. module:: syslog - :platform: Unix :synopsis: An interface to the Unix syslog library routines. -------------- diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index e57a0f121c5..a87ec10b308 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`tarfile` module makes it possible to read and write tar +The :mod:`!tarfile` module makes it possible to read and write tar archives, including those using gzip, bz2 and lzma compression. Use the :mod:`zipfile` module to read or write :file:`.zip` files, or the higher-level functions in :ref:`shutil `. @@ -216,25 +216,25 @@ Some facts and figures: .. function:: is_tarfile(name) - Return :const:`True` if *name* is a tar archive file, that the :mod:`tarfile` + Return :const:`True` if *name* is a tar archive file, that the :mod:`!tarfile` module can read. *name* may be a :class:`str`, file, or file-like object. .. versionchanged:: 3.9 Support for file and file-like objects. -The :mod:`tarfile` module defines the following exceptions: +The :mod:`!tarfile` module defines the following exceptions: .. exception:: TarError - Base class for all :mod:`tarfile` exceptions. + Base class for all :mod:`!tarfile` exceptions. .. exception:: ReadError Is raised when a tar archive is opened, that either cannot be handled by the - :mod:`tarfile` module or is somehow invalid. + :mod:`!tarfile` module or is somehow invalid. .. exception:: CompressionError @@ -355,7 +355,7 @@ The following constants are available at the module level: Each of the following constants defines a tar archive format that the -:mod:`tarfile` module is able to create. See section :ref:`tar-formats` for +:mod:`!tarfile` module is able to create. See section :ref:`tar-formats` for details. @@ -1285,7 +1285,7 @@ Command-Line Interface .. versionadded:: 3.4 -The :mod:`tarfile` module provides a simple command-line interface to interact +The :mod:`!tarfile` module provides a simple command-line interface to interact with tar archives. If you want to create a new tar archive, specify its name after the :option:`-c` @@ -1446,7 +1446,7 @@ parameter in :meth:`TarFile.add`:: Supported tar formats --------------------- -There are three tar formats that can be created with the :mod:`tarfile` module: +There are three tar formats that can be created with the :mod:`!tarfile` module: * The POSIX.1-1988 ustar format (:const:`USTAR_FORMAT`). It supports filenames up to a length of at best 256 characters and linknames up to 100 characters. @@ -1455,7 +1455,7 @@ There are three tar formats that can be created with the :mod:`tarfile` module: * The GNU tar format (:const:`GNU_FORMAT`). It supports long filenames and linknames, files bigger than 8 GiB and sparse files. It is the de facto - standard on GNU/Linux systems. :mod:`tarfile` fully supports the GNU tar + standard on GNU/Linux systems. :mod:`!tarfile` fully supports the GNU tar extensions for long names, sparse file support is read-only. * The POSIX.1-2001 pax format (:const:`PAX_FORMAT`). It is the most flexible @@ -1500,7 +1500,7 @@ Unfortunately, there is no way to autodetect the encoding of an archive. The pax format was designed to solve this problem. It stores non-ASCII metadata using the universal character encoding *UTF-8*. -The details of character conversion in :mod:`tarfile` are controlled by the +The details of character conversion in :mod:`!tarfile` are controlled by the *encoding* and *errors* keyword arguments of the :class:`TarFile` class. *encoding* defines the character encoding to use for the metadata in the diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 9d26f47820b..78d37d8135c 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -386,7 +386,7 @@ not surprise other unsuspecting code by changing global API behavior. Examples -------- -Here are some examples of typical usage of the :mod:`tempfile` module:: +Here are some examples of typical usage of the :mod:`!tempfile` module:: >>> import tempfile diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst index 0c6f3059fe7..537dfcedd8c 100644 --- a/Doc/library/termios.rst +++ b/Doc/library/termios.rst @@ -2,7 +2,6 @@ =========================================== .. module:: termios - :platform: Unix :synopsis: POSIX style tty control. .. index:: @@ -38,7 +37,7 @@ The module defines the following functions: items with indices :const:`VMIN` and :const:`VTIME`, which are integers when these fields are defined). The interpretation of the flags and the speeds as well as the indexing in the *cc* array must be done using the symbolic - constants defined in the :mod:`termios` module. + constants defined in the :mod:`!termios` module. .. function:: tcsetattr(fd, when, attributes) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 44b1d395a27..b5a3005f854 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -7,7 +7,7 @@ .. sectionauthor:: Brett Cannon .. note:: - The :mod:`test` package is meant for internal use by Python only. It is + The :mod:`!test` package is meant for internal use by Python only. It is documented for the benefit of the core developers of Python. Any use of this package outside of Python's standard library is discouraged as code mentioned here can change or be removed without notice between releases of @@ -15,12 +15,12 @@ -------------- -The :mod:`test` package contains all regression tests for Python as well as the +The :mod:`!test` package contains all regression tests for Python as well as the modules :mod:`test.support` and :mod:`test.regrtest`. :mod:`test.support` is used to enhance your tests while :mod:`test.regrtest` drives the testing suite. -Each module in the :mod:`test` package whose name starts with ``test_`` is a +Each module in the :mod:`!test` package whose name starts with ``test_`` is a testing suite for a specific module or feature. All new tests should be written using the :mod:`unittest` or :mod:`doctest` module. Some older tests are written using a "traditional" testing style that compares output printed to @@ -38,8 +38,8 @@ written using a "traditional" testing style that compares output printed to .. _writing-tests: -Writing Unit Tests for the :mod:`test` package ----------------------------------------------- +Writing Unit Tests for the :mod:`!test` package +----------------------------------------------- It is preferred that tests that use the :mod:`unittest` module follow a few guidelines. One is to name the test module by starting it with ``test_`` and end @@ -162,12 +162,12 @@ Running tests using the command-line interface .. module:: test.regrtest :synopsis: Drives the regression test suite. -The :mod:`test` package can be run as a script to drive Python's regression +The :mod:`!test` package can be run as a script to drive Python's regression test suite, thanks to the :option:`-m` option: :program:`python -m test`. Under -the hood, it uses :mod:`test.regrtest`; the call :program:`python -m +the hood, it uses :mod:`!test.regrtest`; the call :program:`python -m test.regrtest` used in previous Python versions still works. Running the script by itself automatically starts running all regression tests in the -:mod:`test` package. It does this by finding all modules in the package whose +:mod:`!test` package. It does this by finding all modules in the package whose name starts with ``test_``, importing them, and executing the function :func:`test_main` if present or loading the tests via unittest.TestLoader.loadTestsFromModule if ``test_main`` does not exist. The @@ -175,14 +175,14 @@ names of tests to execute may also be passed to the script. Specifying a single regression test (:program:`python -m test test_spam`) will minimize output and only print whether the test passed or failed. -Running :mod:`test` directly allows what resources are available for +Running :mod:`!test` directly allows what resources are available for tests to use to be set. You do this by using the ``-u`` command-line option. Specifying ``all`` as the value for the ``-u`` option enables all possible resources: :program:`python -m test -uall`. If all but one resource is desired (a more common case), a comma-separated list of resources that are not desired may be listed after ``all``. The command :program:`python -m test -uall,-audio,-largefile` -will run :mod:`test` with all resources except the ``audio`` and +will run :mod:`!test` with all resources except the ``audio`` and ``largefile`` resources. For a list of all resources and more command-line options, run :program:`python -m test -h`. @@ -197,19 +197,19 @@ regression tests. :ref:`controlled using environment variables `. -:mod:`test.support` --- Utilities for the Python test suite -=========================================================== +:mod:`!test.support` --- Utilities for the Python test suite +============================================================ .. module:: test.support :synopsis: Support for Python's regression test suite. -The :mod:`test.support` module provides support for Python's regression +The :mod:`!test.support` module provides support for Python's regression test suite. .. note:: - :mod:`test.support` is not a public module. It is documented here to help + :mod:`!test.support` is not a public module. It is documented here to help Python developers write tests. The API of this module is subject to change without backwards compatibility concerns between releases. @@ -230,7 +230,7 @@ This module defines the following exceptions: function. -The :mod:`test.support` module defines the following constants: +The :mod:`!test.support` module defines the following constants: .. data:: verbose @@ -363,7 +363,7 @@ The :mod:`test.support` module defines the following constants: .. data:: TEST_SUPPORT_DIR - Set to the top level directory that contains :mod:`test.support`. + Set to the top level directory that contains :mod:`!test.support`. .. data:: TEST_HOME_DIR @@ -438,7 +438,7 @@ The :mod:`test.support` module defines the following constants: Used to test mixed type comparison. -The :mod:`test.support` module defines the following functions: +The :mod:`!test.support` module defines the following functions: .. function:: busy_retry(timeout, err_msg=None, /, *, error=True) @@ -1043,7 +1043,7 @@ The :mod:`test.support` module defines the following functions: .. versionadded:: 3.11 -The :mod:`test.support` module defines the following classes: +The :mod:`!test.support` module defines the following classes: .. class:: SuppressCrashReport() @@ -1089,14 +1089,14 @@ The :mod:`test.support` module defines the following classes: Try to match a single stored value (*dv*) with a supplied value (*v*). -:mod:`test.support.socket_helper` --- Utilities for socket tests -================================================================ +:mod:`!test.support.socket_helper` --- Utilities for socket tests +================================================================= .. module:: test.support.socket_helper :synopsis: Support for socket tests. -The :mod:`test.support.socket_helper` module provides support for socket tests. +The :mod:`!test.support.socket_helper` module provides support for socket tests. .. versionadded:: 3.9 @@ -1167,14 +1167,14 @@ The :mod:`test.support.socket_helper` module provides support for socket tests. exceptions. -:mod:`test.support.script_helper` --- Utilities for the Python execution tests -============================================================================== +:mod:`!test.support.script_helper` --- Utilities for the Python execution tests +=============================================================================== .. module:: test.support.script_helper :synopsis: Support for Python's script execution tests. -The :mod:`test.support.script_helper` module provides support for Python's +The :mod:`!test.support.script_helper` module provides support for Python's script execution tests. .. function:: interpreter_requires_environment() @@ -1278,13 +1278,13 @@ script execution tests. path and the archive name for the zip file. -:mod:`test.support.bytecode_helper` --- Support tools for testing correct bytecode generation -============================================================================================= +:mod:`!test.support.bytecode_helper` --- Support tools for testing correct bytecode generation +============================================================================================== .. module:: test.support.bytecode_helper :synopsis: Support tools for testing correct bytecode generation. -The :mod:`test.support.bytecode_helper` module provides support for testing +The :mod:`!test.support.bytecode_helper` module provides support for testing and inspecting bytecode generation. .. versionadded:: 3.9 @@ -1310,13 +1310,13 @@ The module defines the following class: Throws :exc:`AssertionError` if *opname* is found. -:mod:`test.support.threading_helper` --- Utilities for threading tests -====================================================================== +:mod:`!test.support.threading_helper` --- Utilities for threading tests +======================================================================= .. module:: test.support.threading_helper :synopsis: Support for threading tests. -The :mod:`test.support.threading_helper` module provides support for threading tests. +The :mod:`!test.support.threading_helper` module provides support for threading tests. .. versionadded:: 3.10 @@ -1397,13 +1397,13 @@ The :mod:`test.support.threading_helper` module provides support for threading t finished. -:mod:`test.support.os_helper` --- Utilities for os tests -======================================================================== +:mod:`!test.support.os_helper` --- Utilities for os tests +========================================================= .. module:: test.support.os_helper :synopsis: Support for os tests. -The :mod:`test.support.os_helper` module provides support for os tests. +The :mod:`!test.support.os_helper` module provides support for os tests. .. versionadded:: 3.10 @@ -1592,13 +1592,13 @@ The :mod:`test.support.os_helper` module provides support for os tests. wrapped with a wait loop that checks for the existence of the file. -:mod:`test.support.import_helper` --- Utilities for import tests -================================================================ +:mod:`!test.support.import_helper` --- Utilities for import tests +================================================================= .. module:: test.support.import_helper :synopsis: Support for import tests. -The :mod:`test.support.import_helper` module provides support for import tests. +The :mod:`!test.support.import_helper` module provides support for import tests. .. versionadded:: 3.10 @@ -1706,13 +1706,13 @@ The :mod:`test.support.import_helper` module provides support for import tests. will be reverted at the end of the block. -:mod:`test.support.warnings_helper` --- Utilities for warnings tests -==================================================================== +:mod:`!test.support.warnings_helper` --- Utilities for warnings tests +===================================================================== .. module:: test.support.warnings_helper :synopsis: Support for warnings tests. -The :mod:`test.support.warnings_helper` module provides support for warnings tests. +The :mod:`!test.support.warnings_helper` module provides support for warnings tests. .. versionadded:: 3.10 diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index 3c96c0e9cc0..c9230f7d827 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`textwrap` module provides some convenience functions, +The :mod:`!textwrap` module provides some convenience functions, as well as :class:`TextWrapper`, the class that does all the work. If you're just wrapping or filling one or two text strings, the convenience functions should be good enough; otherwise, you should use an instance of diff --git a/Doc/library/threadsafety.rst b/Doc/library/threadsafety.rst new file mode 100644 index 00000000000..5b5949d4eff --- /dev/null +++ b/Doc/library/threadsafety.rst @@ -0,0 +1,262 @@ +.. _threadsafety: + +************************ +Thread Safety Guarantees +************************ + +This page documents thread-safety guarantees for built-in types in Python's +free-threaded build. The guarantees described here apply when using Python with +the :term:`GIL` disabled (free-threaded mode). When the GIL is enabled, most +operations are implicitly serialized. + +For general guidance on writing thread-safe code in free-threaded Python, see +:ref:`freethreading-python-howto`. + + +.. _thread-safety-list: + +Thread safety for list objects +============================== + +Reading a single element from a :class:`list` is +:term:`atomic `: + +.. code-block:: + :class: good + + lst[i] # list.__getitem__ + +The following methods traverse the list and use :term:`atomic ` +reads of each item to perform their function. That means that they may +return results affected by concurrent modifications: + +.. code-block:: + :class: maybe + + item in lst + lst.index(item) + lst.count(item) + +All of the above operations avoid acquiring :term:`per-object locks +`. They do not block concurrent modifications. Other +operations that hold a lock will not block these from observing intermediate +states. + +All other operations from here on block using the :term:`per-object lock`. + +Writing a single item via ``lst[i] = x`` is safe to call from multiple +threads and will not corrupt the list. + +The following operations return new objects and appear +:term:`atomic ` to other threads: + +.. code-block:: + :class: good + + lst1 + lst2 # concatenates two lists into a new list + x * lst # repeats lst x times into a new list + lst.copy() # returns a shallow copy of the list + +The following methods that only operate on a single element with no shifting +required are :term:`atomic `: + +.. code-block:: + :class: good + + lst.append(x) # append to the end of the list, no shifting required + lst.pop() # pop element from the end of the list, no shifting required + +The :meth:`~list.clear` method is also :term:`atomic `. +Other threads cannot observe elements being removed. + +The :meth:`~list.sort` method is not :term:`atomic `. +Other threads cannot observe intermediate states during sorting, but the +list appears empty for the duration of the sort. + +The following operations may allow :term:`lock-free` operations to observe +intermediate states since they modify multiple elements in place: + +.. code-block:: + :class: maybe + + lst.insert(idx, item) # shifts elements + lst.pop(idx) # idx not at the end of the list, shifts elements + lst *= x # copies elements in place + +The :meth:`~list.remove` method may allow concurrent modifications since +element comparison may execute arbitrary Python code (via +:meth:`~object.__eq__`). + +:meth:`~list.extend` is safe to call from multiple threads. However, its +guarantees depend on the iterable passed to it. If it is a :class:`list`, a +:class:`tuple`, a :class:`set`, a :class:`frozenset`, a :class:`dict` or a +:ref:`dictionary view object ` (but not their subclasses), the +``extend`` operation is safe from concurrent modifications to the iterable. +Otherwise, an iterator is created which can be concurrently modified by +another thread. The same applies to inplace concatenation of a list with +other iterables when using ``lst += iterable``. + +Similarly, assigning to a list slice with ``lst[i:j] = iterable`` is safe +to call from multiple threads, but ``iterable`` is only locked when it is +also a :class:`list` (but not its subclasses). + +Operations that involve multiple accesses, as well as iteration, are never +atomic. For example: + +.. code-block:: + :class: bad + + # NOT atomic: read-modify-write + lst[i] = lst[i] + 1 + + # NOT atomic: check-then-act + if lst: + item = lst.pop() + + # NOT thread-safe: iteration while modifying + for item in lst: + process(item) # another thread may modify lst + +Consider external synchronization when sharing :class:`list` instances +across threads. + + +.. _thread-safety-dict: + +Thread safety for dict objects +============================== + +Creating a dictionary with the :class:`dict` constructor is atomic when the +argument to it is a :class:`dict` or a :class:`tuple`. When using the +:meth:`dict.fromkeys` method, dictionary creation is atomic when the +argument is a :class:`dict`, :class:`tuple`, :class:`set` or +:class:`frozenset`. + +The following operations and functions are :term:`lock-free` and +:term:`atomic `. + +.. code-block:: + :class: good + + d[key] # dict.__getitem__ + d.get(key) # dict.get + key in d # dict.__contains__ + len(d) # dict.__len__ + +All other operations from here on hold the :term:`per-object lock`. + +Writing or removing a single item is safe to call from multiple threads +and will not corrupt the dictionary: + +.. code-block:: + :class: good + + d[key] = value # write + del d[key] # delete + d.pop(key) # remove and return + d.popitem() # remove and return last item + d.setdefault(key, v) # insert if missing + +These operations may compare keys using :meth:`~object.__eq__`, which can +execute arbitrary Python code. During such comparisons, the dictionary may +be modified by another thread. For built-in types like :class:`str`, +:class:`int`, and :class:`float`, that implement :meth:`~object.__eq__` in C, +the underlying lock is not released during comparisons and this is not a +concern. + +The following operations return new objects and hold the :term:`per-object lock` +for the duration of the operation: + +.. code-block:: + :class: good + + d.copy() # returns a shallow copy of the dictionary + d | other # merges two dicts into a new dict + d.keys() # returns a new dict_keys view object + d.values() # returns a new dict_values view object + d.items() # returns a new dict_items view object + +The :meth:`~dict.clear` method holds the lock for its duration. Other +threads cannot observe elements being removed. + +The following operations lock both dictionaries. For :meth:`~dict.update` +and ``|=``, this applies only when the other operand is a :class:`dict` +that uses the standard dict iterator (but not subclasses that override +iteration). For equality comparison, this applies to :class:`dict` and +its subclasses: + +.. code-block:: + :class: good + + d.update(other_dict) # both locked when other_dict is a dict + d |= other_dict # both locked when other_dict is a dict + d == other_dict # both locked for dict and subclasses + +All comparison operations also compare values using :meth:`~object.__eq__`, +so for non-built-in types the lock may be released during comparison. + +:meth:`~dict.fromkeys` locks both the new dictionary and the iterable +when the iterable is exactly a :class:`dict`, :class:`set`, or +:class:`frozenset` (not subclasses): + +.. code-block:: + :class: good + + dict.fromkeys(a_dict) # locks both + dict.fromkeys(a_set) # locks both + dict.fromkeys(a_frozenset) # locks both + +When updating from a non-dict iterable, only the target dictionary is +locked. The iterable may be concurrently modified by another thread: + +.. code-block:: + :class: maybe + + d.update(iterable) # iterable is not a dict: only d locked + d |= iterable # iterable is not a dict: only d locked + dict.fromkeys(iterable) # iterable is not a dict/set/frozenset: only result locked + +Operations that involve multiple accesses, as well as iteration, are never +atomic: + +.. code-block:: + :class: bad + + # NOT atomic: read-modify-write + d[key] = d[key] + 1 + + # NOT atomic: check-then-act (TOCTOU) + if key in d: + del d[key] + + # NOT thread-safe: iteration while modifying + for key, value in d.items(): + process(key) # another thread may modify d + +To avoid time-of-check to time-of-use (TOCTOU) issues, use atomic +operations or handle exceptions: + +.. code-block:: + :class: good + + # Use pop() with default instead of check-then-delete + d.pop(key, None) + + # Or handle the exception + try: + del d[key] + except KeyError: + pass + +To safely iterate over a dictionary that may be modified by another +thread, iterate over a copy: + +.. code-block:: + :class: good + + # Make a copy to iterate safely + for key, value in d.copy().items(): + process(key) + +Consider external synchronization when sharing :class:`dict` instances +across threads. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 548a3ee0540..bc12061a2ae 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -355,7 +355,7 @@ to test for missing and present object attributes: 0.08588060699912603 -To give the :mod:`timeit` module access to functions you define, you can pass a +To give the :mod:`!timeit` module access to functions you define, you can pass a *setup* parameter which contains an import statement:: def test(): diff --git a/Doc/library/tkinter.colorchooser.rst b/Doc/library/tkinter.colorchooser.rst index df2b324fd5d..73f8f76a210 100644 --- a/Doc/library/tkinter.colorchooser.rst +++ b/Doc/library/tkinter.colorchooser.rst @@ -2,14 +2,13 @@ ====================================================== .. module:: tkinter.colorchooser - :platform: Tk :synopsis: Color choosing dialog **Source code:** :source:`Lib/tkinter/colorchooser.py` -------------- -The :mod:`tkinter.colorchooser` module provides the :class:`Chooser` class +The :mod:`!tkinter.colorchooser` module provides the :class:`Chooser` class as an interface to the native color picker dialog. ``Chooser`` implements a modal color choosing dialog window. The ``Chooser`` class inherits from the :class:`~tkinter.commondialog.Dialog` class. diff --git a/Doc/library/tkinter.dnd.rst b/Doc/library/tkinter.dnd.rst index 62298d96c26..48d16ccb204 100644 --- a/Doc/library/tkinter.dnd.rst +++ b/Doc/library/tkinter.dnd.rst @@ -2,7 +2,6 @@ ============================================= .. module:: tkinter.dnd - :platform: Tk :synopsis: Tkinter drag-and-drop interface **Source code:** :source:`Lib/tkinter/dnd.py` @@ -12,7 +11,7 @@ .. note:: This is experimental and due to be deprecated when it is replaced with the Tk DND. -The :mod:`tkinter.dnd` module provides drag-and-drop support for objects within +The :mod:`!tkinter.dnd` module provides drag-and-drop support for objects within a single application, within the same window or between windows. To enable an object to be dragged, you must create an event binding for it that starts the drag-and-drop process. Typically, you bind a ButtonPress event to a callback diff --git a/Doc/library/tkinter.font.rst b/Doc/library/tkinter.font.rst index ed01bd5f483..9eecb803c3a 100644 --- a/Doc/library/tkinter.font.rst +++ b/Doc/library/tkinter.font.rst @@ -2,14 +2,13 @@ ============================================= .. module:: tkinter.font - :platform: Tk :synopsis: Tkinter font-wrapping class **Source code:** :source:`Lib/tkinter/font.py` -------------- -The :mod:`tkinter.font` module provides the :class:`Font` class for creating +The :mod:`!tkinter.font` module provides the :class:`Font` class for creating and using named fonts. The different font weights and slants are: diff --git a/Doc/library/tkinter.messagebox.rst b/Doc/library/tkinter.messagebox.rst index 0dc9632ca73..2a69d282638 100644 --- a/Doc/library/tkinter.messagebox.rst +++ b/Doc/library/tkinter.messagebox.rst @@ -2,14 +2,13 @@ ====================================================== .. module:: tkinter.messagebox - :platform: Tk :synopsis: Various types of alert dialogs **Source code:** :source:`Lib/tkinter/messagebox.py` -------------- -The :mod:`tkinter.messagebox` module provides a template base class as well as +The :mod:`!tkinter.messagebox` module provides a template base class as well as a variety of convenience methods for commonly used configurations. The message boxes are modal and will return a subset of (``True``, ``False``, ``None``, :data:`OK`, :data:`CANCEL`, :data:`YES`, :data:`NO`) based on diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 07ce8c40577..805f619eab8 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -10,12 +10,12 @@ -------------- -The :mod:`tkinter` package ("Tk interface") is the standard Python interface to -the Tcl/Tk GUI toolkit. Both Tk and :mod:`tkinter` are available on most Unix +The :mod:`!tkinter` package ("Tk interface") is the standard Python interface to +the Tcl/Tk GUI toolkit. Both Tk and :mod:`!tkinter` are available on most Unix platforms, including macOS, as well as on Windows systems. Running ``python -m tkinter`` from the command line should open a window -demonstrating a simple Tk interface, letting you know that :mod:`tkinter` is +demonstrating a simple Tk interface, letting you know that :mod:`!tkinter` is properly installed on your system, and also showing what version of Tcl/Tk is installed, so you can read the Tcl/Tk documentation specific to that version. @@ -108,7 +108,7 @@ Internally, Tk and Ttk use facilities of the underlying operating system, i.e., Xlib on Unix/X11, Cocoa on macOS, GDI on Windows. When your Python application uses a class in Tkinter, e.g., to create a widget, -the :mod:`tkinter` module first assembles a Tcl/Tk command string. It passes that +the :mod:`!tkinter` module first assembles a Tcl/Tk command string. It passes that Tcl command string to an internal :mod:`_tkinter` binary module, which then calls the Tcl interpreter to evaluate it. The Tcl interpreter will then call into the Tk and/or Ttk packages, which will in turn make calls to Xlib, Cocoa, or GDI. @@ -118,7 +118,7 @@ Tkinter Modules --------------- Support for Tkinter is spread across several modules. Most applications will need the -main :mod:`tkinter` module, as well as the :mod:`tkinter.ttk` module, which provides +main :mod:`!tkinter` module, as well as the :mod:`tkinter.ttk` module, which provides the modern themed widget set and API:: @@ -204,7 +204,7 @@ the modern themed widget set and API:: The modules that provide Tk support include: -:mod:`tkinter` +:mod:`!tkinter` Main Tkinter module. :mod:`tkinter.colorchooser` @@ -230,7 +230,7 @@ The modules that provide Tk support include: :mod:`tkinter.ttk` Themed widget set introduced in Tk 8.5, providing modern alternatives - for many of the classic widgets in the main :mod:`tkinter` module. + for many of the classic widgets in the main :mod:`!tkinter` module. Additional modules: @@ -239,22 +239,22 @@ Additional modules: :mod:`_tkinter` A binary module that contains the low-level interface to Tcl/Tk. - It is automatically imported by the main :mod:`tkinter` module, + It is automatically imported by the main :mod:`!tkinter` module, and should never be used directly by application programmers. It is usually a shared library (or DLL), but might in some cases be statically linked with the Python interpreter. :mod:`idlelib` Python's Integrated Development and Learning Environment (IDLE). Based - on :mod:`tkinter`. + on :mod:`!tkinter`. :mod:`tkinter.constants` Symbolic constants that can be used in place of strings when passing various parameters to Tkinter calls. Automatically imported by the - main :mod:`tkinter` module. + main :mod:`!tkinter` module. :mod:`tkinter.dnd` - (experimental) Drag-and-drop support for :mod:`tkinter`. This will + (experimental) Drag-and-drop support for :mod:`!tkinter`. This will become deprecated when it is replaced with the Tk DND. :mod:`turtle` @@ -504,7 +504,7 @@ documentation for all of these in the Threading model --------------- -Python and Tcl/Tk have very different threading models, which :mod:`tkinter` +Python and Tcl/Tk have very different threading models, which :mod:`!tkinter` tries to bridge. If you use threads, you may need to be aware of this. A Python interpreter may have many threads associated with it. In Tcl, multiple @@ -512,9 +512,9 @@ threads can be created, but each thread has a separate Tcl interpreter instance associated with it. Threads can also create more than one interpreter instance, though each interpreter instance can be used only by the one thread that created it. -Each :class:`Tk` object created by :mod:`tkinter` contains a Tcl interpreter. +Each :class:`Tk` object created by :mod:`!tkinter` contains a Tcl interpreter. It also keeps track of which thread created that interpreter. Calls to -:mod:`tkinter` can be made from any Python thread. Internally, if a call comes +:mod:`!tkinter` can be made from any Python thread. Internally, if a call comes from a thread other than the one that created the :class:`Tk` object, an event is posted to the interpreter's event queue, and when executed, the result is returned to the calling Python thread. @@ -529,17 +529,17 @@ toolkits where the GUI runs in a completely separate thread from all application code including event handlers. If the Tcl interpreter is not running the event loop and processing events, any -:mod:`tkinter` calls made from threads other than the one running the Tcl +:mod:`!tkinter` calls made from threads other than the one running the Tcl interpreter will fail. A number of special cases exist: * Tcl/Tk libraries can be built so they are not thread-aware. In this case, - :mod:`tkinter` calls the library from the originating Python thread, even + :mod:`!tkinter` calls the library from the originating Python thread, even if this is different than the thread that created the Tcl interpreter. A global lock ensures only one call occurs at a time. -* While :mod:`tkinter` allows you to create more than one instance of a :class:`Tk` +* While :mod:`!tkinter` allows you to create more than one instance of a :class:`Tk` object (with its own interpreter), all interpreters that are part of the same thread share a common event queue, which gets ugly fast. In practice, don't create more than one instance of :class:`Tk` at a time. Otherwise, it's best to create @@ -550,7 +550,7 @@ A number of special cases exist: or abandon the event loop entirely. If you're doing anything tricky when it comes to events or threads, be aware of these possibilities. -* There are a few select :mod:`tkinter` functions that presently work only when +* There are a few select :mod:`!tkinter` functions that presently work only when called from the thread that created the Tcl interpreter. @@ -700,11 +700,11 @@ options are ``variable``, ``textvariable``, ``onvalue``, ``offvalue``, and ``value``. This connection works both ways: if the variable changes for any reason, the widget it's connected to will be updated to reflect the new value. -Unfortunately, in the current implementation of :mod:`tkinter` it is not +Unfortunately, in the current implementation of :mod:`!tkinter` it is not possible to hand over an arbitrary Python variable to a widget through a ``variable`` or ``textvariable`` option. The only kinds of variables for which this works are variables that are subclassed from a class called Variable, -defined in :mod:`tkinter`. +defined in :mod:`!tkinter`. There are many useful subclasses of Variable already defined: :class:`StringVar`, :class:`IntVar`, :class:`DoubleVar`, and @@ -752,7 +752,7 @@ The Window Manager In Tk, there is a utility command, ``wm``, for interacting with the window manager. Options to the ``wm`` command allow you to control things like titles, -placement, icon bitmaps, and the like. In :mod:`tkinter`, these commands have +placement, icon bitmaps, and the like. In :mod:`!tkinter`, these commands have been implemented as methods on the :class:`Wm` class. Toplevel widgets are subclassed from the :class:`Wm` class, and so can call the :class:`Wm` methods directly. @@ -934,7 +934,7 @@ Entry widget, or to particular menu items in a Menu widget. Entry widget indexes (index, view index, etc.) Entry widgets have options that refer to character positions in the text being - displayed. You can use these :mod:`tkinter` functions to access these special + displayed. You can use these :mod:`!tkinter` functions to access these special points in text widgets: Text widget indexes diff --git a/Doc/library/tkinter.scrolledtext.rst b/Doc/library/tkinter.scrolledtext.rst index 763e24929d7..6c3c74afd47 100644 --- a/Doc/library/tkinter.scrolledtext.rst +++ b/Doc/library/tkinter.scrolledtext.rst @@ -2,7 +2,6 @@ ===================================================== .. module:: tkinter.scrolledtext - :platform: Tk :synopsis: Text widget with a vertical scroll bar. .. sectionauthor:: Fred L. Drake, Jr. @@ -11,7 +10,7 @@ -------------- -The :mod:`tkinter.scrolledtext` module provides a class of the same name which +The :mod:`!tkinter.scrolledtext` module provides a class of the same name which implements a basic text widget which has a vertical scroll bar configured to do the "right thing." Using the :class:`ScrolledText` class is a lot easier than setting up a text widget and scroll bar directly. diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index 628e9f945ac..7db57564699 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -12,12 +12,12 @@ -------------- -The :mod:`tkinter.ttk` module provides access to the Tk themed widget set, +The :mod:`!tkinter.ttk` module provides access to the Tk themed widget set, introduced in Tk 8.5. It provides additional benefits including anti-aliased font rendering under X11 and window transparency (requiring a composition window manager on X11). -The basic idea for :mod:`tkinter.ttk` is to separate, to the extent possible, +The basic idea for :mod:`!tkinter.ttk` is to separate, to the extent possible, the code implementing a widget's behavior from the code implementing its appearance. @@ -40,7 +40,7 @@ To override the basic Tk widgets, the import should follow the Tk import:: from tkinter import * from tkinter.ttk import * -That code causes several :mod:`tkinter.ttk` widgets (:class:`Button`, +That code causes several :mod:`!tkinter.ttk` widgets (:class:`Button`, :class:`Checkbutton`, :class:`Entry`, :class:`Frame`, :class:`Label`, :class:`LabelFrame`, :class:`Menubutton`, :class:`PanedWindow`, :class:`Radiobutton`, :class:`Scale` and :class:`Scrollbar`) to diff --git a/Doc/library/token.rst b/Doc/library/token.rst index c228006d4c1..fb826f5465b 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -50,8 +50,7 @@ The token constants are: .. data:: NAME - Token value that indicates an :ref:`identifier `. - Note that keywords are also initially tokenized as ``NAME`` tokens. + Token value that indicates an :ref:`identifier or keyword `. .. data:: NUMBER diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index b80917eae66..cf638f0b095 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`tokenize` module provides a lexical scanner for Python source code, +The :mod:`!tokenize` module provides a lexical scanner for Python source code, implemented in Python. The scanner in this module returns comments as tokens as well, making it useful for implementing "pretty-printers", including colorizers for on-screen displays. @@ -78,7 +78,7 @@ The primary entry point is a :term:`generator`: :func:`.tokenize`. It does not yield an :data:`~token.ENCODING` token. All constants from the :mod:`token` module are also exported from -:mod:`tokenize`. +:mod:`!tokenize`. Another function is provided to reverse the tokenization process. This is useful for creating tools that tokenize a script, modify the token stream, and @@ -154,7 +154,7 @@ Command-Line Usage .. versionadded:: 3.3 -The :mod:`tokenize` module can be executed as a script from the command line. +The :mod:`!tokenize` module can be executed as a script from the command line. It is as simple as: .. code-block:: sh diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index cae94ea08e1..57b8fa29eb3 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`trace` module allows you to trace program execution, generate +The :mod:`!trace` module allows you to trace program execution, generate annotated statement coverage listings, print caller/callee relationships and list functions executed during a program run. It can be used in another program or from the command line. @@ -24,7 +24,7 @@ or from the command line. Command-Line Usage ------------------ -The :mod:`trace` module can be invoked from the command line. It can be as +The :mod:`!trace` module can be invoked from the command line. It can be as simple as :: python -m trace --count -C . somefile.py ... @@ -43,13 +43,13 @@ all Python modules imported during the execution into the current directory. Display the version of the module and exit. .. versionadded:: 3.8 - Added ``--module`` option that allows to run an executable module. + Added ``--module`` option that allows running an executable module. Main options ^^^^^^^^^^^^ At least one of the following options must be specified when invoking -:mod:`trace`. The :option:`--listfuncs <-l>` option is mutually exclusive with +:mod:`!trace`. The :option:`--listfuncs <-l>` option is mutually exclusive with the :option:`--trace <-t>` and :option:`--count <-c>` options. When :option:`--listfuncs <-l>` is provided, neither :option:`--count <-c>` nor :option:`--trace <-t>` are accepted, and vice versa. diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 2370d927292..0fa70389f1f 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -307,7 +307,7 @@ Functions .. function:: get_object_traceback(obj) Get the traceback where the Python object *obj* was allocated. - Return a :class:`Traceback` instance, or ``None`` if the :mod:`tracemalloc` + Return a :class:`Traceback` instance, or ``None`` if the :mod:`!tracemalloc` module is not tracing memory allocations or did not trace the allocation of the object. @@ -318,7 +318,7 @@ Functions Get the maximum number of frames stored in the traceback of a trace. - The :mod:`tracemalloc` module must be tracing memory allocations to + The :mod:`!tracemalloc` module must be tracing memory allocations to get the limit, otherwise an exception is raised. The limit is set by the :func:`start` function. @@ -327,15 +327,15 @@ Functions .. function:: get_traced_memory() Get the current size and peak size of memory blocks traced by the - :mod:`tracemalloc` module as a tuple: ``(current: int, peak: int)``. + :mod:`!tracemalloc` module as a tuple: ``(current: int, peak: int)``. .. function:: reset_peak() - Set the peak size of memory blocks traced by the :mod:`tracemalloc` module + Set the peak size of memory blocks traced by the :mod:`!tracemalloc` module to the current size. - Do nothing if the :mod:`tracemalloc` module is not tracing memory + Do nothing if the :mod:`!tracemalloc` module is not tracing memory allocations. This function only modifies the recorded peak size, and does not modify or @@ -350,14 +350,14 @@ Functions .. function:: get_tracemalloc_memory() - Get the memory usage in bytes of the :mod:`tracemalloc` module used to store + Get the memory usage in bytes of the :mod:`!tracemalloc` module used to store traces of memory blocks. Return an :class:`int`. .. function:: is_tracing() - ``True`` if the :mod:`tracemalloc` module is tracing Python memory + ``True`` if the :mod:`!tracemalloc` module is tracing Python memory allocations, ``False`` otherwise. See also :func:`start` and :func:`stop` functions. @@ -378,8 +378,8 @@ Functions :meth:`Snapshot.compare_to` and :meth:`Snapshot.statistics` methods. Storing more frames increases the memory and CPU overhead of the - :mod:`tracemalloc` module. Use the :func:`get_tracemalloc_memory` function - to measure how much memory is used by the :mod:`tracemalloc` module. + :mod:`!tracemalloc` module. Use the :func:`get_tracemalloc_memory` function + to measure how much memory is used by the :mod:`!tracemalloc` module. The :envvar:`PYTHONTRACEMALLOC` environment variable (``PYTHONTRACEMALLOC=NFRAME``) and the :option:`-X` ``tracemalloc=NFRAME`` @@ -408,12 +408,12 @@ Functions :class:`Snapshot` instance. The snapshot does not include memory blocks allocated before the - :mod:`tracemalloc` module started to trace memory allocations. + :mod:`!tracemalloc` module started to trace memory allocations. Tracebacks of traces are limited to :func:`get_traceback_limit` frames. Use the *nframe* parameter of the :func:`start` function to store more frames. - The :mod:`tracemalloc` module must be tracing memory allocations to take a + The :mod:`!tracemalloc` module must be tracing memory allocations to take a snapshot, see the :func:`start` function. See also the :func:`get_object_traceback` function. @@ -457,7 +457,7 @@ Filter * ``Filter(True, subprocess.__file__)`` only includes traces of the :mod:`subprocess` module * ``Filter(False, tracemalloc.__file__)`` excludes traces of the - :mod:`tracemalloc` module + :mod:`!tracemalloc` module * ``Filter(False, "")`` excludes empty tracebacks @@ -589,7 +589,7 @@ Snapshot If *cumulative* is ``True``, cumulate size and count of memory blocks of all frames of the traceback of a trace, not only the most recent frame. - The cumulative mode can only be used with *key_type* equals to + The cumulative mode can only be used with *key_type* equal to ``'filename'`` and ``'lineno'``. The result is sorted from the biggest to the smallest by: @@ -720,11 +720,10 @@ Traceback When a snapshot is taken, tracebacks of traces are limited to :func:`get_traceback_limit` frames. See the :func:`take_snapshot` function. The original number of frames of the traceback is stored in the - :attr:`Traceback.total_nframe` attribute. That allows to know if a traceback + :attr:`Traceback.total_nframe` attribute. That allows one to know if a traceback has been truncated by the traceback limit. - The :attr:`Trace.traceback` attribute is an instance of :class:`Traceback` - instance. + The :attr:`Trace.traceback` attribute is a :class:`Traceback` instance. .. versionchanged:: 3.7 Frames are now sorted from the oldest to the most recent, instead of most recent to oldest. diff --git a/Doc/library/tty.rst b/Doc/library/tty.rst index 37778bf20bd..fe46be8b321 100644 --- a/Doc/library/tty.rst +++ b/Doc/library/tty.rst @@ -2,7 +2,6 @@ ========================================== .. module:: tty - :platform: Unix :synopsis: Utility functions that perform common terminal control operations. .. moduleauthor:: Steen Lumholt @@ -12,14 +11,14 @@ -------------- -The :mod:`tty` module defines functions for putting the tty into cbreak and raw +The :mod:`!tty` module defines functions for putting the tty into cbreak and raw modes. .. availability:: Unix. Because it requires the :mod:`termios` module, it will work only on Unix. -The :mod:`tty` module defines the following functions: +The :mod:`!tty` module defines the following functions: .. function:: cfmakeraw(mode) diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 95a57c57e71..bfe93bc253d 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -2248,7 +2248,7 @@ Settings and special methods Set turtle mode ("standard", "logo" or "world") and perform reset. If mode is not given, current mode is returned. - Mode "standard" is compatible with old :mod:`turtle`. Mode "logo" is + Mode "standard" is compatible with old :mod:`!turtle`. Mode "logo" is compatible with most Logo turtle graphics. Mode "world" uses user-defined "world coordinates". **Attention**: in this mode angles appear distorted if ``x/y`` unit-ratio doesn't equal 1. @@ -2689,7 +2689,7 @@ Screen and Turtle. Python script :file:`{filename}.py`. It is intended to serve as a template for translation of the docstrings into different languages. -If you (or your students) want to use :mod:`turtle` with online help in your +If you (or your students) want to use :mod:`!turtle` with online help in your native language, you have to translate the docstrings and save the resulting file as e.g. :file:`turtle_docstringdict_german.py`. @@ -2752,7 +2752,7 @@ Short explanation of selected entries: auto``. - If you set e.g. ``language = italian`` the docstringdict :file:`turtle_docstringdict_italian.py` will be loaded at import time (if - present on the import path, e.g. in the same directory as :mod:`turtle`). + present on the import path, e.g. in the same directory as :mod:`!turtle`). - The entries *exampleturtle* and *examplescreen* define the names of these objects as they occur in the docstrings. The transformation of method-docstrings to function-docstrings will delete these names from the @@ -2761,7 +2761,7 @@ Short explanation of selected entries: switch ("no subprocess"). This will prevent :func:`exitonclick` to enter the mainloop. -There can be a :file:`turtle.cfg` file in the directory where :mod:`turtle` is +There can be a :file:`turtle.cfg` file in the directory where :mod:`!turtle` is stored and an additional one in the current working directory. The latter will override the settings of the first one. @@ -2770,13 +2770,13 @@ study it as an example and see its effects when running the demos (preferably not from within the demo-viewer). -:mod:`turtledemo` --- Demo scripts -================================== +:mod:`!turtledemo` --- Demo scripts +=================================== .. module:: turtledemo :synopsis: A viewer for example turtle scripts -The :mod:`turtledemo` package includes a set of demo scripts. These +The :mod:`!turtledemo` package includes a set of demo scripts. These scripts can be run and viewed using the supplied demo viewer as follows:: python -m turtledemo @@ -2785,11 +2785,11 @@ Alternatively, you can run the demo scripts individually. For example, :: python -m turtledemo.bytedesign -The :mod:`turtledemo` package directory contains: +The :mod:`!turtledemo` package directory contains: - A demo viewer :file:`__main__.py` which can be used to view the sourcecode of the scripts and run them at the same time. -- Multiple scripts demonstrating different features of the :mod:`turtle` +- Multiple scripts demonstrating different features of the :mod:`!turtle` module. Examples can be accessed via the Examples menu. They can also be run standalone. - A :file:`turtle.cfg` file which serves as an example of how to write diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 886b230d171..a2625ab2a37 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -813,7 +813,7 @@ For example, this conforms to :pep:`484`:: def __len__(self) -> int: ... def __iter__(self) -> Iterator[int]: ... -:pep:`544` allows to solve this problem by allowing users to write +:pep:`544` solves this problem by allowing users to write the above code without explicit base classes in the class definition, allowing ``Bucket`` to be implicitly considered a subtype of both ``Sized`` and ``Iterable[int]`` by static type checkers. This is known as @@ -3347,8 +3347,8 @@ Introspection helpers .. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False) - Return a dictionary containing type hints for a function, method, module - or class object. + Return a dictionary containing type hints for a function, method, module, + class object, or other callable object. This is often the same as ``obj.__annotations__``, but this function makes the following changes to the annotations dictionary: @@ -3389,6 +3389,13 @@ Introspection helpers :ref:`type aliases ` that include forward references, or with names imported under :data:`if TYPE_CHECKING `. + .. note:: + + Calling :func:`get_type_hints` on an instance is not supported. + To retrieve annotations for an instance, call + :func:`get_type_hints` on the instance's class instead + (for example, ``get_type_hints(type(obj))``). + .. versionchanged:: 3.9 Added ``include_extras`` parameter as part of :pep:`593`. See the documentation on :data:`Annotated` for more information. @@ -3398,6 +3405,11 @@ Introspection helpers if a default value equal to ``None`` was set. Now the annotation is returned unchanged. + .. versionchanged:: 3.14 + Calling :func:`get_type_hints` on instances is no longer supported. + Some instances were accepted in earlier versions as an undocumented + implementation detail. + .. function:: get_origin(tp) Get the unsubscripted version of a type: for a typing object of the form diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 91f90a0726a..549b0499788 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -13,11 +13,11 @@ -------------- -:mod:`unittest.mock` is a library for testing in Python. It allows you to +:mod:`!unittest.mock` is a library for testing in Python. It allows you to replace parts of your system under test with mock objects and make assertions about how they have been used. -:mod:`unittest.mock` provides a core :class:`Mock` class removing the need to +:mod:`!unittest.mock` provides a core :class:`Mock` class removing the need to create a host of stubs throughout your test suite. After performing an action, you can make assertions about which methods / attributes were used and arguments they were called with. You can also specify return values and @@ -33,7 +33,7 @@ Mock is designed for use with :mod:`unittest` and is based on the 'action -> assertion' pattern instead of 'record -> replay' used by many mocking frameworks. -There is a backport of :mod:`unittest.mock` for earlier versions of Python, +There is a backport of :mod:`!unittest.mock` for earlier versions of Python, available as :pypi:`mock` on PyPI. @@ -2638,7 +2638,7 @@ unit tests. Testing everything in isolation is all fine and dandy, but if you don't test how your units are "wired together" there is still lots of room for bugs that tests might have caught. -:mod:`unittest.mock` already provides a feature to help with this, called speccing. If you +:mod:`!unittest.mock` already provides a feature to help with this, called speccing. If you use a class or instance as the :attr:`!spec` for a mock then you can only access attributes on the mock that exist on the real class: diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index d2ac5eedbbc..67ae1d91573 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -16,13 +16,13 @@ (If you are already familiar with the basic concepts of testing, you might want to skip to :ref:`the list of assert methods `.) -The :mod:`unittest` unit testing framework was originally inspired by JUnit +The :mod:`!unittest` unit testing framework was originally inspired by JUnit and has a similar flavor as major unit testing frameworks in other languages. It supports test automation, sharing of setup and shutdown code for tests, aggregation of tests into collections, and independence of the tests from the reporting framework. -To achieve this, :mod:`unittest` supports some important concepts in an +To achieve this, :mod:`!unittest` supports some important concepts in an object-oriented way: test fixture @@ -33,7 +33,7 @@ test fixture test case A :dfn:`test case` is the individual unit of testing. It checks for a specific - response to a particular set of inputs. :mod:`unittest` provides a base class, + response to a particular set of inputs. :mod:`!unittest` provides a base class, :class:`TestCase`, which may be used to create new test cases. test suite @@ -53,7 +53,7 @@ test runner `Simple Smalltalk Testing: With Patterns `_ Kent Beck's original paper on testing frameworks using the pattern shared - by :mod:`unittest`. + by :mod:`!unittest`. `pytest `_ Third-party unittest framework with a lighter-weight syntax for writing @@ -81,7 +81,7 @@ test runner Basic example ------------- -The :mod:`unittest` module provides a rich set of tools for constructing and +The :mod:`!unittest` module provides a rich set of tools for constructing and running tests. This section demonstrates that a small subset of the tools suffice to meet the needs of most users. @@ -147,7 +147,7 @@ to enable a higher level of verbosity, and produce the following output:: OK -The above examples show the most commonly used :mod:`unittest` features which +The above examples show the most commonly used :mod:`!unittest` features which are sufficient to meet many everyday testing needs. The remainder of the documentation explores the full feature set from first principles. @@ -365,7 +365,7 @@ Organizing test code -------------------- The basic building blocks of unit testing are :dfn:`test cases` --- single -scenarios that must be set up and checked for correctness. In :mod:`unittest`, +scenarios that must be set up and checked for correctness. In :mod:`!unittest`, test cases are represented by :class:`unittest.TestCase` instances. To make your own test cases you must write subclasses of :class:`TestCase` or use :class:`FunctionTestCase`. @@ -387,7 +387,7 @@ testing code:: Note that in order to test something, we use one of the :ref:`assert\* methods ` provided by the :class:`TestCase` base class. If the test fails, an -exception will be raised with an explanatory message, and :mod:`unittest` +exception will be raised with an explanatory message, and :mod:`!unittest` will identify the test case as a :dfn:`failure`. Any other exceptions will be treated as :dfn:`errors`. @@ -442,8 +442,8 @@ test fixture used to execute each individual test method. Thus will be called once per test. It is recommended that you use TestCase implementations to group tests together -according to the features they test. :mod:`unittest` provides a mechanism for -this: the :dfn:`test suite`, represented by :mod:`unittest`'s +according to the features they test. :mod:`!unittest` provides a mechanism for +this: the :dfn:`test suite`, represented by :mod:`!unittest`'s :class:`TestSuite` class. In most cases, calling :func:`unittest.main` will do the right thing and collect all the module's test cases for you and execute them. @@ -489,10 +489,10 @@ Re-using old test code ---------------------- Some users will find that they have existing test code that they would like to -run from :mod:`unittest`, without converting every old test function to a +run from :mod:`!unittest`, without converting every old test function to a :class:`TestCase` subclass. -For this reason, :mod:`unittest` provides a :class:`FunctionTestCase` class. +For this reason, :mod:`!unittest` provides a :class:`FunctionTestCase` class. This subclass of :class:`TestCase` can be used to wrap an existing test function. Set-up and tear-down functions can also be provided. @@ -513,7 +513,7 @@ set-up and tear-down methods:: .. note:: Even though :class:`FunctionTestCase` can be used to quickly convert an - existing test base over to a :mod:`unittest`\ -based system, this approach is + existing test base over to a :mod:`!unittest`\ -based system, this approach is not recommended. Taking the time to set up proper :class:`TestCase` subclasses will make future test refactorings infinitely easier. @@ -709,7 +709,7 @@ wouldn't be displayed:: Classes and functions --------------------- -This section describes in depth the API of :mod:`unittest`. +This section describes in depth the API of :mod:`!unittest`. .. _testcase-objects: @@ -720,7 +720,7 @@ Test cases .. class:: TestCase(methodName='runTest') Instances of the :class:`TestCase` class represent the logical test units - in the :mod:`unittest` universe. This class is intended to be used as a base + in the :mod:`!unittest` universe. This class is intended to be used as a base class, with specific tests being implemented by concrete subclasses. This class implements the interface needed by the test runner to allow it to drive the tests, and methods that the test code can use to check for and report various @@ -1727,7 +1727,7 @@ Test cases allows the test runner to drive the test, but does not provide the methods which test code can use to check and report errors. This is used to create test cases using legacy test code, allowing it to be integrated into a - :mod:`unittest`-based test framework. + :mod:`!unittest`-based test framework. .. _testsuite-objects: @@ -1822,7 +1822,7 @@ Loading and running tests The :class:`TestLoader` class is used to create test suites from classes and modules. Normally, there is no need to create an instance of this class; the - :mod:`unittest` module provides an instance that can be shared as + :mod:`!unittest` module provides an instance that can be shared as :data:`unittest.defaultTestLoader`. Using a subclass or instance, however, allows customization of some configurable properties. @@ -2048,7 +2048,7 @@ Loading and running tests properly recorded; test authors do not need to worry about recording the outcome of tests. - Testing frameworks built on top of :mod:`unittest` may want access to the + Testing frameworks built on top of :mod:`!unittest` may want access to the :class:`TestResult` object generated by running a set of tests for reporting purposes; a :class:`TestResult` instance is returned by the :meth:`!TestRunner.run` method for this purpose. diff --git a/Doc/library/urllib.error.rst b/Doc/library/urllib.error.rst index 1686ddd09ca..dd2c9858eaa 100644 --- a/Doc/library/urllib.error.rst +++ b/Doc/library/urllib.error.rst @@ -11,10 +11,10 @@ -------------- -The :mod:`urllib.error` module defines the exception classes for exceptions +The :mod:`!urllib.error` module defines the exception classes for exceptions raised by :mod:`urllib.request`. The base exception class is :exc:`URLError`. -The following exceptions are raised by :mod:`urllib.error` as appropriate: +The following exceptions are raised by :mod:`!urllib.error` as appropriate: .. exception:: URLError diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 44a9c79cba2..f893039c9ac 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -35,7 +35,7 @@ Resource Locators. It supports the following URL schemes: ``file``, ``ftp``, macOS, it *may* be removed if CPython has been built with the :option:`--with-app-store-compliance` option. -The :mod:`urllib.parse` module defines functions that fall into two broad +The :mod:`!urllib.parse` module defines functions that fall into two broad categories: URL parsing and URL quoting. These are covered in detail in the following sections. @@ -50,11 +50,12 @@ URL Parsing The URL parsing functions focus on splitting a URL string into its components, or on combining URL components into a URL string. -.. function:: urlparse(urlstring, scheme='', allow_fragments=True) +.. function:: urlsplit(urlstring, scheme=None, allow_fragments=True) - Parse a URL into six components, returning a 6-item :term:`named tuple`. This - corresponds to the general structure of a URL: - ``scheme://netloc/path;parameters?query#fragment``. + Parse a URL into five components, returning a 5-item :term:`named tuple` + :class:`SplitResult` or :class:`SplitResultBytes`. + This corresponds to the general structure of a URL: + ``scheme://netloc/path?query#fragment``. Each tuple item is a string, possibly empty. The components are not broken up into smaller parts (for example, the network location is a single string), and % escapes are not expanded. The delimiters as shown above are not part of the @@ -64,15 +65,15 @@ or on combining URL components into a URL string. .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> from urllib.parse import urlparse - >>> urlparse("scheme://netloc/path;parameters?query#fragment") - ParseResult(scheme='scheme', netloc='netloc', path='/path;parameters', params='', + >>> from urllib.parse import urlsplit + >>> urlsplit("scheme://netloc/path?query#fragment") + SplitResult(scheme='scheme', netloc='netloc', path='/path', query='query', fragment='fragment') - >>> o = urlparse("http://docs.python.org:80/3/library/urllib.parse.html?" + >>> o = urlsplit("http://docs.python.org:80/3/library/urllib.parse.html?" ... "highlight=params#url-parsing") >>> o - ParseResult(scheme='http', netloc='docs.python.org:80', - path='/3/library/urllib.parse.html', params='', + SplitResult(scheme='http', netloc='docs.python.org:80', + path='/3/library/urllib.parse.html', query='highlight=params', fragment='url-parsing') >>> o.scheme 'http' @@ -85,7 +86,7 @@ or on combining URL components into a URL string. >>> o._replace(fragment="").geturl() 'http://docs.python.org:80/3/library/urllib.parse.html?highlight=params' - Following the syntax specifications in :rfc:`1808`, urlparse recognizes + Following the syntax specifications in :rfc:`1808`, :func:`!urlsplit` recognizes a netloc only if it is properly introduced by '//'. Otherwise the input is presumed to be a relative URL and thus to start with a path component. @@ -93,15 +94,15 @@ or on combining URL components into a URL string. .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> from urllib.parse import urlparse - >>> urlparse('//www.cwi.nl:80/%7Eguido/Python.html') - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') - >>> urlparse('www.cwi.nl/%7Eguido/Python.html') - ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', - params='', query='', fragment='') - >>> urlparse('help/Python.html') - ParseResult(scheme='', netloc='', path='help/Python.html', params='', + >>> from urllib.parse import urlsplit + >>> urlsplit('//www.cwi.nl:80/%7Eguido/Python.html') + SplitResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + query='', fragment='') + >>> urlsplit('www.cwi.nl/%7Eguido/Python.html') + SplitResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', + query='', fragment='') + >>> urlsplit('help/Python.html') + SplitResult(scheme='', netloc='', path='help/Python.html', query='', fragment='') The *scheme* argument gives the default addressing scheme, to be @@ -126,12 +127,9 @@ or on combining URL components into a URL string. +------------------+-------+-------------------------+------------------------+ | :attr:`path` | 2 | Hierarchical path | empty string | +------------------+-------+-------------------------+------------------------+ - | :attr:`params` | 3 | Parameters for last | empty string | - | | | path element | | - +------------------+-------+-------------------------+------------------------+ - | :attr:`query` | 4 | Query component | empty string | + | :attr:`query` | 3 | Query component | empty string | +------------------+-------+-------------------------+------------------------+ - | :attr:`fragment` | 5 | Fragment identifier | empty string | + | :attr:`fragment` | 4 | Fragment identifier | empty string | +------------------+-------+-------------------------+------------------------+ | :attr:`username` | | User name | :const:`None` | +------------------+-------+-------------------------+------------------------+ @@ -155,26 +153,30 @@ or on combining URL components into a URL string. ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is decomposed before parsing, no error will be raised. + Following some of the `WHATWG spec`_ that updates :rfc:`3986`, leading C0 + control and space characters are stripped from the URL. ``\n``, + ``\r`` and tab ``\t`` characters are removed from the URL at any position. + As is the case with all named tuples, the subclass has a few additional methods and attributes that are particularly useful. One such method is :meth:`_replace`. - The :meth:`_replace` method will return a new ParseResult object replacing specified - fields with new values. + The :meth:`_replace` method will return a new :class:`SplitResult` object + replacing specified fields with new values. .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> from urllib.parse import urlparse - >>> u = urlparse('//www.cwi.nl:80/%7Eguido/Python.html') + >>> from urllib.parse import urlsplit + >>> u = urlsplit('//www.cwi.nl:80/%7Eguido/Python.html') >>> u - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') + SplitResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + query='', fragment='') >>> u._replace(scheme='http') - ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') + SplitResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + query='', fragment='') .. warning:: - :func:`urlparse` does not perform validation. See :ref:`URL parsing + :func:`urlsplit` does not perform validation. See :ref:`URL parsing security ` for details. .. versionchanged:: 3.2 @@ -193,6 +195,14 @@ or on combining URL components into a URL string. Characters that affect netloc parsing under NFKC normalization will now raise :exc:`ValueError`. + .. versionchanged:: 3.10 + ASCII newline and tab characters are stripped from the URL. + + .. versionchanged:: 3.12 + Leading WHATWG C0 control and space characters are stripped from the URL. + +.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser + .. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator='&') @@ -287,93 +297,35 @@ or on combining URL components into a URL string. separator key, with ``&`` as the default separator. -.. function:: urlunparse(parts) +.. function:: urlunsplit(parts) - Construct a URL from a tuple as returned by ``urlparse()``. The *parts* - argument can be any six-item iterable. This may result in a slightly + Construct a URL from a tuple as returned by ``urlsplit()``. The *parts* + argument can be any five-item iterable. This may result in a slightly different, but equivalent URL, if the URL that was parsed originally had unnecessary delimiters (for example, a ``?`` with an empty query; the RFC states that these are equivalent). -.. function:: urlsplit(urlstring, scheme='', allow_fragments=True) - - This is similar to :func:`urlparse`, but does not split the params from the URL. - This should generally be used instead of :func:`urlparse` if the more recent URL - syntax allowing parameters to be applied to each segment of the *path* portion - of the URL (see :rfc:`2396`) is wanted. A separate function is needed to - separate the path segments and parameters. This function returns a 5-item - :term:`named tuple`:: - - (addressing scheme, network location, path, query, fragment identifier). - - The return value is a :term:`named tuple`, its items can be accessed by index - or as named attributes: - - +------------------+-------+-------------------------+----------------------+ - | Attribute | Index | Value | Value if not present | - +==================+=======+=========================+======================+ - | :attr:`scheme` | 0 | URL scheme specifier | *scheme* parameter | - +------------------+-------+-------------------------+----------------------+ - | :attr:`netloc` | 1 | Network location part | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`path` | 2 | Hierarchical path | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`query` | 3 | Query component | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`fragment` | 4 | Fragment identifier | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`username` | | User name | :const:`None` | - +------------------+-------+-------------------------+----------------------+ - | :attr:`password` | | Password | :const:`None` | - +------------------+-------+-------------------------+----------------------+ - | :attr:`hostname` | | Host name (lower case) | :const:`None` | - +------------------+-------+-------------------------+----------------------+ - | :attr:`port` | | Port number as integer, | :const:`None` | - | | | if present | | - +------------------+-------+-------------------------+----------------------+ - - Reading the :attr:`port` attribute will raise a :exc:`ValueError` if - an invalid port is specified in the URL. See section - :ref:`urlparse-result-object` for more information on the result object. - - Unmatched square brackets in the :attr:`netloc` attribute will raise a - :exc:`ValueError`. - - Characters in the :attr:`netloc` attribute that decompose under NFKC - normalization (as used by the IDNA encoding) into any of ``/``, ``?``, - ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is - decomposed before parsing, no error will be raised. - - Following some of the `WHATWG spec`_ that updates RFC 3986, leading C0 - control and space characters are stripped from the URL. ``\n``, - ``\r`` and tab ``\t`` characters are removed from the URL at any position. - - .. warning:: - - :func:`urlsplit` does not perform validation. See :ref:`URL parsing - security ` for details. +.. function:: urlparse(urlstring, scheme=None, allow_fragments=True) - .. versionchanged:: 3.6 - Out-of-range port numbers now raise :exc:`ValueError`, instead of - returning :const:`None`. + This is similar to :func:`urlsplit`, but additionally splits the *path* + component on *path* and *params*. + This function returns a 6-item :term:`named tuple` :class:`ParseResult` + or :class:`ParseResultBytes`. + Its items are the same as for the :func:`!urlsplit` result, except that + *params* is inserted at index 3, between *path* and *query*. - .. versionchanged:: 3.8 - Characters that affect netloc parsing under NFKC normalization will - now raise :exc:`ValueError`. + This function is based on obsoleted :rfc:`1738` and :rfc:`1808`, which + listed *params* as the main URL component. + The more recent URL syntax allows parameters to be applied to each segment + of the *path* portion of the URL (see :rfc:`3986`). + :func:`urlsplit` should generally be used instead of :func:`urlparse`. + A separate function is needed to separate the path segments and parameters. - .. versionchanged:: 3.10 - ASCII newline and tab characters are stripped from the URL. - - .. versionchanged:: 3.12 - Leading WHATWG C0 control and space characters are stripped from the URL. - -.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser - -.. function:: urlunsplit(parts) +.. function:: urlunparse(parts) - Combine the elements of a tuple as returned by :func:`urlsplit` into a - complete URL as a string. The *parts* argument can be any five-item + Combine the elements of a tuple as returned by :func:`urlparse` into a + complete URL as a string. The *parts* argument can be any six-item iterable. This may result in a slightly different, but equivalent URL, if the URL that was parsed originally had unnecessary delimiters (for example, a ? with an empty query; the RFC states that these are equivalent). @@ -391,7 +343,7 @@ or on combining URL components into a URL string. 'http://www.cwi.nl/%7Eguido/FAQ.html' The *allow_fragments* argument has the same meaning and default as for - :func:`urlparse`. + :func:`urlsplit`. .. note:: @@ -531,7 +483,7 @@ individual URL quoting functions. Structured Parse Results ------------------------ -The result objects from the :func:`urlparse`, :func:`urlsplit` and +The result objects from the :func:`urlsplit`, :func:`urlparse` and :func:`urldefrag` functions are subclasses of the :class:`tuple` type. These subclasses add the attributes listed in the documentation for those functions, the encoding and decoding support described in the diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 5f796578eaa..b857b2a235e 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -12,7 +12,7 @@ -------------- -The :mod:`urllib.request` module defines functions and classes which help in +The :mod:`!urllib.request` module defines functions and classes which help in opening URLs (mostly HTTP) in a complex world --- basic and digest authentication, redirections, cookies and more. @@ -31,7 +31,7 @@ authentication, redirections, cookies and more. .. include:: ../includes/wasm-notavail.rst -The :mod:`urllib.request` module defines the following functions: +The :mod:`!urllib.request` module defines the following functions: .. function:: urlopen(url, data=None[, timeout], *, context=None) @@ -1485,8 +1485,8 @@ some point in the future. calls to :func:`urlretrieve`. -:mod:`urllib.request` Restrictions ----------------------------------- +:mod:`!urllib.request` Restrictions +----------------------------------- .. index:: pair: HTTP; protocol @@ -1539,15 +1539,15 @@ some point in the future. -:mod:`urllib.response` --- Response classes used by urllib -========================================================== +:mod:`!urllib.response` --- Response classes used by urllib +=========================================================== .. module:: urllib.response :synopsis: Response classes used by urllib. -The :mod:`urllib.response` module defines functions and classes which define a +The :mod:`!urllib.response` module defines functions and classes which define a minimal file-like interface, including ``read()`` and ``readline()``. -Functions defined by this module are used internally by the :mod:`urllib.request` module. +Functions defined by this module are used internally by the :mod:`!urllib.request` module. The typical response object is a :class:`urllib.response.addinfourl` instance: .. class:: addinfourl diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst index 674f646c633..ba719ae084e 100644 --- a/Doc/library/urllib.robotparser.rst +++ b/Doc/library/urllib.robotparser.rst @@ -91,16 +91,16 @@ class:: >>> import urllib.robotparser >>> rp = urllib.robotparser.RobotFileParser() - >>> rp.set_url("http://www.musi-cal.com/robots.txt") + >>> rp.set_url("http://www.pythontest.net/robots.txt") >>> rp.read() >>> rrate = rp.request_rate("*") >>> rrate.requests - 3 + 1 >>> rrate.seconds - 20 + 1 >>> rp.crawl_delay("*") 6 - >>> rp.can_fetch("*", "http://www.musi-cal.com/cgi-bin/search?city=San+Francisco") - False - >>> rp.can_fetch("*", "http://www.musi-cal.com/") + >>> rp.can_fetch("*", "http://www.pythontest.net/") True + >>> rp.can_fetch("*", "http://www.pythontest.net/no-robots-here/") + False diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst index aa4f1bf940b..fe3a3a8e510 100644 --- a/Doc/library/uuid.rst +++ b/Doc/library/uuid.rst @@ -168,7 +168,7 @@ which relays any information about the UUID's safety, using this enumeration: .. versionadded:: 3.7 -The :mod:`uuid` module defines the following functions: +The :mod:`!uuid` module defines the following functions: .. function:: getnode() @@ -273,7 +273,7 @@ The :mod:`uuid` module defines the following functions: .. versionadded:: 3.14 -The :mod:`uuid` module defines the following namespace identifiers for use with +The :mod:`!uuid` module defines the following namespace identifiers for use with :func:`uuid3` or :func:`uuid5`. @@ -298,7 +298,7 @@ The :mod:`uuid` module defines the following namespace identifiers for use with When this namespace is specified, the *name* string is an X.500 DN in DER or a text output format. -The :mod:`uuid` module defines the following constants for the possible values +The :mod:`!uuid` module defines the following constants for the possible values of the :attr:`~UUID.variant` attribute: @@ -324,7 +324,7 @@ of the :attr:`~UUID.variant` attribute: Reserved for future definition. -The :mod:`uuid` module defines the special Nil and Max UUID values: +The :mod:`!uuid` module defines the special Nil and Max UUID values: .. data:: NIL @@ -357,7 +357,7 @@ Command-Line Usage .. versionadded:: 3.12 -The :mod:`uuid` module can be executed as a script from the command line. +The :mod:`!uuid` module can be executed as a script from the command line. .. code-block:: sh @@ -406,7 +406,7 @@ The following options are accepted: Example ------- -Here are some examples of typical usage of the :mod:`uuid` module:: +Here are some examples of typical usage of the :mod:`!uuid` module:: >>> import uuid @@ -473,7 +473,7 @@ Here are some examples of typical usage of the :mod:`uuid` module:: Command-Line Example -------------------- -Here are some examples of typical usage of the :mod:`uuid` command-line interface: +Here are some examples of typical usage of the :mod:`!uuid` command-line interface: .. code-block:: shell diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index b0eb8ee18fa..59ec863c14f 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -545,7 +545,7 @@ subclass which installs setuptools and pip into a created virtual environment:: from subprocess import Popen, PIPE import sys from threading import Thread - from urllib.parse import urlparse + from urllib.parse import urlsplit from urllib.request import urlretrieve import venv @@ -616,7 +616,7 @@ subclass which installs setuptools and pip into a created virtual environment:: stream.close() def install_script(self, context, name, url): - _, _, path, _, _, _ = urlparse(url) + _, _, path, _, _ = urlsplit(url) fn = os.path.split(path)[-1] binpath = context.bin_path distpath = os.path.join(binpath, fn) diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 15b62a0653d..01c316bbfd3 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -203,7 +203,7 @@ Describing Warning Filters The warnings filter is initialized by :option:`-W` options passed to the Python interpreter command line and the :envvar:`PYTHONWARNINGS` environment variable. The interpreter saves the arguments for all supplied entries without -interpretation in :data:`sys.warnoptions`; the :mod:`warnings` module parses these +interpretation in :data:`sys.warnoptions`; the :mod:`!warnings` module parses these when it is first imported (invalid options are ignored, after printing a message to :data:`sys.stderr`). @@ -616,8 +616,8 @@ Available Context Managers :func:`showwarning`. The *module* argument takes a module that will be used instead of the - module returned when you import :mod:`warnings` whose filter will be - protected. This argument exists primarily for testing the :mod:`warnings` + module returned when you import :mod:`!warnings` whose filter will be + protected. This argument exists primarily for testing the :mod:`!warnings` module itself. If the *action* argument is not ``None``, the remaining arguments are @@ -654,7 +654,7 @@ to true for free-threaded builds and false otherwise. If the :data:`~sys.flags.context_aware_warnings` flag is false, then :class:`catch_warnings` will modify the global attributes of the -:mod:`warnings` module. This is not safe if used within a concurrent program +:mod:`!warnings` module. This is not safe if used within a concurrent program (using multiple threads or using asyncio coroutines). For example, if two or more threads use the :class:`catch_warnings` class at the same time, the behavior is undefined. diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst index 36c2bde87fb..38b05f0b4fd 100644 --- a/Doc/library/wave.rst +++ b/Doc/library/wave.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`wave` module provides a convenient interface to the Waveform Audio +The :mod:`!wave` module provides a convenient interface to the Waveform Audio "WAVE" (or "WAV") file format. Only uncompressed PCM encoded wave files are supported. @@ -20,7 +20,7 @@ supported. Support for ``WAVE_FORMAT_EXTENSIBLE`` headers was added, provided that the extended format is ``KSDATAFORMAT_SUBTYPE_PCM``. -The :mod:`wave` module defines the following function and exception: +The :mod:`!wave` module defines the following function and exception: .. function:: open(file, mode=None) @@ -72,7 +72,7 @@ Wave_read Objects .. method:: close() - Close the stream if it was opened by :mod:`wave`, and make the instance + Close the stream if it was opened by :mod:`!wave`, and make the instance unusable. This is called automatically on object collection. @@ -189,7 +189,7 @@ Wave_write Objects .. method:: close() Make sure *nframes* is correct, and close the file if it was opened by - :mod:`wave`. This method is called upon object collection. It will raise + :mod:`!wave`. This method is called upon object collection. It will raise an exception if the output stream is not seekable and *nframes* does not match the number of frames actually written. @@ -199,11 +199,21 @@ Wave_write Objects Set the number of channels. + .. method:: getnchannels() + + Return the number of channels. + + .. method:: setsampwidth(n) Set the sample width to *n* bytes. + .. method:: getsampwidth() + + Return the sample width in bytes. + + .. method:: setframerate(n) Set the frame rate to *n*. @@ -213,6 +223,11 @@ Wave_write Objects integer. + .. method:: getframerate() + + Return the frame rate. + + .. method:: setnframes(n) Set the number of frames to *n*. This will be changed later if the number @@ -220,12 +235,27 @@ Wave_write Objects raise an error if the output stream is not seekable). + .. method:: getnframes() + + Return the number of audio frames written so far. + + .. method:: setcomptype(type, name) Set the compression type and description. At the moment, only compression type ``NONE`` is supported, meaning no compression. + .. method:: getcomptype() + + Return the compression type (``'NONE'``). + + + .. method:: getcompname() + + Return the human-readable compression type name. + + .. method:: setparams(tuple) The *tuple* should be ``(nchannels, sampwidth, framerate, nframes, comptype, @@ -233,6 +263,13 @@ Wave_write Objects parameters. + .. method:: getparams() + + Return a :func:`~collections.namedtuple` + ``(nchannels, sampwidth, framerate, nframes, comptype, compname)`` + containing the current output parameters. + + .. method:: tell() Return current position in the file, with the same disclaimer for the diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 2a25ed045c6..6dc5f90686c 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -1,7 +1,7 @@ .. _mod-weakref: -:mod:`weakref` --- Weak references -================================== +:mod:`!weakref` --- Weak references +=================================== .. module:: weakref :synopsis: Support for weak references and weak dictionaries. @@ -15,7 +15,7 @@ -------------- -The :mod:`weakref` module allows the Python programmer to create :dfn:`weak +The :mod:`!weakref` module allows the Python programmer to create :dfn:`weak references` to objects. .. When making changes to the examples in this file, be sure to update @@ -39,7 +39,7 @@ associate a name with each. If you used a Python dictionary to map names to images, or images to names, the image objects would remain alive just because they appeared as values or keys in the dictionaries. The :class:`WeakKeyDictionary` and :class:`WeakValueDictionary` classes supplied by -the :mod:`weakref` module are an alternative, using weak references to construct +the :mod:`!weakref` module are an alternative, using weak references to construct mappings that don't keep objects alive solely because they appear in the mapping objects. If, for example, an image object is a value in a :class:`WeakValueDictionary`, then when the last remaining references to that @@ -63,7 +63,7 @@ remains alive until the object is collected. Most programs should find that using one of these weak container types or :class:`finalize` is all they need -- it's not usually necessary to create your own weak references directly. The low-level machinery is -exposed by the :mod:`weakref` module for the benefit of advanced uses. +exposed by the :mod:`!weakref` module for the benefit of advanced uses. Not all objects can be weakly referenced. Objects which support weak references include class instances, functions written in Python (but not in C), instance methods, diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index a2103d8fdd8..7b37b270e75 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`webbrowser` module provides a high-level interface to allow displaying +The :mod:`!webbrowser` module provides a high-level interface to allow displaying web-based documents to users. Under most circumstances, simply calling the :func:`.open` function from this module will do the right thing. @@ -46,7 +46,7 @@ On iOS, the :envvar:`BROWSER` environment variable, as well as any arguments controlling autoraise, browser preference, and new tab/window creation will be ignored. Web pages will *always* be opened in the user's preferred browser, in a new tab, with the browser being brought to the foreground. The use of the -:mod:`webbrowser` module on iOS requires the :mod:`ctypes` module. If +:mod:`!webbrowser` module on iOS requires the :mod:`ctypes` module. If :mod:`ctypes` isn't available, calls to :func:`.open` will fail. .. _webbrowser-cli: diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index 6d1e8ecfc17..086f5fcf968 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -2,7 +2,6 @@ ========================================== .. module:: winreg - :platform: Windows :synopsis: Routines and objects for manipulating the Windows registry. .. sectionauthor:: Mark Hammond @@ -538,7 +537,7 @@ This module offers the following functions: Constants ------------------ -The following constants are defined for use in many :mod:`winreg` functions. +The following constants are defined for use in many :mod:`!winreg` functions. .. _hkey-constants: diff --git a/Doc/library/winsound.rst b/Doc/library/winsound.rst index 93c0c025982..70ddfe3bae0 100644 --- a/Doc/library/winsound.rst +++ b/Doc/library/winsound.rst @@ -2,7 +2,6 @@ ======================================================== .. module:: winsound - :platform: Windows :synopsis: Access to the sound-playing machinery for Windows. .. moduleauthor:: Toby Dickenson @@ -10,7 +9,7 @@ -------------- -The :mod:`winsound` module provides access to the basic sound-playing machinery +The :mod:`!winsound` module provides access to the basic sound-playing machinery provided by Windows platforms. It includes functions and several constants. .. availability:: Windows. diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index 381c9938347..0ace7b72d32 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -11,6 +11,11 @@ -------------- +.. warning:: + + :mod:`!wsgiref` is a reference implementation and is not recommended for + production. The module only implements basic security checks. + The Web Server Gateway Interface (WSGI) is a standard interface between web server software and web applications written in Python. Having a standard interface makes it easy to use an application that supports WSGI with a number @@ -21,7 +26,7 @@ and corner case of the WSGI design. You don't need to understand every detail of WSGI just to install a WSGI application or to write a web application using an existing framework. -:mod:`wsgiref` is a reference implementation of the WSGI specification that can +:mod:`!wsgiref` is a reference implementation of the WSGI specification that can be used to add WSGI support to a web server or framework. It provides utilities for manipulating WSGI environment variables and response headers, base classes for implementing WSGI servers, a demo HTTP server that serves WSGI applications, @@ -35,8 +40,8 @@ to tutorials and other resources. .. XXX If you're just trying to write a web application... -:mod:`wsgiref.util` -- WSGI environment utilities -------------------------------------------------- +:mod:`!wsgiref.util` -- WSGI environment utilities +-------------------------------------------------- .. module:: wsgiref.util :synopsis: WSGI environment utilities. @@ -144,7 +149,7 @@ in type annotations. httpd.serve_forever() -In addition to the environment functions above, the :mod:`wsgiref.util` module +In addition to the environment functions above, the :mod:`!wsgiref.util` module also provides these miscellaneous utilities: @@ -184,8 +189,8 @@ also provides these miscellaneous utilities: Support for :meth:`~object.__getitem__` method has been removed. -:mod:`wsgiref.headers` -- WSGI response header tools ----------------------------------------------------- +:mod:`!wsgiref.headers` -- WSGI response header tools +----------------------------------------------------- .. module:: wsgiref.headers :synopsis: WSGI response header tools. @@ -268,8 +273,8 @@ manipulation of WSGI response headers using a mapping-like interface. *headers* parameter is optional. -:mod:`wsgiref.simple_server` -- a simple WSGI HTTP server ---------------------------------------------------------- +:mod:`!wsgiref.simple_server` -- a simple WSGI HTTP server +---------------------------------------------------------- .. module:: wsgiref.simple_server :synopsis: A simple WSGI HTTP server. @@ -310,7 +315,7 @@ request. (E.g., using the :func:`shift_path_info` function from This function is a small but complete WSGI application that returns a text page containing the message "Hello world!" and a list of the key/value pairs provided in the *environ* parameter. It's useful for verifying that a WSGI server (such - as :mod:`wsgiref.simple_server`) is able to run a simple WSGI application + as :mod:`!wsgiref.simple_server`) is able to run a simple WSGI application correctly. The *start_response* callable should follow the :class:`.StartResponse` protocol. @@ -382,8 +387,8 @@ request. (E.g., using the :func:`shift_path_info` function from interface. -:mod:`wsgiref.validate` --- WSGI conformance checker ----------------------------------------------------- +:mod:`!wsgiref.validate` --- WSGI conformance checker +----------------------------------------------------- .. module:: wsgiref.validate :synopsis: WSGI conformance checker. @@ -391,7 +396,7 @@ request. (E.g., using the :func:`shift_path_info` function from When creating new WSGI application objects, frameworks, servers, or middleware, it can be useful to validate the new code's conformance using -:mod:`wsgiref.validate`. This module provides a function that creates WSGI +:mod:`!wsgiref.validate`. This module provides a function that creates WSGI application objects that validate communications between a WSGI server or gateway and a WSGI application object, to check both sides for protocol conformance. @@ -450,8 +455,8 @@ Paste" library. httpd.serve_forever() -:mod:`wsgiref.handlers` -- server/gateway base classes ------------------------------------------------------- +:mod:`!wsgiref.handlers` -- server/gateway base classes +------------------------------------------------------- .. module:: wsgiref.handlers :synopsis: WSGI server/gateway base classes. @@ -622,7 +627,7 @@ input, output, and error streams. The default environment variables to be included in every request's WSGI environment. By default, this is a copy of ``os.environ`` at the time that - :mod:`wsgiref.handlers` was imported, but subclasses can either create their own + :mod:`!wsgiref.handlers` was imported, but subclasses can either create their own at the class or instance level. Note that the dictionary should be considered read-only, since the default value is shared between multiple classes and instances. @@ -773,8 +778,8 @@ input, output, and error streams. .. versionadded:: 3.2 -:mod:`wsgiref.types` -- WSGI types for static type checking ------------------------------------------------------------ +:mod:`!wsgiref.types` -- WSGI types for static type checking +------------------------------------------------------------ .. module:: wsgiref.types :synopsis: WSGI types for static type checking diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 9ffedf7366a..321d93079bc 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -12,7 +12,7 @@ -------------- -:mod:`xml.dom.minidom` is a minimal implementation of the Document Object +:mod:`!xml.dom.minidom` is a minimal implementation of the Document Object Model interface, with an API similar to that in other languages. It is intended to be simpler than the full DOM and also significantly smaller. Users who are not already proficient with the DOM should consider using the @@ -26,7 +26,7 @@ not already proficient with the DOM should consider using the DOM applications typically start by parsing some XML into a DOM. With -:mod:`xml.dom.minidom`, this is done through the parse functions:: +:mod:`!xml.dom.minidom`, this is done through the parse functions:: from xml.dom.minidom import parse, parseString @@ -62,7 +62,7 @@ document. What the :func:`parse` and :func:`parseString` functions do is connect an XML parser with a "DOM builder" that can accept parse events from any SAX parser and -convert them into a DOM tree. The name of the functions are perhaps misleading, +convert them into a DOM tree. The names of the functions are perhaps misleading, but are easy to grasp when learning the interfaces. The parsing of the document will be completed before these functions return; it's simply that these functions do not provide a parser implementation themselves. @@ -70,7 +70,7 @@ functions do not provide a parser implementation themselves. You can also create a :class:`Document` by calling a method on a "DOM Implementation" object. You can get this object either by calling the :func:`getDOMImplementation` function in the :mod:`xml.dom` package or the -:mod:`xml.dom.minidom` module. Once you have a :class:`Document`, you +:mod:`!xml.dom.minidom` module. Once you have a :class:`Document`, you can add child nodes to it to populate the DOM:: from xml.dom.minidom import getDOMImplementation @@ -93,7 +93,7 @@ document: the one that holds all others. Here is an example program:: When you are finished with a DOM tree, you may optionally call the :meth:`unlink` method to encourage early cleanup of the now-unneeded -objects. :meth:`unlink` is an :mod:`xml.dom.minidom`\ -specific +objects. :meth:`unlink` is an :mod:`!xml.dom.minidom`\ -specific extension to the DOM API that renders the node and its descendants essentially useless. Otherwise, Python's garbage collector will eventually take care of the objects in the tree. @@ -101,7 +101,7 @@ eventually take care of the objects in the tree. .. seealso:: `Document Object Model (DOM) Level 1 Specification `_ - The W3C recommendation for the DOM supported by :mod:`xml.dom.minidom`. + The W3C recommendation for the DOM supported by :mod:`!xml.dom.minidom`. .. _minidom-objects: @@ -111,7 +111,7 @@ DOM Objects The definition of the DOM API for Python is given as part of the :mod:`xml.dom` module documentation. This section lists the differences between the API and -:mod:`xml.dom.minidom`. +:mod:`!xml.dom.minidom`. .. method:: Node.unlink() @@ -214,7 +214,7 @@ particular case, we do not take much advantage of the flexibility of the DOM. minidom and the DOM standard ---------------------------- -The :mod:`xml.dom.minidom` module is essentially a DOM 1.0-compatible DOM with +The :mod:`!xml.dom.minidom` module is essentially a DOM 1.0-compatible DOM with some DOM 2 features (primarily namespace features). Usage of the DOM interface in Python is straight-forward. The following mapping @@ -237,7 +237,7 @@ rules apply: * The types ``short int``, ``unsigned int``, ``unsigned long long``, and ``boolean`` all map to Python integer objects. -* The type ``DOMString`` maps to Python strings. :mod:`xml.dom.minidom` supports +* The type ``DOMString`` maps to Python strings. :mod:`!xml.dom.minidom` supports either bytes or strings, but will normally produce strings. Values of type ``DOMString`` may also be ``None`` where allowed to have the IDL ``null`` value by the DOM specification from the W3C. @@ -245,8 +245,8 @@ rules apply: * ``const`` declarations map to variables in their respective scope (e.g. ``xml.dom.minidom.Node.PROCESSING_INSTRUCTION_NODE``); they must not be changed. -* ``DOMException`` is currently not supported in :mod:`xml.dom.minidom`. - Instead, :mod:`xml.dom.minidom` uses standard Python exceptions such as +* ``DOMException`` is currently not supported in :mod:`!xml.dom.minidom`. + Instead, :mod:`!xml.dom.minidom` uses standard Python exceptions such as :exc:`TypeError` and :exc:`AttributeError`. * :class:`NodeList` objects are implemented using Python's built-in list type. @@ -255,7 +255,7 @@ rules apply: however, much more "Pythonic" than the interface defined in the W3C recommendations. -The following interfaces have no implementation in :mod:`xml.dom.minidom`: +The following interfaces have no implementation in :mod:`!xml.dom.minidom`: * :class:`DOMTimeStamp` diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index a21cfaa4645..5027596ed96 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -10,7 +10,7 @@ -------------- -The :mod:`xml.dom.pulldom` module provides a "pull parser" which can also be +The :mod:`!xml.dom.pulldom` module provides a "pull parser" which can also be asked to produce DOM-accessible fragments of the document where necessary. The basic concept involves pulling "events" from a stream of incoming XML and processing them. In contrast to SAX which also employs an event-driven diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index f33b19bc272..8e5a3c13cfd 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -80,7 +80,7 @@ implementations are free to support the strict mapping from IDL). See section Module Contents --------------- -The :mod:`xml.dom` contains the following functions: +The :mod:`!xml.dom` contains the following functions: .. function:: registerDOMImplementation(name, factory) @@ -135,7 +135,7 @@ Some convenience constants are also provided: HyperText Markup Language `_ (section 3.1.1). -In addition, :mod:`xml.dom` contains a base :class:`Node` class and the DOM +In addition, :mod:`!xml.dom` contains a base :class:`Node` class and the DOM exception classes. The :class:`Node` class provided by this module does not implement any of the methods or attributes defined by the DOM specification; concrete DOM implementations must provide those. The :class:`Node` class diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 00075ac2a23..bdd5fd564ee 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -10,7 +10,7 @@ -------------- -The :mod:`xml.etree.ElementTree` module implements a simple and efficient API +The :mod:`!xml.etree.ElementTree` module implements a simple and efficient API for parsing and creating XML data. .. versionchanged:: 3.3 @@ -28,7 +28,7 @@ for parsing and creating XML data. Tutorial -------- -This is a short tutorial for using :mod:`xml.etree.ElementTree` (``ET`` in +This is a short tutorial for using :mod:`!xml.etree.ElementTree` (``ET`` in short). The goal is to demonstrate some of the building blocks and basic concepts of the module. @@ -791,7 +791,7 @@ Here's an example that demonstrates use of the XInclude module. To include an XM By default, the **href** attribute is treated as a file name. You can use custom loaders to override this behaviour. Also note that the standard helper does not support XPointer syntax. -To process this file, load it as usual, and pass the root element to the :mod:`xml.etree.ElementTree` module: +To process this file, load it as usual, and pass the root element to the :mod:`!xml.etree.ElementTree` module: .. code-block:: python diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index acd8d399fe3..81d47147f33 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -20,7 +20,7 @@ Python's interfaces for processing XML are grouped in the ``xml`` package. If you need to parse untrusted or unauthenticated data, see :ref:`xml-security`. -It is important to note that modules in the :mod:`xml` package require that +It is important to note that modules in the :mod:`!xml` package require that there be at least one SAX-compliant XML parser available. The Expat parser is included with Python, so the :mod:`xml.parsers.expat` module will always be available. diff --git a/Doc/library/xml.sax.handler.rst b/Doc/library/xml.sax.handler.rst index f1af7253e43..5079fc0f19e 100644 --- a/Doc/library/xml.sax.handler.rst +++ b/Doc/library/xml.sax.handler.rst @@ -16,7 +16,7 @@ error handlers, entity resolvers and lexical handlers. Applications normally only need to implement those interfaces whose events they are interested in; they can implement the interfaces in a single object or in multiple objects. Handler implementations should inherit from the base classes provided in the -module :mod:`xml.sax.handler`, so that all methods get default implementations. +module :mod:`!xml.sax.handler`, so that all methods get default implementations. .. class:: ContentHandler @@ -53,7 +53,7 @@ module :mod:`xml.sax.handler`, so that all methods get default implementations. Interface used by the parser to represent low frequency events which may not be of interest to many applications. -In addition to these classes, :mod:`xml.sax.handler` provides symbolic constants +In addition to these classes, :mod:`!xml.sax.handler` provides symbolic constants for the feature and property names. diff --git a/Doc/library/xml.sax.rst b/Doc/library/xml.sax.rst index 5fa92645a44..148cb863aca 100644 --- a/Doc/library/xml.sax.rst +++ b/Doc/library/xml.sax.rst @@ -12,7 +12,7 @@ -------------- -The :mod:`xml.sax` package provides a number of modules which implement the +The :mod:`!xml.sax` package provides a number of modules which implement the Simple API for XML (SAX) interface for Python. The package itself provides the SAX exceptions and the convenience functions which will be most used by users of the SAX API. @@ -89,9 +89,9 @@ module :mod:`xml.sax.xmlreader`. The handler interfaces are defined in :mod:`xml.sax.handler`. For convenience, :class:`~xml.sax.xmlreader.InputSource` (which is often instantiated directly) and the handler classes are also available from -:mod:`xml.sax`. These interfaces are described below. +:mod:`!xml.sax`. These interfaces are described below. -In addition to these classes, :mod:`xml.sax` provides the following exception +In addition to these classes, :mod:`!xml.sax` provides the following exception classes. diff --git a/Doc/library/xml.sax.utils.rst b/Doc/library/xml.sax.utils.rst index 7731f03d875..f93fe374e1c 100644 --- a/Doc/library/xml.sax.utils.rst +++ b/Doc/library/xml.sax.utils.rst @@ -11,7 +11,7 @@ -------------- -The module :mod:`xml.sax.saxutils` contains a number of classes and functions +The module :mod:`!xml.sax.saxutils` contains a number of classes and functions that are commonly useful when creating SAX applications, either in direct use, or as base classes. diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index a21c7d3e4e3..8f87a2f52cd 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -23,13 +23,13 @@ between conformable Python objects and XML on the wire. .. warning:: - The :mod:`xmlrpc.client` module is not secure against maliciously + The :mod:`!xmlrpc.client` module is not secure against maliciously constructed data. If you need to parse untrusted or unauthenticated data, see :ref:`xml-security`. .. versionchanged:: 3.5 - For HTTPS URIs, :mod:`xmlrpc.client` now performs all the necessary + For HTTPS URIs, :mod:`!xmlrpc.client` now performs all the necessary certificate and hostname checks by default. .. include:: ../includes/wasm-notavail.rst diff --git a/Doc/library/xmlrpc.server.rst b/Doc/library/xmlrpc.server.rst index 2a8f6f8d5fc..9f16c470567 100644 --- a/Doc/library/xmlrpc.server.rst +++ b/Doc/library/xmlrpc.server.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`xmlrpc.server` module provides a basic server framework for XML-RPC +The :mod:`!xmlrpc.server` module provides a basic server framework for XML-RPC servers written in Python. Servers can either be free standing, using :class:`SimpleXMLRPCServer`, or embedded in a CGI environment, using :class:`CGIXMLRPCRequestHandler`. @@ -19,7 +19,7 @@ servers written in Python. Servers can either be free standing, using .. warning:: - The :mod:`xmlrpc.server` module is not secure against maliciously + The :mod:`!xmlrpc.server` module is not secure against maliciously constructed data. If you need to parse untrusted or unauthenticated data, see :ref:`xml-security`. @@ -230,7 +230,7 @@ a server allowing dotted names and registering a multicall function. Enabling the *allow_dotted_names* option allows intruders to access your module's global variables and may allow intruders to execute arbitrary code on - your machine. Only use this example only within a secure, closed network. + your machine. Only use this example within a secure, closed network. :: diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index cdaba07ab46..2083857312f 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -258,7 +258,7 @@ depending on whether your code is written for Python 2 or 3. Creating Standalone Applications with zipapp -------------------------------------------- -Using the :mod:`zipapp` module, it is possible to create self-contained Python +Using the :mod:`!zipapp` module, it is possible to create self-contained Python programs, which can be distributed to end users who only need to have a suitable version of Python installed on their system. The key to doing this is to bundle all of the application's dependencies into the archive, along diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index ae4e25b13b9..082c4f8d3b4 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -82,7 +82,7 @@ The module defines the following items: Class used to represent information about a member of an archive. Instances of this class are returned by the :meth:`.getinfo` and :meth:`.infolist` - methods of :class:`ZipFile` objects. Most users of the :mod:`zipfile` module + methods of :class:`ZipFile` objects. Most users of the :mod:`!zipfile` module will not need to create these, but only use those created by this module. *filename* should be the full name of the archive member, and *date_time* should be a tuple containing six fields which describe the time @@ -209,7 +209,7 @@ ZipFile objects If *allowZip64* is ``True`` (the default) zipfile will create ZIP files that use the ZIP64 extensions when the zipfile is larger than 4 GiB. If it is - ``false`` :mod:`zipfile` will raise an exception when the ZIP file would + ``false`` :mod:`!zipfile` will raise an exception when the ZIP file would require ZIP64 extensions. The *compresslevel* parameter controls the compression level to use when @@ -957,7 +957,7 @@ Instances have the following methods and attributes: Command-line interface ---------------------- -The :mod:`zipfile` module provides a simple command-line interface to interact +The :mod:`!zipfile` module provides a simple command-line interface to interact with ZIP archives. If you want to create a new ZIP archive, specify its name after the :option:`-c` diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index ead87c2d6ad..7c33db6848a 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -12,7 +12,7 @@ This module adds the ability to import Python modules (:file:`\*.py`, :file:`\*.pyc`) and packages from ZIP-format archives. It is usually not -needed to use the :mod:`zipimport` module explicitly; it is automatically used +needed to use the :mod:`!zipimport` module explicitly; it is automatically used by the built-in :keyword:`import` mechanism for :data:`sys.path` items that are paths to ZIP archives. @@ -184,7 +184,7 @@ Examples -------- Here is an example that imports a module from a ZIP archive - note that the -:mod:`zipimport` module is not explicitly used. +:mod:`!zipimport` module is not explicitly used. .. code-block:: shell-session diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst index efd28b31ef2..6f0b84012f0 100644 --- a/Doc/library/zoneinfo.rst +++ b/Doc/library/zoneinfo.rst @@ -13,9 +13,9 @@ -------------- -The :mod:`zoneinfo` module provides a concrete time zone implementation to +The :mod:`!zoneinfo` module provides a concrete time zone implementation to support the IANA time zone database as originally specified in :pep:`615`. By -default, :mod:`zoneinfo` uses the system's time zone data if available; if no +default, :mod:`!zoneinfo` uses the system's time zone data if available; if no system time zone data is available, the library will fall back to using the first-party :pypi:`tzdata` package available on PyPI. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 52bae88607a..ed73f6c068c 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -544,9 +544,9 @@ The following code:: is semantically equivalent to:: manager = (EXPRESSION) - enter = type(manager).__enter__ - exit = type(manager).__exit__ - value = enter(manager) + enter = manager.__enter__ + exit = manager.__exit__ + value = enter() hit_except = False try: @@ -554,11 +554,14 @@ is semantically equivalent to:: SUITE except: hit_except = True - if not exit(manager, *sys.exc_info()): + if not exit(*sys.exc_info()): raise finally: if not hit_except: - exit(manager, None, None, None) + exit(None, None, None) + +except that implicit :ref:`special method lookup ` is used +for :meth:`~object.__enter__` and :meth:`~object.__exit__`. With more than one item, the context managers are processed as if multiple :keyword:`with` statements were nested:: @@ -1563,13 +1566,12 @@ The following code:: Is semantically equivalent to:: - iter = (ITER) - iter = type(iter).__aiter__(iter) + iter = (ITER).__aiter__() running = True while running: try: - TARGET = await type(iter).__anext__(iter) + TARGET = await iter.__anext__() except StopAsyncIteration: running = False else: @@ -1577,7 +1579,8 @@ Is semantically equivalent to:: else: SUITE2 -See also :meth:`~object.__aiter__` and :meth:`~object.__anext__` for details. +except that implicit :ref:`special method lookup ` is used +for :meth:`~object.__aiter__` and :meth:`~object.__anext__`. It is a :exc:`SyntaxError` to use an ``async for`` statement outside the body of a coroutine function. @@ -1603,9 +1606,9 @@ The following code:: is semantically equivalent to:: manager = (EXPRESSION) - aenter = type(manager).__aenter__ - aexit = type(manager).__aexit__ - value = await aenter(manager) + aenter = manager.__aenter__ + aexit = manager.__aexit__ + value = await aenter() hit_except = False try: @@ -1613,13 +1616,14 @@ is semantically equivalent to:: SUITE except: hit_except = True - if not await aexit(manager, *sys.exc_info()): + if not await aexit(*sys.exc_info()): raise finally: if not hit_except: - await aexit(manager, None, None, None) + await aexit(None, None, None) -See also :meth:`~object.__aenter__` and :meth:`~object.__aexit__` for details. +except that implicit :ref:`special method lookup ` is used +for :meth:`~object.__aenter__` and :meth:`~object.__aexit__`. It is a :exc:`SyntaxError` to use an ``async with`` statement outside the body of a coroutine function. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index ed9389a0d90..e085acded93 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -290,6 +290,7 @@ floating-point numbers. The same caveats apply as for floating-point numbers. The real and imaginary parts of a complex number ``z`` can be retrieved through the read-only attributes ``z.real`` and ``z.imag``. +.. _datamodel-sequences: Sequences --------- @@ -309,12 +310,25 @@ including built-in sequences, interpret negative subscripts by adding the sequence length. For example, ``a[-2]`` equals ``a[n-2]``, the second to last item of sequence a with length ``n``. -.. index:: single: slicing +The resulting value must be a nonnegative integer less than the number of items +in the sequence. If it is not, an :exc:`IndexError` is raised. -Sequences also support slicing: ``a[i:j]`` selects all items with index *k* such -that *i* ``<=`` *k* ``<`` *j*. When used as an expression, a slice is a -sequence of the same type. The comment above about negative indexes also applies +.. index:: + single: slicing + single: start (slice object attribute) + single: stop (slice object attribute) + single: step (slice object attribute) + +Sequences also support slicing: ``a[start:stop]`` selects all items with index *k* such +that *start* ``<=`` *k* ``<`` *stop*. When used as an expression, a slice is a +sequence of the same type. The comment above about negative subscripts also applies to negative slice positions. +Note that no error is raised if a slice position is less than zero or larger +than the length of the sequence. + +If *start* is missing or :data:`None`, slicing behaves as if *start* was zero. +If *stop* is missing or ``None``, slicing behaves as if *stop* was equal to +the length of the sequence. Some sequences also support "extended slicing" with a third "step" parameter: ``a[i:j:k]`` selects all items of *a* with index *x* where ``x = i + n*k``, *n* @@ -345,17 +359,22 @@ Strings pair: built-in function; chr pair: built-in function; ord single: character - single: integer + pair: string; item single: Unicode - A string is a sequence of values that represent Unicode code points. - All the code points in the range ``U+0000 - U+10FFFF`` can be - represented in a string. Python doesn't have a :c:expr:`char` type; - instead, every code point in the string is represented as a string - object with length ``1``. The built-in function :func:`ord` + A string (:class:`str`) is a sequence of values that represent + :dfn:`characters`, or more formally, *Unicode code points*. + All the code points in the range ``0`` to ``0x10FFFF`` can be + represented in a string. + + Python doesn't have a dedicated *character* type. + Instead, every code point in the string is represented as a string + object with length ``1``. + + The built-in function :func:`ord` converts a code point from its string form to an integer in the - range ``0 - 10FFFF``; :func:`chr` converts an integer in the range - ``0 - 10FFFF`` to the corresponding length ``1`` string object. + range ``0`` to ``0x10FFFF``; :func:`chr` converts an integer in the range + ``0`` to ``0x10FFFF`` to the corresponding length ``1`` string object. :meth:`str.encode` can be used to convert a :class:`str` to :class:`bytes` using the given text encoding, and :meth:`bytes.decode` can be used to achieve the opposite. @@ -366,7 +385,7 @@ Tuples pair: singleton; tuple pair: empty; tuple - The items of a tuple are arbitrary Python objects. Tuples of two or + The items of a :class:`tuple` are arbitrary Python objects. Tuples of two or more items are formed by comma-separated lists of expressions. A tuple of one item (a 'singleton') can be formed by affixing a comma to an expression (an expression by itself does not create a tuple, since @@ -376,7 +395,7 @@ Tuples Bytes .. index:: bytes, byte - A bytes object is an immutable array. The items are 8-bit bytes, + A :class:`bytes` object is an immutable array. The items are 8-bit bytes, represented by integers in the range 0 <= x < 256. Bytes literals (like ``b'abc'``) and the built-in :func:`bytes` constructor can be used to create bytes objects. Also, bytes objects can be @@ -461,6 +480,8 @@ Frozen sets a dictionary key. +.. _datamodel-mappings: + Mappings -------- @@ -1396,12 +1417,28 @@ also :func:`os.popen`, :func:`os.fdopen`, and the :meth:`~socket.socket.makefile` method of socket objects (and perhaps by other functions or methods provided by extension modules). +File objects implement common methods, listed below, to simplify usage in +generic code. They are expected to be :ref:`context-managers`. + The objects ``sys.stdin``, ``sys.stdout`` and ``sys.stderr`` are initialized to file objects corresponding to the interpreter's standard input, output and error streams; they are all open in text mode and therefore follow the interface defined by the :class:`io.TextIOBase` abstract class. +.. method:: file.read(size=-1, /) + + Retrieve up to *size* data from the file. As a convenience if *size* is + unspecified or -1 retrieve all data available. + +.. method:: file.write(data, /) + + Store *data* to the file. + +.. method:: file.close() + + Flush any buffers and close the underlying file. + Internal types -------------- @@ -3205,43 +3242,55 @@ through the object's keys; for sequences, it should iterate through the values. .. versionadded:: 3.4 -.. index:: pair: object; slice +.. method:: object.__getitem__(self, subscript) -.. note:: + Called to implement *subscription*, that is, ``self[subscript]``. + See :ref:`subscriptions` for details on the syntax. + + There are two types of built-in objects that support subscription + via :meth:`!__getitem__`: - Slicing is done exclusively with the following three methods. A call like :: + - **sequences**, where *subscript* (also called + :term:`index`) should be an integer or a :class:`slice` object. + See the :ref:`sequence documentation ` for the expected + behavior, including handling :class:`slice` objects and negative indices. + - **mappings**, where *subscript* is also called the :term:`key`. + See :ref:`mapping documentation ` for the expected + behavior. - a[1:2] = b + If *subscript* is of an inappropriate type, :meth:`!__getitem__` + should raise :exc:`TypeError`. + If *subscript* has an inappropriate value, :meth:`!__getitem__` + should raise an :exc:`LookupError` or one of its subclasses + (:exc:`IndexError` for sequences; :exc:`KeyError` for mappings). - is translated to :: + .. index:: pair: object; slice + + .. note:: - a[slice(1, 2, None)] = b + Slicing is handled by :meth:`!__getitem__`, :meth:`~object.__setitem__`, + and :meth:`~object.__delitem__`. + A call like :: - and so forth. Missing slice items are always filled in with ``None``. + a[1:2] = b + is translated to :: -.. method:: object.__getitem__(self, key) + a[slice(1, 2, None)] = b - Called to implement evaluation of ``self[key]``. For :term:`sequence` types, - the accepted keys should be integers. Optionally, they may support - :class:`slice` objects as well. Negative index support is also optional. - If *key* is - of an inappropriate type, :exc:`TypeError` may be raised; if *key* is a value - outside the set of indexes for the sequence (after any special - interpretation of negative values), :exc:`IndexError` should be raised. For - :term:`mapping` types, if *key* is missing (not in the container), - :exc:`KeyError` should be raised. + and so forth. Missing slice items are always filled in with ``None``. .. note:: - :keyword:`for` loops expect that an :exc:`IndexError` will be raised for - illegal indexes to allow proper detection of the end of the sequence. + The sequence iteration protocol (used, for example, in :keyword:`for` + loops), expects that an :exc:`IndexError` will be raised for illegal + indexes to allow proper detection of the end of a sequence. .. note:: - When :ref:`subscripting` a *class*, the special + When :ref:`subscripting ` a *class*, the special class method :meth:`~object.__class_getitem__` may be called instead of - ``__getitem__()``. See :ref:`classgetitem-versus-getitem` for more + :meth:`!__getitem__`. See :ref:`classgetitem-versus-getitem` for more details. diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 165dfa69f88..42b13b61e53 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -9,9 +9,11 @@ Expressions This chapter explains the meaning of the elements of expressions in Python. -**Syntax Notes:** In this and the following chapters, extended BNF notation will -be used to describe syntax, not lexical analysis. When (one alternative of) a -syntax rule has the form +**Syntax Notes:** In this and the following chapters, +:ref:`grammar notation ` will be used to describe syntax, +not lexical analysis. + +When (one alternative of) a syntax rule has the form: .. productionlist:: python-grammar name: othername @@ -29,17 +31,13 @@ Arithmetic conversions When a description of an arithmetic operator below uses the phrase "the numeric arguments are converted to a common real type", this means that the operator -implementation for built-in types works as follows: - -* If both arguments are complex numbers, no conversion is performed; - -* if either argument is a complex or a floating-point number, the other is converted to a floating-point number; +implementation for built-in numeric types works as described in the +:ref:`Numeric Types ` section of the standard +library documentation. -* otherwise, both must be integers and no conversion is necessary. - -Some additional rules apply for certain operators (e.g., a string as a left -argument to the '%' operator). Extensions must define their own conversion -behavior. +Some additional rules apply for certain operators and non-numeric operands +(for example, a string as a left argument to the ``%`` operator). +Extensions must define their own conversion behavior. .. _atoms: @@ -49,15 +47,57 @@ Atoms .. index:: atom -Atoms are the most basic elements of expressions. The simplest atoms are -identifiers or literals. Forms enclosed in parentheses, brackets or braces are -also categorized syntactically as atoms. The syntax for atoms is: +Atoms are the most basic elements of expressions. +The simplest atoms are :ref:`names ` or literals. +Forms enclosed in parentheses, brackets or braces are also categorized +syntactically as atoms. -.. productionlist:: python-grammar - atom: `identifier` | `literal` | `enclosure` - enclosure: `parenth_form` | `list_display` | `dict_display` | `set_display` - : | `generator_expression` | `yield_atom` +Formally, the syntax for atoms is: + +.. grammar-snippet:: + :group: python-grammar + + atom: + | 'True' + | 'False' + | 'None' + | '...' + | `identifier` + | `literal` + | `enclosure` + enclosure: + | `parenth_form` + | `list_display` + | `dict_display` + | `set_display` + | `generator_expression` + | `yield_atom` + + +.. _atom-singletons: +Built-in constants +------------------ + +The keywords ``True``, ``False``, and ``None`` name +:ref:`built-in constants `. +The token ``...`` names the :py:data:`Ellipsis` constant. + +Evaluation of these atoms yields the corresponding value. + +.. note:: + + Several more built-in constants are available as global variables, + but only the ones mentioned here are :ref:`keywords `. + In particular, these names cannot be reassigned or used as attributes: + + .. code-block:: pycon + + >>> False = 123 + File "", line 1 + False = 123 + ^^^^^ + SyntaxError: cannot assign to False .. _atom-identifiers: @@ -131,51 +171,104 @@ Literals .. index:: single: literal -Python supports string and bytes literals and various numeric literals: +A :dfn:`literal` is a textual representation of a value. +Python supports numeric, string and bytes literals. +:ref:`Format strings ` and :ref:`template strings ` +are treated as string literals. + +Numeric literals consist of a single :token:`NUMBER ` +token, which names an integer, floating-point number, or an imaginary number. +See the :ref:`numbers` section in Lexical analysis documentation for details. + +String and bytes literals may consist of several tokens. +See section :ref:`string-concatenation` for details. + +Note that negative and complex numbers, like ``-3`` or ``3+4.2j``, +are syntactically not literals, but :ref:`unary ` or +:ref:`binary ` arithmetic operations involving the ``-`` or ``+`` +operator. + +Evaluation of a literal yields an object of the given type +(:class:`int`, :class:`float`, :class:`complex`, :class:`str`, +:class:`bytes`, or :class:`~string.templatelib.Template`) with the given value. +The value may be approximated in the case of floating-point +and imaginary literals. + +The formal grammar for literals is: .. grammar-snippet:: :group: python-grammar literal: `strings` | `NUMBER` -Evaluation of a literal yields an object of the given type (string, bytes, -integer, floating-point number, complex number) with the given value. The value -may be approximated in the case of floating-point and imaginary (complex) -literals. -See section :ref:`literals` for details. -See section :ref:`string-concatenation` for details on ``strings``. - .. index:: triple: immutable; data; type pair: immutable; object +Literals and object identity +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + All literals correspond to immutable data types, and hence the object's identity is less important than its value. Multiple evaluations of literals with the same value (either the same occurrence in the program text or a different occurrence) may obtain the same object or a different object with the same value. +.. admonition:: CPython implementation detail + + For example, in CPython, *small* integers with the same value evaluate + to the same object:: + + >>> x = 7 + >>> y = 7 + >>> x is y + True + + However, large integers evaluate to different objects:: + + >>> x = 123456789 + >>> y = 123456789 + >>> x is y + False + + This behavior may change in future versions of CPython. + In particular, the boundary between "small" and "large" integers has + already changed in the past. + + CPython will emit a :py:exc:`SyntaxWarning` when you compare literals + using ``is``:: + + >>> x = 7 + >>> x is 7 + :1: SyntaxWarning: "is" with 'int' literal. Did you mean "=="? + True + + See :ref:`faq-identity-with-is` for more information. + +:ref:`Template strings ` are immutable but may reference mutable +objects as :class:`~string.templatelib.Interpolation` values. +For the purposes of this section, two t-strings have the "same value" if +both their structure and the *identity* of the values match. + +.. impl-detail:: + + Currently, each evaluation of a template string results in + a different object. + .. _string-concatenation: String literal concatenation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Multiple adjacent string or bytes literals (delimited by whitespace), possibly +Multiple adjacent string or bytes literals, possibly using different quoting conventions, are allowed, and their meaning is the same as their concatenation:: >>> "hello" 'world' "helloworld" -Formally: - -.. grammar-snippet:: - :group: python-grammar - - strings: ( `STRING` | `fstring`)+ | `tstring`+ - This feature is defined at the syntactical level, so it only works with literals. To concatenate string expressions at run time, the '+' operator may be used:: @@ -208,6 +301,13 @@ string literals:: >>> t"Hello" t"{name}!" Template(strings=('Hello', '!'), interpolations=(...)) +Formally: + +.. grammar-snippet:: + :group: python-grammar + + strings: (`STRING` | `fstring`)+ | `tstring`+ + .. _parenthesized: @@ -893,7 +993,7 @@ Primaries represent the most tightly bound operations of the language. Their syntax is: .. productionlist:: python-grammar - primary: `atom` | `attributeref` | `subscription` | `slicing` | `call` + primary: `atom` | `attributeref` | `subscription` | `call` .. _attribute-references: @@ -932,8 +1032,8 @@ method, that method is called as a fallback. .. _subscriptions: -Subscriptions -------------- +Subscriptions and slicings +-------------------------- .. index:: single: subscription @@ -948,67 +1048,74 @@ Subscriptions pair: object; dictionary pair: sequence; item -The subscription of an instance of a :ref:`container class ` -will generally select an element from the container. The subscription of a -:term:`generic class ` will generally return a -:ref:`GenericAlias ` object. - -.. productionlist:: python-grammar - subscription: `primary` "[" `flexible_expression_list` "]" - -When an object is subscripted, the interpreter will evaluate the primary and -the expression list. +The :dfn:`subscription` syntax is usually used for selecting an element from a +:ref:`container ` -- for example, to get a value from +a :class:`dict`:: -The primary must evaluate to an object that supports subscription. An object -may support subscription through defining one or both of -:meth:`~object.__getitem__` and :meth:`~object.__class_getitem__`. When the -primary is subscripted, the evaluated result of the expression list will be -passed to one of these methods. For more details on when ``__class_getitem__`` -is called instead of ``__getitem__``, see :ref:`classgetitem-versus-getitem`. + >>> digits_by_name = {'one': 1, 'two': 2} + >>> digits_by_name['two'] # Subscripting a dictionary using the key 'two' + 2 -If the expression list contains at least one comma, or if any of the expressions -are starred, the expression list will evaluate to a :class:`tuple` containing -the items of the expression list. Otherwise, the expression list will evaluate -to the value of the list's sole member. +In the subscription syntax, the object being subscribed -- a +:ref:`primary ` -- is followed by a :dfn:`subscript` in +square brackets. +In the simplest case, the subscript is a single expression. + +Depending on the type of the object being subscribed, the subscript is +sometimes called a :term:`key` (for mappings), :term:`index` (for sequences), +or *type argument* (for :term:`generic types `). +Syntactically, these are all equivalent:: + + >>> colors = ['red', 'blue', 'green', 'black'] + >>> colors[3] # Subscripting a list using the index 3 + 'black' + + >>> list[str] # Parameterizing the list type using the type argument str + list[str] + +At runtime, the interpreter will evaluate the primary and +the subscript, and call the primary's :meth:`~object.__getitem__` or +:meth:`~object.__class_getitem__` :term:`special method` with the subscript +as argument. +For more details on which of these methods is called, see +:ref:`classgetitem-versus-getitem`. + +To show how subscription works, we can define a custom object that +implements :meth:`~object.__getitem__` and prints out the value of +the subscript:: + + >>> class SubscriptionDemo: + ... def __getitem__(self, key): + ... print(f'subscripted with: {key!r}') + ... + >>> demo = SubscriptionDemo() + >>> demo[1] + subscripted with: 1 + >>> demo['a' * 3] + subscripted with: 'aaa' -.. versionchanged:: 3.11 - Expressions in an expression list may be starred. See :pep:`646`. - -For built-in objects, there are two types of objects that support subscription -via :meth:`~object.__getitem__`: - -1. Mappings. If the primary is a :term:`mapping`, the expression list must - evaluate to an object whose value is one of the keys of the mapping, and the - subscription selects the value in the mapping that corresponds to that key. - An example of a builtin mapping class is the :class:`dict` class. -2. Sequences. If the primary is a :term:`sequence`, the expression list must - evaluate to an :class:`int` or a :class:`slice` (as discussed in the - following section). Examples of builtin sequence classes include the - :class:`str`, :class:`list` and :class:`tuple` classes. - -The formal syntax makes no special provision for negative indices in -:term:`sequences `. However, built-in sequences all provide a :meth:`~object.__getitem__` -method that interprets negative indices by adding the length of the sequence -to the index so that, for example, ``x[-1]`` selects the last item of ``x``. The -resulting value must be a nonnegative integer less than the number of items in -the sequence, and the subscription selects the item whose index is that value -(counting from zero). Since the support for negative indices and slicing -occurs in the object's :meth:`~object.__getitem__` method, subclasses overriding -this method will need to explicitly add that support. +See :meth:`~object.__getitem__` documentation for how built-in types handle +subscription. -.. index:: - single: character - pair: string; item +Subscriptions may also be used as targets in :ref:`assignment ` or +:ref:`deletion ` statements. +In these cases, the interpreter will call the subscripted object's +:meth:`~object.__setitem__` or :meth:`~object.__delitem__` +:term:`special method`, respectively, instead of :meth:`~object.__getitem__`. -A :class:`string ` is a special kind of sequence whose items are -*characters*. A character is not a separate data type but a -string of exactly one character. +.. code-block:: + >>> colors = ['red', 'blue', 'green', 'black'] + >>> colors[3] = 'white' # Setting item at index + >>> colors + ['red', 'blue', 'green', 'white'] + >>> del colors[3] # Deleting item at index 3 + >>> colors + ['red', 'blue', 'green'] -.. _slicings: +All advanced forms of *subscript* documented in the following sections +are also usable for assignment and deletion. -Slicings --------- .. index:: single: slicing @@ -1022,43 +1129,111 @@ Slicings pair: object; tuple pair: object; list -A slicing selects a range of items in a sequence object (e.g., a string, tuple -or list). Slicings may be used as expressions or as targets in assignment or -:keyword:`del` statements. The syntax for a slicing: +.. _slicings: -.. productionlist:: python-grammar - slicing: `primary` "[" `slice_list` "]" - slice_list: `slice_item` ("," `slice_item`)* [","] - slice_item: `expression` | `proper_slice` - proper_slice: [`lower_bound`] ":" [`upper_bound`] [ ":" [`stride`] ] - lower_bound: `expression` - upper_bound: `expression` - stride: `expression` - -There is ambiguity in the formal syntax here: anything that looks like an -expression list also looks like a slice list, so any subscription can be -interpreted as a slicing. Rather than further complicating the syntax, this is -disambiguated by defining that in this case the interpretation as a subscription -takes priority over the interpretation as a slicing (this is the case if the -slice list contains no proper slice). +Slicings +^^^^^^^^ -.. index:: - single: start (slice object attribute) - single: stop (slice object attribute) - single: step (slice object attribute) - -The semantics for a slicing are as follows. The primary is indexed (using the -same :meth:`~object.__getitem__` method as -normal subscription) with a key that is constructed from the slice list, as -follows. If the slice list contains at least one comma, the key is a tuple -containing the conversion of the slice items; otherwise, the conversion of the -lone slice item is the key. The conversion of a slice item that is an -expression is that expression. The conversion of a proper slice is a slice -object (see section :ref:`types`) whose :attr:`~slice.start`, -:attr:`~slice.stop` and :attr:`~slice.step` attributes are the values of the -expressions given as lower bound, upper bound and stride, respectively, -substituting ``None`` for missing expressions. +A more advanced form of subscription, :dfn:`slicing`, is commonly used +to extract a portion of a :ref:`sequence `. +In this form, the subscript is a :term:`slice`: up to three +expressions separated by colons. +Any of the expressions may be omitted, but a slice must contain at least one +colon:: + + >>> number_names = ['zero', 'one', 'two', 'three', 'four', 'five'] + >>> number_names[1:3] + ['one', 'two'] + >>> number_names[1:] + ['one', 'two', 'three', 'four', 'five'] + >>> number_names[:3] + ['zero', 'one', 'two'] + >>> number_names[:] + ['zero', 'one', 'two', 'three', 'four', 'five'] + >>> number_names[::2] + ['zero', 'two', 'four'] + >>> number_names[:-3] + ['zero', 'one', 'two'] + >>> del number_names[4:] + >>> number_names + ['zero', 'one', 'two', 'three'] + +When a slice is evaluated, the interpreter constructs a :class:`slice` object +whose :attr:`~slice.start`, :attr:`~slice.stop` and +:attr:`~slice.step` attributes, respectively, are the results of the +expressions between the colons. +Any missing expression evaluates to :const:`None`. +This :class:`!slice` object is then passed to the :meth:`~object.__getitem__` +or :meth:`~object.__class_getitem__` :term:`special method`, as above. :: + + # continuing with the SubscriptionDemo instance defined above: + >>> demo[2:3] + subscripted with: slice(2, 3, None) + >>> demo[::'spam'] + subscripted with: slice(None, None, 'spam') + + +Comma-separated subscripts +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The subscript can also be given as two or more comma-separated expressions +or slices:: + + # continuing with the SubscriptionDemo instance defined above: + >>> demo[1, 2, 3] + subscripted with: (1, 2, 3) + >>> demo[1:2, 3] + subscripted with: (slice(1, 2, None), 3) + +This form is commonly used with numerical libraries for slicing +multi-dimensional data. +In this case, the interpreter constructs a :class:`tuple` of the results of the +expressions or slices, and passes this tuple to the :meth:`~object.__getitem__` +or :meth:`~object.__class_getitem__` :term:`special method`, as above. + +The subscript may also be given as a single expression or slice followed +by a comma, to specify a one-element tuple:: + + >>> demo['spam',] + subscripted with: ('spam',) + + +"Starred" subscriptions +^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 3.11 + Expressions in *tuple_slices* may be starred. See :pep:`646`. + +The subscript can also contain a starred expression. +In this case, the interpreter unpacks the result into a tuple, and passes +this tuple to :meth:`~object.__getitem__` or :meth:`~object.__class_getitem__`:: + + # continuing with the SubscriptionDemo instance defined above: + >>> demo[*range(10)] + subscripted with: (0, 1, 2, 3, 4, 5, 6, 7, 8, 9) + +Starred expressions may be combined with comma-separated expressions +and slices:: + + >>> demo['a', 'b', *range(3), 'c'] + subscripted with: ('a', 'b', 0, 1, 2, 'c') + + +Formal subscription grammar +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. grammar-snippet:: + :group: python-grammar + + subscription: `primary` '[' `subscript` ']' + subscript: `single_subscript` | `tuple_subscript` + single_subscript: `proper_slice` | `assignment_expression` + proper_slice: [`expression`] ":" [`expression`] [ ":" [`expression`] ] + tuple_subscript: ','.(`single_subscript` | `starred_expression`)+ [','] +Recall that the ``|`` operator :ref:`denotes ordered choice `. +Specifically, in :token:`!subscript`, if both alternatives would match, the +first (:token:`!single_subscript`) has priority. .. index:: pair: object; callable @@ -1297,8 +1472,9 @@ for the operands): ``-1**2`` results in ``-1``. The power operator has the same semantics as the built-in :func:`pow` function, when called with two arguments: it yields its left argument raised to the power -of its right argument. The numeric arguments are first converted to a common -type, and the result is of that type. +of its right argument. +Numeric arguments are first :ref:`converted to a common type `, +and the result is of that type. For int operands, the result has the same type as the operands unless the second argument is negative; in that case, all arguments are converted to float and a @@ -1384,9 +1560,10 @@ operators and one for additive operators: The ``*`` (multiplication) operator yields the product of its arguments. The arguments must either both be numbers, or one argument must be an integer and -the other must be a sequence. In the former case, the numbers are converted to a -common real type and then multiplied together. In the latter case, sequence -repetition is performed; a negative repetition factor yields an empty sequence. +the other must be a sequence. In the former case, the numbers are +:ref:`converted to a common real type ` and then +multiplied together. In the latter case, sequence repetition is performed; +a negative repetition factor yields an empty sequence. This operation can be customized using the special :meth:`~object.__mul__` and :meth:`~object.__rmul__` methods. @@ -1414,7 +1591,8 @@ This operation can be customized using the special :meth:`~object.__matmul__` an pair: operator; // The ``/`` (division) and ``//`` (floor division) operators yield the quotient of -their arguments. The numeric arguments are first converted to a common type. +their arguments. The numeric arguments are first +:ref:`converted to a common type `. Division of integers yields a float, while floor division of integers results in an integer; the result is that of mathematical division with the 'floor' function applied to the result. Division by zero raises the :exc:`ZeroDivisionError` @@ -1430,8 +1608,9 @@ The floor division operation can be customized using the special pair: operator; % (percent) The ``%`` (modulo) operator yields the remainder from the division of the first -argument by the second. The numeric arguments are first converted to a common -type. A zero right argument raises the :exc:`ZeroDivisionError` exception. The +argument by the second. The numeric arguments are first +:ref:`converted to a common type `. +A zero right argument raises the :exc:`ZeroDivisionError` exception. The arguments may be floating-point numbers, e.g., ``3.14%0.7`` equals ``0.34`` (since ``3.14`` equals ``4*0.7 + 0.34``.) The modulo operator always yields a result with the same sign as its second operand (or zero); the absolute value of @@ -1462,7 +1641,9 @@ floating-point number using the :func:`abs` function if appropriate. The ``+`` (addition) operator yields the sum of its arguments. The arguments must either both be numbers or both be sequences of the same type. In the -former case, the numbers are converted to a common real type and then added together. +former case, the numbers are +:ref:`converted to a common real type ` and then +added together. In the latter case, the sequences are concatenated. This operation can be customized using the special :meth:`~object.__add__` and @@ -1477,8 +1658,9 @@ This operation can be customized using the special :meth:`~object.__add__` and single: operator; - (minus) single: - (minus); binary operator -The ``-`` (subtraction) operator yields the difference of its arguments. The -numeric arguments are first converted to a common real type. +The ``-`` (subtraction) operator yields the difference of its arguments. +The numeric arguments are first +:ref:`converted to a common real type `. This operation can be customized using the special :meth:`~object.__sub__` and :meth:`~object.__rsub__` methods. @@ -2076,7 +2258,7 @@ precedence and have a left-to-right chaining feature as described in the | ``{key: value...}``, | dictionary display, | | ``{expressions...}`` | set display | +-----------------------------------------------+-------------------------------------+ -| ``x[index]``, ``x[index:index]``, | Subscription, slicing, | +| ``x[index]``, ``x[index:index]`` | Subscription (including slicing), | | ``x(arguments...)``, ``x.attribute`` | call, attribute reference | +-----------------------------------------------+-------------------------------------+ | :keyword:`await x ` | Await expression | diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 9c022570e7e..36b30c9b16b 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -91,11 +91,10 @@ attributes or items of mutable objects: : | "[" [`target_list`] "]" : | `attributeref` : | `subscription` - : | `slicing` : | "*" `target` -(See section :ref:`primaries` for the syntax definitions for *attributeref*, -*subscription*, and *slicing*.) +(See section :ref:`primaries` for the syntax definitions for *attributeref* +and *subscription*.) An assignment statement evaluates the expression list (remember that this can be a single expression or a comma-separated list, the latter yielding a tuple) and @@ -107,8 +106,8 @@ right. pair: target; list Assignment is defined recursively depending on the form of the target (list). -When a target is part of a mutable object (an attribute reference, subscription -or slicing), the mutable object must ultimately perform the assignment and +When a target is part of a mutable object (an attribute reference or +subscription), the mutable object must ultimately perform the assignment and decide about its validity, and may raise an exception if the assignment is unacceptable. The rules observed by various types and the exceptions raised are given with the definition of the object types (see section :ref:`types`). @@ -189,9 +188,14 @@ Assignment of an object to a single target is recursively defined as follows. pair: object; mutable * If the target is a subscription: The primary expression in the reference is - evaluated. It should yield either a mutable sequence object (such as a list) - or a mapping object (such as a dictionary). Next, the subscript expression is evaluated. + Next, the subscript expression is evaluated. + Then, the primary's :meth:`~object.__setitem__` method is called with + two arguments: the subscript and the assigned object. + + Typically, :meth:`~object.__setitem__` is defined on mutable sequence objects + (such as lists) and mapping objects (such as dictionaries), and behaves as + follows. .. index:: pair: object; sequence @@ -214,16 +218,13 @@ Assignment of an object to a single target is recursively defined as follows. object. This can either replace an existing key/value pair with the same key value, or insert a new key/value pair (if no key with the same value existed). - For user-defined objects, the :meth:`~object.__setitem__` method is called with - appropriate arguments. - .. index:: pair: slicing; assignment -* If the target is a slicing: The primary expression in the reference is - evaluated. It should yield a mutable sequence object (such as a list). The - assigned object should be a sequence object of the same type. Next, the lower - and upper bound expressions are evaluated, insofar they are present; defaults - are zero and the sequence's length. The bounds should evaluate to integers. + If the target is a slicing: The primary expression should evaluate to + a mutable sequence object (such as a list). + The assigned object should be :term:`iterable`. + The slicing's lower and upper bounds should be integers; if they are ``None`` + (or not present), the defaults are zero and the sequence's length. If either bound is negative, the sequence's length is added to it. The resulting bounds are clipped to lie between zero and the sequence's length, inclusive. Finally, the sequence object is asked to replace the slice with @@ -231,12 +232,6 @@ Assignment of an object to a single target is recursively defined as follows. from the length of the assigned sequence, thus changing the length of the target sequence, if the target sequence allows it. -.. impl-detail:: - - In the current implementation, the syntax for targets is taken to be the same - as for expressions, and invalid syntax is rejected during the code generation - phase, causing less detailed error messages. - Although the definition of assignment implies that overlaps between the left-hand side and the right-hand side are 'simultaneous' (for example ``a, b = b, a`` swaps two variables), overlaps *within* the collection of assigned-to @@ -281,7 +276,7 @@ operation and an assignment statement: .. productionlist:: python-grammar augmented_assignment_stmt: `augtarget` `augop` (`expression_list` | `yield_expression`) - augtarget: `identifier` | `attributeref` | `subscription` | `slicing` + augtarget: `identifier` | `attributeref` | `subscription` augop: "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**=" : | ">>=" | "<<=" | "&=" | "^=" | "|=" @@ -470,7 +465,7 @@ in the same code block. Trying to delete an unbound name raises a .. index:: pair: attribute; deletion -Deletion of attribute references, subscriptions and slicings is passed to the +Deletion of attribute references and subscriptions is passed to the primary object involved; deletion of a slicing is in general equivalent to assignment of an empty slice of the right type (but even this is determined by the sliced object). @@ -836,7 +831,9 @@ where the :keyword:`import` statement occurs. The *public names* defined by a module are determined by checking the module's namespace for a variable named ``__all__``; if defined, it must be a sequence -of strings which are names defined or imported by that module. The names +of strings which are names defined or imported by that module. +Names containing non-ASCII characters must be in the `normalization form`_ +NFKC; see :ref:`lexical-names-nonascii` for details. The names given in ``__all__`` are all considered public and are required to exist. If ``__all__`` is not defined, the set of public names includes all names found in the module's namespace which do not begin with an underscore character @@ -870,6 +867,8 @@ determine dynamically the modules to be loaded. .. audit-event:: import module,filename,sys.path,sys.meta_path,sys.path_hooks import +.. _normalization form: https://www.unicode.org/reports/tr15/#Norm_Forms + .. _future: Future statements diff --git a/Doc/requirements.txt b/Doc/requirements.txt index d0107744ecb..536ae57e4ef 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -18,4 +18,6 @@ sphinx-notfound-page~=1.0.0 # to install that as well. python-docs-theme>=2023.3.1,!=2023.7 +linklint + -c constraints.txt diff --git a/Doc/tools/check-html-ids.py b/Doc/tools/check-html-ids.py new file mode 100644 index 00000000000..8e8e0a581df --- /dev/null +++ b/Doc/tools/check-html-ids.py @@ -0,0 +1,181 @@ +from compression import gzip +import concurrent.futures +from pathlib import Path +import html.parser +import functools +import argparse +import json +import sys +import re + + +IGNORED_ID_RE = re.compile( + r""" + index-\d+ + | id\d+ + | [_a-z]+_\d+ +""", + re.VERBOSE, +) + + +class IDGatherer(html.parser.HTMLParser): + def __init__(self, ids): + super().__init__() + self.__ids = ids + + def handle_starttag(self, tag, attrs): + for name, value in attrs: + if name == 'id': + if not IGNORED_ID_RE.fullmatch(value): + self.__ids.add(value) + + +def get_ids_from_file(path): + ids = set() + gatherer = IDGatherer(ids) + with path.open(encoding='utf-8') as file: + while chunk := file.read(4096): + gatherer.feed(chunk) + return ids + + +def gather_ids(htmldir, *, verbose_print): + if not htmldir.joinpath('objects.inv').exists(): + raise ValueError(f'{htmldir!r} is not a Sphinx HTML output directory') + + if sys._is_gil_enabled: + pool = concurrent.futures.ProcessPoolExecutor() + else: + pool = concurrent.futures.ThreadPoolExecutor() + tasks = {} + for path in htmldir.glob('**/*.html'): + relative_path = path.relative_to(htmldir) + if '_static' in relative_path.parts: + continue + if 'whatsnew' in relative_path.parts: + continue + tasks[relative_path] = pool.submit(get_ids_from_file, path=path) + + ids_by_page = {} + for relative_path, future in tasks.items(): + verbose_print(relative_path) + ids = future.result() + ids_by_page[str(relative_path)] = ids + verbose_print(f' - {len(ids)} ids found') + + common = set.intersection(*ids_by_page.values()) + verbose_print(f'Filtering out {len(common)} common ids') + for key, page_ids in ids_by_page.items(): + ids_by_page[key] = sorted(page_ids - common) + + return ids_by_page + + +def do_check(baseline, checked, excluded, *, verbose_print): + successful = True + for name, baseline_ids in sorted(baseline.items()): + try: + checked_ids = checked[name] + except KeyError: + successful = False + print(f'{name}: (page missing)') + print() + else: + missing_ids = set(baseline_ids) - set(checked_ids) + if missing_ids: + missing_ids = { + a + for a in missing_ids + if not IGNORED_ID_RE.fullmatch(a) + and (name, a) not in excluded + } + if missing_ids: + successful = False + for missing_id in sorted(missing_ids): + print(f'{name}: {missing_id}') + print() + return successful + + +def main(argv): + parser = argparse.ArgumentParser() + parser.add_argument( + '-v', + '--verbose', + action='store_true', + help='print out more information', + ) + subparsers = parser.add_subparsers(dest='command', required=True) + + collect = subparsers.add_parser( + 'collect', help='collect IDs from a set of HTML files' + ) + collect.add_argument( + 'htmldir', type=Path, help='directory with HTML documentation' + ) + collect.add_argument( + '-o', + '--outfile', + help='File to save the result in; default /html-ids.json.gz', + ) + + check = subparsers.add_parser('check', help='check two archives of IDs') + check.add_argument( + 'baseline_file', type=Path, help='file with baseline IDs' + ) + check.add_argument('checked_file', type=Path, help='file with checked IDs') + check.add_argument( + '-x', + '--exclude-file', + type=Path, + help='file with IDs to exclude from the check', + ) + + args = parser.parse_args(argv[1:]) + + if args.verbose: + verbose_print = functools.partial(print, file=sys.stderr) + else: + + def verbose_print(*args, **kwargs): + """do nothing""" + + if args.command == 'collect': + ids = gather_ids(args.htmldir, verbose_print=verbose_print) + if args.outfile is None: + args.outfile = args.htmldir / 'html-ids.json.gz' + with gzip.open(args.outfile, 'wt', encoding='utf-8') as zfile: + json.dump({'ids_by_page': ids}, zfile) + + if args.command == 'check': + with gzip.open(args.baseline_file) as zfile: + baseline = json.load(zfile)['ids_by_page'] + with gzip.open(args.checked_file) as zfile: + checked = json.load(zfile)['ids_by_page'] + excluded = set() + if args.exclude_file: + with open(args.exclude_file, encoding='utf-8') as file: + for line in file: + line = line.strip() + if line and not line.startswith('#'): + name, sep, excluded_id = line.partition(':') + if sep: + excluded.add((name.strip(), excluded_id.strip())) + if do_check(baseline, checked, excluded, verbose_print=verbose_print): + verbose_print('All OK') + else: + sys.stdout.flush() + print( + 'ERROR: Removed IDs found', + 'The above HTML IDs were removed from the documentation, ' + + 'resulting in broken links. Please add them back.', + sep='\n', + file=sys.stderr, + ) + if args.exclude_file: + print(f'Alternatively, add them to {args.exclude_file}.') + + +if __name__ == '__main__': + main(sys.argv) diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html index 0d83ac9f78a..8feeed2fee3 100644 --- a/Doc/tools/templates/customsourcelink.html +++ b/Doc/tools/templates/customsourcelink.html @@ -1,10 +1,25 @@ {%- if show_source and has_source and sourcename %} +

{{ _('This page') }}

  • {% trans %}Report a bug{% endtrans %}
    • +
  • {% trans %}Improve this page{% endtrans %}
  • - {{ _('Show source') }}
    • diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 645acdf20fb..3c690343022 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -420,7 +420,7 @@ of the class:: 'Buddy' As discussed in :ref:`tut-object`, shared data can have possibly surprising -effects with involving :term:`mutable` objects such as lists and dictionaries. +effects involving :term:`mutable` objects such as lists and dictionaries. For example, the *tricks* list in the following code should not be used as a class variable because just a single list would be shared by all *Dog* instances:: diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index f3e69756401..bee6cc39faf 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -155,8 +155,8 @@ that takes an iterable is :func:`sum`:: 6 Later we will see more functions that return iterables and take iterables as -arguments. In chapter :ref:`tut-structures`, we will discuss in more detail about -:func:`list`. +arguments. In chapter :ref:`tut-structures`, we will discuss :func:`list` in more +detail. .. _tut-break: @@ -441,7 +441,7 @@ Several other key features of this statement: ``False`` and ``None`` are compared by identity. - Patterns may use named constants. These must be dotted names - to prevent them from being interpreted as capture variable:: + to prevent them from being interpreted as capture variables:: from enum import Enum class Color(Enum): @@ -1107,7 +1107,7 @@ Intermezzo: Coding Style Now that you are about to write longer, more complex pieces of Python, it is a good time to talk about *coding style*. Most languages can be written (or more -concise, *formatted*) in different styles; some are more readable than others. +concisely, *formatted*) in different styles; some are more readable than others. Making it easy for others to read your code is always a good idea, and adopting a nice coding style helps tremendously for that. diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index deabac52530..7778e37a9ad 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -184,11 +184,11 @@ If you don't want characters prefaced by ``\`` to be interpreted as special characters, you can use *raw strings* by adding an ``r`` before the first quote:: - >>> print('C:\some\name') # here \n means newline! - C:\some + >>> print('C:\this\name') # here \t means tab, \n means newline + C: his ame - >>> print(r'C:\some\name') # note the r before the quote - C:\some\name + >>> print(r'C:\this\name') # note the r before the quote + C:\this\name There is one subtle aspect to raw strings: a raw string may not end in an odd number of ``\`` characters; see diff --git a/Doc/tutorial/whatnow.rst b/Doc/tutorial/whatnow.rst index 359cf80a7b2..aae8f29b007 100644 --- a/Doc/tutorial/whatnow.rst +++ b/Doc/tutorial/whatnow.rst @@ -68,6 +68,6 @@ already contain the solution for your problem. .. rubric:: Footnotes -.. [#] "Cheese Shop" is a Monty Python's sketch: a customer enters a cheese shop, +.. [#] "Cheese Shop" is a Monty Python sketch: a customer enters a cheese shop, but whatever cheese he asks for, the clerk says it's missing. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 24541e84732..09be3af0abe 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -1045,6 +1045,13 @@ conflict. * ``pymalloc_debug``: same as ``pymalloc`` but also install debug hooks. * ``mimalloc_debug``: same as ``mimalloc`` but also install debug hooks. + .. note:: + + In the :term:`free-threaded ` build, the ``malloc``, + ``malloc_debug``, ``pymalloc``, and ``pymalloc_debug`` values are not + supported. Only ``default``, ``debug``, ``mimalloc``, and + ``mimalloc_debug`` are accepted. + .. versionadded:: 3.6 .. versionchanged:: 3.7 @@ -1054,12 +1061,13 @@ conflict. .. envvar:: PYTHONMALLOCSTATS If set to a non-empty string, Python will print statistics of the - :ref:`pymalloc memory allocator ` every time a new pymalloc object - arena is created, and on shutdown. + :ref:`pymalloc memory allocator ` or the + :ref:`mimalloc memory allocator ` (whichever is in use) + every time a new object arena is created, and on shutdown. This variable is ignored if the :envvar:`PYTHONMALLOC` environment variable is used to force the :c:func:`malloc` allocator of the C library, or if - Python is configured without ``pymalloc`` support. + Python is configured without both ``pymalloc`` and ``mimalloc`` support. .. versionchanged:: 3.6 This variable can now also be used on Python compiled in release mode. diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 61e21fba74c..7372ae71a4b 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -747,6 +747,9 @@ also be used to improve performance. Disable the fast :ref:`mimalloc ` allocator (enabled by default). + This option cannot be used together with :option:`--disable-gil` + because the :term:`free-threaded ` build requires mimalloc. + See also :envvar:`PYTHONMALLOC` environment variable. .. option:: --without-pymalloc diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 0e728e5dd9c..38e6111c765 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -287,6 +287,12 @@ work. Passing ``--dry-run`` will generate output and logs, but will not modify any installs. +Passing ``--refresh`` will update all registrations for installed runtimes. This +will recreate Start menu shortcuts, registry keys, and global aliases (such as +``python3.14.exe`` or for any installed scripts). These are automatically +refreshed on installation of any runtime, but may need to be manually refreshed +after installing packages. + In addition to the above options, the ``--target`` option will extract the runtime to the specified directory instead of doing a normal install. This is useful for embedding runtimes into larger applications. @@ -469,6 +475,14 @@ customization. - ``PYTHON_MANAGER_SOURCE_URL`` - Override the index feed to obtain new installs from. + * - ``install.enable_entrypoints`` + - (none) + - True to generate global commands for installed packages (such as + ``pip.exe``). These are defined by the packages themselves. + If set to false, only the Python interpreter has global commands created. + By default, true. You should run ``py install --refresh`` after changing + this setting. + * - ``list.format`` - ``PYTHON_MANAGER_LIST_FORMAT`` - Specify the default format used by the ``py list`` command. @@ -482,8 +496,8 @@ customization. * - ``global_dir`` - (none) - - Specify the directory where global commands (such as ``python3.14.exe``) - are stored. + - Specify the directory where global commands (such as ``python3.14.exe`` + and ``pip.exe``) are stored. This directory should be added to your :envvar:`PATH` to make the commands available from your terminal. @@ -493,6 +507,7 @@ customization. This directory is a temporary cache, and can be cleaned up from time to time. + Dotted names should be nested inside JSON objects, for example, ``list.format`` would be specified as ``{"list": {"format": "table"}}``. @@ -739,6 +754,14 @@ directory containing the configuration file that specified them. (e.g. ``"pep514,start"``). Disabled shortcuts are not reactivated by ``enable_shortcut_kinds``. + * - ``install.hard_link_entrypoints`` + - True to use hard links for global shortcuts to save disk space. If false, + each shortcut executable is copied instead. After changing this setting, + you must run ``py install --refresh --force`` to update existing + commands. + By default, true. Disabling this may be necessary for troubleshooting or + systems that have issues with file links. + * - ``pep514_root`` - Registry location to read and write PEP 514 entries into. By default, :file:`HKEY_CURRENT_USER\\Software\\Python`. @@ -878,12 +901,22 @@ default). * - - The package may be available but missing the generated executable. - We recommend using the ``python -m pip`` command instead, - or alternatively the ``python -m pip install --force pip`` command - will recreate the executables and show you the path to - add to :envvar:`PATH`. - These scripts are separated for each runtime, and so you may need to - add multiple paths. + We recommend using the ``python -m pip`` command instead. + Running ``py install --refresh`` and ensuring that the global shortcuts + directory is on :envvar:`PATH` (it will be shown in the command output if + it is not) should make commands such as ``pip`` (and other installed + packages) available. + + * - I installed a package with ``pip`` but its command is not found. + - Have you activated a virtual environment? + Run the ``.venv\Scripts\activate`` script in your terminal to activate. + + * - + - New packages do not automatically have global shortcuts created by the + Python install manager. Similarly, uninstalled packages do not have their + shortcuts removed. + Run ``py install --refresh`` to update the global shortcuts for newly + installed packages. * - Typing ``script-name.py`` in the terminal opens in a new window. - This is a known limitation of the operating system. Either specify ``py`` diff --git a/Include/internal/pycore_cell.h b/Include/internal/pycore_cell.h index cef01e80514..d0d45a23436 100644 --- a/Include/internal/pycore_cell.h +++ b/Include/internal/pycore_cell.h @@ -53,7 +53,7 @@ _PyCell_GetStackRef(PyCellObject *cell) { PyObject *value; #ifdef Py_GIL_DISABLED - value = _Py_atomic_load_ptr(&cell->ob_ref); + value = _PyObject_CAST(_Py_atomic_load_ptr(&cell->ob_ref)); if (value == NULL) { return PyStackRef_NULL; } diff --git a/Include/internal/pycore_emscripten_trampoline.h b/Include/internal/pycore_emscripten_trampoline.h index 16916f1a8eb..e37c53a64f4 100644 --- a/Include/internal/pycore_emscripten_trampoline.h +++ b/Include/internal/pycore_emscripten_trampoline.h @@ -27,9 +27,6 @@ #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) -void -_Py_EmscriptenTrampoline_Init(_PyRuntimeState *runtime); - PyObject* _PyEM_TrampolineCall(PyCFunctionWithKeywords func, PyObject* self, diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 4187f46995e..f4c699616a1 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -99,6 +99,8 @@ extern "C" { _Py_atomic_store_ulong_relaxed(&value, new_value) #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ _Py_atomic_store_ssize_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) \ + _Py_atomic_store_ssize_release(&value, new_value) #define FT_ATOMIC_STORE_FLOAT_RELAXED(value, new_value) \ _Py_atomic_store_float_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) \ @@ -117,6 +119,8 @@ extern "C" { _Py_atomic_load_ullong_relaxed(&value) #define FT_ATOMIC_ADD_SSIZE(value, new_value) \ (void)_Py_atomic_add_ssize(&value, new_value) +#define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) +#define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -138,6 +142,7 @@ extern "C" { #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value @@ -168,6 +173,8 @@ extern "C" { #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_ADD_SSIZE(value, new_value) (void)(value += new_value) +#define FT_MUTEX_LOCK(lock) do {} while (0) +#define FT_MUTEX_UNLOCK(lock) do {} while (0) #endif diff --git a/Include/internal/pycore_runtime_structs.h b/Include/internal/pycore_runtime_structs.h index c34d7e7edc0..78f236a51a1 100644 --- a/Include/internal/pycore_runtime_structs.h +++ b/Include/internal/pycore_runtime_structs.h @@ -279,6 +279,16 @@ struct pyruntimestate { struct _types_runtime_state types; struct _Py_time_runtime_state time; +#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) + // Used in "Python/emscripten_trampoline.c" to choose between wasm-gc + // trampoline and JavaScript trampoline. + PyObject* (*emscripten_trampoline)(int* success, + PyCFunctionWithKeywords func, + PyObject* self, + PyObject* args, + PyObject* kw); +#endif + /* All the objects that are shared by the runtime's interpreters. */ struct _Py_cached_objects cached_objects; struct _Py_static_objects static_objects; diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 52acd918c9b..76f6333739d 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -735,6 +735,13 @@ _PyThreadState_PushCStackRef(PyThreadState *tstate, _PyCStackRef *ref) ref->ref = PyStackRef_NULL; } +static inline void +_PyThreadState_PushCStackRefNew(PyThreadState *tstate, _PyCStackRef *ref, PyObject *obj) +{ + _PyThreadState_PushCStackRef(tstate, ref); + ref->ref = PyStackRef_FromPyObjectNew(obj); +} + static inline void _PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref) { diff --git a/Include/object.h b/Include/object.h index 4d577d0e32d..50a41e355ed 100644 --- a/Include/object.h +++ b/Include/object.h @@ -199,11 +199,11 @@ _Py_ThreadId(void) #elif defined(__MINGW32__) && defined(_M_ARM64) tid = __getReg(18); #elif defined(__i386__) - __asm__("movl %%gs:0, %0" : "=r" (tid)); // 32-bit always uses GS + __asm__("{movl %%gs:0, %0|mov %0, dword ptr gs:[0]}" : "=r" (tid)); // 32-bit always uses GS #elif defined(__MACH__) && defined(__x86_64__) - __asm__("movq %%gs:0, %0" : "=r" (tid)); // x86_64 macOSX uses GS + __asm__("{movq %%gs:0, %0|mov %0, qword ptr gs:[0]}" : "=r" (tid)); // x86_64 macOSX uses GS #elif defined(__x86_64__) - __asm__("movq %%fs:0, %0" : "=r" (tid)); // x86_64 Linux, BSD uses FS + __asm__("{movq %%fs:0, %0|mov %0, qword ptr fs:[0]}" : "=r" (tid)); // x86_64 Linux, BSD uses FS #elif defined(__arm__) && __ARM_ARCH >= 7 __asm__ ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tid)); #elif defined(__aarch64__) && defined(__APPLE__) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 8ad65ac59f4..786157e4eac 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -24,7 +24,7 @@ #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.14.3" +#define PY_VERSION "3.14.3+" /*--end constants--*/ diff --git a/Lib/argparse.py b/Lib/argparse.py index 1d7d34f9924..8cf85694300 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -149,6 +149,10 @@ def _copy_items(items): return copy.copy(items) +def _identity(value): + return value + + # =============== # Formatting Help # =============== @@ -200,7 +204,7 @@ def _set_color(self, color): self._decolor = decolor else: self._theme = get_theme(force_no_color=True).argparse - self._decolor = lambda text: text + self._decolor = _identity # =============================== # Section and indentation methods @@ -1903,9 +1907,7 @@ def __init__(self, self._subparsers = None # register types - def identity(string): - return string - self.register('type', None, identity) + self.register('type', None, _identity) # add help argument if necessary # (using explicit default to override global argument_default) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 8cbb71f7085..b83b84181fd 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -1345,6 +1345,17 @@ async def start_tls(self, transport, protocol, sslcontext, *, # have a chance to get called before "ssl_protocol.connection_made()". transport.pause_reading() + # gh-142352: move buffered StreamReader data to SSLProtocol + if server_side: + from .streams import StreamReaderProtocol + if isinstance(protocol, StreamReaderProtocol): + stream_reader = getattr(protocol, '_stream_reader', None) + if stream_reader is not None: + buffer = stream_reader._buffer + if buffer: + ssl_protocol._incoming.write(buffer) + buffer.clear() + transport.set_protocol(ssl_protocol) conmade_cb = self.call_soon(ssl_protocol.connection_made, transport) resume_cb = self.call_soon(transport.resume_reading) diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 321a4e5d5d1..224b1883808 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -265,7 +265,7 @@ def _try_finish(self): # to avoid hanging forever in self._wait as otherwise _exit_waiters # would never be woken up, we wake them up here. for waiter in self._exit_waiters: - if not waiter.cancelled(): + if not waiter.done(): waiter.set_result(self._returncode) if all(p is not None and p.disconnected for p in self._pipes.values()): @@ -278,7 +278,7 @@ def _call_connection_lost(self, exc): finally: # wake up futures waiting for wait() for waiter in self._exit_waiters: - if not waiter.cancelled(): + if not waiter.done(): waiter.set_result(self._returncode) self._exit_waiters = None self._loop = None diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py index ef277fac3e2..acd49441131 100644 --- a/Lib/asyncio/windows_utils.py +++ b/Lib/asyncio/windows_utils.py @@ -10,7 +10,6 @@ import msvcrt import os import subprocess -import tempfile import warnings @@ -24,6 +23,7 @@ PIPE = subprocess.PIPE STDOUT = subprocess.STDOUT _mmap_counter = itertools.count() +_MAX_PIPE_ATTEMPTS = 20 # Replacement for os.pipe() using handles instead of fds @@ -31,10 +31,6 @@ def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE): """Like os.pipe() but with overlapped support and using handles not fds.""" - address = tempfile.mktemp( - prefix=r'\\.\pipe\python-pipe-{:d}-{:d}-'.format( - os.getpid(), next(_mmap_counter))) - if duplex: openmode = _winapi.PIPE_ACCESS_DUPLEX access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE @@ -56,9 +52,20 @@ def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE): h1 = h2 = None try: - h1 = _winapi.CreateNamedPipe( - address, openmode, _winapi.PIPE_WAIT, - 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL) + for attempts in itertools.count(): + address = r'\\.\pipe\python-pipe-{:d}-{:d}-{}'.format( + os.getpid(), next(_mmap_counter), os.urandom(8).hex()) + try: + h1 = _winapi.CreateNamedPipe( + address, openmode, _winapi.PIPE_WAIT, + 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL) + break + except OSError as e: + if attempts >= _MAX_PIPE_ATTEMPTS: + raise + if e.winerror not in (_winapi.ERROR_PIPE_BUSY, + _winapi.ERROR_ACCESS_DENIED): + raise h2 = _winapi.CreateFile( address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING, diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 04ec0270148..7223fc49977 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -470,6 +470,8 @@ def _load_library(self, name, mode, handle, winmode): if name and name.endswith(")") and ".a(" in name: mode |= _os.RTLD_MEMBER | _os.RTLD_NOW self._name = name + if handle is not None: + return handle return _dlopen(name, mode) def __repr__(self): diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 378f12167c6..3b21658433b 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -85,15 +85,10 @@ def find_library(name): wintypes.DWORD, ) - _psapi = ctypes.WinDLL('psapi', use_last_error=True) - _enum_process_modules = _psapi["EnumProcessModules"] - _enum_process_modules.restype = wintypes.BOOL - _enum_process_modules.argtypes = ( - wintypes.HANDLE, - ctypes.POINTER(wintypes.HMODULE), - wintypes.DWORD, - wintypes.LPDWORD, - ) + # gh-145307: We defer loading psapi.dll until _get_module_handles is called. + # Loading additional DLLs at startup for functionality that may never be + # used is wasteful. + _enum_process_modules = None def _get_module_filename(module: wintypes.HMODULE): name = (wintypes.WCHAR * 32767)() # UNICODE_STRING_MAX_CHARS @@ -101,8 +96,19 @@ def _get_module_filename(module: wintypes.HMODULE): return name.value return None - def _get_module_handles(): + global _enum_process_modules + if _enum_process_modules is None: + _psapi = ctypes.WinDLL('psapi', use_last_error=True) + _enum_process_modules = _psapi["EnumProcessModules"] + _enum_process_modules.restype = wintypes.BOOL + _enum_process_modules.argtypes = ( + wintypes.HANDLE, + ctypes.POINTER(wintypes.HMODULE), + wintypes.DWORD, + wintypes.LPDWORD, + ) + process = _get_current_process() space_needed = wintypes.DWORD() n = 1024 diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 51727688c05..03fedd99539 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -80,7 +80,8 @@ # Useful constants and functions # -WSP = set(' \t') +_WSP = ' \t' +WSP = set(_WSP) CFWS_LEADER = WSP | set('(') SPECIALS = set(r'()<>@,:;.\"[]') ATOM_ENDS = SPECIALS | WSP @@ -2831,6 +2832,7 @@ def _steal_trailing_WSP_if_exists(lines): lines.pop() return wsp + def _refold_parse_tree(parse_tree, *, policy): """Return string of contents of parse_tree folded according to RFC rules. @@ -2839,11 +2841,9 @@ def _refold_parse_tree(parse_tree, *, policy): maxlen = policy.max_line_length or sys.maxsize encoding = 'utf-8' if policy.utf8 else 'us-ascii' lines = [''] # Folded lines to be output - leading_whitespace = '' # When we have whitespace between two encoded - # words, we may need to encode the whitespace - # at the beginning of the second word. - last_ew = None # Points to the last encoded character if there's an ew on - # the line + last_word_is_ew = False + last_ew = None # if there is an encoded word in the last line of lines, + # points to the encoded word's first character last_charset = None wrap_as_ew_blocked = 0 want_encoding = False # This is set to True if we need to encode this part @@ -2878,6 +2878,7 @@ def _refold_parse_tree(parse_tree, *, policy): if part.token_type == 'mime-parameters': # Mime parameter folding (using RFC2231) is extra special. _fold_mime_parameters(part, lines, maxlen, encoding) + last_word_is_ew = False continue if want_encoding and not wrap_as_ew_blocked: @@ -2894,6 +2895,7 @@ def _refold_parse_tree(parse_tree, *, policy): # XXX what if encoded_part has no leading FWS? lines.append(newline) lines[-1] += encoded_part + last_word_is_ew = False continue # Either this is not a major syntactic break, so we don't # want it on a line by itself even if it fits, or it @@ -2912,11 +2914,16 @@ def _refold_parse_tree(parse_tree, *, policy): (last_charset == 'unknown-8bit' or last_charset == 'utf-8' and charset != 'us-ascii')): last_ew = None - last_ew = _fold_as_ew(tstr, lines, maxlen, last_ew, - part.ew_combine_allowed, charset, leading_whitespace) - # This whitespace has been added to the lines in _fold_as_ew() - # so clear it now. - leading_whitespace = '' + last_ew = _fold_as_ew( + tstr, + lines, + maxlen, + last_ew, + part.ew_combine_allowed, + charset, + last_word_is_ew, + ) + last_word_is_ew = True last_charset = charset want_encoding = False continue @@ -2929,28 +2936,19 @@ def _refold_parse_tree(parse_tree, *, policy): if len(tstr) <= maxlen - len(lines[-1]): lines[-1] += tstr + last_word_is_ew = last_word_is_ew and not bool(tstr.strip(_WSP)) continue # This part is too long to fit. The RFC wants us to break at # "major syntactic breaks", so unless we don't consider this # to be one, check if it will fit on the next line by itself. - leading_whitespace = '' if (part.syntactic_break and len(tstr) + 1 <= maxlen): newline = _steal_trailing_WSP_if_exists(lines) if newline or part.startswith_fws(): - # We're going to fold the data onto a new line here. Due to - # the way encoded strings handle continuation lines, we need to - # be prepared to encode any whitespace if the next line turns - # out to start with an encoded word. lines.append(newline + tstr) - - whitespace_accumulator = [] - for char in lines[-1]: - if char not in WSP: - break - whitespace_accumulator.append(char) - leading_whitespace = ''.join(whitespace_accumulator) + last_word_is_ew = (last_word_is_ew + and not bool(lines[-1].strip(_WSP))) last_ew = None continue if not hasattr(part, 'encode'): @@ -2990,10 +2988,11 @@ def _refold_parse_tree(parse_tree, *, policy): else: # We can't fold it onto the next line either... lines[-1] += tstr + last_word_is_ew = last_word_is_ew and not bool(tstr.strip(_WSP)) return policy.linesep.join(lines) + policy.linesep -def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, leading_whitespace): +def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, last_word_is_ew): """Fold string to_encode into lines as encoded word, combining if allowed. Return the new value for last_ew, or None if ew_combine_allowed is False. @@ -3008,6 +3007,16 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, to_encode = str( get_unstructured(lines[-1][last_ew:] + to_encode)) lines[-1] = lines[-1][:last_ew] + elif last_word_is_ew: + # If we are following up an encoded word with another encoded word, + # any white space between the two will be ignored when decoded. + # Therefore, we encode all to-be-displayed whitespace in the second + # encoded word. + len_without_wsp = len(lines[-1].rstrip(_WSP)) + leading_whitespace = lines[-1][len_without_wsp:] + lines[-1] = (lines[-1][:len_without_wsp] + + (' ' if leading_whitespace else '')) + to_encode = leading_whitespace + to_encode elif to_encode[0] in WSP: # We're joining this to non-encoded text, so don't encode # the leading blank. @@ -3036,20 +3045,13 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, while to_encode: remaining_space = maxlen - len(lines[-1]) - text_space = remaining_space - chrome_len - len(leading_whitespace) + text_space = remaining_space - chrome_len if text_space <= 0: - lines.append(' ') + newline = _steal_trailing_WSP_if_exists(lines) + lines.append(newline or ' ') + new_last_ew = len(lines[-1]) continue - # If we are at the start of a continuation line, prepend whitespace - # (we only want to do this when the line starts with an encoded word - # but if we're folding in this helper function, then we know that we - # are going to be writing out an encoded word.) - if len(lines) > 1 and len(lines[-1]) == 1 and leading_whitespace: - encoded_word = _ew.encode(leading_whitespace, charset=encode_as) - lines[-1] += encoded_word - leading_whitespace = '' - to_encode_word = to_encode[:text_space] encoded_word = _ew.encode(to_encode_word, charset=encode_as) excess = len(encoded_word) - remaining_space @@ -3061,7 +3063,6 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset, excess = len(encoded_word) - remaining_space lines[-1] += encoded_word to_encode = to_encode[len(to_encode_word):] - leading_whitespace = '' if to_encode: lines.append(' ') diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 21bbfad0fe6..3d5641e3576 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] -_PIP_VERSION = "25.3" +_PIP_VERSION = "26.0.1" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora diff --git a/Lib/glob.py b/Lib/glob.py index f1a87c82fc5..7ce3998c27c 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -15,7 +15,7 @@ def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, include_hidden=False): - """Return a list of paths matching a pathname pattern. + """Return a list of paths matching a `pathname` pattern. The pattern may contain simple shell-style wildcards a la fnmatch. Unlike fnmatch, filenames starting with a @@ -25,6 +25,15 @@ def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, The order of the returned list is undefined. Sort it if you need a particular order. + If `root_dir` is not None, it should be a path-like object specifying the + root directory for searching. It has the same effect as changing the + current directory before calling it (without actually + changing it). If pathname is relative, the result will contain + paths relative to `root_dir`. + + If `dir_fd` is not None, it should be a file descriptor referring to a + directory, and paths will then be relative to that directory. + If `include_hidden` is true, the patterns '*', '?', '**' will match hidden directories. @@ -36,7 +45,7 @@ def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False, include_hidden=False): - """Return an iterator which yields the paths matching a pathname pattern. + """Return an iterator which yields the paths matching a `pathname` pattern. The pattern may contain simple shell-style wildcards a la fnmatch. However, unlike fnmatch, filenames starting with a @@ -46,7 +55,19 @@ def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False, The order of the returned paths is undefined. Sort them if you need a particular order. - If recursive is true, the pattern '**' will match any files and + If `root_dir` is not None, it should be a path-like object specifying + the root directory for searching. It has the same effect as changing + the current directory before calling it (without actually + changing it). If pathname is relative, the result will contain + paths relative to `root_dir`. + + If `dir_fd` is not None, it should be a file descriptor referring to a + directory, and paths will then be relative to that directory. + + If `include_hidden` is true, the patterns '*', '?', '**' will match hidden + directories. + + If `recursive` is true, the pattern '**' will match any files and zero or more directories and subdirectories. """ sys.audit("glob.glob", pathname, recursive) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 499da1e04ef..9d911e1dcab 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1375,6 +1375,14 @@ def _find_and_load(name, import_): # NOTE: because of this, initializing must be set *before* # putting the new module in sys.modules. _lock_unlock_module(name) + else: + # Verify the module is still in sys.modules. Another thread may have + # removed it (due to import failure) between our sys.modules.get() + # above and the _initializing check. If removed, we retry the import + # to preserve normal semantics: the caller gets the exception from + # the actual import failure rather than a synthetic error. + if sys.modules.get(name) is not module: + return _find_and_load(name, import_) if module is None: message = f'import of {name} halted; None in sys.modules' diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 95ce14b2c39..6a828ae75ed 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -946,7 +946,7 @@ def get_filename(self, fullname): def get_data(self, path): """Return the data from path as raw bytes.""" - if isinstance(self, (SourceLoader, ExtensionFileLoader)): + if isinstance(self, (SourceLoader, SourcelessFileLoader, ExtensionFileLoader)): with _io.open_code(str(path)) as file: return file.read() else: diff --git a/Lib/inspect.py b/Lib/inspect.py index 3cee85f39a6..2d229051b4d 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2660,11 +2660,12 @@ class Parameter: The annotation for the parameter if specified. If the parameter has no annotation, this attribute is set to `Parameter.empty`. - * kind : str + * kind Describes how argument values are bound to the parameter. Possible values: `Parameter.POSITIONAL_ONLY`, `Parameter.POSITIONAL_OR_KEYWORD`, `Parameter.VAR_POSITIONAL`, `Parameter.KEYWORD_ONLY`, `Parameter.VAR_KEYWORD`. + Every value has a `description` attribute describing meaning. """ __slots__ = ('_name', '_kind', '_default', '_annotation') diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index fc00d286126..a6e1b0c7862 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -46,6 +46,7 @@ CONNECTION_TIMEOUT = 20. _mmap_counter = itertools.count() +_MAX_PIPE_ATTEMPTS = 100 default_family = 'AF_INET' families = ['AF_INET'] @@ -78,8 +79,8 @@ def arbitrary_address(family): elif family == 'AF_UNIX': return tempfile.mktemp(prefix='sock-', dir=util.get_temp_dir()) elif family == 'AF_PIPE': - return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' % - (os.getpid(), next(_mmap_counter)), dir="") + return (r'\\.\pipe\pyc-%d-%d-%s' % + (os.getpid(), next(_mmap_counter), os.urandom(8).hex())) else: raise ValueError('unrecognized family') @@ -472,17 +473,29 @@ class Listener(object): def __init__(self, address=None, family=None, backlog=1, authkey=None): family = family or (address and address_type(address)) \ or default_family - address = address or arbitrary_address(family) - _validate_family(family) + if authkey is not None and not isinstance(authkey, bytes): + raise TypeError('authkey should be a byte string') + if family == 'AF_PIPE': - self._listener = PipeListener(address, backlog) + if address: + self._listener = PipeListener(address, backlog) + else: + for attempts in itertools.count(): + address = arbitrary_address(family) + try: + self._listener = PipeListener(address, backlog) + break + except OSError as e: + if attempts >= _MAX_PIPE_ATTEMPTS: + raise + if e.winerror not in (_winapi.ERROR_PIPE_BUSY, + _winapi.ERROR_ACCESS_DENIED): + raise else: + address = address or arbitrary_address(family) self._listener = SocketListener(address, family, backlog) - if authkey is not None and not isinstance(authkey, bytes): - raise TypeError('authkey should be a byte string') - self._authkey = authkey def accept(self): @@ -570,7 +583,6 @@ def Pipe(duplex=True): ''' Returns pair of connection objects at either end of a pipe ''' - address = arbitrary_address('AF_PIPE') if duplex: openmode = _winapi.PIPE_ACCESS_DUPLEX access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE @@ -580,15 +592,25 @@ def Pipe(duplex=True): access = _winapi.GENERIC_WRITE obsize, ibsize = 0, BUFSIZE - h1 = _winapi.CreateNamedPipe( - address, openmode | _winapi.FILE_FLAG_OVERLAPPED | - _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE, - _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | - _winapi.PIPE_WAIT, - 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, - # default security descriptor: the handle cannot be inherited - _winapi.NULL - ) + for attempts in itertools.count(): + address = arbitrary_address('AF_PIPE') + try: + h1 = _winapi.CreateNamedPipe( + address, openmode | _winapi.FILE_FLAG_OVERLAPPED | + _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE, + _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | + _winapi.PIPE_WAIT, + 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, + # default security descriptor: the handle cannot be inherited + _winapi.NULL + ) + break + except OSError as e: + if attempts >= _MAX_PIPE_ATTEMPTS: + raise + if e.winerror not in (_winapi.ERROR_PIPE_BUSY, + _winapi.ERROR_ACCESS_DENIED): + raise h2 = _winapi.CreateFile( address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING, _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 5e3ccab5f48..a34e062f839 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -57,10 +57,11 @@ if hasattr(_os, 'O_BINARY'): _bin_openflags |= _os.O_BINARY -if hasattr(_os, 'TMP_MAX'): - TMP_MAX = _os.TMP_MAX -else: - TMP_MAX = 10000 +# This is more than enough. +# Each name contains over 40 random bits. Even with a million temporary +# files, the chance of a conflict is less than 1 in a million, and with +# 20 attempts, it is less than 1e-120. +TMP_MAX = 20 # This variable _was_ unused for legacy reasons, see issue 10354. # But as of 3.5 we actually use it at runtime so changing it would @@ -196,8 +197,7 @@ def _get_default_tempdir(dirlist=None): for dir in dirlist: if dir != _os.curdir: dir = _os.path.abspath(dir) - # Try only a few names per directory. - for seq in range(100): + for seq in range(TMP_MAX): name = next(namer) filename = _os.path.join(dir, name) try: @@ -213,10 +213,8 @@ def _get_default_tempdir(dirlist=None): except FileExistsError: pass except PermissionError: - # This exception is thrown when a directory with the chosen name - # already exists on windows. - if (_os.name == 'nt' and _os.path.isdir(dir) and - _os.access(dir, _os.W_OK)): + # See the comment in mkdtemp(). + if _os.name == 'nt' and _os.path.isdir(dir): continue break # no point trying more names in this directory except OSError: @@ -258,10 +256,8 @@ def _mkstemp_inner(dir, pre, suf, flags, output_type): except FileExistsError: continue # try again except PermissionError: - # This exception is thrown when a directory with the chosen name - # already exists on windows. - if (_os.name == 'nt' and _os.path.isdir(dir) and - _os.access(dir, _os.W_OK)): + # See the comment in mkdtemp(). + if _os.name == 'nt' and _os.path.isdir(dir) and seq < TMP_MAX - 1: continue else: raise @@ -386,10 +382,14 @@ def mkdtemp(suffix=None, prefix=None, dir=None): except FileExistsError: continue # try again except PermissionError: - # This exception is thrown when a directory with the chosen name - # already exists on windows. - if (_os.name == 'nt' and _os.path.isdir(dir) and - _os.access(dir, _os.W_OK)): + # On Posix, this exception is raised when the user has no + # write access to the parent directory. + # On Windows, it is also raised when a directory with + # the chosen name already exists, or if the parent directory + # is not a directory. + # We cannot distinguish between "directory-exists-error" and + # "access-denied-error". + if _os.name == 'nt' and _os.path.isdir(dir) and seq < TMP_MAX - 1: continue else: raise diff --git a/Lib/test/.ruff.toml b/Lib/test/.ruff.toml index f800dc03dce..f6a4dc631c7 100644 --- a/Lib/test/.ruff.toml +++ b/Lib/test/.ruff.toml @@ -1,6 +1,7 @@ extend = "../../.ruff.toml" # Inherit the project-wide settings -target-version = "py312" +# Unlike Tools/, tests can use newer syntax than PYTHON_FOR_REGEN +target-version = "py314" extend-exclude = [ # Excluded (run with the other AC files in its own separate ruff job in pre-commit) @@ -15,15 +16,6 @@ extend-exclude = [ "test_grammar.py", ] -[per-file-target-version] -# Type parameter defaults -"test_type_params.py" = "py313" - -# Template string literals -"test_annotationlib.py" = "py314" -"test_string/test_templatelib.py" = "py314" -"test_tstring.py" = "py314" - [lint] select = [ "F811", # Redefinition of unused variable (useful for finding test methods with the same name) diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 6f6e0903d86..0d537c0b2cc 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -739,6 +739,10 @@ def collect_test_socket(info_add): if name.startswith('HAVE_')] copy_attributes(info_add, test_socket, 'test_socket.%s', attributes) + # Get IOCTL_VM_SOCKETS_GET_LOCAL_CID of /dev/vsock + cid = test_socket.get_cid() + info_add('test_socket.get_cid', cid) + def collect_support(info_add): try: diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 2d14cade188..6635ec3474e 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -3034,6 +3034,10 @@ def get_signal_name(exitcode): except KeyError: pass + # Format Windows exit status as hexadecimal + if 0xC0000000 <= exitcode: + return f"0x{exitcode:X}" + return None class BrokenIter: diff --git a/Lib/test/support/threading_helper.py b/Lib/test/support/threading_helper.py index 3e04c344a0d..cf87233f0e2 100644 --- a/Lib/test/support/threading_helper.py +++ b/Lib/test/support/threading_helper.py @@ -250,21 +250,32 @@ def requires_working_threading(*, module=False): return unittest.skipUnless(can_start_thread, msg) -def run_concurrently(worker_func, nthreads, args=(), kwargs={}): +def run_concurrently(worker_func, nthreads=None, args=(), kwargs={}): """ - Run the worker function concurrently in multiple threads. + Run the worker function(s) concurrently in multiple threads. + + If `worker_func` is a single callable, it is used for all threads. + If it is a list of callables, each callable is used for one thread. """ + from collections.abc import Iterable + + if nthreads is None: + nthreads = len(worker_func) + if not isinstance(worker_func, Iterable): + worker_func = [worker_func] * nthreads + assert len(worker_func) == nthreads + barrier = threading.Barrier(nthreads) - def wrapper_func(*args, **kwargs): + def wrapper_func(func, *args, **kwargs): # Wait for all threads to reach this point before proceeding. barrier.wait() - worker_func(*args, **kwargs) + func(*args, **kwargs) with catch_threading_exception() as cm: workers = [ - threading.Thread(target=wrapper_func, args=args, kwargs=kwargs) - for _ in range(nthreads) + threading.Thread(target=wrapper_func, args=(func, *args), kwargs=kwargs) + for func in worker_func ] with start_threads(workers): pass diff --git a/Lib/test/test__interpchannels.py b/Lib/test/test__interpchannels.py index d7cf77368ef..2b0aba42896 100644 --- a/Lib/test/test__interpchannels.py +++ b/Lib/test/test__interpchannels.py @@ -382,6 +382,38 @@ def test_sequential_ids(self): self.assertEqual(id3, int(id2) + 1) self.assertEqual(set(after) - set(before), {id1, id2, id3}) + def test_channel_list_all_closed(self): + id1 = _channels.create() + id2 = _channels.create() + id3 = _channels.create() + before = _channels.list_all() + expected = [info for info in before if info[0] != id2] + _channels.close(id2, force=True) + after = _channels.list_all() + self.assertEqual(set(after), set(expected)) + self.assertEqual(len(after), len(before) - 1) + + def test_channel_list_all_destroyed(self): + id1 = _channels.create() + id2 = _channels.create() + id3 = _channels.create() + before = _channels.list_all() + expected = [info for info in before if info[0] != id2] + _channels.destroy(id2) + after = _channels.list_all() + self.assertEqual(set(after), set(expected)) + self.assertEqual(len(after), len(before) - 1) + + def test_channel_list_all_released(self): + id1 = _channels.create() + id2 = _channels.create() + id3 = _channels.create() + before = _channels.list_all() + _channels.release(id2, send=True, recv=True) + after = _channels.list_all() + self.assertEqual(set(after), set(before)) + self.assertEqual(len(after), len(before)) + def test_ids_global(self): id1 = _interpreters.create() out = _run_output(id1, dedent(""" diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index f48fb765bb3..8331d021813 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -81,6 +81,27 @@ def test_skip_invalid_stdout(self): self.assertRegex(mocked_stderr.getvalue(), r'usage:') +class TestArgumentParserPickleable(unittest.TestCase): + + @force_not_colorized + def test_pickle_roundtrip(self): + import pickle + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo', type=int, default=42) + parser.add_argument('bar', nargs='?', default='baz') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + # Try to pickle and unpickle the parser + parser2 = pickle.loads(pickle.dumps(parser, protocol=proto)) + # Check that the round-tripped parser still works + ns = parser2.parse_args(['--foo', '123', 'quux']) + self.assertEqual(ns.foo, 123) + self.assertEqual(ns.bar, 'quux') + ns2 = parser2.parse_args([]) + self.assertEqual(ns2.foo, 42) + self.assertEqual(ns2.bar, 'baz') + + class TestCase(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index f93ee54abc6..cae8c7c6f7c 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -819,6 +819,48 @@ async def client(addr): self.assertEqual(msg1, b"hello world 1!\n") self.assertEqual(msg2, b"hello world 2!\n") + @unittest.skipIf(ssl is None, 'No ssl module') + def test_start_tls_buffered_data(self): + # gh-142352: test start_tls() with buffered data + + async def server_handler(client_reader, client_writer): + # Wait for TLS ClientHello to be buffered before start_tls(). + await client_reader._wait_for_data('test_start_tls_buffered_data'), + self.assertTrue(client_reader._buffer) + await client_writer.start_tls(test_utils.simple_server_sslcontext()) + + line = await client_reader.readline() + self.assertEqual(line, b"ping\n") + client_writer.write(b"pong\n") + await client_writer.drain() + client_writer.close() + await client_writer.wait_closed() + + async def client(addr): + reader, writer = await asyncio.open_connection(*addr) + await writer.start_tls(test_utils.simple_client_sslcontext()) + + writer.write(b"ping\n") + await writer.drain() + line = await reader.readline() + self.assertEqual(line, b"pong\n") + writer.close() + await writer.wait_closed() + + async def run_test(): + server = await asyncio.start_server( + server_handler, socket_helper.HOSTv4, 0) + server_addr = server.sockets[0].getsockname() + + await client(server_addr) + server.close() + await server.wait_closed() + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + self.loop.run_until_complete(run_test()) + self.assertEqual(messages, []) + def test_streamreader_constructor_without_loop(self): with self.assertRaisesRegex(RuntimeError, 'no current event loop'): asyncio.StreamReader() diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index bf301740741..c08eb7cf261 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -111,6 +111,37 @@ def test_subprocess_repr(self): ) transport.close() + def test_proc_exited_no_invalid_state_error_on_exit_waiters(self): + # gh-145541: when _connect_pipes hasn't completed (so + # _pipes_connected is False) and the process exits, _try_finish() + # sets the result on exit waiters. Then _call_connection_lost() must + # not call set_result() again on the same waiters. + self.loop.set_exception_handler( + lambda loop, context: self.fail( + f"unexpected exception: {context}") + ) + waiter = self.loop.create_future() + transport, protocol = self.create_transport(waiter) + + # Simulate a waiter registered via _wait() before the process exits. + exit_waiter = self.loop.create_future() + transport._exit_waiters.append(exit_waiter) + + # _connect_pipes hasn't completed, so _pipes_connected is False. + self.assertFalse(transport._pipes_connected) + + # Simulate process exit. _try_finish() will set the result on + # exit_waiter because _pipes_connected is False, and then schedule + # _call_connection_lost() because _pipes is empty (vacuously all + # disconnected). _call_connection_lost() must skip exit_waiter + # because it's already done. + transport._process_exited(6) + self.loop.run_until_complete(waiter) + + self.assertEqual(exit_waiter.result(), 6) + + transport.close() + class SubprocessMixin: diff --git a/Lib/test/test_bisect.py b/Lib/test/test_bisect.py index 97204d4cad3..a7e1f533ff2 100644 --- a/Lib/test/test_bisect.py +++ b/Lib/test/test_bisect.py @@ -391,9 +391,9 @@ class TestErrorHandlingC(TestErrorHandling, unittest.TestCase): class TestDocExample: def test_grades(self): - def grade(score, breakpoints=[60, 70, 80, 90], grades='FDCBA'): - i = self.module.bisect(breakpoints, score) - return grades[i] + def grade(score): + i = self.module.bisect([60, 70, 80, 90], score) + return "FDCBA"[i] result = [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]] self.assertEqual(result, ['F', 'A', 'C', 'C', 'B', 'A', 'A']) diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index 3b7897b8a88..d8e3b671ec2 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -66,18 +66,28 @@ class BaseTest(unittest.TestCase): EMPTY_DATA = b'BZh9\x17rE8P\x90\x00\x00\x00\x00' BAD_DATA = b'this is not a valid bzip2 file' - # Some tests need more than one block of uncompressed data. Since one block - # is at least 100,000 bytes, we gather some data dynamically and compress it. - # Note that this assumes that compression works correctly, so we cannot - # simply use the bigger test data for all tests. + # Some tests need more than one block of data. The bz2 module does not + # support flushing a block during compression, so we must read in data until + # there are at least 2 blocks. Since different orderings of Python files may + # be compressed differently, we need to check the compression output for + # more than one bzip2 block header magic, a hex encoding of Pi + # (0x314159265359) + bz2_block_magic = bytes.fromhex('314159265359') test_size = 0 - BIG_TEXT = bytearray(128*1024) + BIG_TEXT = b'' + BIG_DATA = b'' + compressor = BZ2Compressor(1) for fname in glob.glob(os.path.join(glob.escape(os.path.dirname(__file__)), '*.py')): with open(fname, 'rb') as fh: - test_size += fh.readinto(memoryview(BIG_TEXT)[test_size:]) - if test_size > 128*1024: + data = fh.read() + BIG_DATA += compressor.compress(data) + BIG_TEXT += data + # TODO(emmatyping): if it is impossible for a block header to cross + # multiple outputs, we can just search the output of each compress call + # which should be more efficient + if BIG_DATA.count(bz2_block_magic) > 1: + BIG_DATA += compressor.flush() break - BIG_DATA = bz2.compress(BIG_TEXT, compresslevel=1) def setUp(self): fd, self.filename = tempfile.mkstemp() diff --git a/Lib/test/test_capi/test_function.py b/Lib/test/test_capi/test_function.py index 9dca377e28b..c1a278e5d4d 100644 --- a/Lib/test/test_capi/test_function.py +++ b/Lib/test/test_capi/test_function.py @@ -307,10 +307,27 @@ def function_without_closure(): ... _testcapi.function_get_closure(function_without_closure), (1, 2)) self.assertEqual(function_without_closure.__closure__, (1, 2)) + def test_function_get_annotations(self): + # Test PyFunction_GetAnnotations() + def normal(): + pass + + def annofn(arg: int) -> str: + return f'arg = {arg}' + + annotations = _testcapi.function_get_annotations(normal) + self.assertIsNone(annotations) + + annotations = _testcapi.function_get_annotations(annofn) + self.assertIsInstance(annotations, dict) + self.assertEqual(annotations, annofn.__annotations__) + + with self.assertRaises(SystemError): + _testcapi.function_get_annotations(None) + # TODO: test PyFunction_New() # TODO: test PyFunction_NewWithQualName() # TODO: test PyFunction_SetVectorcall() - # TODO: test PyFunction_GetAnnotations() # TODO: test PyFunction_SetAnnotations() # TODO: test PyClassMethod_New() # TODO: test PyStaticMethod_New() diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index 46fde541494..f75257f3d88 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -12,7 +12,9 @@ from test import support -SOURCE = os.path.join(os.path.dirname(__file__), 'extension.c') +SOURCES = [ + os.path.join(os.path.dirname(__file__), 'extension.c'), +] SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') @@ -28,29 +30,13 @@ @support.requires_venv_with_pip() @support.requires_subprocess() @support.requires_resource('cpu') -class TestExt(unittest.TestCase): +class BaseTests: + TEST_INTERNAL_C_API = False + # Default build with no options def test_build(self): self.check_build('_test_cext') - def test_build_c11(self): - self.check_build('_test_c11_cext', std='c11') - - @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99") - def test_build_c99(self): - # In public docs, we say C API is compatible with C11. However, - # in practice we do maintain C99 compatibility in public headers. - # Please ask the C API WG before adding a new C11-only feature. - self.check_build('_test_c99_cext', std='c99') - - @support.requires_gil_enabled('incompatible with Free Threading') - def test_build_limited(self): - self.check_build('_test_limited_cext', limited=True) - - @support.requires_gil_enabled('broken for now with Free Threading') - def test_build_limited_c11(self): - self.check_build('_test_limited_c11_cext', limited=True, std='c11') - def check_build(self, extension_name, std=None, limited=False): venv_dir = 'env' with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe: @@ -61,7 +47,9 @@ def _check_build(self, extension_name, python_exe, std, limited): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) - shutil.copy(SOURCE, os.path.join(pkg_dir, os.path.basename(SOURCE))) + for source in SOURCES: + dest = os.path.join(pkg_dir, os.path.basename(source)) + shutil.copy(source, dest) def run_cmd(operation, cmd): env = os.environ.copy() @@ -70,6 +58,7 @@ def run_cmd(operation, cmd): if limited: env['CPYTHON_TEST_LIMITED'] = '1' env['CPYTHON_TEST_EXT_NAME'] = extension_name + env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API)) if support.verbose: print('Run:', ' '.join(map(shlex.quote, cmd))) subprocess.run(cmd, check=True, env=env) @@ -110,5 +99,29 @@ def run_cmd(operation, cmd): run_cmd('Import', cmd) +class TestPublicCAPI(BaseTests, unittest.TestCase): + @support.requires_gil_enabled('incompatible with Free Threading') + def test_build_limited(self): + self.check_build('_test_limited_cext', limited=True) + + @support.requires_gil_enabled('broken for now with Free Threading') + def test_build_limited_c11(self): + self.check_build('_test_limited_c11_cext', limited=True, std='c11') + + def test_build_c11(self): + self.check_build('_test_c11_cext', std='c11') + + @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99") + def test_build_c99(self): + # In public docs, we say C API is compatible with C11. However, + # in practice we do maintain C99 compatibility in public headers. + # Please ask the C API WG before adding a new C11-only feature. + self.check_build('_test_c99_cext', std='c99') + + +class TestInteralCAPI(BaseTests, unittest.TestCase): + TEST_INTERNAL_C_API = True + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index 64629c5a6da..56f40b354c6 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -1,10 +1,34 @@ // gh-116869: Basic C test extension to check that the Python C API // does not emit C compiler warnings. +// +// Test also the internal C API if the TEST_INTERNAL_C_API macro is defined. // Always enable assertions #undef NDEBUG +#ifdef TEST_INTERNAL_C_API +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" +#include "datetime.h" + +#ifdef TEST_INTERNAL_C_API + // gh-135906: Check for compiler warnings in the internal C API. + // - Cython uses pycore_critical_section.h, pycore_frame.h and + // pycore_template.h. + // - greenlet uses pycore_frame.h, pycore_interpframe_structs.h and + // pycore_interpframe.h. +# include "internal/pycore_critical_section.h" +# include "internal/pycore_frame.h" +# include "internal/pycore_gc.h" +# include "internal/pycore_interp.h" +# include "internal/pycore_interpframe.h" +# include "internal/pycore_interpframe_structs.h" +# include "internal/pycore_object.h" +# include "internal/pycore_pystate.h" +# include "internal/pycore_template.h" +#endif #ifndef MODULE_NAME # error "MODULE_NAME macro must be defined" @@ -30,27 +54,43 @@ _testcext_add(PyObject *Py_UNUSED(module), PyObject *args) } +static PyObject * +test_datetime(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + // datetime.h is excluded from the limited C API +#ifndef Py_LIMITED_API + PyDateTime_IMPORT; + if (PyErr_Occurred()) { + return NULL; + } +#endif + + Py_RETURN_NONE; +} + + static PyMethodDef _testcext_methods[] = { {"add", _testcext_add, METH_VARARGS, _testcext_add_doc}, + {"test_datetime", test_datetime, METH_NOARGS, NULL}, {NULL, NULL, 0, NULL} // sentinel }; static int -_testcext_exec( -#ifdef __STDC_VERSION__ - PyObject *module -#else - PyObject *Py_UNUSED(module) -#endif - ) +_testcext_exec(PyObject *module) { + PyObject *result; + #ifdef __STDC_VERSION__ if (PyModule_AddIntMacro(module, __STDC_VERSION__) < 0) { return -1; } #endif + result = PyObject_CallMethod(module, "test_datetime", ""); + if (!result) return -1; + Py_DECREF(result); + // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0); diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 1275282983f..f2a8c9f9381 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -14,10 +14,15 @@ if not support.MS_WINDOWS: # C compiler flags for GCC and clang - CFLAGS = [ + BASE_CFLAGS = [ # The purpose of test_cext extension is to check that building a C # extension using the Python C API does not emit C compiler warnings. '-Werror', + ] + + # C compiler flags for GCC and clang + PUBLIC_CFLAGS = [ + *BASE_CFLAGS, # gh-120593: Check the 'const' qualifier '-Wcast-qual', @@ -26,27 +31,42 @@ '-pedantic-errors', ] if not support.Py_GIL_DISABLED: - CFLAGS.append( + PUBLIC_CFLAGS.append( # gh-116869: The Python C API must be compatible with building # with the -Werror=declaration-after-statement compiler flag. '-Werror=declaration-after-statement', ) + INTERNAL_CFLAGS = [*BASE_CFLAGS] else: # MSVC compiler flags - CFLAGS = [ - # Display warnings level 1 to 4 - '/W4', + BASE_CFLAGS = [ # Treat all compiler warnings as compiler errors '/WX', ] + PUBLIC_CFLAGS = [ + *BASE_CFLAGS, + # Display warnings level 1 to 4 + '/W4', + ] + INTERNAL_CFLAGS = [ + *BASE_CFLAGS, + # Display warnings level 1 to 3 + '/W3', + ] def main(): std = os.environ.get("CPYTHON_TEST_STD", "") module_name = os.environ["CPYTHON_TEST_EXT_NAME"] limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) + internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0"))) - cflags = list(CFLAGS) + sources = [SOURCE] + + if not internal: + cflags = list(PUBLIC_CFLAGS) + else: + cflags = list(INTERNAL_CFLAGS) cflags.append(f'-DMODULE_NAME={module_name}') # Add -std=STD or /std:STD (MSVC) compiler flag @@ -75,6 +95,9 @@ def main(): version = sys.hexversion cflags.append(f'-DPy_LIMITED_API={version:#x}') + if internal: + cflags.append('-DTEST_INTERNAL_C_API=1') + # On Windows, add PCbuild\amd64\ to include and library directories include_dirs = [] library_dirs = [] @@ -90,7 +113,7 @@ def main(): print(f"Add PCbuild directory: {pcbuild}") # Display information to help debugging - for env_name in ('CC', 'CFLAGS'): + for env_name in ('CC', 'CFLAGS', 'CPPFLAGS'): if env_name in os.environ: print(f"{env_name} env var: {os.environ[env_name]!r}") else: @@ -99,7 +122,7 @@ def main(): ext = Extension( module_name, - sources=[SOURCE], + sources=sources, extra_compile_args=cflags, include_dirs=include_dirs, library_dirs=library_dirs) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index c17d749d4a1..ab65342f620 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -9,7 +9,6 @@ import tempfile import textwrap import unittest -import warnings from test import support from test.support import os_helper from test.support import force_not_colorized @@ -937,21 +936,15 @@ def test_python_asyncio_debug(self): @unittest.skipUnless(sysconfig.get_config_var('Py_TRACE_REFS'), "Requires --with-trace-refs build option") def test_python_dump_refs(self): - code = 'import sys; sys._clear_type_cache()' - # TODO: Remove warnings context manager once sys._clear_type_cache is removed - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFS='1') + code = 'import sys; sys._clear_internal_caches()' + rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFS='1') self.assertEqual(rc, 0) @unittest.skipUnless(sysconfig.get_config_var('Py_TRACE_REFS'), "Requires --with-trace-refs build option") def test_python_dump_refs_file(self): with tempfile.NamedTemporaryFile() as dump_file: - code = 'import sys; sys._clear_type_cache()' - # TODO: Remove warnings context manager once sys._clear_type_cache is removed - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFSFILE=dump_file.name) + code = 'import sys; sys._clear_internal_caches()' + rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFSFILE=dump_file.name) self.assertEqual(rc, 0) with open(dump_file.name, 'r') as file: contents = file.read() diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index d37a9db8c83..a6542b396cc 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -249,8 +249,8 @@ def test_32_63_bit_values(self): d = -281474976710656 # 1 << 48 e = +4611686018427387904 # 1 << 62 f = -4611686018427387904 # 1 << 62 - g = +9223372036854775807 # 1 << 63 - 1 - h = -9223372036854775807 # 1 << 63 - 1 + g = +9223372036854775807 # (1 << 63) - 1 + h = -9223372036854775807 # (1 << 63) - 1 for variable in self.test_32_63_bit_values.__code__.co_consts: if variable is not None: diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index 0c7e7341f13..bee2aceb187 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -72,8 +72,8 @@ def assertAlmostEqual(self, a, b): else: unittest.TestCase.assertAlmostEqual(self, a, b) - def assertCloseAbs(self, x, y, eps=1e-9): - """Return true iff floats x and y "are close".""" + def assertClose(self, x, y, eps=1e-9): + """Return true iff complexes x and y "are close".""" # put the one with larger magnitude second if abs(x) > abs(y): x, y = y, x @@ -82,26 +82,15 @@ def assertCloseAbs(self, x, y, eps=1e-9): if x == 0: return abs(y) < eps # check that relative difference < eps - self.assertTrue(abs((x-y)/y) < eps) - - def assertClose(self, x, y, eps=1e-9): - """Return true iff complexes x and y "are close".""" - self.assertCloseAbs(x.real, y.real, eps) - self.assertCloseAbs(x.imag, y.imag, eps) + self.assertTrue(abs(x-y)/abs(y) < eps) def check_div(self, x, y): """Compute complex z=x*y, and check that z/x==y and z/y==x.""" z = x * y - if x != 0: - q = z / x - self.assertClose(q, y) - q = z.__truediv__(x) - self.assertClose(q, y) - if y != 0: - q = z / y - self.assertClose(q, x) - q = z.__truediv__(y) - self.assertClose(q, x) + if x: + self.assertClose(z / x, y) + if y: + self.assertClose(z / y, x) def test_truediv(self): simple_real = [float(i) for i in range(-5, 6)] @@ -115,10 +104,20 @@ def test_truediv(self): self.check_div(complex(1e200, 1e200), 1+0j) self.check_div(complex(1e-200, 1e-200), 1+0j) + # Smith's algorithm has several sources of inaccuracy + # for components of the result. In examples below, + # it's cancellation of digits in computation of sum. + self.check_div(1e-09+1j, 1+1j) + self.check_div(8.289760544677449e-09+0.13257307440728516j, + 0.9059966714925808+0.5054864708672686j) + # Just for fun. for i in range(100): - self.check_div(complex(random(), random()), - complex(random(), random())) + x = complex(random(), random()) + y = complex(random(), random()) + self.check_div(x, y) + y = complex(1e10*y.real, y.imag) + self.check_div(x, y) self.assertAlmostEqual(complex.__truediv__(2+0j, 1+1j), 1-1j) self.assertRaises(TypeError, operator.truediv, 1j, None) @@ -454,7 +453,7 @@ def test_boolcontext(self): self.assertTrue(1j) def test_conjugate(self): - self.assertClose(complex(5.3, 9.8).conjugate(), 5.3-9.8j) + self.assertEqual(complex(5.3, 9.8).conjugate(), 5.3-9.8j) def test_constructor(self): def check(z, x, y): diff --git a/Lib/test/test_cppext/__init__.py b/Lib/test/test_cppext/__init__.py index 2b7adac4bcc..5b4c97c181b 100644 --- a/Lib/test/test_cppext/__init__.py +++ b/Lib/test/test_cppext/__init__.py @@ -1,9 +1,11 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. import os.path +import platform import shlex import shutil import subprocess +import sys import unittest from test import support @@ -24,41 +26,19 @@ @support.requires_venv_with_pip() @support.requires_subprocess() @support.requires_resource('cpu') -class TestCPPExt(unittest.TestCase): - def test_build(self): - self.check_build('_testcppext') - - def test_build_cpp03(self): - # In public docs, we say C API is compatible with C++11. However, - # in practice we do maintain C++03 compatibility in public headers. - # Please ask the C API WG before adding a new C++11-only feature. - self.check_build('_testcpp03ext', std='c++03') - - @support.requires_gil_enabled('incompatible with Free Threading') - def test_build_limited_cpp03(self): - self.check_build('_test_limited_cpp03ext', std='c++03', limited=True) - - @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c++11") - def test_build_cpp11(self): - self.check_build('_testcpp11ext', std='c++11') - - # Only test C++14 on MSVC. - # On s390x RHEL7, GCC 4.8.5 doesn't support C++14. - @unittest.skipIf(not support.MS_WINDOWS, "need Windows") - def test_build_cpp14(self): - self.check_build('_testcpp14ext', std='c++14') +class BaseTests: + TEST_INTERNAL_C_API = False - @support.requires_gil_enabled('incompatible with Free Threading') - def test_build_limited(self): - self.check_build('_testcppext_limited', limited=True) - - def check_build(self, extension_name, std=None, limited=False): + def check_build(self, extension_name, std=None, limited=False, + extra_cflags=None): venv_dir = 'env' with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe: self._check_build(extension_name, python_exe, - std=std, limited=limited) + std=std, limited=limited, + extra_cflags=extra_cflags) - def _check_build(self, extension_name, python_exe, std, limited): + def _check_build(self, extension_name, python_exe, std, limited, + extra_cflags=None): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) @@ -71,6 +51,9 @@ def run_cmd(operation, cmd): if limited: env['CPYTHON_TEST_LIMITED'] = '1' env['CPYTHON_TEST_EXT_NAME'] = extension_name + env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API)) + if extra_cflags: + env['CPYTHON_TEST_EXTRA_CFLAGS'] = extra_cflags if support.verbose: print('Run:', ' '.join(map(shlex.quote, cmd))) subprocess.run(cmd, check=True, env=env) @@ -111,5 +94,53 @@ def run_cmd(operation, cmd): run_cmd('Import', cmd) +class TestPublicCAPI(BaseTests, unittest.TestCase): + def test_build(self): + self.check_build('_testcppext') + + @support.requires_gil_enabled('incompatible with Free Threading') + def test_build_limited_cpp03(self): + self.check_build('_test_limited_cpp03ext', std='c++03', limited=True) + + @support.requires_gil_enabled('incompatible with Free Threading') + def test_build_limited(self): + self.check_build('_testcppext_limited', limited=True) + + def test_build_cpp03(self): + # In public docs, we say C API is compatible with C++11. However, + # in practice we do maintain C++03 compatibility in public headers. + # Please ask the C API WG before adding a new C++11-only feature. + self.check_build('_testcpp03ext', std='c++03') + + @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c++11") + def test_build_cpp11(self): + self.check_build('_testcpp11ext', std='c++11') + + # Only test C++14 on MSVC. + # On s390x RHEL7, GCC 4.8.5 doesn't support C++14. + @unittest.skipIf(not support.MS_WINDOWS, "need Windows") + def test_build_cpp14(self): + self.check_build('_testcpp14ext', std='c++14') + + # Test that headers compile with Intel asm syntax, which may conflict + # with inline assembly in free-threading headers that use AT&T syntax. + @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support -masm=intel") + @unittest.skipUnless(platform.machine() in ('x86_64', 'i686', 'AMD64'), + "x86-specific flag") + def test_build_intel_asm(self): + self.check_build('_testcppext_asm', extra_cflags='-masm=intel') + + +class TestInteralCAPI(BaseTests, unittest.TestCase): + TEST_INTERNAL_C_API = True + + def test_build(self): + kwargs = {} + if sys.platform == 'darwin': + # Old Apple clang++ default C++ std is gnu++98 + kwargs['std'] = 'c++11' + self.check_build('_testcppext_internal', **kwargs) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_cppext/extension.cpp b/Lib/test/test_cppext/extension.cpp index 5b3571b295b..4db63df94f5 100644 --- a/Lib/test/test_cppext/extension.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -6,7 +6,31 @@ // Always enable assertions #undef NDEBUG +#ifdef TEST_INTERNAL_C_API +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" +#include "datetime.h" + +#ifdef TEST_INTERNAL_C_API + // gh-135906: Check for compiler warnings in the internal C API + // - Cython uses pycore_critical_section.h, pycore_frame.h and + // pycore_template.h. + // - greenlet uses pycore_frame.h, pycore_interpframe_structs.h and + // pycore_interpframe.h. +# include "internal/pycore_frame.h" +# include "internal/pycore_interpframe_structs.h" +# include "internal/pycore_template.h" + + // mimalloc emits compiler warnings on Windows. +# if !defined(MS_WINDOWS) +# include "internal/pycore_backoff.h" +# include "internal/pycore_cell.h" +# include "internal/pycore_critical_section.h" +# include "internal/pycore_interpframe.h" +# endif +#endif #ifndef MODULE_NAME # error "MODULE_NAME macro must be defined" @@ -214,11 +238,26 @@ test_virtual_object(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) Py_RETURN_NONE; } +static PyObject * +test_datetime(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + // datetime.h is excluded from the limited C API +#ifndef Py_LIMITED_API + PyDateTime_IMPORT; + if (PyErr_Occurred()) { + return NULL; + } +#endif + + Py_RETURN_NONE; +} + static PyMethodDef _testcppext_methods[] = { {"add", _testcppext_add, METH_VARARGS, _testcppext_add_doc}, {"test_api_casts", test_api_casts, METH_NOARGS, _Py_NULL}, {"test_unicode", test_unicode, METH_NOARGS, _Py_NULL}, {"test_virtual_object", test_virtual_object, METH_NOARGS, _Py_NULL}, + {"test_datetime", test_datetime, METH_NOARGS, _Py_NULL}, // Note: _testcppext_exec currently runs all test functions directly. // When adding a new one, add a call there. @@ -247,6 +286,10 @@ _testcppext_exec(PyObject *module) if (!result) return -1; Py_DECREF(result); + result = PyObject_CallMethod(module, "test_datetime", ""); + if (!result) return -1; + Py_DECREF(result); + // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0); diff --git a/Lib/test/test_cppext/setup.py b/Lib/test/test_cppext/setup.py index ea1ed64bf7a..14aeafefcaa 100644 --- a/Lib/test/test_cppext/setup.py +++ b/Lib/test/test_cppext/setup.py @@ -47,6 +47,7 @@ def main(): std = os.environ.get("CPYTHON_TEST_CPP_STD", "") module_name = os.environ["CPYTHON_TEST_EXT_NAME"] limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) + internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0"))) cppflags = list(CPPFLAGS) cppflags.append(f'-DMODULE_NAME={module_name}') @@ -58,7 +59,7 @@ def main(): else: cppflags.append(f'-std={std}') - if limited or (std != 'c++03'): + if limited or (std != 'c++03') and not internal: # See CPPFLAGS_PEDANTIC docstring cppflags.extend(CPPFLAGS_PEDANTIC) @@ -82,6 +83,13 @@ def main(): version = sys.hexversion cppflags.append(f'-DPy_LIMITED_API={version:#x}') + if internal: + cppflags.append('-DTEST_INTERNAL_C_API=1') + + extra_cflags = os.environ.get("CPYTHON_TEST_EXTRA_CFLAGS", "") + if extra_cflags: + cppflags.extend(shlex.split(extra_cflags)) + # On Windows, add PCbuild\amd64\ to include and library directories include_dirs = [] library_dirs = [] @@ -97,7 +105,7 @@ def main(): print(f"Add PCbuild directory: {pcbuild}") # Display information to help debugging - for env_name in ('CC', 'CFLAGS', 'CPPFLAGS'): + for env_name in ('CC', 'CXX', 'CFLAGS', 'CPPFLAGS', 'CXXFLAGS'): if env_name in os.environ: print(f"{env_name} env var: {os.environ[env_name]!r}") else: diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py index 3b8332fbb30..343f6a07c0a 100644 --- a/Lib/test/test_ctypes/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -106,6 +106,14 @@ def test_load_without_name_and_with_handle(self): lib = ctypes.WinDLL(name=None, handle=handle) self.assertIs(handle, lib._handle) + @unittest.skipIf(os.name == "nt", 'POSIX-specific test') + @unittest.skipIf(libc_name is None, 'could not find libc') + def test_load_without_name_and_with_handle_posix(self): + lib1 = CDLL(libc_name) + handle = lib1._handle + lib2 = CDLL(name=None, handle=handle) + self.assertIs(lib2._handle, handle) + @unittest.skipUnless(os.name == "nt", 'Windows-specific test') def test_1703286_A(self): # On winXP 64-bit, advapi32 loads at an address that does diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index d420f097e74..99f3f1ba999 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5138,6 +5138,26 @@ def foo(self): with self.assertRaisesRegex(NotImplementedError, "BAR"): B().foo + def test_staticmethod_new(self): + class MyStaticMethod(staticmethod): + def __init__(self, func): + pass + def func(): pass + sm = MyStaticMethod(func) + self.assertEqual(repr(sm), '') + self.assertIsNone(sm.__func__) + self.assertIsNone(sm.__wrapped__) + + def test_classmethod_new(self): + class MyClassMethod(classmethod): + def __init__(self, func): + pass + def func(): pass + cm = MyClassMethod(func) + self.assertEqual(repr(cm), '') + self.assertIsNone(cm.__func__) + self.assertIsNone(cm.__wrapped__) + class DictProxyTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 1c5a1d20daf..c1e64a5d99b 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1569,6 +1569,26 @@ def make_pairs(): self.assertEqual(d.get(key3_3), 44) self.assertGreaterEqual(eq_count, 1) + def test_overwrite_managed_dict(self): + # GH-130327: Overwriting an object's managed dictionary with another object's + # skipped traversal in favor of inline values, causing the GC to believe that + # the __dict__ wasn't reachable. + import gc + + class Shenanigans: + pass + + to_be_deleted = Shenanigans() + to_be_deleted.attr = "whatever" + holds_reference = Shenanigans() + holds_reference.__dict__ = to_be_deleted.__dict__ + holds_reference.ref = {"circular": to_be_deleted, "data": 42} + + del to_be_deleted + gc.collect() + self.assertEqual(holds_reference.ref['data'], 42) + self.assertEqual(holds_reference.attr, "whatever") + def test_unhashable_key(self): d = {'a': 1} key = [1, 2, 3] @@ -1680,6 +1700,69 @@ def test_hash_collision_remove_add(self): self.assertEqual(len(d), len(items), d) self.assertEqual(d, dict(items)) + def test_clear_reentrant_embedded(self): + # gh-130555: dict.clear() must be safe when values are embedded + # in an object and a destructor mutates the dict. + class MyObj: pass + class ClearOnDelete: + def __del__(self): + nonlocal x + del x + + x = MyObj() + x.a = ClearOnDelete() + + d = x.__dict__ + d.clear() + + def test_clear_reentrant_cycle(self): + # gh-130555: dict.clear() must be safe for embedded dicts when the + # object is part of a reference cycle and the last reference to the + # dict is via the cycle. + class MyObj: pass + obj = MyObj() + obj.f = obj + obj.attr = "attr" + + d = obj.__dict__ + del obj + + d.clear() + + def test_clear_reentrant_force_combined(self): + # gh-130555: dict.clear() must be safe when a destructor forces the + # dict from embedded/split to combined (setting ma_values to NULL). + class MyObj: pass + class ForceConvert: + def __del__(self): + d[1] = "trigger" + + x = MyObj() + x.a = ForceConvert() + x.b = "other" + + d = x.__dict__ + d.clear() + + def test_clear_reentrant_delete(self): + # gh-130555: dict.clear() must be safe when a destructor deletes + # a key from the same embedded dict. + class MyObj: pass + class DelKey: + def __del__(self): + try: + del d['b'] + except KeyError: + pass + + x = MyObj() + x.a = DelKey() + x.b = "value_b" + x.c = "value_c" + + d = x.__dict__ + d.clear() + class CAPITest(unittest.TestCase): diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py index e1adf8e9748..ba2fa99707c 100644 --- a/Lib/test/test_dtrace.py +++ b/Lib/test/test_dtrace.py @@ -8,7 +8,7 @@ import unittest from test import support -from test.support import findfile +from test.support import findfile, MS_WINDOWS if not support.has_subprocess_support: @@ -103,6 +103,7 @@ class SystemTapBackend(TraceBackend): COMMAND = ["stap", "-g"] +@unittest.skipIf(MS_WINDOWS, "Tests not compliant with trace on Windows.") class TraceTests: # unittest.TestCase options maxDiff = None diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py index 3ca79edf6a6..c2d7d09d591 100644 --- a/Lib/test/test_email/test_generator.py +++ b/Lib/test/test_email/test_generator.py @@ -393,6 +393,50 @@ def test_defaults_handle_spaces_at_start_of_continuation_line(self): g.flatten(msg) self.assertEqual(s.getvalue(), expected) + # gh-144156: fold between non-encoded and encoded words don't need to encoded + # the separating space + def test_defaults_handle_spaces_at_start_of_continuation_line_2(self): + source = ("Re: [SOS-1495488] Commande et livraison - Demande de retour - " + "bibijolie - 251210-AABBCC - Abo actualités digitales 20 semaines " + "d’abonnement à 24 heures, Bilan, Tribune de Genève et tous les titres Tamedia") + expected = ( + b"Subject: " + b"Re: [SOS-1495488] Commande et livraison - Demande de retour -\n" + b" bibijolie - 251210-AABBCC - Abo =?utf-8?q?actualit=C3=A9s?= digitales 20\n" + b" semaines =?utf-8?q?d=E2=80=99abonnement_=C3=A0?= 24 heures, Bilan, Tribune de\n" + b" =?utf-8?q?Gen=C3=A8ve?= et tous les titres Tamedia\n\n" + ) + msg = EmailMessage() + msg['Subject'] = source + s = io.BytesIO() + g = BytesGenerator(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), expected) + + def test_ew_folding_round_trip_1(self): + print() + source = "aaaaaaaaa фффффффф " + msg = EmailMessage() + msg['Subject'] = source + s = io.BytesIO() + g = BytesGenerator(s, maxheaderlen=30) + g.flatten(msg) + flat = s.getvalue() + reparsed = message_from_bytes(flat, policy=policy.default)['Subject'] + self.assertMultiLineEqual(reparsed, source) + + def test_ew_folding_round_trip_2(self): + print() + source = "aaa aaaaaaa aaa ффф фффф " + msg = EmailMessage() + msg['Subject'] = source + s = io.BytesIO() + g = BytesGenerator(s, maxheaderlen=30) + g.flatten(msg) + flat = s.getvalue() + reparsed = message_from_bytes(flat, policy=policy.default)['Subject'] + self.assertMultiLineEqual(reparsed, source) + def test_cte_type_7bit_handles_unknown_8bit(self): source = ("Subject: Maintenant je vous présente mon " "collègue\n\n").encode('utf-8') diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index df34ec70504..52be7bdbb12 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -1702,7 +1702,7 @@ def test_fold_unstructured_with_overlong_word(self): 'singlewordthatwontfit') self.assertEqual( h.fold(policy=policy.default.clone(max_line_length=20)), - 'Subject: \n' + 'Subject:\n' ' =?utf-8?q?thisisa?=\n' ' =?utf-8?q?verylon?=\n' ' =?utf-8?q?glineco?=\n' @@ -1718,7 +1718,7 @@ def test_fold_unstructured_with_two_overlong_words(self): 'singlewordthatwontfit plusanotherverylongwordthatwontfit') self.assertEqual( h.fold(policy=policy.default.clone(max_line_length=20)), - 'Subject: \n' + 'Subject:\n' ' =?utf-8?q?thisisa?=\n' ' =?utf-8?q?verylon?=\n' ' =?utf-8?q?glineco?=\n' diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py index 71ec0febb0f..90e8e558029 100644 --- a/Lib/test/test_email/test_policy.py +++ b/Lib/test/test_email/test_policy.py @@ -273,7 +273,7 @@ def test_non_ascii_chars_do_not_cause_inf_loop(self): actual = policy.fold('Subject', 'ą' * 12) self.assertEqual( actual, - 'Subject: \n' + + 'Subject:\n' + 12 * ' =?utf-8?q?=C4=85?=\n') def test_short_maxlen_error(self): diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index a709b837161..08779bdb008 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -14,6 +14,7 @@ busy_retry, requires_gil_enabled, ) +from test.support.import_helper import import_module from test.support.script_helper import make_script from test.support.socket_helper import find_unused_port @@ -150,6 +151,48 @@ def foo(): else: self.fail("Main thread stack trace not found in result") + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_self_trace_after_ctypes_import(self): + """Test that RemoteUnwinder works on the same process after _ctypes import. + + When _ctypes is imported, it may call dlopen on the libpython shared + library, creating a duplicate mapping in the process address space. + The remote debugging code must skip these uninitialized duplicate + mappings and find the real PyRuntime. See gh-144563. + """ + + # Skip the test if the _ctypes module is missing. + import_module("_ctypes") + + # Run the test in a subprocess to avoid side effects + script = textwrap.dedent("""\ + import os + import _remote_debugging + + # Should work before _ctypes import + unwinder = _remote_debugging.RemoteUnwinder(os.getpid()) + + import _ctypes + + # Should still work after _ctypes import (gh-144563) + unwinder = _remote_debugging.RemoteUnwinder(os.getpid()) + """) + + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=SHORT_TIMEOUT, + ) + self.assertEqual( + result.returncode, 0, + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + @skip_if_not_supported @unittest.skipIf( sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, diff --git a/Lib/test/test_free_threading/test_code.py b/Lib/test/test_free_threading/test_code.py index a5136a3ba4e..2fc5eea3773 100644 --- a/Lib/test/test_free_threading/test_code.py +++ b/Lib/test/test_free_threading/test_code.py @@ -1,9 +1,41 @@ import unittest +try: + import ctypes +except ImportError: + ctypes = None + from threading import Thread from unittest import TestCase from test.support import threading_helper +from test.support.threading_helper import run_concurrently + +if ctypes is not None: + capi = ctypes.pythonapi + + freefunc = ctypes.CFUNCTYPE(None, ctypes.c_voidp) + + RequestCodeExtraIndex = capi.PyUnstable_Eval_RequestCodeExtraIndex + RequestCodeExtraIndex.argtypes = (freefunc,) + RequestCodeExtraIndex.restype = ctypes.c_ssize_t + + SetExtra = capi.PyUnstable_Code_SetExtra + SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp) + SetExtra.restype = ctypes.c_int + + GetExtra = capi.PyUnstable_Code_GetExtra + GetExtra.argtypes = ( + ctypes.py_object, + ctypes.c_ssize_t, + ctypes.POINTER(ctypes.c_voidp), + ) + GetExtra.restype = ctypes.c_int + +# Note: each call to RequestCodeExtraIndex permanently allocates a slot +# (the counter is monotonically increasing), up to MAX_CO_EXTRA_USERS (255). +NTHREADS = 20 + @threading_helper.requires_working_threading() class TestCode(TestCase): @@ -25,6 +57,83 @@ def run_in_thread(): for thread in threads: thread.join() + @unittest.skipUnless(ctypes, "ctypes is required") + def test_request_code_extra_index_concurrent(self): + """Test concurrent calls to RequestCodeExtraIndex""" + results = [] + + def worker(): + idx = RequestCodeExtraIndex(freefunc(0)) + self.assertGreaterEqual(idx, 0) + results.append(idx) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + + # Every thread must get a unique index. + self.assertEqual(len(results), NTHREADS) + self.assertEqual(len(set(results)), NTHREADS) + + @unittest.skipUnless(ctypes, "ctypes is required") + def test_code_extra_all_ops_concurrent(self): + """Test concurrent RequestCodeExtraIndex + SetExtra + GetExtra""" + LOOP = 100 + + def f(): + pass + + code = f.__code__ + + def worker(): + idx = RequestCodeExtraIndex(freefunc(0)) + self.assertGreaterEqual(idx, 0) + + for i in range(LOOP): + ret = SetExtra(code, idx, ctypes.c_voidp(i + 1)) + self.assertEqual(ret, 0) + + for _ in range(LOOP): + extra = ctypes.c_voidp() + ret = GetExtra(code, idx, extra) + self.assertEqual(ret, 0) + # The slot was set by this thread, so the value must + # be the last one written. + self.assertEqual(extra.value, LOOP) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + + @unittest.skipUnless(ctypes, "ctypes is required") + def test_code_extra_set_get_concurrent(self): + """Test concurrent SetExtra + GetExtra on a shared index""" + LOOP = 100 + + def f(): + pass + + code = f.__code__ + + idx = RequestCodeExtraIndex(freefunc(0)) + self.assertGreaterEqual(idx, 0) + + def worker(): + for i in range(LOOP): + ret = SetExtra(code, idx, ctypes.c_voidp(i + 1)) + self.assertEqual(ret, 0) + + for _ in range(LOOP): + extra = ctypes.c_voidp() + ret = GetExtra(code, idx, extra) + self.assertEqual(ret, 0) + # Value is set by any writer thread. + self.assertTrue(1 <= extra.value <= LOOP) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + + # Every thread's last write is LOOP, so the final value must be LOOP. + extra = ctypes.c_voidp() + ret = GetExtra(code, idx, extra) + self.assertEqual(ret, 0) + self.assertEqual(extra.value, LOOP) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_collections.py b/Lib/test/test_free_threading/test_collections.py new file mode 100644 index 00000000000..3a413ccf396 --- /dev/null +++ b/Lib/test/test_free_threading/test_collections.py @@ -0,0 +1,29 @@ +import unittest +from collections import deque +from copy import copy +from test.support import threading_helper + +threading_helper.requires_working_threading(module=True) + + +class TestDeque(unittest.TestCase): + def test_copy_race(self): + # gh-144809: Test that deque copy is thread safe. It previously + # could raise a "deque mutated during iteration" error. + d = deque(range(100)) + + def mutate(): + for i in range(1000): + d.append(i) + if len(d) > 200: + d.popleft() + + def copy_loop(): + for _ in range(1000): + copy(d) + + threading_helper.run_concurrently([mutate, copy_loop]) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_frame.py b/Lib/test/test_free_threading/test_frame.py new file mode 100644 index 00000000000..bea49df557a --- /dev/null +++ b/Lib/test/test_free_threading/test_frame.py @@ -0,0 +1,151 @@ +import functools +import sys +import threading +import unittest + +from test.support import threading_helper + +threading_helper.requires_working_threading(module=True) + + +def run_with_frame(funcs, runner=None, iters=10): + """Run funcs with a frame from another thread that is currently executing. + + Args: + funcs: A function or list of functions that take a frame argument + runner: Optional function to run in the executor thread. If provided, + it will be called and should return eventually. The frame + passed to funcs will be the runner's frame. + iters: Number of iterations each func should run + """ + if not isinstance(funcs, list): + funcs = [funcs] + + frame_var = None + e = threading.Event() + b = threading.Barrier(len(funcs) + 1) + + if runner is None: + def runner(): + j = 0 + for i in range(100): + j += i + + def executor(): + nonlocal frame_var + frame_var = sys._getframe() + e.set() + b.wait() + runner() + + def func_wrapper(func): + e.wait() + frame = frame_var + b.wait() + for _ in range(iters): + func(frame) + + test_funcs = [functools.partial(func_wrapper, f) for f in funcs] + threading_helper.run_concurrently([executor] + test_funcs) + + +class TestFrameRaces(unittest.TestCase): + def test_concurrent_f_lasti(self): + run_with_frame(lambda frame: frame.f_lasti) + + def test_concurrent_f_lineno(self): + run_with_frame(lambda frame: frame.f_lineno) + + def test_concurrent_f_code(self): + run_with_frame(lambda frame: frame.f_code) + + def test_concurrent_f_back(self): + run_with_frame(lambda frame: frame.f_back) + + def test_concurrent_f_globals(self): + run_with_frame(lambda frame: frame.f_globals) + + def test_concurrent_f_builtins(self): + run_with_frame(lambda frame: frame.f_builtins) + + def test_concurrent_f_locals(self): + run_with_frame(lambda frame: frame.f_locals) + + def test_concurrent_f_trace_read(self): + run_with_frame(lambda frame: frame.f_trace) + + def test_concurrent_f_trace_opcodes_read(self): + run_with_frame(lambda frame: frame.f_trace_opcodes) + + def test_concurrent_repr(self): + run_with_frame(lambda frame: repr(frame)) + + def test_concurrent_f_trace_write(self): + def trace_func(frame, event, arg): + return trace_func + + def writer(frame): + frame.f_trace = trace_func + frame.f_trace = None + + run_with_frame(writer) + + def test_concurrent_f_trace_read_write(self): + # Test concurrent reads and writes of f_trace on a live frame. + def trace_func(frame, event, arg): + return trace_func + + def reader(frame): + _ = frame.f_trace + + def writer(frame): + frame.f_trace = trace_func + frame.f_trace = None + + run_with_frame([reader, writer, reader, writer]) + + def test_concurrent_f_trace_opcodes_write(self): + def writer(frame): + frame.f_trace_opcodes = True + frame.f_trace_opcodes = False + + run_with_frame(writer) + + def test_concurrent_f_trace_opcodes_read_write(self): + # Test concurrent reads and writes of f_trace_opcodes on a live frame. + def reader(frame): + _ = frame.f_trace_opcodes + + def writer(frame): + frame.f_trace_opcodes = True + frame.f_trace_opcodes = False + + run_with_frame([reader, writer, reader, writer]) + + def test_concurrent_frame_clear(self): + # Test race between frame.clear() and attribute reads. + def create_frame(): + x = 1 + y = 2 + return sys._getframe() + + frame = create_frame() + + def reader(): + for _ in range(10): + try: + _ = frame.f_locals + _ = frame.f_code + _ = frame.f_lineno + except ValueError: + # Frame may be cleared + pass + + def clearer(): + frame.clear() + + threading_helper.run_concurrently([reader, reader, clearer]) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_io.py b/Lib/test/test_free_threading/test_io.py index 4e903928b49..6c98e500658 100644 --- a/Lib/test/test_free_threading/test_io.py +++ b/Lib/test/test_free_threading/test_io.py @@ -1,10 +1,15 @@ +import codecs +import io import threading from unittest import TestCase from test.support import threading_helper +from test.support.threading_helper import run_concurrently from random import randint from io import BytesIO from sys import getsizeof +threading_helper.requires_working_threading(module=True) + class TestBytesIO(TestCase): # Test pretty much everything that can break under free-threading. @@ -107,3 +112,54 @@ def sizeof(barrier, b, *ignore): self.check([truncate] + [sizeof] * 10, BytesIO(b'0\n'*204800)) # no tests for seek or tell because they don't break anything + + +class IncrementalNewlineDecoderTest(TestCase): + def make_decoder(self): + utf8_decoder = codecs.getincrementaldecoder('utf-8')() + return io.IncrementalNewlineDecoder(utf8_decoder, translate=True) + + def test_concurrent_reset(self): + decoder = self.make_decoder() + + def worker(): + for _ in range(100): + decoder.reset() + + run_concurrently(worker_func=worker, nthreads=2) + + def test_concurrent_decode(self): + decoder = self.make_decoder() + + def worker(): + for _ in range(100): + decoder.decode(b"line\r\n", final=False) + + run_concurrently(worker_func=worker, nthreads=2) + + def test_concurrent_getstate_setstate(self): + decoder = self.make_decoder() + state = decoder.getstate() + + def getstate_worker(): + for _ in range(100): + decoder.getstate() + + def setstate_worker(): + for _ in range(100): + decoder.setstate(state) + + run_concurrently([getstate_worker] * 2 + [setstate_worker] * 2) + + def test_concurrent_decode_and_reset(self): + decoder = self.make_decoder() + + def decode_worker(): + for _ in range(100): + decoder.decode(b"line\r\n", final=False) + + def reset_worker(): + for _ in range(100): + decoder.reset() + + run_concurrently([decode_worker] * 2 + [reset_worker] * 2) diff --git a/Lib/test/test_free_threading/test_str.py b/Lib/test/test_free_threading/test_str.py index 72044e979b0..9a1ce3620ac 100644 --- a/Lib/test/test_free_threading/test_str.py +++ b/Lib/test/test_free_threading/test_str.py @@ -69,6 +69,22 @@ def reader_func(): for reader in readers: reader.join() + def test_maketrans_dict_concurrent_modification(self): + for _ in range(5): + d = {2000: 'a'} + + def work(dct): + for i in range(100): + str.maketrans(dct) + dct[2000 + i] = chr(i % 16) + dct.pop(2000 + i, None) + + threading_helper.run_concurrently( + work, + nthreads=5, + args=(d,), + ) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 9d0cebd62ba..16adb8dad7d 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -514,6 +514,58 @@ def test_partial_genericalias(self): self.assertEqual(alias.__args__, (int,)) self.assertEqual(alias.__parameters__, ()) + # GH-144475: Tests that the partial object does not change until repr finishes + def test_repr_safety_against_reentrant_mutation(self): + g_partial = None + + class Function: + def __init__(self, name): + self.name = name + + def __call__(self): + return None + + def __repr__(self): + return f"Function({self.name})" + + class EvilObject: + def __init__(self): + self.triggered = False + + def __repr__(self): + if not self.triggered and g_partial is not None: + self.triggered = True + new_args_tuple = (None,) + new_keywords_dict = {"keyword": None} + new_tuple_state = (Function("new_function"), new_args_tuple, new_keywords_dict, None) + g_partial.__setstate__(new_tuple_state) + gc.collect() + return f"EvilObject" + + trigger = EvilObject() + func = Function("old_function") + + g_partial = functools.partial(func, None, trigger=trigger) + self.assertEqual(repr(g_partial),"functools.partial(Function(old_function), None, trigger=EvilObject)") + + trigger.triggered = False + g_partial = functools.partial(func, trigger, arg=None) + self.assertEqual(repr(g_partial),"functools.partial(Function(old_function), EvilObject, arg=None)") + + + trigger.triggered = False + g_partial = functools.partial(func, trigger, None) + self.assertEqual(repr(g_partial),"functools.partial(Function(old_function), EvilObject, None)") + + trigger.triggered = False + g_partial = functools.partial(func, trigger=trigger, arg=None) + self.assertEqual(repr(g_partial),"functools.partial(Function(old_function), trigger=EvilObject, arg=None)") + + trigger.triggered = False + g_partial = functools.partial(func, trigger, None, None, None, None, arg=None) + self.assertEqual(repr(g_partial),"functools.partial(Function(old_function), EvilObject, None, None, None, None, arg=None)") + + @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestPartialC(TestPartial, unittest.TestCase): diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 14a79a1c698..c7d7b801ac7 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -127,8 +127,11 @@ def __init__(self, *args, **kwargs): algorithms.add(algorithm.lower()) _blake2 = self._conditional_import_module('_blake2') + blake2_hashes = {'blake2b', 'blake2s'} if _blake2: - algorithms.update({'blake2b', 'blake2s'}) + algorithms.update(blake2_hashes) + else: + algorithms.difference_update(blake2_hashes) self.constructors_to_test = {} for algorithm in algorithms: @@ -220,7 +223,12 @@ def test_algorithms_available(self): # all available algorithms must be loadable, bpo-47101 self.assertNotIn("undefined", hashlib.algorithms_available) for name in hashlib.algorithms_available: - digest = hashlib.new(name, usedforsecurity=False) + with self.subTest(name): + try: + _ = hashlib.new(name, usedforsecurity=False) + except ValueError as exc: + self.skip_if_blake2_not_builtin(name, exc) + raise def test_usedforsecurity_true(self): hashlib.new("sha256", usedforsecurity=True) @@ -443,6 +451,7 @@ def test_sha3_256_update_over_4gb(self): self.assertEqual(h.hexdigest(), "e2d4535e3b613135c14f2fe4e026d7ad8d569db44901740beffa30d430acb038") @requires_resource('cpu') + @requires_blake2 def test_blake2_update_over_4gb(self): # blake2s or blake2b doesn't matter based on how our C code is structured, this tests the # common loop macro logic. @@ -733,6 +742,12 @@ def test_case_sha512_3(self): "e718483d0ce769644e2e42c7bc15b4638e1f98b13b2044285632a803afa973eb"+ "de0ff244877ea60a4cb0432ce577c31beb009c5c2c49aa2e4eadb217ad8cc09b") + def skip_if_blake2_not_builtin(self, name, skip_reason): + # blake2 builtins may be absent if python built with + # a subset of --with-builtin-hashlib-hashes or none. + if "blake2" in name and "blake2" not in builtin_hashes: + self.skipTest(skip_reason) + def check_blake2(self, constructor, salt_size, person_size, key_size, digest_size, max_offset): self.assertEqual(constructor.SALT_SIZE, salt_size) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 95121debbbf..2e1c6d72f54 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -43,6 +43,7 @@ Py_GIL_DISABLED, no_rerun, force_not_colorized_test_class, + catch_unraisable_exception ) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, ready_to_import, @@ -2540,6 +2541,32 @@ def test_disallowed_reimport(self): excsnap = _interpreters.run_string(interpid, script) self.assertIsNot(excsnap, None) + @requires_subinterpreters + def test_pyinit_function_raises_exception(self): + # gh-144601: PyInit functions that raised exceptions would cause a + # crash when imported from a subinterpreter. + import _testsinglephase + filename = _testsinglephase.__file__ + script = f"""if True: + from test.test_import import import_extension_from_file + + import_extension_from_file('_testsinglephase_raise_exception', {filename!r})""" + + interp = _interpreters.create() + try: + with catch_unraisable_exception() as cm: + exception = _interpreters.run_string(interp, script) + unraisable = cm.unraisable + finally: + _interpreters.destroy(interp) + + self.assertIsNotNone(exception) + self.assertIsNotNone(exception.type.__name__, "ImportError") + self.assertIsNotNone(exception.msg, "failed to import from subinterpreter due to exception") + self.assertIsNotNone(unraisable) + self.assertIs(unraisable.exc_type, RuntimeError) + self.assertEqual(str(unraisable.exc_value), "evil") + class TestSinglePhaseSnapshot(ModuleSnapshot): """A representation of a single-phase init module for testing. diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index f78dc399720..8b793ebf29b 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -259,6 +259,71 @@ def test_multiprocessing_pool_circular_import(self, size): 'partial', 'pool_in_threads.py') script_helper.assert_python_ok(fn) + def test_import_failure_race_condition(self): + # Regression test for race condition where a thread could receive + # a partially-initialized module when another thread's import fails. + # The race occurs when: + # 1. Thread 1 starts importing, adds module to sys.modules + # 2. Thread 2 sees the module in sys.modules + # 3. Thread 1's import fails, removes module from sys.modules + # 4. Thread 2 should NOT return the stale module reference + os.mkdir(TESTFN) + self.addCleanup(shutil.rmtree, TESTFN) + sys.path.insert(0, TESTFN) + self.addCleanup(sys.path.remove, TESTFN) + + # Create a module that partially initializes then fails + modname = 'failing_import_module' + with open(os.path.join(TESTFN, modname + '.py'), 'w') as f: + f.write(''' +import time +PARTIAL_ATTR = 'initialized' +time.sleep(0.05) # Widen race window +raise RuntimeError("Intentional import failure") +''') + self.addCleanup(forget, modname) + importlib.invalidate_caches() + + errors = [] + results = [] + + def do_import(delay=0): + time.sleep(delay) + try: + mod = __import__(modname) + # If we got a module, verify it's in sys.modules + if modname not in sys.modules: + errors.append( + f"Got module {mod!r} but {modname!r} not in sys.modules" + ) + elif sys.modules[modname] is not mod: + errors.append( + f"Got different module than sys.modules[{modname!r}]" + ) + else: + results.append(('success', mod)) + except RuntimeError: + results.append(('RuntimeError',)) + except Exception as e: + errors.append(f"Unexpected exception: {e}") + + # Run multiple iterations to increase chance of hitting the race + for _ in range(10): + errors.clear() + results.clear() + if modname in sys.modules: + del sys.modules[modname] + + t1 = threading.Thread(target=do_import, args=(0,)) + t2 = threading.Thread(target=do_import, args=(0.01,)) + t1.start() + t2.start() + t1.join() + t2.join() + + # Neither thread should have errors about stale modules + self.assertEqual(errors, [], f"Race condition detected: {errors}") + def setUpModule(): thread_info = threading_helper.threading_setup() diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index edbe78545a2..bd64b03b75f 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -15,7 +15,7 @@ import tempfile import types -_testsinglephase = import_helper.import_module("_testsinglephase") +import_helper.import_module("_testmultiphase") BUILTINS = types.SimpleNamespace() diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 35e31cd5ed1..28acb2a45a3 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -6203,8 +6203,7 @@ def test_operator_module_has_signatures(self): def test_os_module_has_signatures(self): unsupported_signature = {'chmod', 'utime'} unsupported_signature |= {name for name in - ['get_terminal_size', 'link', 'posix_spawn', 'posix_spawnp', - 'register_at_fork', 'startfile'] + ['get_terminal_size', 'link', 'register_at_fork', 'startfile'] if hasattr(os, name)} self._test_module_has_signatures(os, unsupported_signature=unsupported_signature) diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 52827357078..5437792b5a7 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -47,6 +47,12 @@ def test_list_all(self): after = set(channels.list_all()) self.assertEqual(after, created) + def test_list_all_closed(self): + created = [channels.create() for _ in range(3)] + rch, sch = created.pop(1) + rch.close() + self.assertEqual(set(channels.list_all()), set(created)) + def test_shareable(self): interp = interpreters.create() rch, sch = channels.create() diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 61bea9dba07..dc64288085f 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -733,6 +733,27 @@ def keyfunc(obj): keyfunc.skip = 1 self.assertRaises(ExpectedError, gulp, [None, None], keyfunc) + def test_groupby_reentrant_eq_does_not_crash(self): + # regression test for gh-143543 + class Key: + def __init__(self, do_advance): + self.do_advance = do_advance + + def __eq__(self, other): + if self.do_advance: + self.do_advance = False + next(g) + return NotImplemented + return False + + def keys(): + yield Key(True) + yield Key(False) + + g = itertools.groupby([None, None], keys().send) + next(g) + next(g) # must pass with address sanitizer + def test_filter(self): self.assertEqual(list(filter(isEven, range(6))), [0,2,4]) self.assertEqual(list(filter(None, [0,1,0,2,0])), [1,2]) diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py index caa1603c78e..c522bc1c2c0 100644 --- a/Lib/test/test_launcher.py +++ b/Lib/test/test_launcher.py @@ -227,6 +227,8 @@ def run_py(self, args, env=None, allow_fail=False, expect_returncode=0, argv=Non "PYLAUNCHER_LIMIT_TO_COMPANY": "", **{k.upper(): v for k, v in (env or {}).items()}, } + if ini_dir := getattr(self, '_ini_dir', None): + env.setdefault("_PYLAUNCHER_INIDIR", ini_dir) if not argv: argv = [self.py_exe, *args] with subprocess.Popen( @@ -262,11 +264,14 @@ def run_py(self, args, env=None, allow_fail=False, expect_returncode=0, argv=Non return data def py_ini(self, content): - local_appdata = os.environ.get("LOCALAPPDATA") - if not local_appdata: - raise unittest.SkipTest("LOCALAPPDATA environment variable is " - "missing or empty") - return PreservePyIni(Path(local_appdata) / "py.ini", content) + ini_dir = getattr(self, '_ini_dir', None) + if not ini_dir: + local_appdata = os.environ.get("LOCALAPPDATA") + if not local_appdata: + raise unittest.SkipTest("LOCALAPPDATA environment variable is " + "missing or empty") + ini_dir = local_appdata + return PreservePyIni(Path(ini_dir) / "py.ini", content) @contextlib.contextmanager def script(self, content, encoding="utf-8"): @@ -302,6 +307,8 @@ def setUpClass(cls): p = subprocess.check_output("reg query HKCU\\Software\\Python /s") #print(p.decode('mbcs')) + cls._ini_dir = tempfile.mkdtemp() + cls.addClassCleanup(shutil.rmtree, cls._ini_dir, ignore_errors=True) @classmethod def tearDownClass(cls): diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index ad7f62fbf78..442075b47c8 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -180,6 +180,18 @@ def test_references___class___defined(self): code, outputs={"res": [2]}, scopes=["module", "function"]) self._check_in_scopes(code, raises=NameError, scopes=["class"]) + def test_references___classdict__(self): + code = """ + class i: [__classdict__ for x in y] + """ + self._check_in_scopes(code, raises=NameError) + + def test_references___conditional_annotations__(self): + code = """ + class i: [__conditional_annotations__ for x in y] + """ + self._check_in_scopes(code, raises=NameError) + def test_references___class___enclosing(self): code = """ __class__ = 2 diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 7a28e7b4598..555c85f4857 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2607,6 +2607,16 @@ def test_fpathconf_bad_fd(self): self.check(os.pathconf, "PC_NAME_MAX") self.check(os.fpathconf, "PC_NAME_MAX") + @unittest.skipUnless(hasattr(os, 'pathconf'), 'test needs os.pathconf()') + @unittest.skipIf( + support.linked_to_musl(), + 'musl fpathconf ignores the file descriptor and returns a constant', + ) + def test_pathconf_negative_fd_uses_fd_semantics(self): + with self.assertRaises(OSError) as ctx: + os.pathconf(-1, 1) + self.assertEqual(ctx.exception.errno, errno.EBADF) + @unittest.skipUnless(hasattr(os, 'ftruncate'), 'test needs os.ftruncate()') def test_ftruncate(self): self.check(os.truncate, 0) diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 1e1b0522787..7824897dd29 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -170,6 +170,47 @@ def baz(): self.assertNotIn(f"py::bar:{script}", child_perf_file_contents) self.assertNotIn(f"py::baz:{script}", child_perf_file_contents) + @unittest.skipIf(support.check_bolt_optimized(), "fails on BOLT instrumented binaries") + def test_trampoline_works_after_fork_with_many_code_objects(self): + code = """if 1: + import gc, os, sys, signal + + # Create many code objects so trampoline_refcount > 1 + for i in range(50): + exec(compile(f"def _dummy_{i}(): pass", f"", "exec")) + + pid = os.fork() + if pid == 0: + # Child: create and destroy new code objects, + # then collect garbage. If the old code watcher + # survived the fork, the double-decrement of + # trampoline_refcount will cause a SIGSEGV. + for i in range(50): + exec(compile(f"def _child_{i}(): pass", f"", "exec")) + gc.collect() + os._exit(0) + else: + _, status = os.waitpid(pid, 0) + if os.WIFSIGNALED(status): + print(f"FAIL: child killed by signal {os.WTERMSIG(status)}", file=sys.stderr) + sys.exit(1) + sys.exit(os.WEXITSTATUS(status)) + """ + with temp_dir() as script_dir: + script = make_script(script_dir, "perftest", code) + env = {**os.environ, "PYTHON_JIT": "0"} + with subprocess.Popen( + [sys.executable, "-Xperf", script], + text=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + env=env, + ) as process: + stdout, stderr = process.communicate() + + self.assertEqual(process.returncode, 0, stderr) + self.assertEqual(stderr, "") + @unittest.skipIf(support.check_bolt_optimized(), "fails on BOLT instrumented binaries") def test_sys_api(self): code = """if 1: diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index f51cdd26df7..a895d57d1ff 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1933,6 +1933,11 @@ def test_setpgroup(self): ) support.wait_process(pid, exitcode=0) + def test_setpgroup_allow_none(self): + path, args = self.NOOP_PROGRAM[0], self.NOOP_PROGRAM + pid = self.spawn_func(path, args, os.environ, setpgroup=None) + support.wait_process(pid, exitcode=0) + def test_setpgroup_wrong_type(self): with self.assertRaises(TypeError): self.spawn_func(sys.executable, @@ -2033,6 +2038,20 @@ def test_setsigdef_wrong_type(self): [sys.executable, "-c", "pass"], os.environ, setsigdef=[signal.NSIG, signal.NSIG+1]) + def test_scheduler_allow_none(self): + path, args = self.NOOP_PROGRAM[0], self.NOOP_PROGRAM + pid = self.spawn_func(path, args, os.environ, scheduler=None) + support.wait_process(pid, exitcode=0) + + @support.subTests("scheduler", [object(), 1, [1, 2]]) + def test_scheduler_wrong_type(self, scheduler): + path, args = self.NOOP_PROGRAM[0], self.NOOP_PROGRAM + with self.assertRaisesRegex( + TypeError, + "scheduler must be a tuple or None", + ): + self.spawn_func(path, args, os.environ, scheduler=scheduler) + @requires_sched @unittest.skipIf(sys.platform.startswith(('freebsd', 'netbsd')), "bpo-34685: test can fail on BSD") diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index 64387296e84..749a877d013 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -207,6 +207,14 @@ def test_quiet(self): with self.assertRaises(py_compile.PyCompileError): py_compile.compile(bad_coding, doraise=True, quiet=1) + def test_utf7_decoded_cr_compiles(self): + with open(self.source_path, 'wb') as file: + file.write(b"#coding=U7+AA0''\n") + + pyc_path = py_compile.compile(self.source_path, self.pyc_path, doraise=True) + self.assertEqual(pyc_path, self.pyc_path) + self.assertTrue(os.path.exists(self.pyc_path)) + class PyCompileTestsWithSourceEpoch(PyCompileTestsBase, unittest.TestCase, diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index daeaa38a3c5..3e9015910e0 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -999,7 +999,9 @@ def test_set_maximum_amplification__fail_for_subparser(self): self.assert_root_parser_failure(setter, 123.45) -@unittest.skipIf(expat.version_info < (2, 7, 2), "requires Expat >= 2.7.2") +@unittest.skipIf(not hasattr(expat.XMLParserType, + "SetAllocTrackerMaximumAmplification"), + "requires Python compiled with Expat >= 2.7.2") class MemoryProtectionTest(AttackProtectionTestBase, unittest.TestCase): # NOTE: with the default Expat configuration, the billion laughs protection diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index b55a5180c67..855dca2258d 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -143,6 +143,22 @@ def test_multiline_string_parsing(self): output = kill_python(p) self.assertEqual(p.returncode, 0) + @cpython_only + def test_lexer_buffer_realloc_with_null_start(self): + # gh-144759: NULL pointer arithmetic in the lexer when start and + # multi_line_start are NULL (uninitialized in tok_mode_stack[0]) + # and the lexer buffer is reallocated while parsing long input. + long_value = "a" * 2000 + user_input = dedent(f"""\ + x = f'{{{long_value!r}}}' + print(x) + """) + p = spawn_repl() + p.stdin.write(user_input) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + self.assertIn(long_value, output) + def test_close_stdin(self): user_input = dedent(''' import os diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index e66160c5236..249bfeaa14c 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -561,8 +561,8 @@ def clientTearDown(self): @unittest.skipIf(WSL, 'VSOCK does not work on Microsoft WSL') @unittest.skipUnless(HAVE_SOCKET_VSOCK, 'VSOCK sockets required for this test.') -@unittest.skipUnless(get_cid() != 2, # VMADDR_CID_HOST - "This test can only be run on a virtual guest.") +@unittest.skipIf(get_cid() == getattr(socket, 'VMADDR_CID_HOST', 2), + "This test can only be run on a virtual guest.") class ThreadedVSOCKSocketStreamTest(unittest.TestCase, ThreadableTest): def __init__(self, methodName='runTest'): @@ -572,7 +572,16 @@ def __init__(self, methodName='runTest'): def setUp(self): self.serv = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) self.addCleanup(self.serv.close) - self.serv.bind((socket.VMADDR_CID_ANY, VSOCKPORT)) + cid = get_cid() + if cid in (socket.VMADDR_CID_HOST, socket.VMADDR_CID_ANY): + cid = socket.VMADDR_CID_LOCAL + try: + self.serv.bind((cid, VSOCKPORT)) + except OSError as exc: + if exc.errno == errno.EADDRNOTAVAIL: + self.skipTest(f"bind() failed with {exc!r}") + else: + raise self.serv.listen() self.serverExplicitReady() self.serv.settimeout(support.LOOPBACK_TIMEOUT) @@ -2176,6 +2185,24 @@ def test_addressinfo_enum(self): source=_socket) enum._test_simple_enum(CheckedAddressInfo, socket.AddressInfo) + @unittest.skipUnless(hasattr(socket.socket, "sendmsg"),"sendmsg not supported") + def test_sendmsg_reentrant_ancillary_mutation(self): + + class Mut: + def __index__(self): + seq.clear() + return socket.SCM_RIGHTS + + seq = [ + (socket.SOL_SOCKET, Mut(), b'xxxx'), + (socket.SOL_SOCKET, socket.SCM_RIGHTS, b'xxxx'), + ] + + left, right = socket.socketpair() + self.addCleanup(left.close) + self.addCleanup(right.close) + self.assertRaises(OSError, left.sendmsg, [b'x'], seq) + @unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') class BasicCANTest(unittest.TestCase): diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index 36c8d87182d..5fae8a7c5bf 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -65,6 +65,23 @@ def test_issue7820(self): # two bytes in common with the UTF-8 BOM self.assertRaises(SyntaxError, eval, b'\xef\xbb\x20') + def test_truncated_utf8_at_eof(self): + # Regression test for https://issues.oss-fuzz.com/issues/451112368 + # Truncated multi-byte UTF-8 sequences at end of input caused an + # out-of-bounds read in Parser/tokenizer/helpers.c:valid_utf8(). + truncated = [ + b'\xc2', # 2-byte lead, missing 1 continuation + b'\xdf', # 2-byte lead, missing 1 continuation + b'\xe0', # 3-byte lead, missing 2 continuations + b'\xe0\xa0', # 3-byte lead, missing 1 continuation + b'\xf0\x90', # 4-byte lead, missing 2 continuations + b'\xf0\x90\x80', # 4-byte lead, missing 1 continuation + b'\xf3', # 4-byte lead, missing 3 (the oss-fuzz reproducer) + ] + for seq in truncated: + with self.subTest(seq=seq): + self.assertRaises(SyntaxError, compile, seq, '', 'exec') + @support.requires_subprocess() def test_20731(self): sub = subprocess.Popen([sys.executable, diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 1e595bdbd64..7e55785bd4a 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1387,6 +1387,11 @@ def test_blob_get_slice(self): def test_blob_get_empty_slice(self): self.assertEqual(self.blob[5:5], b"") + def test_blob_get_empty_slice_oob_indices(self): + self.cx.execute("insert into test(b) values (?)", (b"abc",)) + with self.cx.blobopen("test", "b", 2) as blob: + self.assertEqual(blob[5:-5], b"") + def test_blob_get_slice_negative_index(self): self.assertEqual(self.blob[5:-5], self.data[5:-5]) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 67a63907293..423d0292b4e 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -50,6 +50,16 @@ IS_OPENSSL_3_0_0 = ssl.OPENSSL_VERSION_INFO >= (3, 0, 0) PY_SSL_DEFAULT_CIPHERS = sysconfig.get_config_var('PY_SSL_DEFAULT_CIPHERS') +HAS_KEYLOG = hasattr(ssl.SSLContext, 'keylog_filename') +requires_keylog = unittest.skipUnless( + HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') +CAN_SET_KEYLOG = HAS_KEYLOG and os.name != "nt" +requires_keylog_setter = unittest.skipUnless( + CAN_SET_KEYLOG, + "cannot set 'keylog_filename' on Windows" +) + + PROTOCOL_TO_TLS_VERSION = {} for proto, ver in ( ("PROTOCOL_SSLv3", "SSLv3"), @@ -258,26 +268,67 @@ def utc_offset(): #NOTE: ignore issues like #1647654 ) -def test_wrap_socket(sock, *, - cert_reqs=ssl.CERT_NONE, ca_certs=None, - ciphers=None, certfile=None, keyfile=None, - **kwargs): - if not kwargs.get("server_side"): - kwargs["server_hostname"] = SIGNED_CERTFILE_HOSTNAME - context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - else: +def make_test_context( + *, + server_side=False, + check_hostname=None, + cert_reqs=ssl.CERT_NONE, + ca_certs=None, certfile=None, keyfile=None, + ciphers=None, + min_version=None, max_version=None, +): + if server_side: context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - if cert_reqs is not None: + else: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + + if check_hostname is None: if cert_reqs == ssl.CERT_NONE: context.check_hostname = False + else: + context.check_hostname = check_hostname + + if cert_reqs is not None: context.verify_mode = cert_reqs + if ca_certs is not None: context.load_verify_locations(ca_certs) if certfile is not None or keyfile is not None: context.load_cert_chain(certfile, keyfile) + if ciphers is not None: context.set_ciphers(ciphers) - return context.wrap_socket(sock, **kwargs) + + if min_version is not None: + context.minimum_version = min_version + if max_version is not None: + context.maximum_version = max_version + + return context + + +def test_wrap_socket( + sock, + *, + server_side=False, + check_hostname=None, + cert_reqs=ssl.CERT_NONE, + ca_certs=None, certfile=None, keyfile=None, + ciphers=None, + min_version=None, max_version=None, + **kwargs, +): + context = make_test_context( + server_side=server_side, + check_hostname=check_hostname, + cert_reqs=cert_reqs, + ca_certs=ca_certs, certfile=certfile, keyfile=keyfile, + ciphers=ciphers, + min_version=min_version, max_version=max_version, + ) + if not server_side: + kwargs.setdefault("server_hostname", SIGNED_CERTFILE_HOSTNAME) + return context.wrap_socket(sock, server_side=server_side, **kwargs) USE_SAME_TEST_CONTEXT = False @@ -1085,7 +1136,12 @@ def test_min_max_version(self): ctx.maximum_version = ssl.TLSVersion.MINIMUM_SUPPORTED self.assertIn( ctx.maximum_version, - {ssl.TLSVersion.TLSv1, ssl.TLSVersion.TLSv1_1, ssl.TLSVersion.SSLv3} + { + ssl.TLSVersion.TLSv1, + ssl.TLSVersion.TLSv1_1, + ssl.TLSVersion.TLSv1_2, + ssl.TLSVersion.SSLv3, + } ) ctx.minimum_version = ssl.TLSVersion.MAXIMUM_SUPPORTED @@ -1660,6 +1716,39 @@ def test_num_tickest(self): with self.assertRaises(ValueError): ctx.num_tickets = 1 + @support.cpython_only + def test_refcycle_msg_callback(self): + # See https://github.com/python/cpython/issues/142516. + ctx = make_test_context() + def msg_callback(*args, _=ctx, **kwargs): ... + ctx._msg_callback = msg_callback + + @support.cpython_only + @requires_keylog_setter + def test_refcycle_keylog_filename(self): + # See https://github.com/python/cpython/issues/142516. + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + ctx = make_test_context() + class KeylogFilename(str): ... + ctx.keylog_filename = KeylogFilename(os_helper.TESTFN) + ctx.keylog_filename._ = ctx + + @support.cpython_only + @unittest.skipUnless(ssl.HAS_PSK, 'requires TLS-PSK') + def test_refcycle_psk_client_callback(self): + # See https://github.com/python/cpython/issues/142516. + ctx = make_test_context() + def psk_client_callback(*args, _=ctx, **kwargs): ... + ctx.set_psk_client_callback(psk_client_callback) + + @support.cpython_only + @unittest.skipUnless(ssl.HAS_PSK, 'requires TLS-PSK') + def test_refcycle_psk_server_callback(self): + # See https://github.com/python/cpython/issues/142516. + ctx = make_test_context(server_side=True) + def psk_server_callback(*args, _=ctx, **kwargs): ... + ctx.set_psk_server_callback(psk_server_callback) + class SSLErrorTests(unittest.TestCase): @@ -4909,10 +4998,6 @@ def test_internal_chain_server(self): self.assertEqual(res, b'\x02\n') -HAS_KEYLOG = hasattr(ssl.SSLContext, 'keylog_filename') -requires_keylog = unittest.skipUnless( - HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') - class TestSSLDebug(unittest.TestCase): def keylog_lines(self, fname=os_helper.TESTFN): diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 59133e24e64..598f10b1862 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -552,6 +552,15 @@ def test_count_overflow(self): hugecount2 = '{}b{}H'.format(sys.maxsize//2, sys.maxsize//2) self.assertRaises(struct.error, struct.calcsize, hugecount2) + hugecount3 = '{}i{}q'.format(sys.maxsize // 4, sys.maxsize // 8) + self.assertRaises(struct.error, struct.calcsize, hugecount3) + + hugecount4 = '{}?s'.format(sys.maxsize) + self.assertRaises(struct.error, struct.calcsize, hugecount4) + + hugecount5 = '{}?p'.format(sys.maxsize) + self.assertRaises(struct.error, struct.calcsize, hugecount5) + def test_trailing_counter(self): store = array.array('b', b' '*100) @@ -581,8 +590,24 @@ def test_Struct_reinitialization(self): # Issue 9422: there was a memory leak when reinitializing a # Struct instance. This test can be used to detect the leak # when running with regrtest -L. - s = struct.Struct('i') - s.__init__('ii') + s = struct.Struct('>h') + s.__init__('>hh') + self.assertEqual(s.format, '>hh') + packed = b'\x00\x01\x00\x02' + self.assertEqual(s.pack(1, 2), packed) + self.assertEqual(s.unpack(packed), (1, 2)) + + with self.assertRaises(UnicodeEncodeError): + s.__init__('\udc00') + self.assertEqual(s.format, '>hh') + self.assertEqual(s.pack(1, 2), packed) + self.assertEqual(s.unpack(packed), (1, 2)) + + with self.assertRaises(struct.error): + s.__init__('$') + self.assertEqual(s.format, '>hh') + self.assertEqual(s.pack(1, 2), packed) + self.assertEqual(s.unpack(packed), (1, 2)) def check_sizeof(self, format_str, number_of_codes): # The size of 'PyStructObject' @@ -833,6 +858,8 @@ def test_operations_on_half_initialized_Struct(self): self.assertRaises(RuntimeError, S.unpack, spam) self.assertRaises(RuntimeError, S.unpack_from, spam) self.assertRaises(RuntimeError, getattr, S, 'format') + self.assertRaises(RuntimeError, S.__sizeof__) + self.assertRaises(RuntimeError, repr, S) self.assertEqual(S.size, -1) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 27cd5afb390..79f0e2eb29f 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -778,6 +778,7 @@ def test_get_signal_name(self): (128 + int(signal.SIGABRT), 'SIGABRT'), (3221225477, "STATUS_ACCESS_VIOLATION"), (0xC00000FD, "STATUS_STACK_OVERFLOW"), + (0xC0000906, "0xC0000906"), ): self.assertEqual(support.get_signal_name(exitcode), expected, exitcode) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 52b13b98cbc..2c524635b57 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -330,17 +330,36 @@ def _mock_candidate_names(*names): class TestBadTempdir: def test_read_only_directory(self): with _inside_empty_temp_dir(): - oldmode = mode = os.stat(tempfile.tempdir).st_mode - mode &= ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) - os.chmod(tempfile.tempdir, mode) + probe = os.path.join(tempfile.tempdir, 'probe') + if os.name == 'nt': + cmd = ['icacls', tempfile.tempdir, '/deny', 'Everyone:(W)'] + stdout = None if support.verbose > 1 else subprocess.DEVNULL + subprocess.run(cmd, check=True, stdout=stdout) + else: + oldmode = mode = os.stat(tempfile.tempdir).st_mode + mode &= ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) + mode = stat.S_IREAD + os.chmod(tempfile.tempdir, mode) try: - if os.access(tempfile.tempdir, os.W_OK): + # Check that the directory is read-only. + try: + os.mkdir(probe) + except PermissionError: + pass + else: + os.rmdir(probe) self.skipTest("can't set the directory read-only") + # gh-66305: Now it takes a split second, but previously + # it took about 10 days on Windows. with self.assertRaises(PermissionError): self.make_temp() - self.assertEqual(os.listdir(tempfile.tempdir), []) finally: - os.chmod(tempfile.tempdir, oldmode) + if os.name == 'nt': + cmd = ['icacls', tempfile.tempdir, '/grant:r', 'Everyone:(M)'] + subprocess.run(cmd, check=True, stdout=stdout) + else: + os.chmod(tempfile.tempdir, oldmode) + self.assertEqual(os.listdir(tempfile.tempdir), []) def test_nonexisting_directory(self): with _inside_empty_temp_dir(): diff --git a/Lib/test/test_tools/test_compute_changes.py b/Lib/test/test_tools/test_compute_changes.py new file mode 100644 index 00000000000..b20ff975fc2 --- /dev/null +++ b/Lib/test/test_tools/test_compute_changes.py @@ -0,0 +1,144 @@ +"""Tests to cover the Tools/build/compute-changes.py script.""" + +import importlib +import os +import unittest +from pathlib import Path +from unittest.mock import patch + +from test.test_tools import skip_if_missing, imports_under_tool + +skip_if_missing("build") + +with patch.dict(os.environ, {"GITHUB_DEFAULT_BRANCH": "main"}): + with imports_under_tool("build"): + compute_changes = importlib.import_module("compute-changes") + +process_changed_files = compute_changes.process_changed_files +Outputs = compute_changes.Outputs +ANDROID_DIRS = compute_changes.ANDROID_DIRS +IOS_DIRS = compute_changes.IOS_DIRS +MACOS_DIRS = compute_changes.MACOS_DIRS +WASI_DIRS = compute_changes.WASI_DIRS +RUN_TESTS_IGNORE = compute_changes.RUN_TESTS_IGNORE +UNIX_BUILD_SYSTEM_FILE_NAMES = compute_changes.UNIX_BUILD_SYSTEM_FILE_NAMES +LIBRARY_FUZZER_PATHS = compute_changes.LIBRARY_FUZZER_PATHS + + +class TestProcessChangedFiles(unittest.TestCase): + + def test_windows(self): + f = {Path(".github/workflows/reusable-windows.yml")} + result = process_changed_files(f) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_windows_tests) + + def test_docs(self): + for f in ( + ".github/workflows/reusable-docs.yml", + "Doc/library/datetime.rst", + "Doc/Makefile", + ): + with self.subTest(f=f): + result = process_changed_files({Path(f)}) + self.assertTrue(result.run_docs) + self.assertFalse(result.run_tests) + + def test_ci_fuzz_stdlib(self): + for p in LIBRARY_FUZZER_PATHS: + with self.subTest(p=p): + if p.is_dir(): + f = p / "file" + elif p.is_file(): + f = p + else: + continue + result = process_changed_files({f}) + self.assertTrue(result.run_ci_fuzz_stdlib) + + def test_android(self): + for d in ANDROID_DIRS: + with self.subTest(d=d): + result = process_changed_files({Path(d) / "file"}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_android) + self.assertFalse(result.run_windows_tests) + + def test_ios(self): + for d in IOS_DIRS: + with self.subTest(d=d): + result = process_changed_files({Path(d) / "file"}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_ios) + self.assertFalse(result.run_windows_tests) + + def test_macos(self): + f = {Path(".github/workflows/reusable-macos.yml")} + result = process_changed_files(f) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_macos) + + for d in MACOS_DIRS: + with self.subTest(d=d): + result = process_changed_files({Path(d) / "file"}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_macos) + self.assertFalse(result.run_windows_tests) + + def test_wasi(self): + f = {Path(".github/workflows/reusable-wasi.yml")} + result = process_changed_files(f) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_wasi) + + for d in WASI_DIRS: + with self.subTest(d=d): + result = process_changed_files({d / "file"}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_wasi) + self.assertFalse(result.run_windows_tests) + + def test_unix(self): + for f in UNIX_BUILD_SYSTEM_FILE_NAMES: + with self.subTest(f=f): + result = process_changed_files({f}) + self.assertTrue(result.run_tests) + self.assertFalse(result.run_windows_tests) + + def test_msi(self): + for f in ( + ".github/workflows/reusable-windows-msi.yml", + "Tools/msi/build.bat", + ): + with self.subTest(f=f): + result = process_changed_files({Path(f)}) + self.assertTrue(result.run_windows_msi) + + def test_all_run(self): + for f in ( + ".github/workflows/some-new-workflow.yml", + ".github/workflows/build.yml", + ): + with self.subTest(f=f): + result = process_changed_files({Path(f)}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_android) + self.assertTrue(result.run_ios) + self.assertTrue(result.run_macos) + self.assertTrue(result.run_ubuntu) + self.assertTrue(result.run_wasi) + + def test_all_ignored(self): + for f in RUN_TESTS_IGNORE: + with self.subTest(f=f): + self.assertEqual(process_changed_files({Path(f)}), Outputs()) + + def test_wasi_and_android(self): + f = {Path(".github/workflows/reusable-wasi.yml"), Path("Android/file")} + result = process_changed_files(f) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_wasi) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index c87ad67ed05..db6063b8650 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -4168,6 +4168,27 @@ def method(self, name): self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_luch'))) self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, 'bluch'))) + def test_getattr_suggestions_with_custom___dir__(self): + class M(type): + def __dir__(cls): + return [None, "fox"] + + class C0: + def __dir__(self): + return [..., "bluch"] + + class C1(C0, metaclass=M): + pass + + self.assertNotIn("'bluch'", self.get_suggestion(C0, "blach")) + self.assertIn("'bluch'", self.get_suggestion(C0(), "blach")) + + self.assertIn("'fox'", self.get_suggestion(C1, "foo")) + self.assertNotIn("'fox'", self.get_suggestion(C1(), "foo")) + + self.assertNotIn("'bluch'", self.get_suggestion(C1, "blach")) + self.assertIn("'bluch'", self.get_suggestion(C1(), "blach")) + def test_getattr_suggestions_do_not_trigger_for_long_attributes(self): class A: blech = None diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 0f393def827..84c1b954136 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -152,6 +152,13 @@ def test_incorrect_mro_explicit_object(self): with self.assertRaisesRegex(TypeError, r"\(MRO\) for bases object, Generic"): class My[X](object): ... + def test_compile_error_in_type_param_bound(self): + # This should not crash, see gh-145187 + check_syntax_error( + self, + "if True:\n class h[l:{7for*()in 0}]:2" + ) + class TypeParamsNonlocalTest(unittest.TestCase): def test_nonlocal_disallowed_01(self): diff --git a/Lib/test/test_ucn.py b/Lib/test/test_ucn.py index 0e2c25aaff2..fb8e98af25b 100644 --- a/Lib/test/test_ucn.py +++ b/Lib/test/test_ucn.py @@ -88,6 +88,9 @@ def test_hangul_syllables(self): self.checkletter("HANGUL SYLLABLE HWEOK", "\ud6f8") self.checkletter("HANGUL SYLLABLE HIH", "\ud7a3") + self.checkletter("haNGul SYllABle WAe", '\uc65c') + self.checkletter("HAngUL syLLabLE waE", '\uc65c') + self.assertRaises(ValueError, unicodedata.name, "\ud7a4") def test_cjk_unified_ideographs(self): @@ -103,6 +106,35 @@ def test_cjk_unified_ideographs(self): self.checkletter("CJK UNIFIED IDEOGRAPH-2B81D", "\U0002B81D") self.checkletter("CJK UNIFIED IDEOGRAPH-3134A", "\U0003134A") + self.checkletter("cjK UniFIeD idEogRAph-3aBc", "\u3abc") + self.checkletter("CJk uNIfiEd IDeOGraPH-3AbC", "\u3abc") + self.checkletter("cjK UniFIeD idEogRAph-2aBcD", "\U0002abcd") + self.checkletter("CJk uNIfiEd IDeOGraPH-2AbCd", "\U0002abcd") + + def test_tangut_ideographs(self): + self.checkletter("TANGUT IDEOGRAPH-17000", "\U00017000") + self.checkletter("TANGUT IDEOGRAPH-187F7", "\U000187f7") + self.checkletter("TANGUT IDEOGRAPH-18D00", "\U00018D00") + self.checkletter("TANGUT IDEOGRAPH-18D08", "\U00018d08") + self.checkletter("tangut ideograph-18d08", "\U00018d08") + + def test_egyptian_hieroglyphs(self): + self.checkletter("EGYPTIAN HIEROGLYPH-13460", "\U00013460") + self.checkletter("EGYPTIAN HIEROGLYPH-143FA", "\U000143fa") + self.checkletter("egyptian hieroglyph-143fa", "\U000143fa") + + def test_khitan_small_script_characters(self): + self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18B00", "\U00018b00") + self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18CD5", "\U00018cd5") + self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18CFF", "\U00018cff") + self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18CFF", "\U00018cff") + self.checkletter("khitan small script character-18cff", "\U00018cff") + + def test_nushu_characters(self): + self.checkletter("NUSHU CHARACTER-1B170", "\U0001b170") + self.checkletter("NUSHU CHARACTER-1B2FB", "\U0001b2fb") + self.checkletter("nushu character-1b2fb", "\U0001b2fb") + def test_bmp_characters(self): for code in range(0x10000): char = chr(code) diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index e98539deb99..93d57399640 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -89,9 +89,9 @@ class UnicodeFunctionsTest(unittest.TestCase): # Update this if the database changes. Make sure to do a full rebuild # (e.g. 'make distclean && make') to get the correct checksum. - expectedchecksum = ('35e842600fa7ae2db93739db08ef201b726a2374' + expectedchecksum = ('1ba453ec456896f1190d849b6e9b7c2e1a4128e0' if quicktest else - '23ab09ed4abdf93db23b97359108ed630dd8311d') + '46ca89d9fe34881d0be3a4a4b29f5aa8c019640c') def test_function_checksum(self): db = self.db @@ -116,6 +116,56 @@ def test_function_checksum(self): result = h.hexdigest() self.assertEqual(result, self.expectedchecksum) + def test_name(self): + name = self.db.name + self.assertRaises(ValueError, name, '\0') + self.assertRaises(ValueError, name, '\n') + self.assertRaises(ValueError, name, '\x1F') + self.assertRaises(ValueError, name, '\x7F') + self.assertRaises(ValueError, name, '\x9F') + self.assertRaises(ValueError, name, '\uFFFE') + self.assertRaises(ValueError, name, '\uFFFF') + self.assertRaises(ValueError, name, '\U0010FFFF') + self.assertEqual(name('\U0010FFFF', 42), 42) + + self.assertEqual(name(' '), 'SPACE') + self.assertEqual(name('1'), 'DIGIT ONE') + self.assertEqual(name('A'), 'LATIN CAPITAL LETTER A') + self.assertEqual(name('\xA0'), 'NO-BREAK SPACE') + self.assertEqual(name('\u0221', None), None if self.old else + 'LATIN SMALL LETTER D WITH CURL') + self.assertEqual(name('\u3400'), 'CJK UNIFIED IDEOGRAPH-3400') + self.assertEqual(name('\u9FA5'), 'CJK UNIFIED IDEOGRAPH-9FA5') + self.assertEqual(name('\uAC00'), 'HANGUL SYLLABLE GA') + self.assertEqual(name('\uD7A3'), 'HANGUL SYLLABLE HIH') + self.assertEqual(name('\uF900'), 'CJK COMPATIBILITY IDEOGRAPH-F900') + self.assertEqual(name('\uFA6A'), 'CJK COMPATIBILITY IDEOGRAPH-FA6A') + self.assertEqual(name('\uFBF9'), + 'ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH HAMZA ' + 'ABOVE WITH ALEF MAKSURA ISOLATED FORM') + self.assertEqual(name('\U00013460', None), None if self.old else + 'EGYPTIAN HIEROGLYPH-13460') + self.assertEqual(name('\U000143FA', None), None if self.old else + 'EGYPTIAN HIEROGLYPH-143FA') + self.assertEqual(name('\U00018B00', None), None if self.old else + 'KHITAN SMALL SCRIPT CHARACTER-18B00') + self.assertEqual(name('\U00018CD5', None), None if self.old else + 'KHITAN SMALL SCRIPT CHARACTER-18CD5') + self.assertEqual(name('\U00018CFF', None), None if self.old else + 'KHITAN SMALL SCRIPT CHARACTER-18CFF') + self.assertEqual(name('\U0001B170', None), None if self.old else + 'NUSHU CHARACTER-1B170') + self.assertEqual(name('\U0001B2FB', None), None if self.old else + 'NUSHU CHARACTER-1B2FB') + self.assertEqual(name('\U0001FBA8', None), None if self.old else + 'BOX DRAWINGS LIGHT DIAGONAL UPPER CENTRE TO ' + 'MIDDLE LEFT AND MIDDLE RIGHT TO LOWER CENTRE') + self.assertEqual(name('\U0002A6D6'), 'CJK UNIFIED IDEOGRAPH-2A6D6') + self.assertEqual(name('\U0002FA1D'), 'CJK COMPATIBILITY IDEOGRAPH-2FA1D') + self.assertEqual(name('\U000323AF', None), None if self.old else + 'CJK UNIFIED IDEOGRAPH-323AF') + + @requires_resource('cpu') def test_name_inverse_lookup(self): for char in iterallchars(): looked_name = self.db.name(char, None) @@ -139,6 +189,17 @@ def test_lookup_nonexistant(self): "HANDBUG", "MODIFIER LETTER CYRILLIC SMALL QUESTION MARK", "???", + "CJK UNIFIED IDEOGRAPH-03400", + "CJK UNIFIED IDEOGRAPH-020000", + "CJK UNIFIED IDEOGRAPH-33FF", + "CJK UNIFIED IDEOGRAPH-F900", + "CJK UNIFIED IDEOGRAPH-13460", + "CJK UNIFIED IDEOGRAPH-17000", + "CJK UNIFIED IDEOGRAPH-18B00", + "CJK UNIFIED IDEOGRAPH-1B170", + "CJK COMPATIBILITY IDEOGRAPH-3400", + "TANGUT IDEOGRAPH-3400", + "HANGUL SYLLABLE AC00", ]: self.assertRaises(KeyError, self.db.lookup, nonexistent) @@ -170,10 +231,14 @@ def test_numeric(self): # New in 4.1.0 self.assertEqual(self.db.numeric('\U0001012A', None), None if self.old else 9000) + # Changed in 4.1.0 + self.assertEqual(self.db.numeric('\u5793', None), 1e20 if self.old else None) # New in 5.0.0 self.assertEqual(self.db.numeric('\u07c0', None), None if self.old else 0.0) # New in 5.1.0 self.assertEqual(self.db.numeric('\ua627', None), None if self.old else 7.0) + # Changed in 5.2.0 + self.assertEqual(self.db.numeric('\u09f6'), 3.0 if self.old else 3/16) # New in 6.0.0 self.assertEqual(self.db.numeric('\u0b72', None), None if self.old else 0.25) # New in 12.0.0 @@ -281,6 +346,12 @@ def test_decomposition(self): # New in 16.0.0 self.assertEqual(self.db.decomposition('\U0001CCD6'), '' if self.old else ' 0041') + # Hangul characters + self.assertEqual(self.db.decomposition('\uAC00'), '1100 1161') + self.assertEqual(self.db.decomposition('\uD4DB'), '1111 1171 11B6') + self.assertEqual(self.db.decomposition('\uC2F8'), '110A 1161') + self.assertEqual(self.db.decomposition('\uD7A3'), '1112 1175 11C2') + self.assertRaises(TypeError, self.db.decomposition) self.assertRaises(TypeError, self.db.decomposition, 'xx') @@ -584,9 +655,9 @@ def test_east_asian_width_unassigned(self): class Unicode_3_2_0_FunctionsTest(UnicodeFunctionsTest): db = unicodedata.ucd_3_2_0 old = True - expectedchecksum = ('76b126d719d52ba11788a627d058163106da7d56' + expectedchecksum = ('883824cb6c0ccf994e4451ebf281e2d6d479af47' if quicktest else - 'ed843cb7ab5aaf149466498db27fefce81c4214c') + 'caf1a7f2f380f927461837f1901ef20683f98683') class UnicodeMiscTest(unittest.TestCase): diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py index 9e2d54bd3a8..c948d51452d 100644 --- a/Lib/test/test_unpack_ex.py +++ b/Lib/test/test_unpack_ex.py @@ -383,13 +383,13 @@ Some size constraints (all fail.) - >>> s = ", ".join("a%d" % i for i in range(1<<8)) + ", *rest = range(1<<8 + 1)" + >>> s = ", ".join("a%d" % i for i in range(1<<8)) + ", *rest = range((1<<8) + 1)" >>> compile(s, 'test', 'exec') # doctest:+ELLIPSIS Traceback (most recent call last): ... SyntaxError: too many expressions in star-unpacking assignment - >>> s = ", ".join("a%d" % i for i in range(1<<8 + 1)) + ", *rest = range(1<<8 + 2)" + >>> s = ", ".join("a%d" % i for i in range((1<<8) + 1)) + ", *rest = range((1<<8) + 2)" >>> compile(s, 'test', 'exec') # doctest:+ELLIPSIS Traceback (most recent call last): ... diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index d108165be51..e63b9dfc182 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -11,13 +11,13 @@ import os.path import pathlib import re +import shlex import shutil import struct import subprocess import sys import sysconfig import tempfile -import shlex from test.support import (captured_stdout, captured_stderr, skip_if_broken_multiprocessing_synchronize, verbose, requires_subprocess, is_android, is_apple_mobile, @@ -379,6 +379,16 @@ def create_contents(self, paths, filename): with open(fn, 'wb') as f: f.write(b'Still here?') + @unittest.skipUnless(hasattr(os, 'listxattr'), 'test requires os.listxattr') + def test_install_scripts_selinux(self): + """ + gh-145417: Test that install_scripts does not copy SELinux context + when copying scripts. + """ + with patch('os.listxattr') as listxattr_mock: + venv.create(self.env_dir) + listxattr_mock.assert_not_called() + def test_overwrite_existing(self): """ Test creating environment in an existing directory. diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py index 0bf9e947b5e..0b33db93780 100644 --- a/Lib/test/test_wsgiref.py +++ b/Lib/test/test_wsgiref.py @@ -504,14 +504,20 @@ def testExtras(self): ) def testRaisesControlCharacters(self): - headers = Headers() for c0 in control_characters_c0(): - self.assertRaises(ValueError, headers.__setitem__, f"key{c0}", "val") - self.assertRaises(ValueError, headers.__setitem__, "key", f"val{c0}") - self.assertRaises(ValueError, headers.add_header, f"key{c0}", "val", param="param") - self.assertRaises(ValueError, headers.add_header, "key", f"val{c0}", param="param") - self.assertRaises(ValueError, headers.add_header, "key", "val", param=f"param{c0}") - + with self.subTest(c0): + headers = Headers() + self.assertRaises(ValueError, headers.__setitem__, f"key{c0}", "val") + self.assertRaises(ValueError, headers.add_header, f"key{c0}", "val", param="param") + # HTAB (\x09) is allowed in values, not names. + if c0 == "\t": + headers["key"] = f"val{c0}" + headers.add_header("key", f"val{c0}") + headers.setdefault(f"key", f"val{c0}") + else: + self.assertRaises(ValueError, headers.__setitem__, "key", f"val{c0}") + self.assertRaises(ValueError, headers.add_header, "key", f"val{c0}", param="param") + self.assertRaises(ValueError, headers.add_header, "key", "val", param=f"param{c0}") class ErrorHandler(BaseCGIHandler): """Simple handler subclass for testing BaseHandler""" @@ -849,6 +855,25 @@ def write(self, b): self.assertIsNotNone(h.status) self.assertIsNotNone(h.environ) + def testRaisesControlCharacters(self): + for c0 in control_characters_c0(): + with self.subTest(c0): + base = BaseHandler() + with self.assertRaises(ValueError): + base.start_response(c0, [('x', 'y')]) + + base = BaseHandler() + with self.assertRaises(ValueError): + base.start_response('200 OK', [(c0, 'y')]) + + # HTAB (\x09) is allowed in header values, but not in names. + base = BaseHandler() + if c0 != "\t": + with self.assertRaises(ValueError): + base.start_response('200 OK', [('x', c0)]) + else: + base.start_response('200 OK', [('x', c0)]) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index 88d79b258cd..8a58c7d68ac 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -1577,6 +1577,44 @@ class EvilZoneInfo(self.klass): class CZoneInfoCacheTest(ZoneInfoCacheTest): module = c_zoneinfo + def test_inconsistent_weak_cache_get(self): + class Cache: + def get(self, key, default=None): + return 1337 + + class ZI(self.klass): + pass + # Class attribute must be set after class creation + # to override zoneinfo.ZoneInfo.__init_subclass__. + ZI._weak_cache = Cache() + + with self.assertRaises(RuntimeError) as te: + ZI("America/Los_Angeles") + self.assertEqual( + str(te.exception), + "Unexpected instance of int in ZI weak cache for key 'America/Los_Angeles'" + ) + + def test_inconsistent_weak_cache_setdefault(self): + class Cache: + def get(self, key, default=None): + return default + def setdefault(self, key, value): + return 1337 + + class ZI(self.klass): + pass + # Class attribute must be set after class creation + # to override zoneinfo.ZoneInfo.__init_subclass__. + ZI._weak_cache = Cache() + + with self.assertRaises(RuntimeError) as te: + ZI("America/Los_Angeles") + self.assertEqual( + str(te.exception), + "Unexpected instance of int in ZI weak cache for key 'America/Los_Angeles'" + ) + class ZoneInfoPickleTest(TzPathUserMixin, ZoneInfoTestBase): module = py_zoneinfo diff --git a/Lib/traceback.py b/Lib/traceback.py index 5a34a2b87b6..79f67b9878b 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1589,17 +1589,23 @@ def _substitution_cost(ch_a, ch_b): return _MOVE_COST +def _get_safe___dir__(obj): + # Use obj.__dir__() to avoid a TypeError when calling dir(obj). + # See gh-131001 and gh-139933. + try: + d = obj.__dir__() + except TypeError: # when obj is a class + d = type(obj).__dir__(obj) + return sorted(x for x in d if isinstance(x, str)) + + def _compute_suggestion_error(exc_value, tb, wrong_name): if wrong_name is None or not isinstance(wrong_name, str): return None if isinstance(exc_value, AttributeError): obj = exc_value.obj try: - try: - d = dir(obj) - except TypeError: # Attributes are unsortable, e.g. int and str - d = list(obj.__class__.__dict__.keys()) + list(obj.__dict__.keys()) - d = sorted([x for x in d if isinstance(x, str)]) + d = _get_safe___dir__(obj) hide_underscored = (wrong_name[:1] != '_') if hide_underscored and tb is not None: while tb.tb_next is not None: @@ -1614,11 +1620,7 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): elif isinstance(exc_value, ImportError): try: mod = __import__(exc_value.name) - try: - d = dir(mod) - except TypeError: # Attributes are unsortable, e.g. int and str - d = list(mod.__dict__.keys()) - d = sorted([x for x in d if isinstance(x, str)]) + d = _get_safe___dir__(mod) if wrong_name[:1] != '_': d = [x for x in d if x[:1] != '_'] except Exception: diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index feb9730d6e9..545b0730d2e 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -1184,10 +1184,16 @@ def _increment_mock_call(self, /, *args, **kwargs): # handle call_args # needs to be set here so assertions on call arguments pass before # execution in the case of awaited calls - _call = _Call((args, kwargs), two=True) - self.call_args = _call - self.call_args_list.append(_call) - self.call_count = len(self.call_args_list) + with NonCallableMock._lock: + # Lock is used here so that call_args_list and call_count are + # set atomically otherwise it is possible that by the time call_count + # is set another thread may have appended to call_args_list. + # The rest of this function relies on list.append being atomic and + # skips locking. + _call = _Call((args, kwargs), two=True) + self.call_args = _call + self.call_args_list.append(_call) + self.call_count = len(self.call_args_list) # initial stuff for method_calls: do_method_calls = self._mock_parent is not None diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 67d9bbea0d3..a651e815ddc 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -1,6 +1,6 @@ """Parse (absolute and relative) URLs. -urlparse module is based upon the following RFC specifications. +urllib.parse module is based upon the following RFC specifications. RFC 3986 (STD66): "Uniform Resource Identifiers" by T. Berners-Lee, R. Fielding and L. Masinter, January 2005. @@ -20,7 +20,7 @@ McCahill, December 1994 RFC 3986 is considered the current standard and any future changes to -urlparse module should conform with it. The urlparse module is +urllib.parse module should conform with it. The urllib.parse module is currently not entirely compliant with this RFC due to defacto scenarios for parsing, and for backward compatibility purposes, some parsing quirks from older RFCs are retained. The testcases in @@ -390,6 +390,8 @@ def urlparse(url, scheme='', allow_fragments=True): path or query. Note that % escapes are not expanded. + + urlsplit() should generally be used instead of urlparse(). """ url, scheme, _coerce_result = _coerce_args(url, scheme) scheme, netloc, url, params, query, fragment = _urlparse(url, scheme, allow_fragments) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 17ee28e826d..88f3340af41 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -588,7 +588,7 @@ def skip_file(f): 'may be binary: %s', srcfile, e) continue if new_data == data: - shutil.copy2(srcfile, dstfile) + shutil.copy(srcfile, dstfile) else: with open(dstfile, 'wb') as f: f.write(new_data) diff --git a/Lib/wsgiref/handlers.py b/Lib/wsgiref/handlers.py index cafe872c7aa..f8fe89f6e13 100644 --- a/Lib/wsgiref/handlers.py +++ b/Lib/wsgiref/handlers.py @@ -1,7 +1,7 @@ """Base classes for server/gateway implementations""" from .util import FileWrapper, guess_scheme, is_hop_by_hop -from .headers import Headers +from .headers import Headers, _name_disallowed_re import sys, os, time @@ -249,6 +249,8 @@ def start_response(self, status, headers,exc_info=None): return self.write def _validate_status(self, status): + if _name_disallowed_re.search(status): + raise ValueError("Control characters are not allowed in status") if len(status) < 4: raise AssertionError("Status must be at least 4 characters") if not status[:3].isdigit(): diff --git a/Lib/wsgiref/headers.py b/Lib/wsgiref/headers.py index e180a623cb2..eb6ea6a412d 100644 --- a/Lib/wsgiref/headers.py +++ b/Lib/wsgiref/headers.py @@ -9,7 +9,11 @@ # existence of which force quoting of the parameter value. import re tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]') -_control_chars_re = re.compile(r'[\x00-\x1F\x7F]') +# Disallowed characters for headers and values. +# HTAB (\x09) is allowed in header values, but +# not in header names. (RFC 9110 Section 5.5) +_name_disallowed_re = re.compile(r'[\x00-\x1F\x7F]') +_value_disallowed_re = re.compile(r'[\x00-\x08\x0A-\x1F\x7F]') def _formatparam(param, value=None, quote=1): """Convenience function to format and return a key=value pair. @@ -36,13 +40,14 @@ def __init__(self, headers=None): self._headers = headers if __debug__: for k, v in headers: - self._convert_string_type(k) - self._convert_string_type(v) + self._convert_string_type(k, name=True) + self._convert_string_type(v, name=False) - def _convert_string_type(self, value): + def _convert_string_type(self, value, *, name): """Convert/check value type.""" if type(value) is str: - if _control_chars_re.search(value): + regex = (_name_disallowed_re if name else _value_disallowed_re) + if regex.search(value): raise ValueError("Control characters not allowed in headers") return value raise AssertionError("Header names/values must be" @@ -56,14 +61,14 @@ def __setitem__(self, name, val): """Set the value of a header.""" del self[name] self._headers.append( - (self._convert_string_type(name), self._convert_string_type(val))) + (self._convert_string_type(name, name=True), self._convert_string_type(val, name=False))) def __delitem__(self,name): """Delete all occurrences of a header, if present. Does *not* raise an exception if the header is missing. """ - name = self._convert_string_type(name.lower()) + name = self._convert_string_type(name.lower(), name=True) self._headers[:] = [kv for kv in self._headers if kv[0].lower() != name] def __getitem__(self,name): @@ -90,13 +95,13 @@ def get_all(self, name): fields deleted and re-inserted are always appended to the header list. If no fields exist with the given name, returns an empty list. """ - name = self._convert_string_type(name.lower()) + name = self._convert_string_type(name.lower(), name=True) return [kv[1] for kv in self._headers if kv[0].lower()==name] def get(self,name,default=None): """Get the first header value for 'name', or return 'default'""" - name = self._convert_string_type(name.lower()) + name = self._convert_string_type(name.lower(), name=True) for k,v in self._headers: if k.lower()==name: return v @@ -151,8 +156,8 @@ def setdefault(self,name,value): and value 'value'.""" result = self.get(name) if result is None: - self._headers.append((self._convert_string_type(name), - self._convert_string_type(value))) + self._headers.append((self._convert_string_type(name, name=True), + self._convert_string_type(value, name=False))) return value else: return result @@ -175,13 +180,13 @@ def add_header(self, _name, _value, **_params): """ parts = [] if _value is not None: - _value = self._convert_string_type(_value) + _value = self._convert_string_type(_value, name=False) parts.append(_value) for k, v in _params.items(): - k = self._convert_string_type(k) + k = self._convert_string_type(k, name=True) if v is None: parts.append(k.replace('_', '-')) else: - v = self._convert_string_type(v) + v = self._convert_string_type(v, name=False) parts.append(_formatparam(k.replace('_', '-'), v)) - self._headers.append((self._convert_string_type(_name), "; ".join(parts))) + self._headers.append((self._convert_string_type(_name, name=True), "; ".join(parts))) diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index ac2332e5846..19aea290b58 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -950,7 +950,7 @@ class ZipExtFile(io.BufferedIOBase): """ # Max size supported by decompressor. - MAX_N = 1 << 31 - 1 + MAX_N = (1 << 31) - 1 # Read from compressed files in 4k blocks. MIN_READ_SIZE = 4096 diff --git a/Lib/zoneinfo/_zoneinfo.py b/Lib/zoneinfo/_zoneinfo.py index 3ffdb4c8371..bd3fefc6c9d 100644 --- a/Lib/zoneinfo/_zoneinfo.py +++ b/Lib/zoneinfo/_zoneinfo.py @@ -47,7 +47,11 @@ def __new__(cls, key): cls._strong_cache[key] = cls._strong_cache.pop(key, instance) if len(cls._strong_cache) > cls._strong_cache_size: - cls._strong_cache.popitem(last=False) + try: + cls._strong_cache.popitem(last=False) + except KeyError: + # another thread may have already emptied the cache + pass return instance diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 4fd8d55f35a..e7732981d38 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.18", - url="https://github.com/openssl/openssl/releases/download/openssl-3.0.18/openssl-3.0.18.tar.gz", - checksum='d80c34f5cf902dccf1f1b5df5ebb86d0392e37049e5d73df1b3abae72e4ffe8b', + name="OpenSSL 3.0.19", + url="https://github.com/openssl/openssl/releases/download/openssl-3.0.19/openssl-3.0.19.tar.gz", + checksum='fa5a4143b8aae18be53ef2f3caf29a2e0747430b8bc74d32d88335b94ab63072', buildrecipe=build_universal_openssl, configure=None, install=None, diff --git a/Mac/BuildScript/resources/install_certificates.command b/Mac/BuildScript/resources/install_certificates.command index 19b4adac07b..700eb462b68 100755 --- a/Mac/BuildScript/resources/install_certificates.command +++ b/Mac/BuildScript/resources/install_certificates.command @@ -25,7 +25,8 @@ def main(): print(" -- pip install --upgrade certifi") subprocess.check_call([sys.executable, - "-E", "-s", "-m", "pip", "install", "--upgrade", "certifi"]) + "-E", "-s", "-m", "pip", "install", "--upgrade", "certifi", + "--disable-pip-version-check"]) import certifi diff --git a/Misc/ACKS b/Misc/ACKS index 949df55ea39..e17b83ae973 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1071,6 +1071,7 @@ Wolfgang Langner Detlef Lannert Rémi Lapeyre Soren Larsen +Seth Michael Larson Amos Latteier Keenan Lau Piers Lauder diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 59aceedb94d..3b37e125cae 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -70,21 +70,21 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "9b07560b6c1afa666bd78b8d3aa5c83fdda02149afdf048596d5b0e0dac1ee55" + "checksumValue": "c6ea8a5423f3966923060db2089f869017dfb10bcf2037394146e7a74caec0a8" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.18.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.19.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.18:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.19:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "openssl", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.0.18" + "versionInfo": "3.0.19" }, { "SPDXID": "SPDXRef-PACKAGE-sqlite", diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index d64857ce233..2c06a7d3745 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -6,11 +6,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "39e6f567a10e36b2e77727e98e60bbcb3eb3af0b" + "checksumValue": "f1b1126ed7da8f2068302e7a692b0600e6f94b07" }, { "algorithm": "SHA256", - "checksumValue": "122f2c27000472a201d337b9b31f7eb2b52d091b02857061a8880371612d9534" + "checksumValue": "31b15de82aa19a845156169a17a5488bf597e561b2c318d159ed583139b25e87" } ], "fileName": "Modules/expat/COPYING" @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "a4395dd0589a97aab0904f7a5f5dc5781a086aa2" + "checksumValue": "9bd33bd279c0d7ea37b0f2d7e07c7c53b7053507" }, { "algorithm": "SHA256", - "checksumValue": "610b844bbfa3ec955772cc825db4d4db470827d57adcb214ad372d0eaf00e591" + "checksumValue": "d20997001462356b5ce3810ebf5256c8205f58462c64f21eb9bf80f8d1822b08" } ], "fileName": "Modules/expat/expat.h" @@ -62,11 +62,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "c22196e3d8bee88fcdda715623b3b9d2119d2fb3" + "checksumValue": "e658ee5d638ab326109282ff09f1541e27fff8c2" }, { "algorithm": "SHA256", - "checksumValue": "f2c2283ba03b057e92beefc7f81ba901ebb6dfc1a45b036c8a7d65808eb77a84" + "checksumValue": "dbe0582b8f8a8140aca97009e8760105ceed9e7df01ea9d8b3fe47cebf2e5b2d" } ], "fileName": "Modules/expat/expat_external.h" @@ -90,11 +90,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7dce7d98943c5db33ae05e54801dcafb4547b9dd" + "checksumValue": "6a4a232233ba1034c3f2b459159d502e9b2d413b" }, { "algorithm": "SHA256", - "checksumValue": "6bfe307d52e7e4c71dbc30d3bd902a4905cdd83bbe4226a7e8dfa8e4c462a157" + "checksumValue": "c803935722f0dbdeeede7f040028fb119135e96dfad949479f8a5304b885bdd6" } ], "fileName": "Modules/expat/internal.h" @@ -174,11 +174,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "4c81a1f04fc653877c63c834145c18f93cd95f3e" + "checksumValue": "7d3d7d72aa56c53fb5b9e10c0e74e161381f0255" }, { "algorithm": "SHA256", - "checksumValue": "04a379615f476d55f95ca1853107e20627b48ca4afe8d0fd5981ac77188bf0a6" + "checksumValue": "f4f87aa0268d92f2b8f5e663788bfadd2e926477d0b061ed4463c02ad29a3e25" } ], "fileName": "Modules/expat/xmlparse.c" @@ -188,11 +188,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "ef767128d2dda99436712dcf3465dde5dbaab876" + "checksumValue": "c8769fcb93f00272a6e6ca560be633649c817ff7" }, { "algorithm": "SHA256", - "checksumValue": "71fb52aa302cf6f56e41943009965804f49ff2210d9bd15b258f70aaf70db772" + "checksumValue": "5b81f0eb0e144b611dbd1bc9e6037075a16bff94f823d57a81eb2a3e4999e91a" } ], "fileName": "Modules/expat/xmlrole.c" @@ -216,11 +216,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "1e2d35d90a1c269217f83d3bdf3c71cc22cb4c3f" + "checksumValue": "63e4766a09e63760c6518670509198f8d638f4ad" }, { "algorithm": "SHA256", - "checksumValue": "98d0fc735041956cc2e7bbbe2fb8f03130859410e0aee5e8015f406a37c02a3c" + "checksumValue": "0ad3f915f2748dc91bf4e4b4a50cf40bf2c95769d0eca7e3b293a230d82bb779" } ], "fileName": "Modules/expat/xmltok.c" @@ -272,11 +272,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "2d82d0a1201f78d478b30d108ff8fc27ee3e2672" + "checksumValue": "41b8c8fc275882c76d4210b7d40a18e506b07147" }, { "algorithm": "SHA256", - "checksumValue": "6ce6d03193279078d55280150fe91e7370370b504a6c123a79182f28341f3e90" + "checksumValue": "b2188c7e5fa5b33e355cf6cf342dfb8f6e23859f2a6b1ddf79841d7f84f7b196" } ], "fileName": "Modules/expat/xmltok_ns.c" @@ -1730,14 +1730,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "821ac9710d2c073eaf13e1b1895a9c9aa66c1157a99635c639fbff65cdbdd732" + "checksumValue": "461ecc8aa98ab1a68c2db788175665d1a4db640dc05bf0e289b6ea17122144ec" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_7_3/expat-2.7.3.tar.gz", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_7_4/expat-2.7.4.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.7.3:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.7.4:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1745,7 +1745,7 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.7.3" + "versionInfo": "2.7.4" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 7ff3aeff2c7..6ec50b15200 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -4075,30 +4075,44 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) return NULL; } - PyInterpreterState *interp = PyInterpreterState_Get(); - // Stop the world and traverse the per-thread linked list - // of asyncio tasks for every thread, as well as the - // interpreter's linked list, and add them to `tasks`. - // The interpreter linked list is used for any lingering tasks - // whose thread state has been deallocated while the task was - // still alive. This can happen if a task is referenced by - // a different thread, in which case the task is moved to - // the interpreter's linked list from the thread's linked - // list before deallocation. See PyThreadState_Clear. - // - // The stop-the-world pause is required so that no thread - // modifies its linked list while being iterated here - // in parallel. This design allows for lock-free - // register_task/unregister_task for loops running in parallel - // in different threads (the general case). - _PyEval_StopTheWorld(interp); - int ret = add_tasks_interp(interp, (PyListObject *)tasks); - _PyEval_StartTheWorld(interp); - if (ret < 0) { - // call any escaping calls after starting the world to avoid any deadlocks. - Py_DECREF(tasks); - Py_DECREF(loop); - return NULL; + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (ts->asyncio_running_loop == loop) { + // Fast path for the current running loop of current thread + // no locking or stop the world pause is required + struct llist_node *head = &ts->asyncio_tasks_head; + if (add_tasks_llist(head, (PyListObject *)tasks) < 0) { + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } + } + else { + // Slow path for loop running in different thread + PyInterpreterState *interp = ts->base.interp; + // Stop the world and traverse the per-thread linked list + // of asyncio tasks for every thread, as well as the + // interpreter's linked list, and add them to `tasks`. + // The interpreter linked list is used for any lingering tasks + // whose thread state has been deallocated while the task was + // still alive. This can happen if a task is referenced by + // a different thread, in which case the task is moved to + // the interpreter's linked list from the thread's linked + // list before deallocation. See PyThreadState_Clear. + // + // The stop-the-world pause is required so that no thread + // modifies its linked list while being iterated here + // in parallel. This design allows for lock-free + // register_task/unregister_task for loops running in parallel + // in different threads (the general case). + _PyEval_StopTheWorld(interp); + int ret = add_tasks_interp(interp, (PyListObject *)tasks); + _PyEval_StartTheWorld(interp); + if (ret < 0) { + // call any escaping calls after starting the world to avoid any deadlocks. + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } } // All the tasks are now in the list, now filter the tasks which are done diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 45ca63e6d7c..72865f87fc4 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -605,29 +605,42 @@ deque_copy_impl(dequeobject *deque) collections_state *state = find_module_state_by_def(Py_TYPE(deque)); if (Py_IS_TYPE(deque, state->deque_type)) { dequeobject *new_deque; - PyObject *rv; + Py_ssize_t n = Py_SIZE(deque); new_deque = (dequeobject *)deque_new(state->deque_type, NULL, NULL); if (new_deque == NULL) return NULL; new_deque->maxlen = old_deque->maxlen; - /* Fast path for the deque_repeat() common case where len(deque) == 1 - * - * It's safe to not acquire the per-object lock for new_deque; it's - * invisible to other threads. + + /* Copy elements directly by walking the block structure. + * This is safe because the caller holds the deque lock and + * the new deque is not yet visible to other threads. */ - if (Py_SIZE(deque) == 1) { - PyObject *item = old_deque->leftblock->data[old_deque->leftindex]; - rv = deque_append_impl(new_deque, item); - } else { - rv = deque_extend_impl(new_deque, (PyObject *)deque); - } - if (rv != NULL) { - Py_DECREF(rv); - return (PyObject *)new_deque; + if (n > 0) { + block *b = old_deque->leftblock; + Py_ssize_t index = old_deque->leftindex; + + /* Space saving heuristic. Start filling from the left */ + assert(new_deque->leftblock == new_deque->rightblock); + assert(new_deque->leftindex == new_deque->rightindex + 1); + new_deque->leftindex = 1; + new_deque->rightindex = 0; + + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *item = b->data[index]; + if (deque_append_lock_held(new_deque, Py_NewRef(item), + new_deque->maxlen) < 0) { + Py_DECREF(new_deque); + return NULL; + } + index++; + if (index == BLOCKLEN) { + b = b->rightlink; + index = 0; + } + } } - Py_DECREF(new_deque); - return NULL; + return (PyObject *)new_deque; } if (old_deque->maxlen < 0) result = PyObject_CallOneArg((PyObject *)(Py_TYPE(deque)), diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 490c6b83d21..d779376b191 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -611,65 +611,72 @@ partial_repr(PyObject *self) { partialobject *pto = partialobject_CAST(self); PyObject *result = NULL; - PyObject *arglist; - PyObject *mod; - PyObject *name; + PyObject *arglist = NULL; + PyObject *mod = NULL; + PyObject *name = NULL; Py_ssize_t i, n; PyObject *key, *value; int status; status = Py_ReprEnter(self); if (status != 0) { - if (status < 0) + if (status < 0) { return NULL; + } return PyUnicode_FromString("..."); } + /* Reference arguments in case they change */ + PyObject *fn = Py_NewRef(pto->fn); + PyObject *args = Py_NewRef(pto->args); + PyObject *kw = Py_NewRef(pto->kw); + assert(PyTuple_Check(args)); + assert(PyDict_Check(kw)); arglist = Py_GetConstant(Py_CONSTANT_EMPTY_STR); - if (arglist == NULL) + if (arglist == NULL) { goto done; + } /* Pack positional arguments */ - assert(PyTuple_Check(pto->args)); - n = PyTuple_GET_SIZE(pto->args); + n = PyTuple_GET_SIZE(args); for (i = 0; i < n; i++) { Py_SETREF(arglist, PyUnicode_FromFormat("%U, %R", arglist, - PyTuple_GET_ITEM(pto->args, i))); - if (arglist == NULL) + PyTuple_GET_ITEM(args, i))); + if (arglist == NULL) { goto done; + } } /* Pack keyword arguments */ - assert (PyDict_Check(pto->kw)); - for (i = 0; PyDict_Next(pto->kw, &i, &key, &value);) { + for (i = 0; PyDict_Next(kw, &i, &key, &value);) { /* Prevent key.__str__ from deleting the value. */ Py_INCREF(value); Py_SETREF(arglist, PyUnicode_FromFormat("%U, %S=%R", arglist, key, value)); Py_DECREF(value); - if (arglist == NULL) + if (arglist == NULL) { goto done; + } } mod = PyType_GetModuleName(Py_TYPE(pto)); if (mod == NULL) { - goto error; + goto done; } + name = PyType_GetQualName(Py_TYPE(pto)); if (name == NULL) { - Py_DECREF(mod); - goto error; + goto done; } - result = PyUnicode_FromFormat("%S.%S(%R%U)", mod, name, pto->fn, arglist); - Py_DECREF(mod); - Py_DECREF(name); - Py_DECREF(arglist); - done: + result = PyUnicode_FromFormat("%S.%S(%R%U)", mod, name, fn, arglist); +done: + Py_XDECREF(name); + Py_XDECREF(mod); + Py_XDECREF(arglist); + Py_DECREF(fn); + Py_DECREF(args); + Py_DECREF(kw); Py_ReprLeave(self); return result; - error: - Py_DECREF(arglist); - Py_ReprLeave(self); - return NULL; } /* Pickle strategy: diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index c8a76e14990..e7cb315f160 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -238,7 +238,7 @@ py_hashentry_table_new(void) { if (h->py_alias != NULL) { if (_Py_hashtable_set(ht, (const void*)entry->py_alias, (void*)entry) < 0) { - PyMem_Free(entry); + /* entry is already in ht, will be freed by _Py_hashtable_destroy() */ goto error; } entry->refcnt++; diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index ef9cf01ecbe..2933332ad46 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -1644,14 +1644,16 @@ _channels_list_all(_channels *channels, int64_t *count) if (ids == NULL) { goto done; } - _channelref *ref = channels->head; - for (int64_t i=0; ref != NULL; ref = ref->next, i++) { - ids[i] = (struct channel_id_and_info){ - .id = ref->cid, - .defaults = ref->chan->defaults, - }; + int64_t i = 0; + for (_channelref *ref = channels->head; ref != NULL; ref = ref->next) { + if (ref->chan != NULL) { + ids[i++] = (struct channel_id_and_info){ + .id = ref->cid, + .defaults = ref->chan->defaults, + }; + } } - *count = channels->numopen; + *count = i; cids = ids; done: diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 128a5ad1678..3898a9c2982 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -430,7 +430,9 @@ _io_IncrementalNewlineDecoder_decode(PyObject *self, PyObject *const *args, Py_s goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_IncrementalNewlineDecoder_decode_impl((nldecoder_object *)self, input, final); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -450,7 +452,13 @@ _io_IncrementalNewlineDecoder_getstate_impl(nldecoder_object *self); static PyObject * _io_IncrementalNewlineDecoder_getstate(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_IncrementalNewlineDecoder_getstate_impl((nldecoder_object *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_IncrementalNewlineDecoder_getstate_impl((nldecoder_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_IncrementalNewlineDecoder_setstate__doc__, @@ -470,7 +478,9 @@ _io_IncrementalNewlineDecoder_setstate(PyObject *self, PyObject *state) { PyObject *return_value = NULL; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_IncrementalNewlineDecoder_setstate_impl((nldecoder_object *)self, state); + Py_END_CRITICAL_SECTION(); return return_value; } @@ -489,7 +499,13 @@ _io_IncrementalNewlineDecoder_reset_impl(nldecoder_object *self); static PyObject * _io_IncrementalNewlineDecoder_reset(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_IncrementalNewlineDecoder_reset_impl((nldecoder_object *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_IncrementalNewlineDecoder_reset_impl((nldecoder_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_TextIOWrapper___init____doc__, @@ -1312,4 +1328,4 @@ _io_TextIOWrapper__CHUNK_SIZE_set(PyObject *self, PyObject *value, void *Py_UNUS return return_value; } -/*[clinic end generated code: output=30404271a1151056 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c38e6cd5ff4b7eea input=a9049054013a1b77]*/ diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 9945febea80..99180017a3c 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -518,6 +518,7 @@ _PyIncrementalNewlineDecoder_decode(PyObject *myself, } /*[clinic input] +@critical_section _io.IncrementalNewlineDecoder.decode input: object final: bool = False @@ -526,18 +527,19 @@ _io.IncrementalNewlineDecoder.decode static PyObject * _io_IncrementalNewlineDecoder_decode_impl(nldecoder_object *self, PyObject *input, int final) -/*[clinic end generated code: output=0d486755bb37a66e input=90e223c70322c5cd]*/ +/*[clinic end generated code: output=0d486755bb37a66e input=9475d16a73168504]*/ { return _PyIncrementalNewlineDecoder_decode((PyObject *) self, input, final); } /*[clinic input] +@critical_section _io.IncrementalNewlineDecoder.getstate [clinic start generated code]*/ static PyObject * _io_IncrementalNewlineDecoder_getstate_impl(nldecoder_object *self) -/*[clinic end generated code: output=f0d2c9c136f4e0d0 input=f8ff101825e32e7f]*/ +/*[clinic end generated code: output=f0d2c9c136f4e0d0 input=dc3e1f27aa850f12]*/ { PyObject *buffer; unsigned long long flag; @@ -575,6 +577,7 @@ _io_IncrementalNewlineDecoder_getstate_impl(nldecoder_object *self) } /*[clinic input] +@critical_section _io.IncrementalNewlineDecoder.setstate state: object / @@ -583,7 +586,7 @@ _io.IncrementalNewlineDecoder.setstate static PyObject * _io_IncrementalNewlineDecoder_setstate_impl(nldecoder_object *self, PyObject *state) -/*[clinic end generated code: output=09135cb6e78a1dc8 input=c53fb505a76dbbe2]*/ +/*[clinic end generated code: output=09135cb6e78a1dc8 input=275fd3982d2b08cb]*/ { PyObject *buffer; unsigned long long flag; @@ -613,12 +616,13 @@ _io_IncrementalNewlineDecoder_setstate_impl(nldecoder_object *self, } /*[clinic input] +@critical_section _io.IncrementalNewlineDecoder.reset [clinic start generated code]*/ static PyObject * _io_IncrementalNewlineDecoder_reset_impl(nldecoder_object *self) -/*[clinic end generated code: output=32fa40c7462aa8ff input=728678ddaea776df]*/ +/*[clinic end generated code: output=32fa40c7462aa8ff input=31bd8ae4e36cec83]*/ { CHECK_INITIALIZED_DECODER(self); diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index cb448b14d8c..b1d9e74db62 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -579,7 +579,6 @@ _locale__getdefaultlocale_impl(PyObject *module) } /* cannot determine the language code (very unlikely) */ - Py_INCREF(Py_None); return Py_BuildValue("Os", Py_None, encoding); } #endif diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 01235c77bd7..9e3e5d251aa 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -165,6 +165,7 @@ RingBuf_Put(RingBuf *buf, PyObject *item) // Buffer is full, grow it. if (resize_ringbuf(buf, buf->items_cap * 2) < 0) { PyErr_NoMemory(); + Py_DECREF(item); return -1; } } diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index b46538b76df..a26e6820f55 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -77,6 +77,8 @@ offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)) #endif #define INTERP_STATE_BUFFER_SIZE MAX(INTERP_STATE_MIN_SIZE, 256) +#define MAX_STACK_CHUNK_SIZE (16 * 1024 * 1024) /* 16 MB max for stack chunks */ +#define MAX_SET_TABLE_SIZE (1 << 20) /* 1 million entries max for set iteration */ @@ -318,10 +320,13 @@ static int append_awaited_by(RemoteUnwinderObject *unwinder, unsigned long tid, * UTILITY FUNCTIONS AND HELPERS * ============================================================================ */ -#define set_exception_cause(unwinder, exc_type, message) \ - if (unwinder->debug) { \ - _set_debug_exception_cause(exc_type, message); \ - } +#define set_exception_cause(unwinder, exc_type, message) \ + do { \ + assert(PyErr_Occurred() && "function returned -1 without setting exception"); \ + if (unwinder->debug) { \ + _set_debug_exception_cause(exc_type, message); \ + } \ + } while (0) static void cached_code_metadata_destroy(void *ptr) @@ -498,8 +503,15 @@ iterate_set_entries( } Py_ssize_t num_els = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.used); - Py_ssize_t set_len = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.mask) + 1; + Py_ssize_t mask = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.mask); uintptr_t table_ptr = GET_MEMBER(uintptr_t, set_object, unwinder->debug_offsets.set_object.table); + if (mask < 0 || mask >= MAX_SET_TABLE_SIZE || num_els < 0 || num_els > mask + 1) { + PyErr_SetString(PyExc_RuntimeError, "Invalid set object (corrupted remote memory)"); + set_exception_cause(unwinder, PyExc_RuntimeError, + "Invalid set object (corrupted remote memory)"); + return -1; + } + Py_ssize_t set_len = mask + 1; Py_ssize_t i = 0; Py_ssize_t els = 0; @@ -805,7 +817,7 @@ _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle) #ifdef MS_WINDOWS // On Windows, search for asyncio debug in executable or DLL - address = search_windows_map_for_section(handle, "AsyncioD", L"_asyncio"); + address = search_windows_map_for_section(handle, "AsyncioD", L"_asyncio", NULL); if (address == 0) { // Error out: 'python' substring covers both executable and DLL PyObject *exc = PyErr_GetRaisedException(); @@ -814,7 +826,7 @@ _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle) } #elif defined(__linux__) // On Linux, search for asyncio debug in executable or DLL - address = search_linux_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); + address = search_linux_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython", NULL); if (address == 0) { // Error out: 'python' substring covers both executable and DLL PyObject *exc = PyErr_GetRaisedException(); @@ -823,10 +835,10 @@ _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle) } #elif defined(__APPLE__) && TARGET_OS_OSX // On macOS, try libpython first, then fall back to python - address = search_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); + address = search_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython", NULL); if (address == 0) { PyErr_Clear(); - address = search_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); + address = search_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython", NULL); } if (address == 0) { // Error out: 'python' substring covers both executable and DLL @@ -1825,7 +1837,15 @@ parse_code_object(RemoteUnwinderObject *unwinder, tlbc_entry = get_tlbc_cache_entry(unwinder, real_address, unwinder->tlbc_generation); } - if (tlbc_entry && tlbc_index < tlbc_entry->tlbc_array_size) { + if (tlbc_entry) { + if (tlbc_index < 0 || tlbc_index >= tlbc_entry->tlbc_array_size) { + PyErr_Format(PyExc_RuntimeError, + "Invalid tlbc_index %d (array size %zd, corrupted remote memory)", + tlbc_index, tlbc_entry->tlbc_array_size); + set_exception_cause(unwinder, PyExc_RuntimeError, + "Invalid tlbc_index (corrupted remote memory)"); + goto error; + } // Use cached TLBC data uintptr_t *entries = (uintptr_t *)((char *)tlbc_entry->tlbc_array + sizeof(Py_ssize_t)); uintptr_t tlbc_bytecode_addr = entries[tlbc_index]; @@ -1924,6 +1944,15 @@ process_single_stack_chunk( // Check actual size and reread if necessary size_t actual_size = GET_MEMBER(size_t, this_chunk, offsetof(_PyStackChunk, size)); if (actual_size != current_size) { + if (actual_size <= offsetof(_PyStackChunk, data) || actual_size > MAX_STACK_CHUNK_SIZE) { + PyMem_RawFree(this_chunk); + PyErr_Format(PyExc_RuntimeError, + "Invalid stack chunk size %zu (corrupted remote memory)", actual_size); + set_exception_cause(unwinder, PyExc_RuntimeError, + "Invalid stack chunk size (corrupted remote memory)"); + return -1; + } + this_chunk = PyMem_RawRealloc(this_chunk, actual_size); if (!this_chunk) { PyErr_NoMemory(); @@ -2027,6 +2056,7 @@ parse_frame_from_chunks( ) { void *frame_ptr = find_frame_in_chunks(chunks, address); if (!frame_ptr) { + PyErr_Format(PyExc_RuntimeError, "Frame at address 0x%lx not found in stack chunks", address); set_exception_cause(unwinder, PyExc_RuntimeError, "Frame not found in stack chunks"); return -1; } @@ -2736,6 +2766,7 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self } while (current_tstate != 0) { + uintptr_t prev_tstate = current_tstate; PyObject* frame_info = unwind_stack_for_thread(self, ¤t_tstate); if (!frame_info) { Py_CLEAR(result); @@ -2751,6 +2782,16 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self } Py_DECREF(frame_info); + if (current_tstate == prev_tstate) { + PyErr_Format(PyExc_RuntimeError, + "Thread list cycle detected at address 0x%lx (corrupted remote memory)", + current_tstate); + set_exception_cause(self, PyExc_RuntimeError, + "Thread list cycle detected (corrupted remote memory)"); + Py_CLEAR(result); + goto exit; + } + // We are targeting a single tstate, break here if (self->tstate_addr) { break; diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index aafefbf316e..16143106953 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -434,6 +434,10 @@ subscript_slice(pysqlite_Blob *self, PyObject *item) return NULL; } + if (len == 0) { + return PyBytes_FromStringAndSize(NULL, 0); + } + if (step == 1) { return read_multiple(self, len, start); } diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 417d5ca5931..a24b6b30b2f 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -301,7 +301,7 @@ typedef struct { int post_handshake_auth; #endif PyObject *msg_cb; - PyObject *keylog_filename; + PyObject *keylog_filename; // can be anything accepted by Py_fopen() BIO *keylog_bio; /* Cached module state, also used in SSLSocket and SSLSession code. */ _sslmodulestate *state; @@ -331,7 +331,7 @@ typedef struct { PySSLContext *ctx; /* weakref to SSL context */ char shutdown_seen_zero; enum py_ssl_server_or_client socket_type; - PyObject *owner; /* Python level "owner" passed to servername callback */ + PyObject *owner; /* weakref to Python level "owner" passed to servername callback */ PyObject *server_hostname; _PySSLError err; /* last seen error from various sources */ /* Some SSL callbacks don't have error reporting. Callback wrappers @@ -896,8 +896,8 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, self->ssl = SSL_new(ctx); PySSL_END_ALLOW_THREADS(sslctx) if (self->ssl == NULL) { + _setSSLError(get_state_ctx(sslctx), NULL, 0, __FILE__, __LINE__); Py_DECREF(self); - _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); return NULL; } @@ -2345,6 +2345,10 @@ static int PySSL_clear(PyObject *op) { PySSLSocket *self = PySSLSocket_CAST(op); + Py_CLEAR(self->Socket); + Py_CLEAR(self->ctx); + Py_CLEAR(self->owner); + Py_CLEAR(self->server_hostname); Py_CLEAR(self->exc); return 0; } @@ -2369,10 +2373,7 @@ PySSL_dealloc(PyObject *op) SSL_set_shutdown(self->ssl, SSL_SENT_SHUTDOWN | SSL_get_shutdown(self->ssl)); SSL_free(self->ssl); } - Py_XDECREF(self->Socket); - Py_XDECREF(self->ctx); - Py_XDECREF(self->server_hostname); - Py_XDECREF(self->owner); + (void)PySSL_clear(op); PyObject_GC_Del(self); Py_DECREF(tp); } @@ -3309,6 +3310,11 @@ context_traverse(PyObject *op, visitproc visit, void *arg) PySSLContext *self = PySSLContext_CAST(op); Py_VISIT(self->set_sni_cb); Py_VISIT(self->msg_cb); + Py_VISIT(self->keylog_filename); +#ifndef OPENSSL_NO_PSK + Py_VISIT(self->psk_client_callback); + Py_VISIT(self->psk_server_callback); +#endif Py_VISIT(Py_TYPE(self)); return 0; } diff --git a/Modules/_ssl_data_35.h b/Modules/_ssl_data_35.h index e4919b550e3..8a9fef87b2a 100644 --- a/Modules/_ssl_data_35.h +++ b/Modules/_ssl_data_35.h @@ -1,6 +1,6 @@ /* File generated by Tools/ssl/make_ssl_data.py */ -/* Generated on 2025-10-04T17:49:19.148321+00:00 */ -/* Generated from Git commit openssl-3.5.4-0-gc1eeb9406 */ +/* Generated on 2026-02-13T19:18:20.130102+00:00 */ +/* Generated from Git commit openssl-3.5.5-0-g67b5686b4 */ /* generated from args.lib2errnum */ static struct py_ssl_library_code library_codes[] = { @@ -1668,6 +1668,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"CERTIFICATE_VERIFY_ERROR", 46, 100}, #endif + #ifdef CMS_R_CIPHER_AEAD_IN_ENVELOPED_DATA + {"CIPHER_AEAD_IN_ENVELOPED_DATA", ERR_LIB_CMS, CMS_R_CIPHER_AEAD_IN_ENVELOPED_DATA}, + #else + {"CIPHER_AEAD_IN_ENVELOPED_DATA", 46, 200}, + #endif #ifdef CMS_R_CIPHER_AEAD_SET_TAG_ERROR {"CIPHER_AEAD_SET_TAG_ERROR", ERR_LIB_CMS, CMS_R_CIPHER_AEAD_SET_TAG_ERROR}, #else diff --git a/Modules/_struct.c b/Modules/_struct.c index 87014a4a1e3..fe8c689b629 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1620,11 +1620,11 @@ align(Py_ssize_t size, char c, const formatdef *e) /* calculate the size of a format string */ static int -prepare_s(PyStructObject *self) +prepare_s(PyStructObject *self, PyObject *format) { const formatdef *f; const formatdef *e; - formatcode *codes; + formatcode *codes, *codes0; const char *s; const char *fmt; @@ -1634,8 +1634,8 @@ prepare_s(PyStructObject *self) _structmodulestate *state = get_struct_state_structinst(self); - fmt = PyBytes_AS_STRING(self->s_format); - if (strlen(fmt) != (size_t)PyBytes_GET_SIZE(self->s_format)) { + fmt = PyBytes_AS_STRING(format); + if (strlen(fmt) != (size_t)PyBytes_GET_SIZE(format)) { PyErr_SetString(state->StructError, "embedded null character"); return -1; @@ -1676,9 +1676,23 @@ prepare_s(PyStructObject *self) switch (c) { case 's': _Py_FALLTHROUGH; - case 'p': len++; ncodes++; break; + case 'p': + if (len == PY_SSIZE_T_MAX) { + goto overflow; + } + len++; + ncodes++; + break; case 'x': break; - default: len += num; if (num) ncodes++; break; + default: + if (num > PY_SSIZE_T_MAX - len) { + goto overflow; + } + len += num; + if (num) { + ncodes++; + } + break; } itemsize = e->size; @@ -1703,13 +1717,7 @@ prepare_s(PyStructObject *self) PyErr_NoMemory(); return -1; } - /* Free any s_codes value left over from a previous initialization. */ - if (self->s_codes != NULL) - PyMem_Free(self->s_codes); - self->s_codes = codes; - self->s_size = size; - self->s_len = len; - + codes0 = codes; s = fmt; size = 0; while ((c = *s++) != '\0') { @@ -1749,6 +1757,14 @@ prepare_s(PyStructObject *self) codes->size = 0; codes->repeat = 0; + /* Free any s_codes value left over from a previous initialization. */ + if (self->s_codes != NULL) + PyMem_Free(self->s_codes); + self->s_codes = codes0; + self->s_size = size; + self->s_len = len; + Py_XSETREF(self->s_format, Py_NewRef(format)); + return 0; overflow: @@ -1814,9 +1830,8 @@ Struct___init___impl(PyStructObject *self, PyObject *format) return -1; } - Py_SETREF(self->s_format, format); - - ret = prepare_s(self); + ret = prepare_s(self, format); + Py_DECREF(format); return ret; } @@ -2402,6 +2417,7 @@ static PyObject * s_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy)) { PyStructObject *self = PyStructObject_CAST(op); + ENSURE_STRUCT_IS_READY(self); size_t size = _PyObject_SIZE(Py_TYPE(self)) + sizeof(formatcode); for (formatcode *code = self->s_codes; code->fmtdef != NULL; code++) { size += sizeof(formatcode); @@ -2413,6 +2429,7 @@ static PyObject * s_repr(PyObject *op) { PyStructObject *self = PyStructObject_CAST(op); + ENSURE_STRUCT_IS_READY(self); PyObject* fmt = PyUnicode_FromStringAndSize( PyBytes_AS_STRING(self->s_format), PyBytes_GET_SIZE(self->s_format)); if (fmt == NULL) { diff --git a/Modules/_testcapi/function.c b/Modules/_testcapi/function.c index ec1ba508df2..40767adbd3f 100644 --- a/Modules/_testcapi/function.c +++ b/Modules/_testcapi/function.c @@ -123,6 +123,13 @@ function_set_closure(PyObject *self, PyObject *args) } +static PyObject * +function_get_annotations(PyObject *self, PyObject *func) +{ + return Py_XNewRef(PyFunction_GetAnnotations(func)); +} + + static PyMethodDef test_methods[] = { {"function_get_code", function_get_code, METH_O, NULL}, {"function_get_globals", function_get_globals, METH_O, NULL}, @@ -133,6 +140,7 @@ static PyMethodDef test_methods[] = { {"function_set_kw_defaults", function_set_kw_defaults, METH_VARARGS, NULL}, {"function_get_closure", function_get_closure, METH_O, NULL}, {"function_set_closure", function_set_closure, METH_VARARGS, NULL}, + {"function_get_annotations", function_get_annotations, METH_O, NULL}, {NULL}, }; diff --git a/Modules/_testinternalcapi/test_critical_sections.c b/Modules/_testinternalcapi/test_critical_sections.c index e3b2fe716d4..72a1fa2cdc7 100644 --- a/Modules/_testinternalcapi/test_critical_sections.c +++ b/Modules/_testinternalcapi/test_critical_sections.c @@ -4,6 +4,8 @@ #include "parts.h" #include "pycore_critical_section.h" +#include "pycore_pystate.h" +#include "pycore_pythread.h" #ifdef MS_WINDOWS # include // Sleep() @@ -381,6 +383,87 @@ test_critical_section2_reacquisition(PyObject *self, PyObject *Py_UNUSED(args)) #endif // Py_GIL_DISABLED +#ifdef Py_CAN_START_THREADS + +// gh-144513: Test that critical sections don't deadlock with stop-the-world. +// This test is designed to deadlock (timeout) on builds without the fix. +struct test_data_stw { + PyObject *obj; + Py_ssize_t num_threads; + Py_ssize_t started; + PyEvent ready; +}; + +static void +thread_stw(void *arg) +{ + struct test_data_stw *test_data = arg; + PyGILState_STATE gil = PyGILState_Ensure(); + + if (_Py_atomic_add_ssize(&test_data->started, 1) == test_data->num_threads - 1) { + _PyEvent_Notify(&test_data->ready); + } + + // All threads: acquire critical section and hold it long enough to + // trigger TIME_TO_BE_FAIR_NS (1 ms), which causes direct handoff on unlock. + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + pysleep(10); // 10 ms = 10 x TIME_TO_BE_FAIR_NS + Py_END_CRITICAL_SECTION(); + + PyGILState_Release(gil); +} + +static PyObject * +test_critical_sections_stw(PyObject *self, PyObject *Py_UNUSED(args)) +{ + // gh-144513: Test that critical sections don't deadlock during STW. + // + // The deadlock occurs when lock ownership is handed off (due to fairness + // after TIME_TO_BE_FAIR_NS) to a thread that has already suspended for + // stop-the-world. The STW requester then cannot acquire the lock. + // + // With the fix, the STW requester detects world_stopped and skips locking. + + #define STW_NUM_THREADS 2 + struct test_data_stw test_data = { + .obj = PyDict_New(), + .num_threads = STW_NUM_THREADS, + }; + if (test_data.obj == NULL) { + return NULL; + } + + PyThread_handle_t handles[STW_NUM_THREADS]; + PyThread_ident_t idents[STW_NUM_THREADS]; + for (Py_ssize_t i = 0; i < STW_NUM_THREADS; i++) { + PyThread_start_joinable_thread(&thread_stw, &test_data, + &idents[i], &handles[i]); + } + + // Wait for threads to start, then let them compete for the lock + PyEvent_Wait(&test_data.ready); + pysleep(5); + + // Request stop-the-world and try to acquire the critical section. + // Without the fix, this may deadlock. + PyInterpreterState *interp = PyInterpreterState_Get(); + _PyEval_StopTheWorld(interp); + + Py_BEGIN_CRITICAL_SECTION(test_data.obj); + Py_END_CRITICAL_SECTION(); + + _PyEval_StartTheWorld(interp); + + for (Py_ssize_t i = 0; i < STW_NUM_THREADS; i++) { + PyThread_join_thread(handles[i]); + } + #undef STW_NUM_THREADS + Py_DECREF(test_data.obj); + Py_RETURN_NONE; +} + +#endif // Py_CAN_START_THREADS + static PyMethodDef test_methods[] = { {"test_critical_sections", test_critical_sections, METH_NOARGS}, {"test_critical_sections_nest", test_critical_sections_nest, METH_NOARGS}, @@ -392,6 +475,7 @@ static PyMethodDef test_methods[] = { #ifdef Py_CAN_START_THREADS {"test_critical_sections_threads", test_critical_sections_threads, METH_NOARGS}, {"test_critical_sections_gc", test_critical_sections_gc, METH_NOARGS}, + {"test_critical_sections_stw", test_critical_sections_stw, METH_NOARGS}, #endif {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 2c59085d15b..f74b964faf3 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -799,3 +799,11 @@ PyInit__testsinglephase_circular(void) } return Py_NewRef(static_module_circular); } + + +PyMODINIT_FUNC +PyInit__testsinglephase_raise_exception(void) +{ + PyErr_SetString(PyExc_RuntimeError, "evil"); + return NULL; +} diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index bfb86bdbf1a..25f7eb7a44d 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -335,6 +335,7 @@ zoneinfo_ZoneInfo_impl(PyTypeObject *type, PyObject *key) return NULL; } + ((PyZoneInfo_ZoneInfo *)tmp)->source = SOURCE_CACHE; instance = PyObject_CallMethod(weak_cache, "setdefault", "OO", key, tmp); Py_DECREF(tmp); @@ -342,7 +343,15 @@ zoneinfo_ZoneInfo_impl(PyTypeObject *type, PyObject *key) Py_DECREF(weak_cache); return NULL; } - ((PyZoneInfo_ZoneInfo *)instance)->source = SOURCE_CACHE; + } + + if (!PyObject_TypeCheck(instance, type)) { + PyErr_Format(PyExc_RuntimeError, + "Unexpected instance of %T in %s weak cache for key %R", + instance, _PyType_Name(type), key); + Py_DECREF(instance); + Py_DECREF(weak_cache); + return NULL; } update_strong_cache(state, type, key, instance); diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 87a17935507..1c7a860e1ec 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -3845,8 +3845,8 @@ exit: PyDoc_STRVAR(os_posix_spawn__doc__, "posix_spawn($module, path, argv, env, /, *, file_actions=(),\n" -" setpgroup=, resetids=False, setsid=False,\n" -" setsigmask=(), setsigdef=(), scheduler=)\n" +" setpgroup=None, resetids=False, setsid=False,\n" +" setsigmask=(), setsigdef=(), scheduler=None)\n" "--\n" "\n" "Execute the program specified by path in a new process.\n" @@ -3998,8 +3998,8 @@ exit: PyDoc_STRVAR(os_posix_spawnp__doc__, "posix_spawnp($module, path, argv, env, /, *, file_actions=(),\n" -" setpgroup=, resetids=False, setsid=False,\n" -" setsigmask=(), setsigdef=(), scheduler=)\n" +" setpgroup=None, resetids=False, setsid=False,\n" +" setsigmask=(), setsigdef=(), scheduler=None)\n" "--\n" "\n" "Execute the program specified by path in a new process.\n" @@ -13476,4 +13476,4 @@ exit: #ifndef OS__EMSCRIPTEN_LOG_METHODDEF #define OS__EMSCRIPTEN_LOG_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ -/*[clinic end generated code: output=9e5f9b9ce732a534 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=291f607b7a26ca5e input=a9049054013a1b77]*/ diff --git a/Modules/expat/COPYING b/Modules/expat/COPYING index ce9e5939291..c6d184a8aae 100644 --- a/Modules/expat/COPYING +++ b/Modules/expat/COPYING @@ -1,5 +1,5 @@ Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper -Copyright (c) 2001-2022 Expat maintainers +Copyright (c) 2001-2025 Expat maintainers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 290dfeb0f6d..6c7c4186927 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -11,7 +11,7 @@ Copyright (c) 2000-2005 Fred L. Drake, Jr. Copyright (c) 2001-2002 Greg Stein Copyright (c) 2002-2016 Karl Waclawek - Copyright (c) 2016-2025 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2016 Cristian Rodríguez Copyright (c) 2016 Thomas Beutlich Copyright (c) 2017 Rhodri James @@ -1082,7 +1082,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ # define XML_MAJOR_VERSION 2 # define XML_MINOR_VERSION 7 -# define XML_MICRO_VERSION 3 +# define XML_MICRO_VERSION 4 # ifdef __cplusplus } diff --git a/Modules/expat/expat_config.h b/Modules/expat/expat_config.h index e7d9499d907..09d3161dbc0 100644 --- a/Modules/expat/expat_config.h +++ b/Modules/expat/expat_config.h @@ -3,7 +3,7 @@ * distribution. */ #ifndef EXPAT_CONFIG_H -#define EXPAT_CONFIG_H +#define EXPAT_CONFIG_H 1 #include #ifdef WORDS_BIGENDIAN diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index 0f01a05d0e9..6f3f3c48ce9 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -12,7 +12,7 @@ Copyright (c) 2001-2002 Greg Stein Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2016 Cristian Rodríguez - Copyright (c) 2016-2019 Sebastian Pipping + Copyright (c) 2016-2025 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2018 Yury Gribov Licensed under the MIT license: @@ -91,8 +91,7 @@ # ifndef XML_BUILDING_EXPAT /* using Expat from an application */ -# if defined(_MSC_EXTENSIONS) && ! defined(__BEOS__) \ - && ! defined(__CYGWIN__) +# if defined(_MSC_VER) && ! defined(__BEOS__) && ! defined(__CYGWIN__) # define XMLIMPORT __declspec(dllimport) # endif diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index 8f5edf48ef7..61266ebb772 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -128,7 +128,7 @@ # elif ULONG_MAX == 18446744073709551615u // 2^64-1 # define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "ld" # define EXPAT_FMT_SIZE_T(midpart) "%" midpart "lu" -# elif defined(EMSCRIPTEN) // 32bit mode Emscripten +# elif defined(__wasm32__) // 32bit mode Emscripten or WASI SDK # define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "ld" # define EXPAT_FMT_SIZE_T(midpart) "%" midpart "zu" # else diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh index bb1a805277c..54d58d09b90 100755 --- a/Modules/expat/refresh.sh +++ b/Modules/expat/refresh.sh @@ -12,9 +12,9 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. These values are used for verifying the SBOM, too. -expected_libexpat_tag="R_2_7_3" -expected_libexpat_version="2.7.3" -expected_libexpat_sha256="821ac9710d2c073eaf13e1b1895a9c9aa66c1157a99635c639fbff65cdbdd732" +expected_libexpat_tag="R_2_7_4" +expected_libexpat_version="2.7.4" +expected_libexpat_sha256="461ecc8aa98ab1a68c2db788175665d1a4db640dc05bf0e289b6ea17122144ec" expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" cd ${expat_dir} @@ -24,6 +24,9 @@ curl --location "https://github.com/libexpat/libexpat/releases/download/${expect echo "${expected_libexpat_sha256} libexpat.tar.gz" | sha256sum --check # Step 2: Pull files from the libexpat distribution + +tar xzvf libexpat.tar.gz "expat-${expected_libexpat_version}/COPYING" --strip-components 2 + declare -a lib_files lib_files=( ascii.h diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index a187a3a18f1..086fca59112 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* 28bcd8b1ba7eb595d82822908257fd9c3589b4243e3c922d0369f35bfcd7b506 (2.7.3+) +/* fab937ab8b186d7d296013669c332e6dfce2f99567882cff1f8eb24223c524a7 (2.7.4+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -13,7 +13,7 @@ Copyright (c) 2002-2016 Karl Waclawek Copyright (c) 2005-2009 Steven Solie Copyright (c) 2016 Eric Rahm - Copyright (c) 2016-2025 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2016 Gaurav Copyright (c) 2016 Thomas Beutlich Copyright (c) 2016 Gustavo Grieco @@ -42,6 +42,9 @@ Copyright (c) 2024-2025 Berkay Eren Ürün Copyright (c) 2024 Hanno Böck Copyright (c) 2025 Matthew Fernandez + Copyright (c) 2025 Atrem Borovik + Copyright (c) 2025 Alfonso Gregory + Copyright (c) 2026 Rosen Penev Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -101,7 +104,7 @@ #include /* INT_MAX, UINT_MAX */ #include /* fprintf */ #include /* getenv, rand_s */ -#include /* uintptr_t */ +#include /* SIZE_MAX, uintptr_t */ #include /* isnan */ #ifdef _WIN32 @@ -134,11 +137,6 @@ # endif /* defined(GRND_NONBLOCK) */ #endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ -#if defined(HAVE_LIBBSD) \ - && (defined(HAVE_ARC4RANDOM_BUF) || defined(HAVE_ARC4RANDOM)) -# include -#endif - #if defined(_WIN32) && ! defined(LOAD_LIBRARY_SEARCH_SYSTEM32) # define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 #endif @@ -155,8 +153,6 @@ * Linux >=3.17 + glibc (including <2.25) (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ * BSD / macOS >=10.7 / glibc >=2.36 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ * BSD / macOS (including <10.7) / glibc >=2.36 (arc4random): HAVE_ARC4RANDOM, \ - * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \ - * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \ * Linux (including <3.17) / BSD / macOS (including <10.7) / Solaris >=8 (/dev/urandom): XML_DEV_URANDOM, \ * Windows >=Vista (rand_s): _WIN32. \ \ @@ -311,8 +307,11 @@ typedef struct tag { const char *rawName; /* tagName in the original encoding */ int rawNameLength; TAG_NAME name; /* tagName in the API encoding */ - char *buf; /* buffer for name components */ - char *bufEnd; /* end of the buffer */ + union { + char *raw; /* for byte-level access (rawName storage) */ + XML_Char *str; /* for character-level access (converted name) */ + } buf; /* buffer for name components */ + char *bufEnd; /* end of the buffer */ BINDING *bindings; } TAG; @@ -349,7 +348,7 @@ typedef struct { typedef struct block { struct block *next; int size; - XML_Char s[1]; + XML_Char s[]; } BLOCK; typedef struct { @@ -1230,8 +1229,11 @@ generate_hash_secret_salt(XML_Parser parser) { # endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ /* .. and self-made low quality for backup: */ + entropy = gather_time_entropy(); +# if ! defined(__wasi__) /* Process ID is 0 bits entropy if attacker has local access */ - entropy = gather_time_entropy() ^ getpid(); + entropy ^= getpid(); +# endif /* Factors are 2^31-1 and 2^61-1 (Mersenne primes M31 and M61) */ if (sizeof(unsigned long) == 4) { @@ -1754,6 +1756,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, XML_ExternalEntityRefHandler oldExternalEntityRefHandler; XML_SkippedEntityHandler oldSkippedEntityHandler; XML_UnknownEncodingHandler oldUnknownEncodingHandler; + void *oldUnknownEncodingHandlerData; XML_ElementDeclHandler oldElementDeclHandler; XML_AttlistDeclHandler oldAttlistDeclHandler; XML_EntityDeclHandler oldEntityDeclHandler; @@ -1799,6 +1802,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, oldExternalEntityRefHandler = parser->m_externalEntityRefHandler; oldSkippedEntityHandler = parser->m_skippedEntityHandler; oldUnknownEncodingHandler = parser->m_unknownEncodingHandler; + oldUnknownEncodingHandlerData = parser->m_unknownEncodingHandlerData; oldElementDeclHandler = parser->m_elementDeclHandler; oldAttlistDeclHandler = parser->m_attlistDeclHandler; oldEntityDeclHandler = parser->m_entityDeclHandler; @@ -1859,6 +1863,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, parser->m_externalEntityRefHandler = oldExternalEntityRefHandler; parser->m_skippedEntityHandler = oldSkippedEntityHandler; parser->m_unknownEncodingHandler = oldUnknownEncodingHandler; + parser->m_unknownEncodingHandlerData = oldUnknownEncodingHandlerData; parser->m_elementDeclHandler = oldElementDeclHandler; parser->m_attlistDeclHandler = oldAttlistDeclHandler; parser->m_entityDeclHandler = oldEntityDeclHandler; @@ -1934,7 +1939,7 @@ XML_ParserFree(XML_Parser parser) { } p = tagList; tagList = tagList->parent; - FREE(parser, p->buf); + FREE(parser, p->buf.raw); destroyBindings(p->bindings, parser); FREE(parser, p); } @@ -2599,7 +2604,7 @@ XML_GetBuffer(XML_Parser parser, int len) { // NOTE: We are avoiding MALLOC(..) here to leave limiting // the input size to the application using Expat. newBuf = parser->m_mem.malloc_fcn(bufferSize); - if (newBuf == 0) { + if (newBuf == NULL) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } @@ -3126,7 +3131,7 @@ storeRawNames(XML_Parser parser) { size_t bufSize; size_t nameLen = sizeof(XML_Char) * (tag->name.strLen + 1); size_t rawNameLen; - char *rawNameBuf = tag->buf + nameLen; + char *rawNameBuf = tag->buf.raw + nameLen; /* Stop if already stored. Since m_tagStack is a stack, we can stop at the first entry that has already been copied; everything below it in the stack is already been accounted for in a @@ -3142,22 +3147,22 @@ storeRawNames(XML_Parser parser) { if (rawNameLen > (size_t)INT_MAX - nameLen) return XML_FALSE; bufSize = nameLen + rawNameLen; - if (bufSize > (size_t)(tag->bufEnd - tag->buf)) { - char *temp = REALLOC(parser, tag->buf, bufSize); + if (bufSize > (size_t)(tag->bufEnd - tag->buf.raw)) { + char *temp = REALLOC(parser, tag->buf.raw, bufSize); if (temp == NULL) return XML_FALSE; - /* if tag->name.str points to tag->buf (only when namespace + /* if tag->name.str points to tag->buf.str (only when namespace processing is off) then we have to update it */ - if (tag->name.str == (XML_Char *)tag->buf) + if (tag->name.str == tag->buf.str) tag->name.str = (XML_Char *)temp; /* if tag->name.localPart is set (when namespace processing is on) then update it as well, since it will always point into tag->buf */ if (tag->name.localPart) tag->name.localPart - = (XML_Char *)temp + (tag->name.localPart - (XML_Char *)tag->buf); - tag->buf = temp; + = (XML_Char *)temp + (tag->name.localPart - tag->buf.str); + tag->buf.raw = temp; tag->bufEnd = temp + bufSize; rawNameBuf = temp + nameLen; } @@ -3472,12 +3477,12 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, tag = MALLOC(parser, sizeof(TAG)); if (! tag) return XML_ERROR_NO_MEMORY; - tag->buf = MALLOC(parser, INIT_TAG_BUF_SIZE); - if (! tag->buf) { + tag->buf.raw = MALLOC(parser, INIT_TAG_BUF_SIZE); + if (! tag->buf.raw) { FREE(parser, tag); return XML_ERROR_NO_MEMORY; } - tag->bufEnd = tag->buf + INIT_TAG_BUF_SIZE; + tag->bufEnd = tag->buf.raw + INIT_TAG_BUF_SIZE; } tag->bindings = NULL; tag->parent = parser->m_tagStack; @@ -3490,31 +3495,32 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, { const char *rawNameEnd = tag->rawName + tag->rawNameLength; const char *fromPtr = tag->rawName; - toPtr = (XML_Char *)tag->buf; + toPtr = tag->buf.str; for (;;) { - int bufSize; int convLen; const enum XML_Convert_Result convert_res = XmlConvert(enc, &fromPtr, rawNameEnd, (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1); - convLen = (int)(toPtr - (XML_Char *)tag->buf); + convLen = (int)(toPtr - tag->buf.str); if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) { tag->name.strLen = convLen; break; } - bufSize = (int)(tag->bufEnd - tag->buf) << 1; + if (SIZE_MAX / 2 < (size_t)(tag->bufEnd - tag->buf.raw)) + return XML_ERROR_NO_MEMORY; + const size_t bufSize = (size_t)(tag->bufEnd - tag->buf.raw) * 2; { - char *temp = REALLOC(parser, tag->buf, bufSize); + char *temp = REALLOC(parser, tag->buf.raw, bufSize); if (temp == NULL) return XML_ERROR_NO_MEMORY; - tag->buf = temp; + tag->buf.raw = temp; tag->bufEnd = temp + bufSize; toPtr = (XML_Char *)temp + convLen; } } } - tag->name.str = (XML_Char *)tag->buf; + tag->name.str = tag->buf.str; *toPtr = XML_T('\0'); result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings), account); @@ -3878,7 +3884,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if ((unsigned)parser->m_attsSize > (size_t)(-1) / sizeof(ATTRIBUTE)) { + if ((unsigned)parser->m_attsSize > SIZE_MAX / sizeof(ATTRIBUTE)) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } @@ -3897,7 +3903,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ # if UINT_MAX >= SIZE_MAX - if ((unsigned)parser->m_attsSize > (size_t)(-1) / sizeof(XML_AttrInfo)) { + if ((unsigned)parser->m_attsSize > SIZE_MAX / sizeof(XML_AttrInfo)) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } @@ -4073,7 +4079,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if (nsAttsSize > (size_t)(-1) / sizeof(NS_ATT)) { + if (nsAttsSize > SIZE_MAX / sizeof(NS_ATT)) { /* Restore actual size of memory in m_nsAtts */ parser->m_nsAttsPower = oldNsAttsPower; return XML_ERROR_NO_MEMORY; @@ -4256,7 +4262,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if ((unsigned)(n + EXPAND_SPARE) > (size_t)(-1) / sizeof(XML_Char)) { + if ((unsigned)(n + EXPAND_SPARE) > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } #endif @@ -4502,7 +4508,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if ((unsigned)(len + EXPAND_SPARE) > (size_t)(-1) / sizeof(XML_Char)) { + if ((unsigned)(len + EXPAND_SPARE) > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } #endif @@ -4529,7 +4535,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if ((unsigned)(len + EXPAND_SPARE) > (size_t)(-1) / sizeof(XML_Char)) { + if ((unsigned)(len + EXPAND_SPARE) > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } #endif @@ -5920,15 +5926,18 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if (parser->m_groupSize > (size_t)(-1) / sizeof(int)) { + if (parser->m_groupSize > SIZE_MAX / sizeof(int)) { + parser->m_groupSize /= 2; return XML_ERROR_NO_MEMORY; } #endif int *const new_scaff_index = REALLOC( parser, dtd->scaffIndex, parser->m_groupSize * sizeof(int)); - if (new_scaff_index == NULL) + if (new_scaff_index == NULL) { + parser->m_groupSize /= 2; return XML_ERROR_NO_MEMORY; + } dtd->scaffIndex = new_scaff_index; } } else { @@ -7190,7 +7199,7 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if ((unsigned)count > (size_t)(-1) / sizeof(DEFAULT_ATTRIBUTE)) { + if ((unsigned)count > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { return 0; } #endif @@ -7666,8 +7675,7 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, * from -Wtype-limits on platforms where * sizeof(int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if ((size_t)oldE->nDefaultAtts - > ((size_t)(-1) / sizeof(DEFAULT_ATTRIBUTE))) { + if ((size_t)oldE->nDefaultAtts > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { return 0; } #endif @@ -7869,7 +7877,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { unsigned long newMask = (unsigned long)newSize - 1; /* Detect and prevent integer overflow */ - if (newSize > (size_t)(-1) / sizeof(NAMED *)) { + if (newSize > SIZE_MAX / sizeof(NAMED *)) { return NULL; } @@ -8105,7 +8113,7 @@ poolBytesToAllocateFor(int blockSize) { static XML_Bool FASTCALL poolGrow(STRING_POOL *pool) { if (pool->freeBlocks) { - if (pool->start == 0) { + if (pool->start == NULL) { pool->blocks = pool->freeBlocks; pool->freeBlocks = pool->freeBlocks->next; pool->blocks->next = NULL; @@ -8217,7 +8225,7 @@ nextScaffoldPart(XML_Parser parser) { * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if (parser->m_groupSize > ((size_t)(-1) / sizeof(int))) { + if (parser->m_groupSize > SIZE_MAX / sizeof(int)) { return -1; } #endif @@ -8244,7 +8252,7 @@ nextScaffoldPart(XML_Parser parser) { * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if (dtd->scaffSize > (size_t)(-1) / 2u / sizeof(CONTENT_SCAFFOLD)) { + if (dtd->scaffSize > SIZE_MAX / 2u / sizeof(CONTENT_SCAFFOLD)) { return -1; } #endif @@ -8294,15 +8302,15 @@ build_model(XML_Parser parser) { * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if (dtd->scaffCount > (size_t)(-1) / sizeof(XML_Content)) { + if (dtd->scaffCount > SIZE_MAX / sizeof(XML_Content)) { return NULL; } - if (dtd->contentStringLen > (size_t)(-1) / sizeof(XML_Char)) { + if (dtd->contentStringLen > SIZE_MAX / sizeof(XML_Char)) { return NULL; } #endif if (dtd->scaffCount * sizeof(XML_Content) - > (size_t)(-1) - dtd->contentStringLen * sizeof(XML_Char)) { + > SIZE_MAX - dtd->contentStringLen * sizeof(XML_Char)) { return NULL; } diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index 2c48bf40867..d56bee82dd2 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -16,6 +16,7 @@ Copyright (c) 2017 Rhodri James Copyright (c) 2019 David Loffredo Copyright (c) 2021 Donghee Na + Copyright (c) 2025 Alfonso Gregory Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -46,7 +47,6 @@ # include "winconfig.h" #endif -#include "expat_external.h" #include "internal.h" #include "xmlrole.h" #include "ascii.h" diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index 95d5e84b67f..32cd5f147e9 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -24,6 +24,7 @@ Copyright (c) 2022 Martin Ettl Copyright (c) 2022 Sean McBride Copyright (c) 2023 Hanno Böck + Copyright (c) 2025 Alfonso Gregory Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -56,7 +57,6 @@ # include "winconfig.h" #endif -#include "expat_external.h" #include "internal.h" #include "xmltok.h" #include "nametab.h" diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c index fbdd3e3c7b7..810ca2c6d04 100644 --- a/Modules/expat/xmltok_ns.c +++ b/Modules/expat/xmltok_ns.c @@ -12,6 +12,7 @@ Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2017-2021 Sebastian Pipping + Copyright (c) 2025 Alfonso Gregory Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -98,13 +99,13 @@ NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end) { int i; XmlUtf8Convert(enc, &ptr, end, &p, p + ENCODING_MAX - 1); if (ptr != end) - return 0; + return NULL; *p = 0; if (streqci(buf, KW_UTF_16) && enc->minBytesPerChar == 2) return enc; i = getEncodingIndex(buf); if (i == UNKNOWN_ENC) - return 0; + return NULL; return NS(encodings)[i]; } diff --git a/Modules/hmacmodule.c b/Modules/hmacmodule.c index 8cd470f4f80..8a0b3496b1a 100644 --- a/Modules/hmacmodule.c +++ b/Modules/hmacmodule.c @@ -1529,7 +1529,6 @@ static void py_hmac_hinfo_ht_free(void *hinfo) { py_hmac_hinfo *entry = (py_hmac_hinfo *)hinfo; - assert(entry->display_name != NULL); if (--(entry->refcnt) == 0) { Py_CLEAR(entry->display_name); PyMem_Free(hinfo); @@ -1604,16 +1603,19 @@ py_hmac_hinfo_ht_new(void) assert(value->display_name == NULL); value->refcnt = 0; -#define Py_HMAC_HINFO_LINK(KEY) \ - do { \ - int rc = py_hmac_hinfo_ht_add(table, KEY, value); \ - if (rc < 0) { \ - PyMem_Free(value); \ - goto error; \ - } \ - else if (rc == 1) { \ - value->refcnt++; \ - } \ +#define Py_HMAC_HINFO_LINK(KEY) \ + do { \ + int rc = py_hmac_hinfo_ht_add(table, (KEY), value); \ + if (rc < 0) { \ + /* entry may already be in ht, freed upon exit */ \ + if (value->refcnt == 0) { \ + PyMem_Free(value); \ + } \ + goto error; \ + } \ + else if (rc == 1) { \ + value->refcnt++; \ + } \ } while (0) Py_HMAC_HINFO_LINK(e->name); Py_HMAC_HINFO_LINK(e->hashlib_name); @@ -1625,7 +1627,8 @@ py_hmac_hinfo_ht_new(void) e->hashlib_name == NULL ? e->name : e->hashlib_name ); if (value->display_name == NULL) { - PyMem_Free(value); + /* 'value' is owned by the table (refcnt > 0), + so _Py_hashtable_destroy() will free it. */ goto error; } } diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index a48d2fe76d0..fd03fae03f9 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -544,9 +544,19 @@ groupby_next(PyObject *op) else if (gbo->tgtkey == NULL) break; else { - int rcmp; + /* A user-defined __eq__ can re-enter groupby and advance the iterator, + mutating gbo->tgtkey / gbo->currkey while we are comparing them. + Take local snapshots and hold strong references so INCREF/DECREF + apply to the same objects even under re-entrancy. */ + PyObject *tgtkey = gbo->tgtkey; + PyObject *currkey = gbo->currkey; + + Py_INCREF(tgtkey); + Py_INCREF(currkey); + int rcmp = PyObject_RichCompareBool(tgtkey, currkey, Py_EQ); + Py_DECREF(tgtkey); + Py_DECREF(currkey); - rcmp = PyObject_RichCompareBool(gbo->tgtkey, gbo->currkey, Py_EQ); if (rcmp == -1) return NULL; else if (rcmp == 0) diff --git a/Modules/main.c b/Modules/main.c index 9b2ee103d52..15f51acbcf5 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -509,6 +509,7 @@ pymain_run_interactive_hook(int *exitcode) } if (PySys_Audit("cpython.run_interactivehook", "O", hook) < 0) { + Py_DECREF(hook); goto error; } diff --git a/Modules/md5module.c b/Modules/md5module.c index 9b5ea2d6e02..f3855ec3f37 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -87,7 +87,10 @@ static void MD5_dealloc(PyObject *op) { MD5object *ptr = _MD5object_CAST(op); - Hacl_Hash_MD5_free(ptr->hash_state); + if (ptr->hash_state != NULL) { + Hacl_Hash_MD5_free(ptr->hash_state); + ptr->hash_state = NULL; + } PyTypeObject *tp = Py_TYPE(op); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index cb3a62d211a..bb9ef0e6da6 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -1251,6 +1251,8 @@ get_posix_state(PyObject *module) * Contains a file descriptor if path.accept_fd was true * and the caller provided a signed integer instead of any * sort of string. + * path.is_fd + * True if path was provided as a file descriptor. * * WARNING: if your "path" parameter is optional, and is * unspecified, path_converter will never get called. @@ -1303,6 +1305,7 @@ typedef struct { const wchar_t *wide; const char *narrow; int fd; + bool is_fd; int value_error; Py_ssize_t length; PyObject *object; @@ -1312,7 +1315,7 @@ typedef struct { #define PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, \ make_wide, suppress_value_error, allow_fd) \ {function_name, argument_name, nullable, nonstrict, make_wide, \ - suppress_value_error, allow_fd, NULL, NULL, -1, 0, 0, NULL, NULL} + suppress_value_error, allow_fd, NULL, NULL, -1, false, 0, 0, NULL, NULL} #ifdef MS_WINDOWS #define PATH_T_INITIALIZE_P(function_name, argument_name, nullable, \ nonstrict, suppress_value_error, allow_fd) \ @@ -1446,6 +1449,7 @@ path_converter(PyObject *o, void *p) } path->wide = NULL; path->narrow = NULL; + path->is_fd = true; goto success_exit; } else { @@ -7155,6 +7159,7 @@ parse_posix_spawn_flags(PyObject *module, const char *func_name, PyObject *setpg PyObject *setsigdef, PyObject *scheduler, posix_spawnattr_t *attrp) { + assert(scheduler == NULL || scheduler == Py_None || PyTuple_Check(scheduler)); long all_flags = 0; errno = posix_spawnattr_init(attrp); @@ -7163,7 +7168,7 @@ parse_posix_spawn_flags(PyObject *module, const char *func_name, PyObject *setpg return -1; } - if (setpgroup) { + if (setpgroup && setpgroup != Py_None) { pid_t pgid = PyLong_AsPid(setpgroup); if (pgid == (pid_t)-1 && PyErr_Occurred()) { goto fail; @@ -7236,7 +7241,7 @@ parse_posix_spawn_flags(PyObject *module, const char *func_name, PyObject *setpg } #endif - if (scheduler) { + if (scheduler && scheduler != Py_None) { #ifdef POSIX_SPAWN_SETSCHEDULER PyObject *py_schedpolicy; PyObject *schedparam_obj; @@ -7461,6 +7466,12 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a goto exit; } + if (scheduler && !PyTuple_Check(scheduler) && scheduler != Py_None) { + PyErr_Format(PyExc_TypeError, + "%s: scheduler must be a tuple or None", func_name); + goto exit; + } + argvlist = parse_arglist(argv, &argc); if (argvlist == NULL) { goto exit; @@ -7572,7 +7583,7 @@ os.posix_spawn * file_actions: object(c_default='NULL') = () A sequence of file action tuples. - setpgroup: object = NULL + setpgroup: object(c_default='NULL') = None The pgroup to use with the POSIX_SPAWN_SETPGROUP flag. resetids: bool = False If the value is `true` the POSIX_SPAWN_RESETIDS will be activated. @@ -7582,7 +7593,7 @@ os.posix_spawn The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. setsigdef: object(c_default='NULL') = () The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag. - scheduler: object = NULL + scheduler: object(c_default='NULL') = None A tuple with the scheduler policy (optional) and parameters. Execute the program specified by path in a new process. @@ -7594,7 +7605,7 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, PyObject *setpgroup, int resetids, int setsid, PyObject *setsigmask, PyObject *setsigdef, PyObject *scheduler) -/*[clinic end generated code: output=14a1098c566bc675 input=808aed1090d84e33]*/ +/*[clinic end generated code: output=14a1098c566bc675 input=69e7c9ebbdcf94a5]*/ { return py_posix_spawn(0, module, path, argv, env, file_actions, setpgroup, resetids, setsid, setsigmask, setsigdef, @@ -7618,7 +7629,7 @@ os.posix_spawnp * file_actions: object(c_default='NULL') = () A sequence of file action tuples. - setpgroup: object = NULL + setpgroup: object(c_default='NULL') = None The pgroup to use with the POSIX_SPAWN_SETPGROUP flag. resetids: bool = False If the value is `True` the POSIX_SPAWN_RESETIDS will be activated. @@ -7628,7 +7639,7 @@ os.posix_spawnp The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. setsigdef: object(c_default='NULL') = () The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag. - scheduler: object = NULL + scheduler: object(c_default='NULL') = None A tuple with the scheduler policy (optional) and parameters. Execute the program specified by path in a new process. @@ -7640,7 +7651,7 @@ os_posix_spawnp_impl(PyObject *module, path_t *path, PyObject *argv, PyObject *setpgroup, int resetids, int setsid, PyObject *setsigmask, PyObject *setsigdef, PyObject *scheduler) -/*[clinic end generated code: output=7b9aaefe3031238d input=9e89e616116752a1]*/ +/*[clinic end generated code: output=7b9aaefe3031238d input=a5c057527c6881a5]*/ { return py_posix_spawn(1, module, path, argv, env, file_actions, setpgroup, resetids, setsid, setsigmask, setsigdef, @@ -13834,8 +13845,9 @@ os_pathconf_impl(PyObject *module, path_t *path, int name) errno = 0; #ifdef HAVE_FPATHCONF - if (path->fd != -1) + if (path->is_fd) { limit = fpathconf(path->fd, name); + } else #endif limit = pathconf(path->narrow, name); diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 8c8a5efa8c9..563a930bbcd 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -4952,11 +4952,13 @@ sock_sendmsg(PyObject *self, PyObject *args) if (cmsg_arg == NULL) ncmsgs = 0; else { - if ((cmsg_fast = PySequence_Fast(cmsg_arg, - "sendmsg() argument 2 must be an " - "iterable")) == NULL) + cmsg_fast = PySequence_Tuple(cmsg_arg); + if (cmsg_fast == NULL) { + PyErr_SetString(PyExc_TypeError, + "sendmsg() argument 2 must be an iterable"); goto finally; - ncmsgs = PySequence_Fast_GET_SIZE(cmsg_fast); + } + ncmsgs = PyTuple_GET_SIZE(cmsg_fast); } #ifndef CMSG_SPACE @@ -4976,8 +4978,9 @@ sock_sendmsg(PyObject *self, PyObject *args) controllen = controllen_last = 0; while (ncmsgbufs < ncmsgs) { size_t bufsize, space; + PyObject *item = PyTuple_GET_ITEM(cmsg_fast, ncmsgbufs); - if (!PyArg_Parse(PySequence_Fast_GET_ITEM(cmsg_fast, ncmsgbufs), + if (!PyArg_Parse(item, "(iiy*):[sendmsg() ancillary data items]", &cmsgs[ncmsgbufs].level, &cmsgs[ncmsgbufs].type, diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index ef8cf3d0d27..83de1be56a7 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -230,9 +230,9 @@ unicodedata_UCD_numeric_impl(PyObject *self, int chr, have_old = 1; rc = -1.0; } - else if (old->decimal_changed != 0xFF) { + else if (old->numeric_changed != 0.0) { have_old = 1; - rc = old->decimal_changed; + rc = old->numeric_changed; } } @@ -388,6 +388,17 @@ unicodedata_UCD_east_asian_width_impl(PyObject *self, int chr) return PyUnicode_FromString(_PyUnicode_EastAsianWidthNames[index]); } +// For Hangul decomposition +#define SBase 0xAC00 +#define LBase 0x1100 +#define VBase 0x1161 +#define TBase 0x11A7 +#define LCount 19 +#define VCount 21 +#define TCount 28 +#define NCount (VCount*TCount) +#define SCount (LCount*NCount) + /*[clinic input] unicodedata.UCD.decomposition @@ -418,6 +429,25 @@ unicodedata_UCD_decomposition_impl(PyObject *self, int chr) return Py_GetConstant(Py_CONSTANT_EMPTY_STR); /* unassigned */ } + // Hangul Decomposition. + // See section 3.12.2, "Hangul Syllable Decomposition" + // https://www.unicode.org/versions/latest/core-spec/chapter-3/#G56669 + if (SBase <= code && code < (SBase + SCount)) { + int SIndex = code - SBase; + int L = LBase + SIndex / NCount; + int V = VBase + (SIndex % NCount) / TCount; + int T = TBase + SIndex % TCount; + if (T != TBase) { + PyOS_snprintf(decomp, sizeof(decomp), + "%04X %04X %04X", L, V, T); + } + else { + PyOS_snprintf(decomp, sizeof(decomp), + "%04X %04X", L, V); + } + return PyUnicode_FromString(decomp); + } + if (code < 0 || code >= 0x110000) index = 0; else { @@ -480,16 +510,6 @@ get_decomp_record(PyObject *self, Py_UCS4 code, (*index)++; } -#define SBase 0xAC00 -#define LBase 0x1100 -#define VBase 0x1161 -#define TBase 0x11A7 -#define LCount 19 -#define VCount 21 -#define TCount 28 -#define NCount (VCount*TCount) -#define SCount (LCount*NCount) - static PyObject* nfd_nfkd(PyObject *self, PyObject *input, int k) { @@ -543,7 +563,9 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) } output = new_output; } - /* Hangul Decomposition. */ + // Hangul Decomposition. + // See section 3.12.2, "Hangul Syllable Decomposition" + // https://www.unicode.org/versions/latest/core-spec/chapter-3/#G56669 if (SBase <= code && code < (SBase+SCount)) { int SIndex = code - SBase; int L = LBase + SIndex / NCount; @@ -1010,21 +1032,18 @@ static const char * const hangul_syllables[][3] = { { 0, 0, "H" } }; -/* These ranges need to match makeunicodedata.py:cjk_ranges. */ static int -is_unified_ideograph(Py_UCS4 code) +find_prefix_id(Py_UCS4 code) { - return - (0x3400 <= code && code <= 0x4DBF) || /* CJK Ideograph Extension A */ - (0x4E00 <= code && code <= 0x9FFF) || /* CJK Ideograph */ - (0x20000 <= code && code <= 0x2A6DF) || /* CJK Ideograph Extension B */ - (0x2A700 <= code && code <= 0x2B739) || /* CJK Ideograph Extension C */ - (0x2B740 <= code && code <= 0x2B81D) || /* CJK Ideograph Extension D */ - (0x2B820 <= code && code <= 0x2CEA1) || /* CJK Ideograph Extension E */ - (0x2CEB0 <= code && code <= 0x2EBE0) || /* CJK Ideograph Extension F */ - (0x2EBF0 <= code && code <= 0x2EE5D) || /* CJK Ideograph Extension I */ - (0x30000 <= code && code <= 0x3134A) || /* CJK Ideograph Extension G */ - (0x31350 <= code && code <= 0x323AF); /* CJK Ideograph Extension H */ + for (int i = 0; i < (int)Py_ARRAY_LENGTH(derived_name_ranges); i++) { + if (code < derived_name_ranges[i].first) { + return -1; + } + if (code <= derived_name_ranges[i].last) { + return derived_name_ranges[i].prefixid; + } + } + return -1; } /* macros used to determine if the given code point is in the PUA range that @@ -1302,7 +1321,9 @@ _getucname(PyObject *self, } } - if (SBase <= code && code < SBase+SCount) { + int prefixid = find_prefix_id(code); + if (prefixid == 0) { + assert(SBase <= code && code < SBase+SCount); /* Hangul syllable. */ int SIndex = code - SBase; int L = SIndex / NCount; @@ -1324,11 +1345,13 @@ _getucname(PyObject *self, return 1; } - if (is_unified_ideograph(code)) { - if (buflen < 28) - /* Worst case: CJK UNIFIED IDEOGRAPH-20000 */ + /* Only support CJK unified ideographs. + * Support for Tangut ideographs is a new feature in 3.15. */ + if (prefixid == 1) { + const char *prefix = derived_name_prefixes[prefixid]; + if (snprintf(buffer, buflen, "%s%04X", prefix, code) >= buflen) { return 0; - sprintf(buffer, "CJK UNIFIED IDEOGRAPH-%X", code); + } return 1; } @@ -1362,7 +1385,7 @@ find_syllable(const char *str, int *len, int *pos, int count, int column) len1 = Py_SAFE_DOWNCAST(strlen(s), size_t, int); if (len1 <= *len) continue; - if (strncmp(str, s, len1) == 0) { + if (PyOS_strnicmp(str, s, len1) == 0) { *len = len1; *pos = i; } @@ -1385,6 +1408,35 @@ _check_alias_and_seq(Py_UCS4* code, int with_named_seq) return 1; } +static Py_UCS4 +parse_hex_code(const char *name, int namelen) +{ + if (namelen < 4 || namelen > 6) { + return (Py_UCS4)-1; + } + if (*name == '0') { + return (Py_UCS4)-1; + } + int v = 0; + while (namelen--) { + v *= 16; + Py_UCS1 c = Py_TOUPPER(*name); + if (c >= '0' && c <= '9') { + v += c - '0'; + } + else if (c >= 'A' && c <= 'F') { + v += c - 'A' + 10; + } + else { + return (Py_UCS4)-1; + } + name++; + } + if (v > 0x10ffff) { + return (Py_UCS4)-1; + } + return v; +} static int _getcode(const char* name, int namelen, Py_UCS4* code) @@ -1393,8 +1445,19 @@ _getcode(const char* name, int namelen, Py_UCS4* code) * Named aliases are not resolved, they are returned as a code point in the * PUA */ - /* Check for hangul syllables. */ - if (strncmp(name, "HANGUL SYLLABLE ", 16) == 0) { + int i = 0; + size_t prefixlen; + for (; i < (int)Py_ARRAY_LENGTH(derived_name_prefixes); i++) { + const char *prefix = derived_name_prefixes[i]; + prefixlen = strlen(derived_name_prefixes[i]); + if (PyOS_strnicmp(name, prefix, prefixlen) == 0) { + break; + } + } + + if (i == 0) { + /* Hangul syllables. */ + assert(PyOS_strnicmp(name, "HANGUL SYLLABLE ", 16) == 0); int len, L = -1, V = -1, T = -1; const char *pos = name + 16; find_syllable(pos, &len, &L, LCount, 0); @@ -1411,27 +1474,11 @@ _getcode(const char* name, int namelen, Py_UCS4* code) return 0; } - /* Check for unified ideographs. */ - if (strncmp(name, "CJK UNIFIED IDEOGRAPH-", 22) == 0) { - /* Four or five hexdigits must follow. */ - unsigned int v; - v = 0; - name += 22; - namelen -= 22; - if (namelen != 4 && namelen != 5) + if (i < (int)Py_ARRAY_LENGTH(derived_name_prefixes)) { + Py_UCS4 v = parse_hex_code(name + prefixlen, namelen - prefixlen); + if (find_prefix_id(v) != i) { return 0; - while (namelen--) { - v *= 16; - if (*name >= '0' && *name <= '9') - v += *name - '0'; - else if (*name >= 'A' && *name <= 'F') - v += *name - 'A' + 10; - else - return 0; - name++; } - if (!is_unified_ideograph(v)) - return 0; *code = v; return 1; } diff --git a/Modules/unicodename_db.h b/Modules/unicodename_db.h index 0697e259b39..acc85cb0b4e 100644 --- a/Modules/unicodename_db.h +++ b/Modules/unicodename_db.h @@ -19473,3 +19473,30 @@ static const named_sequence named_sequences[] = { {2, {0x02E5, 0x02E9}}, {2, {0x02E9, 0x02E5}}, }; + +typedef struct { + Py_UCS4 first; + Py_UCS4 last; + int prefixid; +} derived_name_range; + +static const derived_name_range derived_name_ranges[] = { + {0x3400, 0x4DBF, 1}, + {0x4E00, 0x9FFF, 1}, + {0xAC00, 0xD7A3, 0}, + {0x17000, 0x187F7, 2}, + {0x18D00, 0x18D08, 2}, + {0x20000, 0x2A6DF, 1}, + {0x2A700, 0x2B739, 1}, + {0x2B740, 0x2B81D, 1}, + {0x2B820, 0x2CEA1, 1}, + {0x2CEB0, 0x2EBE0, 1}, + {0x2EBF0, 0x2EE5D, 1}, + {0x30000, 0x3134A, 1}, + {0x31350, 0x323AF, 1}, +}; +static const char * const derived_name_prefixes[] = { + "HANGUL SYLLABLE ", + "CJK UNIFIED IDEOGRAPH-", + "TANGUT IDEOGRAPH-", +}; diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 287558302aa..f0b2e5f36c6 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1582,6 +1582,67 @@ typedef struct { } _PyCodeObjectExtra; +static inline size_t +code_extra_size(Py_ssize_t n) +{ + return sizeof(_PyCodeObjectExtra) + (n - 1) * sizeof(void *); +} + +#ifdef Py_GIL_DISABLED +static int +code_extra_grow_ft(PyCodeObject *co, _PyCodeObjectExtra *old_co_extra, + Py_ssize_t old_ce_size, Py_ssize_t new_ce_size, + Py_ssize_t index, void *extra) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(co); + _PyCodeObjectExtra *new_co_extra = PyMem_Malloc( + code_extra_size(new_ce_size)); + if (new_co_extra == NULL) { + PyErr_NoMemory(); + return -1; + } + + if (old_ce_size > 0) { + memcpy(new_co_extra->ce_extras, old_co_extra->ce_extras, + old_ce_size * sizeof(void *)); + } + for (Py_ssize_t i = old_ce_size; i < new_ce_size; i++) { + new_co_extra->ce_extras[i] = NULL; + } + new_co_extra->ce_size = new_ce_size; + new_co_extra->ce_extras[index] = extra; + + // Publish new buffer and its contents to lock-free readers. + FT_ATOMIC_STORE_PTR_RELEASE(co->co_extra, new_co_extra); + if (old_co_extra != NULL) { + // QSBR: defer old-buffer free until lock-free readers quiesce. + _PyMem_FreeDelayed(old_co_extra, code_extra_size(old_ce_size)); + } + return 0; +} +#else +static int +code_extra_grow_gil(PyCodeObject *co, _PyCodeObjectExtra *old_co_extra, + Py_ssize_t old_ce_size, Py_ssize_t new_ce_size, + Py_ssize_t index, void *extra) +{ + _PyCodeObjectExtra *new_co_extra = PyMem_Realloc( + old_co_extra, code_extra_size(new_ce_size)); + if (new_co_extra == NULL) { + PyErr_NoMemory(); + return -1; + } + + for (Py_ssize_t i = old_ce_size; i < new_ce_size; i++) { + new_co_extra->ce_extras[i] = NULL; + } + new_co_extra->ce_size = new_ce_size; + new_co_extra->ce_extras[index] = extra; + co->co_extra = new_co_extra; + return 0; +} +#endif + int PyUnstable_Code_GetExtra(PyObject *code, Py_ssize_t index, void **extra) { @@ -1590,15 +1651,19 @@ PyUnstable_Code_GetExtra(PyObject *code, Py_ssize_t index, void **extra) return -1; } - PyCodeObject *o = (PyCodeObject*) code; - _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) o->co_extra; + PyCodeObject *co = (PyCodeObject *)code; + *extra = NULL; - if (co_extra == NULL || index < 0 || co_extra->ce_size <= index) { - *extra = NULL; + if (index < 0) { return 0; } - *extra = co_extra->ce_extras[index]; + // Lock-free read; pairs with release stores in SetExtra. + _PyCodeObjectExtra *co_extra = FT_ATOMIC_LOAD_PTR_ACQUIRE(co->co_extra); + if (co_extra != NULL && index < co_extra->ce_size) { + *extra = FT_ATOMIC_LOAD_PTR_ACQUIRE(co_extra->ce_extras[index]); + } + return 0; } @@ -1608,40 +1673,59 @@ PyUnstable_Code_SetExtra(PyObject *code, Py_ssize_t index, void *extra) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (!PyCode_Check(code) || index < 0 || - index >= interp->co_extra_user_count) { + // co_extra_user_count is monotonically increasing and published with + // release store in RequestCodeExtraIndex, so once an index is valid + // it stays valid. + Py_ssize_t user_count = FT_ATOMIC_LOAD_SSIZE_ACQUIRE( + interp->co_extra_user_count); + + if (!PyCode_Check(code) || index < 0 || index >= user_count) { PyErr_BadInternalCall(); return -1; } - PyCodeObject *o = (PyCodeObject*) code; - _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra *) o->co_extra; + PyCodeObject *co = (PyCodeObject *)code; + int result = 0; + void *old_slot_value = NULL; - if (co_extra == NULL || co_extra->ce_size <= index) { - Py_ssize_t i = (co_extra == NULL ? 0 : co_extra->ce_size); - co_extra = PyMem_Realloc( - co_extra, - sizeof(_PyCodeObjectExtra) + - (interp->co_extra_user_count-1) * sizeof(void*)); - if (co_extra == NULL) { - return -1; - } - for (; i < interp->co_extra_user_count; i++) { - co_extra->ce_extras[i] = NULL; - } - co_extra->ce_size = interp->co_extra_user_count; - o->co_extra = co_extra; + Py_BEGIN_CRITICAL_SECTION(co); + + _PyCodeObjectExtra *old_co_extra = (_PyCodeObjectExtra *)co->co_extra; + Py_ssize_t old_ce_size = (old_co_extra == NULL) + ? 0 : old_co_extra->ce_size; + + // Fast path: slot already exists, update in place. + if (index < old_ce_size) { + old_slot_value = old_co_extra->ce_extras[index]; + FT_ATOMIC_STORE_PTR_RELEASE(old_co_extra->ce_extras[index], extra); + goto done; } - if (co_extra->ce_extras[index] != NULL) { - freefunc free = interp->co_extra_freefuncs[index]; - if (free != NULL) { - free(co_extra->ce_extras[index]); + // Slow path: buffer needs to grow. + Py_ssize_t new_ce_size = user_count; +#ifdef Py_GIL_DISABLED + // FT build: allocate new buffer and swap; QSBR reclaims the old one. + result = code_extra_grow_ft( + co, old_co_extra, old_ce_size, new_ce_size, index, extra); +#else + // GIL build: grow with realloc. + result = code_extra_grow_gil( + co, old_co_extra, old_ce_size, new_ce_size, index, extra); +#endif + +done:; + Py_END_CRITICAL_SECTION(); + if (old_slot_value != NULL) { + // Free the old slot value if a free function was registered. + // The caller must ensure no other thread can still access the old + // value after this overwrite. + freefunc free_extra = interp->co_extra_freefuncs[index]; + if (free_extra != NULL) { + free_extra(old_slot_value); } } - co_extra->ce_extras[index] = extra; - return 0; + return result; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index c2223af9500..b41d2548bec 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2878,6 +2878,21 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, return res; } +static void +clear_embedded_values(PyDictValues *values, Py_ssize_t nentries) +{ + PyObject *refs[SHARED_KEYS_MAX_SIZE]; + assert(nentries <= SHARED_KEYS_MAX_SIZE); + for (Py_ssize_t i = 0; i < nentries; i++) { + refs[i] = values->values[i]; + values->values[i] = NULL; + } + values->size = 0; + for (Py_ssize_t i = 0; i < nentries; i++) { + Py_XDECREF(refs[i]); + } +} + static void clear_lock_held(PyObject *op) { @@ -2907,20 +2922,18 @@ clear_lock_held(PyObject *op) assert(oldkeys->dk_refcnt == 1); dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); } + else if (oldvalues->embedded) { + clear_embedded_values(oldvalues, oldkeys->dk_nentries); + } else { + set_values(mp, NULL); + set_keys(mp, Py_EMPTY_KEYS); n = oldkeys->dk_nentries; for (i = 0; i < n; i++) { Py_CLEAR(oldvalues->values[i]); } - if (oldvalues->embedded) { - oldvalues->size = 0; - } - else { - set_values(mp, NULL); - set_keys(mp, Py_EMPTY_KEYS); - free_values(oldvalues, IS_DICT_SHARED(mp)); - dictkeys_decref(interp, oldkeys, false); - } + free_values(oldvalues, IS_DICT_SHARED(mp)); + dictkeys_decref(interp, oldkeys, false); } ASSERT_CONSISTENT(mp); } @@ -4611,10 +4624,8 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) if (DK_IS_UNICODE(keys)) { if (_PyDict_HasSplitTable(mp)) { - if (!mp->ma_values->embedded) { - for (i = 0; i < n; i++) { - Py_VISIT(mp->ma_values->values[i]); - } + for (i = 0; i < n; i++) { + Py_VISIT(mp->ma_values->values[i]); } } else { @@ -7175,16 +7186,21 @@ PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { return 0; } - if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict; + if (dict != NULL) { + // GH-130327: If there's a managed dictionary available, we should + // *always* traverse it. The dict is responsible for traversing the + // inline values if it points to them. + Py_VISIT(dict); + } + else if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { PyDictValues *values = _PyObject_InlineValues(obj); if (values->valid) { for (Py_ssize_t i = 0; i < values->capacity; i++) { Py_VISIT(values->values[i]); } - return 0; } } - Py_VISIT(_PyObject_ManagedDictPointer(obj)->dict); return 0; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 944e98e062d..7c2773085f4 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1036,11 +1036,11 @@ static PyObject * frame_lasti_get_impl(PyFrameObject *self) /*[clinic end generated code: output=03275b4f0327d1a2 input=0225ed49cb1fbeeb]*/ { - int lasti = _PyInterpreterFrame_LASTI(self->f_frame); + int lasti = PyUnstable_InterpreterFrame_GetLasti(self->f_frame); if (lasti < 0) { return PyLong_FromLong(-1); } - return PyLong_FromLong(lasti * sizeof(_Py_CODEUNIT)); + return PyLong_FromLong(lasti); } /*[clinic input] @@ -2044,11 +2044,15 @@ static PyObject * frame_repr(PyObject *op) { PyFrameObject *f = PyFrameObject_CAST(op); + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(f); int lineno = PyFrame_GetLineNumber(f); PyCodeObject *code = _PyFrame_GetCode(f->f_frame); - return PyUnicode_FromFormat( + result = PyUnicode_FromFormat( "", f, code->co_filename, lineno, code->co_name); + Py_END_CRITICAL_SECTION(); + return result; } static PyMethodDef frame_methods[] = { diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 9532c21fc70..b870106479a 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -1470,6 +1470,18 @@ cm_descr_get(PyObject *self, PyObject *obj, PyObject *type) return PyMethod_New(cm->cm_callable, type); } +static PyObject * +cm_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + classmethod *cm = (classmethod *)PyType_GenericAlloc(type, 0); + if (cm == NULL) { + return NULL; + } + cm->cm_callable = Py_None; + cm->cm_dict = NULL; + return (PyObject *)cm; +} + static int cm_init(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1616,7 +1628,7 @@ PyTypeObject PyClassMethod_Type = { offsetof(classmethod, cm_dict), /* tp_dictoffset */ cm_init, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ + cm_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ }; @@ -1701,6 +1713,18 @@ sm_descr_get(PyObject *self, PyObject *obj, PyObject *type) return Py_NewRef(sm->sm_callable); } +static PyObject * +sm_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + staticmethod *sm = (staticmethod *)PyType_GenericAlloc(type, 0); + if (sm == NULL) { + return NULL; + } + sm->sm_callable = Py_None; + sm->sm_dict = NULL; + return (PyObject *)sm; +} + static int sm_init(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1851,7 +1875,7 @@ PyTypeObject PyStaticMethod_Type = { offsetof(staticmethod, sm_dict), /* tp_dictoffset */ sm_init, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ + sm_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ }; diff --git a/Objects/mimalloc/heap.c b/Objects/mimalloc/heap.c index d92dc768e5e..5fbfb82baa0 100644 --- a/Objects/mimalloc/heap.c +++ b/Objects/mimalloc/heap.c @@ -100,7 +100,10 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t // note: this will free retired pages as well. bool freed = _PyMem_mi_page_maybe_free(page, pq, collect >= MI_FORCE); if (!freed && collect == MI_ABANDON) { - _mi_page_abandon(page, pq); + // _PyMem_mi_page_maybe_free may have moved the page to a different + // page queue, so we need to re-fetch the correct queue. + uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : _mi_bin(page->xblock_size)); + _mi_page_abandon(page, &heap->pages[bin]); } } else if (collect == MI_ABANDON) { diff --git a/Objects/mimalloc/page.c b/Objects/mimalloc/page.c index ff7444cce10..ded59f8eb1c 100644 --- a/Objects/mimalloc/page.c +++ b/Objects/mimalloc/page.c @@ -213,6 +213,13 @@ static void _mi_page_thread_free_collect(mi_page_t* page) // update counts now page->used -= count; + + if (page->used == 0) { + // The page may have had a QSBR goal set from a previous point when it + // was all-free. That goal is no longer valid because the page was + // allocated from and then freed again by other threads. + _PyMem_mi_page_clear_qsbr(page); + } } void _mi_page_free_collect(mi_page_t* page, bool force) { @@ -225,9 +232,6 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { // and the local free list if (page->local_free != NULL) { - // any previous QSBR goals are no longer valid because we reused the page - _PyMem_mi_page_clear_qsbr(page); - if mi_likely(page->free == NULL) { // usual case page->free = page->local_free; diff --git a/Objects/mimalloc/segment.c b/Objects/mimalloc/segment.c index 9b092b9b734..9dad69c995e 100644 --- a/Objects/mimalloc/segment.c +++ b/Objects/mimalloc/segment.c @@ -1286,6 +1286,7 @@ static bool mi_segment_check_free(mi_segment_t* segment, size_t slices_needed, s _mi_stat_decrease(&tld->stats->pages_abandoned, 1); #ifdef Py_GIL_DISABLED page->qsbr_goal = 0; + mi_assert_internal(page->qsbr_node.next == NULL); #endif segment->abandoned--; slice = mi_segment_page_clear(page, tld); // re-assign slice due to coalesce! @@ -1361,6 +1362,7 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, // if everything free by now, free the page #ifdef Py_GIL_DISABLED page->qsbr_goal = 0; + mi_assert_internal(page->qsbr_node.next == NULL); #endif slice = mi_segment_page_clear(page, tld); // set slice again due to coalesceing } diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 092be84d2b9..9af59f13dc8 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -149,6 +149,12 @@ should_advance_qsbr_for_page(struct _qsbr_thread_state *qsbr, mi_page_t *page) } return false; } + +static _PyThreadStateImpl * +tstate_from_heap(mi_heap_t *heap) +{ + return _Py_CONTAINER_OF(heap->tld, _PyThreadStateImpl, mimalloc.tld); +} #endif static bool @@ -157,23 +163,35 @@ _PyMem_mi_page_maybe_free(mi_page_t *page, mi_page_queue_t *pq, bool force) #ifdef Py_GIL_DISABLED assert(mi_page_all_free(page)); if (page->use_qsbr) { - _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)PyThreadState_GET(); - if (page->qsbr_goal != 0 && _Py_qbsr_goal_reached(tstate->qsbr, page->qsbr_goal)) { + struct _qsbr_thread_state *qsbr = ((_PyThreadStateImpl *)PyThreadState_GET())->qsbr; + if (page->qsbr_goal != 0 && _Py_qbsr_goal_reached(qsbr, page->qsbr_goal)) { _PyMem_mi_page_clear_qsbr(page); _mi_page_free(page, pq, force); return true; } + // gh-145615: since we are not freeing this page yet, we want to + // make it available for allocations. Note that the QSBR goal and + // linked list node remain set even if the page is later used for + // an allocation. We only detect and clear the QSBR goal when the + // page becomes empty again (used == 0). + if (mi_page_is_in_full(page)) { + _mi_page_unfull(page); + } + _PyMem_mi_page_clear_qsbr(page); page->retire_expire = 0; - if (should_advance_qsbr_for_page(tstate->qsbr, page)) { - page->qsbr_goal = _Py_qsbr_advance(tstate->qsbr->shared); + if (should_advance_qsbr_for_page(qsbr, page)) { + page->qsbr_goal = _Py_qsbr_advance(qsbr->shared); } else { - page->qsbr_goal = _Py_qsbr_shared_next(tstate->qsbr->shared); + page->qsbr_goal = _Py_qsbr_shared_next(qsbr->shared); } + // We may be freeing a page belonging to a different thread during a + // stop-the-world event. Find the _PyThreadStateImpl for the page. + _PyThreadStateImpl *tstate = tstate_from_heap(mi_page_heap(page)); llist_insert_tail(&tstate->mimalloc.page_list, &page->qsbr_node); return false; } @@ -190,7 +208,8 @@ _PyMem_mi_page_reclaimed(mi_page_t *page) if (page->qsbr_goal != 0) { if (mi_page_all_free(page)) { assert(page->qsbr_node.next == NULL); - _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)PyThreadState_GET(); + _PyThreadStateImpl *tstate = tstate_from_heap(mi_page_heap(page)); + assert(tstate == (_PyThreadStateImpl *)_PyThreadState_GET()); page->retire_expire = 0; llist_insert_tail(&tstate->mimalloc.page_list, &page->qsbr_node); } diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h index b447865c986..26bb0555ca9 100644 --- a/Objects/stringlib/fastsearch.h +++ b/Objects/stringlib/fastsearch.h @@ -69,8 +69,8 @@ STRINGLIB(find_char)(const STRINGLIB_CHAR* s, Py_ssize_t n, STRINGLIB_CHAR ch) and UCS4 representations. */ if (needle != 0) { do { - void *candidate = memchr(p, needle, - (e - p) * sizeof(STRINGLIB_CHAR)); + const void *candidate = memchr(p, needle, + (e - p) * sizeof(STRINGLIB_CHAR)); if (candidate == NULL) return -1; s1 = p; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 03d5cfa4ca5..9777055c5f2 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -76,6 +76,62 @@ class object "PyObject *" "&PyBaseObject_Type" #define ASSERT_TYPE_LOCK_HELD() \ _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(TYPE_LOCK) +static void +types_stop_world(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); +} + +static void +types_start_world(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StartTheWorld(interp); +} + +// This is used to temporarily prevent the TYPE_LOCK from being suspended +// when held by the topmost critical section. +static void +type_lock_prevent_release(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + uintptr_t *tagptr = &tstate->critical_section; + PyCriticalSection *c = (PyCriticalSection *)(*tagptr & ~_Py_CRITICAL_SECTION_MASK); + if (!(*tagptr & _Py_CRITICAL_SECTION_TWO_MUTEXES)) { + assert(c->_cs_mutex == TYPE_LOCK); + c->_cs_mutex = NULL; + } + else { + PyCriticalSection2 *c2 = (PyCriticalSection2 *)c; + if (c->_cs_mutex == TYPE_LOCK) { + c->_cs_mutex = c2->_cs_mutex2; + c2->_cs_mutex2 = NULL; + } + else { + assert(c2->_cs_mutex2 == TYPE_LOCK); + c2->_cs_mutex2 = NULL; + } + } +} + +static void +type_lock_allow_release(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + uintptr_t *tagptr = &tstate->critical_section; + PyCriticalSection *c = (PyCriticalSection *)(*tagptr & ~_Py_CRITICAL_SECTION_MASK); + if (!(*tagptr & _Py_CRITICAL_SECTION_TWO_MUTEXES)) { + assert(c->_cs_mutex == NULL); + c->_cs_mutex = TYPE_LOCK; + } + else { + PyCriticalSection2 *c2 = (PyCriticalSection2 *)c; + assert(c2->_cs_mutex2 == NULL); + c2->_cs_mutex2 = TYPE_LOCK; + } +} + #else #define BEGIN_TYPE_LOCK() @@ -83,6 +139,10 @@ class object "PyObject *" "&PyBaseObject_Type" #define BEGIN_TYPE_DICT_LOCK(d) #define END_TYPE_DICT_LOCK() #define ASSERT_TYPE_LOCK_HELD() +#define types_stop_world() +#define types_start_world() +#define type_lock_prevent_release() +#define type_lock_allow_release() #endif @@ -541,7 +601,6 @@ clear_tp_bases(PyTypeObject *self, int final) static inline PyObject * lookup_tp_mro(PyTypeObject *self) { - ASSERT_TYPE_LOCK_HELD(); return self->tp_mro; } @@ -580,8 +639,19 @@ set_tp_mro(PyTypeObject *self, PyObject *mro, int initial) /* Other checks are done via set_tp_bases. */ _Py_SetImmortal(mro); } + else { + PyUnstable_Object_EnableDeferredRefcount(mro); + } + } + if (!initial) { + type_lock_prevent_release(); + types_stop_world(); } self->tp_mro = mro; + if (!initial) { + types_start_world(); + type_lock_allow_release(); + } } static inline void @@ -1627,18 +1697,11 @@ static PyObject * type_get_mro(PyObject *tp, void *Py_UNUSED(closure)) { PyTypeObject *type = PyTypeObject_CAST(tp); - PyObject *mro; - - BEGIN_TYPE_LOCK(); - mro = lookup_tp_mro(type); + PyObject *mro = lookup_tp_mro(type); if (mro == NULL) { - mro = Py_None; - } else { - Py_INCREF(mro); + Py_RETURN_NONE; } - - END_TYPE_LOCK(); - return mro; + return Py_NewRef(mro); } static PyTypeObject *best_base(PyObject *); @@ -5639,17 +5702,18 @@ PyObject_GetItemData(PyObject *obj) } /* Internal API to look for a name through the MRO, bypassing the method cache. - This returns a borrowed reference, and might set an exception. - 'error' is set to: -1: error with exception; 1: error without exception; 0: ok */ -static PyObject * -find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) + The result is stored as a _PyStackRef in `out`. It never set an exception. + Returns -1 if there was an error, 0 if the name was not found, and 1 if + the name was found. */ +static int +find_name_in_mro(PyTypeObject *type, PyObject *name, _PyStackRef *out) { ASSERT_TYPE_LOCK_HELD(); Py_hash_t hash = _PyObject_HashFast(name); if (hash == -1) { - *error = -1; - return NULL; + PyErr_Clear(); + return -1; } /* Look in tp_dict of types in MRO */ @@ -5657,37 +5721,42 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) if (mro == NULL) { if (!is_readying(type)) { if (PyType_Ready(type) < 0) { - *error = -1; - return NULL; + PyErr_Clear(); + return -1; } mro = lookup_tp_mro(type); } if (mro == NULL) { - *error = 1; - return NULL; + return -1; } } - PyObject *res = NULL; + int res = 0; + PyThreadState *tstate = _PyThreadState_GET(); /* Keep a strong reference to mro because type->tp_mro can be replaced during dict lookup, e.g. when comparing to non-string keys. */ - Py_INCREF(mro); + _PyCStackRef mro_ref; + _PyThreadState_PushCStackRef(tstate, &mro_ref); + mro_ref.ref = PyStackRef_FromPyObjectNew(mro); Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 0; i < n; i++) { PyObject *base = PyTuple_GET_ITEM(mro, i); PyObject *dict = lookup_tp_dict(_PyType_CAST(base)); assert(dict && PyDict_Check(dict)); - if (_PyDict_GetItemRef_KnownHash((PyDictObject *)dict, name, hash, &res) < 0) { - *error = -1; + Py_ssize_t ix = _Py_dict_lookup_threadsafe_stackref( + (PyDictObject *)dict, name, hash, out); + if (ix == DKIX_ERROR) { + PyErr_Clear(); + res = -1; goto done; } - if (res != NULL) { + if (!PyStackRef_IsNull(*out)) { + res = 1; break; } } - *error = 0; done: - Py_DECREF(mro); + _PyThreadState_PopCStackRef(tstate, &mro_ref); return res; } @@ -5842,11 +5911,11 @@ _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef // We need to atomically do the lookup and capture the version before // anyone else can modify our mro or mutate the type. - PyObject *res; - int error; + int res; PyInterpreterState *interp = _PyInterpreterState_GET(); int has_version = 0; unsigned int assigned_version = 0; + BEGIN_TYPE_LOCK(); // We must assign the version before doing the lookup. If // find_name_in_mro() blocks and releases the critical section @@ -5855,35 +5924,24 @@ _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef has_version = assign_version_tag(interp, type); assigned_version = type->tp_version_tag; } - res = find_name_in_mro(type, name, &error); + res = find_name_in_mro(type, name, out); END_TYPE_LOCK(); /* Only put NULL results into cache if there was no error. */ - if (error) { - /* It's not ideal to clear the error condition, - but this function is documented as not setting - an exception, and I don't want to change that. - E.g., when PyType_Ready() can't proceed, it won't - set the "ready" flag, so future attempts to ready - the same type will call it again -- hopefully - in a context that propagates the exception out. - */ - if (error == -1) { - PyErr_Clear(); - } + if (res < 0) { *out = PyStackRef_NULL; return 0; } if (has_version) { + PyObject *res_obj = PyStackRef_AsPyObjectBorrow(*out); #if Py_GIL_DISABLED - update_cache_gil_disabled(entry, name, assigned_version, res); + update_cache_gil_disabled(entry, name, assigned_version, res_obj); #else - PyObject *old_value = update_cache(entry, name, assigned_version, res); + PyObject *old_value = update_cache(entry, name, assigned_version, res_obj); Py_DECREF(old_value); #endif } - *out = res ? PyStackRef_FromPyObjectSteal(res) : PyStackRef_NULL; return has_version ? assigned_version : 0; } @@ -7120,7 +7178,11 @@ object_set_class_world_stopped(PyObject *self, PyTypeObject *newto) assert(_PyObject_GetManagedDict(self) == dict); - if (_PyDict_DetachFromObject(dict, self) < 0) { + int err; + Py_BEGIN_CRITICAL_SECTION(dict); + err = _PyDict_DetachFromObject(dict, self); + Py_END_CRITICAL_SECTION(); + if (err < 0) { return -1; } @@ -7162,12 +7224,17 @@ object_set_class(PyObject *self, PyObject *value, void *closure) #ifdef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyEval_StopTheWorld(interp); + int unique = _PyObject_IsUniquelyReferenced(self); + if (!unique) { + _PyEval_StopTheWorld(interp); + } #endif PyTypeObject *oldto = Py_TYPE(self); int res = object_set_class_world_stopped(self, newto); #ifdef Py_GIL_DISABLED - _PyEval_StartTheWorld(interp); + if (!unique) { + _PyEval_StartTheWorld(interp); + } #endif if (res == 0) { if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE) { @@ -11234,7 +11301,6 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) int use_generic = 0; int offset = p->offset; - int error; void **ptr = slotptr(type, offset); if (ptr == NULL) { @@ -11247,19 +11313,15 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) assert(!PyErr_Occurred()); do { /* Use faster uncached lookup as we won't get any cache hits during type setup. */ - descr = find_name_in_mro(type, p->name_strobj, &error); - if (descr == NULL) { - if (error == -1) { - /* It is unlikely but not impossible that there has been an exception - during lookup. Since this function originally expected no errors, - we ignore them here in order to keep up the interface. */ - PyErr_Clear(); - } + _PyStackRef descr_ref; + int res = find_name_in_mro(type, p->name_strobj, &descr_ref); + if (res <= 0) { if (ptr == (void**)&type->tp_iternext) { specific = (void *)_PyObject_NextNotImplemented; } continue; } + descr = PyStackRef_AsPyObjectBorrow(descr_ref); if (Py_IS_TYPE(descr, &PyWrapperDescr_Type) && ((PyWrapperDescrObject *)descr)->d_base->name_strobj == p->name_strobj) { void **tptr = resolve_slotdups(type, p->name_strobj); @@ -11327,7 +11389,7 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) type_clear_flags(type, Py_TPFLAGS_HAVE_VECTORCALL); } } - Py_DECREF(descr); + PyStackRef_CLOSE(descr_ref); } while ((++p)->offset == offset); if (specific && !use_generic) *ptr = specific; @@ -11758,18 +11820,16 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * PyObject *mro, *res; Py_ssize_t i, n; - BEGIN_TYPE_LOCK(); mro = lookup_tp_mro(su_obj_type); - /* keep a strong reference to mro because su_obj_type->tp_mro can be - replaced during PyDict_GetItemRef(dict, name, &res) and because - another thread can modify it after we end the critical section - below */ - Py_XINCREF(mro); - END_TYPE_LOCK(); - if (mro == NULL) return NULL; + /* Keep a strong reference to mro because su_obj_type->tp_mro can be + replaced during PyDict_GetItemRef(dict, name, &res). */ + PyThreadState *tstate = _PyThreadState_GET(); + _PyCStackRef mro_ref; + _PyThreadState_PushCStackRefNew(tstate, &mro_ref, mro); + assert(PyTuple_Check(mro)); n = PyTuple_GET_SIZE(mro); @@ -11780,7 +11840,7 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * } i++; /* skip su->type (if any) */ if (i >= n) { - Py_DECREF(mro); + _PyThreadState_PopCStackRef(tstate, &mro_ref); return NULL; } @@ -11791,13 +11851,13 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * if (PyDict_GetItemRef(dict, name, &res) != 0) { // found or error - Py_DECREF(mro); + _PyThreadState_PopCStackRef(tstate, &mro_ref); return res; } i++; } while (i < n); - Py_DECREF(mro); + _PyThreadState_PopCStackRef(tstate, &mro_ref); return NULL; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index c71c5720ea0..3835b8d462a 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13366,6 +13366,45 @@ unicode_swapcase_impl(PyObject *self) return case_operation(self, do_swapcase); } +static int +unicode_maketrans_from_dict(PyObject *x, PyObject *newdict) +{ + PyObject *key, *value; + Py_ssize_t i = 0; + int res; + while (PyDict_Next(x, &i, &key, &value)) { + if (PyUnicode_Check(key)) { + PyObject *newkey; + int kind; + const void *data; + if (PyUnicode_GET_LENGTH(key) != 1) { + PyErr_SetString(PyExc_ValueError, "string keys in translate" + "table must be of length 1"); + return -1; + } + kind = PyUnicode_KIND(key); + data = PyUnicode_DATA(key); + newkey = PyLong_FromLong(PyUnicode_READ(kind, data, 0)); + if (!newkey) + return -1; + res = PyDict_SetItem(newdict, newkey, value); + Py_DECREF(newkey); + if (res < 0) + return -1; + } + else if (PyLong_Check(key)) { + if (PyDict_SetItem(newdict, key, value) < 0) + return -1; + } + else { + PyErr_SetString(PyExc_TypeError, "keys in translate table must" + "be strings or integers"); + return -1; + } + } + return 0; +} + /*[clinic input] @staticmethod @@ -13451,9 +13490,6 @@ unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z) } } } else { - int kind; - const void *data; - /* x must be a dict */ if (!PyDict_CheckExact(x)) { PyErr_SetString(PyExc_TypeError, "if you give only one argument " @@ -13461,34 +13497,12 @@ unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z) goto err; } /* copy entries into the new dict, converting string keys to int keys */ - while (PyDict_Next(x, &i, &key, &value)) { - if (PyUnicode_Check(key)) { - /* convert string keys to integer keys */ - PyObject *newkey; - if (PyUnicode_GET_LENGTH(key) != 1) { - PyErr_SetString(PyExc_ValueError, "string keys in translate " - "table must be of length 1"); - goto err; - } - kind = PyUnicode_KIND(key); - data = PyUnicode_DATA(key); - newkey = PyLong_FromLong(PyUnicode_READ(kind, data, 0)); - if (!newkey) - goto err; - res = PyDict_SetItem(new, newkey, value); - Py_DECREF(newkey); - if (res < 0) - goto err; - } else if (PyLong_Check(key)) { - /* just keep integer keys */ - if (PyDict_SetItem(new, key, value) < 0) - goto err; - } else { - PyErr_SetString(PyExc_TypeError, "keys in translate table must " - "be strings or integers"); - goto err; - } - } + int errcode; + Py_BEGIN_CRITICAL_SECTION(x); + errcode = unicode_maketrans_from_dict(x, new); + Py_END_CRITICAL_SECTION(); + if (errcode < 0) + goto err; } return new; err: diff --git a/PC/launcher2.c b/PC/launcher2.c index 832935c5cc6..4dd18c8eb54 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -922,6 +922,20 @@ _readIni(const wchar_t *section, const wchar_t *settingName, wchar_t *buffer, in { wchar_t iniPath[MAXLEN]; int n; + // Check for _PYLAUNCHER_INIDIR override (used for test isolation) + DWORD len = GetEnvironmentVariableW(L"_PYLAUNCHER_INIDIR", iniPath, MAXLEN); + if (len && len < MAXLEN) { + if (join(iniPath, MAXLEN, L"py.ini")) { + debug(L"# Reading from %s for %s/%s\n", iniPath, section, settingName); + n = GetPrivateProfileStringW(section, settingName, NULL, buffer, bufferLength, iniPath); + if (n) { + debug(L"# Found %s in %s\n", settingName, iniPath); + return n; + } + } + // When _PYLAUNCHER_INIDIR is set, skip the default locations + return 0; + } if (SUCCEEDED(SHGetFolderPathW(NULL, CSIDL_LOCAL_APPDATA, NULL, 0, iniPath)) && join(iniPath, MAXLEN, L"py.ini")) { debug(L"# Reading from %s for %s/%s\n", iniPath, section, settingName); diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c index 63aa1ea2ad4..e122fd0d987 100644 --- a/Parser/lexer/buffer.c +++ b/Parser/lexer/buffer.c @@ -13,8 +13,8 @@ _PyLexer_remember_fstring_buffers(struct tok_state *tok) for (index = tok->tok_mode_stack_index; index >= 0; --index) { mode = &(tok->tok_mode_stack[index]); - mode->start_offset = mode->start - tok->buf; - mode->multi_line_start_offset = mode->multi_line_start - tok->buf; + mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf; + mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf; } } @@ -27,8 +27,8 @@ _PyLexer_restore_fstring_buffers(struct tok_state *tok) for (index = tok->tok_mode_stack_index; index >= 0; --index) { mode = &(tok->tok_mode_stack[index]); - mode->start = tok->buf + mode->start_offset; - mode->multi_line_start = tok->buf + mode->multi_line_start_offset; + mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset; + mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset; } } diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index e5e2eed2d34..7bdf6367671 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -494,9 +494,11 @@ valid_utf8(const unsigned char* s) return 0; } length = expected + 1; - for (; expected; expected--) - if (s[expected] < 0x80 || s[expected] >= 0xC0) + for (int i = 1; i <= expected; i++) { + if (s[i] < 0x80 || s[i] >= 0xC0) { return 0; + } + } return length; } diff --git a/Parser/tokenizer/string_tokenizer.c b/Parser/tokenizer/string_tokenizer.c index 7299ecf483c..7f07cca37ee 100644 --- a/Parser/tokenizer/string_tokenizer.c +++ b/Parser/tokenizer/string_tokenizer.c @@ -108,6 +108,19 @@ decode_str(const char *input, int single, struct tok_state *tok, int preserve_cr else if (!_PyTokenizer_ensure_utf8(str, tok, 1)) { return _PyTokenizer_error_ret(tok); } + if (utf8 != NULL) { + char *translated = _PyTokenizer_translate_newlines( + str, single, preserve_crlf, tok); + if (translated == NULL) { + Py_DECREF(utf8); + return _PyTokenizer_error_ret(tok); + } + PyMem_Free(tok->input); + tok->input = translated; + str = translated; + Py_CLEAR(utf8); + } + tok->str = str; assert(tok->decoding_buffer == NULL); tok->decoding_buffer = utf8; /* CAUTION */ return str; diff --git a/Python/ceval.c b/Python/ceval.c index 87cf01730b4..e6d82f249ef 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3486,11 +3486,27 @@ PyUnstable_Eval_RequestCodeExtraIndex(freefunc free) PyInterpreterState *interp = _PyInterpreterState_GET(); Py_ssize_t new_index; - if (interp->co_extra_user_count == MAX_CO_EXTRA_USERS - 1) { +#ifdef Py_GIL_DISABLED + struct _py_code_state *state = &interp->code_state; + FT_MUTEX_LOCK(&state->mutex); +#endif + + if (interp->co_extra_user_count >= MAX_CO_EXTRA_USERS - 1) { +#ifdef Py_GIL_DISABLED + FT_MUTEX_UNLOCK(&state->mutex); +#endif return -1; } - new_index = interp->co_extra_user_count++; + + new_index = interp->co_extra_user_count; interp->co_extra_freefuncs[new_index] = free; + + // Publish freefuncs[new_index] before making the index visible. + FT_ATOMIC_STORE_SSIZE_RELEASE(interp->co_extra_user_count, new_index + 1); + +#ifdef Py_GIL_DISABLED + FT_MUTEX_UNLOCK(&state->mutex); +#endif return new_index; } diff --git a/Python/codegen.c b/Python/codegen.c index bacd3460f2f..085ebc391a1 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -1200,11 +1200,11 @@ codegen_type_param_bound_or_default(compiler *c, expr_ty e, ADDOP_LOAD_CONST_NEW(c, LOC(e), defaults); RETURN_IF_ERROR(codegen_setup_annotations_scope(c, LOC(e), key, name)); if (allow_starred && e->kind == Starred_kind) { - VISIT(c, expr, e->v.Starred.value); - ADDOP_I(c, LOC(e), UNPACK_SEQUENCE, (Py_ssize_t)1); + VISIT_IN_SCOPE(c, expr, e->v.Starred.value); + ADDOP_I_IN_SCOPE(c, LOC(e), UNPACK_SEQUENCE, (Py_ssize_t)1); } else { - VISIT(c, expr, e); + VISIT_IN_SCOPE(c, expr, e); } ADDOP_IN_SCOPE(c, LOC(e), RETURN_VALUE); PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 1); diff --git a/Python/critical_section.c b/Python/critical_section.c index 218b580e951..11a3f027547 100644 --- a/Python/critical_section.c +++ b/Python/critical_section.c @@ -1,7 +1,8 @@ #include "Python.h" -#include "pycore_lock.h" #include "pycore_critical_section.h" +#include "pycore_interp.h" +#include "pycore_lock.h" #ifdef Py_GIL_DISABLED static_assert(_Alignof(PyCriticalSection) >= 4, @@ -43,6 +44,15 @@ _PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m) } } } + // If the world is stopped, we don't need to acquire the lock because + // there are no other threads that could be accessing the object. + // Without this check, acquiring a critical section while the world is + // stopped could lead to a deadlock. + if (tstate->interp->stoptheworld.world_stopped) { + c->_cs_mutex = NULL; + c->_cs_prev = 0; + return; + } c->_cs_mutex = NULL; c->_cs_prev = (uintptr_t)tstate->critical_section; tstate->critical_section = (uintptr_t)c; @@ -58,6 +68,12 @@ _PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, { #ifdef Py_GIL_DISABLED PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->interp->stoptheworld.world_stopped) { + c->_cs_base._cs_mutex = NULL; + c->_cs_mutex2 = NULL; + c->_cs_base._cs_prev = 0; + return; + } c->_cs_base._cs_mutex = NULL; c->_cs_mutex2 = NULL; c->_cs_base._cs_prev = tstate->critical_section; diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 80772fcedb4..3b6c0c6e1ed 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -609,6 +609,7 @@ check_missing___main___attr(PyObject *exc) // Get the error message. PyObject *args = PyException_GetArgs(exc); if (args == NULL || args == Py_None || PyObject_Size(args) < 1) { + Py_XDECREF(args); assert(!PyErr_Occurred()); return 0; } diff --git a/Python/emscripten_trampoline.c b/Python/emscripten_trampoline.c index d61146504d0..1833311ca74 100644 --- a/Python/emscripten_trampoline.c +++ b/Python/emscripten_trampoline.c @@ -2,20 +2,59 @@ #include // EM_JS, EM_JS_DEPS #include +#include "pycore_runtime.h" // _PyRuntime -EM_JS( -PyObject*, -_PyEM_TrampolineCall_inner, (int* success, - PyCFunctionWithKeywords func, - PyObject *arg1, - PyObject *arg2, - PyObject *arg3), { - // JavaScript fallback trampoline +// We use the _PyRuntime.emscripten_trampoline field to store a function pointer +// for a wasm-gc based trampoline if it works. Otherwise fall back to JS +// trampoline. The JS trampoline breaks stack switching but every runtime that +// supports stack switching also supports wasm-gc. +// +// We'd like to make the trampoline call into a direct call but currently we +// need to import the wasmTable to compile trampolineModule. emcc >= 4.0.19 +// defines the table in WebAssembly and exports it so we won't have access to it +// until after the main module is compiled. +// +// To fix this, one natural solution would be to pass a funcref to the +// trampoline instead of a table index. Several PRs would be needed to fix +// things in llvm and emscripten in order to make this possible. +// +// The performance costs of an extra call_indirect aren't that large anyways. +// The JIT should notice that the target is always the same and turn into a +// check +// +// if (call_target != expected) deoptimize; +// direct_call(call_target, args); + +// Offset of emscripten_trampoline in _PyRuntimeState. There's a couple of +// alternatives: +// +// 1. Just make emscripten_trampoline a real C global variable instead of a +// field of _PyRuntimeState. This would violate our rule against mutable +// globals. +// +// 2. #define a preprocessor constant equal to a hard coded number and make a +// _Static_assert(offsetof(_PyRuntimeState, emscripten_trampoline) == OURCONSTANT) +// This has the disadvantage that we have to update the hard coded constant +// when _PyRuntimeState changes +// +// So putting the mutable constant in _PyRuntime and using a immutable global to +// record the offset so we can access it from JS is probably the best way. +EMSCRIPTEN_KEEPALIVE const int _PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET = offsetof(_PyRuntimeState, emscripten_trampoline); + +typedef PyObject* (*TrampolineFunc)(int* success, + PyCFunctionWithKeywords func, + PyObject* self, + PyObject* args, + PyObject* kw); + +/** + * Backwards compatible trampoline works with all JS runtimes + */ +EM_JS(PyObject*, _PyEM_TrampolineCall_JS, (PyCFunctionWithKeywords func, PyObject *arg1, PyObject *arg2, PyObject *arg3), { return wasmTable.get(func)(arg1, arg2, arg3); } -// Try to replace the JS definition of _PyEM_TrampolineCall_inner with a wasm -// version. -(function () { +// Try to compile wasm-gc trampoline if possible. +function getPyEMTrampolinePtr() { // Starting with iOS 18.3.1, WebKit on iOS has an issue with the garbage // collector that breaks the call trampoline. See #130418 and // https://bugs.webkit.org/show_bug.cgi?id=293113 for details. @@ -27,19 +66,32 @@ _PyEM_TrampolineCall_inner, (int* success, (navigator.platform === 'MacIntel' && typeof navigator.maxTouchPoints !== 'undefined' && navigator.maxTouchPoints > 1) ); if (isIOS) { - return; + return 0; } + let trampolineModule; try { - const trampolineModule = getWasmTrampolineModule(); - const trampolineInstance = new WebAssembly.Instance(trampolineModule, { - env: { __indirect_function_table: wasmTable, memory: wasmMemory }, - }); - _PyEM_TrampolineCall_inner = trampolineInstance.exports.trampoline_call; + trampolineModule = getWasmTrampolineModule(); } catch (e) { // Compilation error due to missing wasm-gc support, fall back to JS // trampoline + return 0; } -})(); + const trampolineInstance = new WebAssembly.Instance(trampolineModule, { + env: { __indirect_function_table: wasmTable, memory: wasmMemory }, + }); + return addFunction(trampolineInstance.exports.trampoline_call); +} +// We have to be careful to work correctly with memory snapshots -- the value of +// _PyRuntimeState.emscripten_trampoline needs to reflect whether wasm-gc is +// available in the current runtime, not in the runtime the snapshot was taken +// in. This writes the appropriate value to +// _PyRuntimeState.emscripten_trampoline from JS startup code that runs every +// time, whether we are restoring a snapshot or not. +addOnPreRun(function setEmscriptenTrampoline() { + const ptr = getPyEMTrampolinePtr(); + const offset = HEAP32[__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET / 4]; + HEAP32[(__PyRuntime + offset) / 4] = ptr; +}); ); PyObject* @@ -48,12 +100,19 @@ _PyEM_TrampolineCall(PyCFunctionWithKeywords func, PyObject* args, PyObject* kw) { + TrampolineFunc trampoline = _PyRuntime.emscripten_trampoline; + if (trampoline == 0) { + return _PyEM_TrampolineCall_JS(func, self, args, kw); + } int success = 1; - PyObject *result = _PyEM_TrampolineCall_inner(&success, func, self, args, kw); + PyObject *result = trampoline(&success, func, self, args, kw); if (!success) { PyErr_SetString(PyExc_SystemError, "Handler takes too many arguments"); } return result; } +#else +// This is exported so we need to define it even when it isn't used +__attribute__((used)) const int _PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET = 0; #endif diff --git a/Python/frame.c b/Python/frame.c index ce216797e47..1196154d894 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -54,7 +54,7 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) _PyFrame_Copy(frame, new_frame); // _PyFrame_Copy takes the reference to the executable, // so we need to restore it. - frame->f_executable = PyStackRef_DUP(new_frame->f_executable); + new_frame->f_executable = PyStackRef_DUP(new_frame->f_executable); f->f_frame = new_frame; new_frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; if (_PyFrame_IsIncomplete(new_frame)) { @@ -135,14 +135,14 @@ PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame) return PyStackRef_AsPyObjectNew(frame->f_executable); } -int +// NOTE: We allow racy accesses to the instruction pointer from other threads +// for sys._current_frames() and similar APIs. +int _Py_NO_SANITIZE_THREAD PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame) { return _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); } -// NOTE: We allow racy accesses to the instruction pointer from other threads -// for sys._current_frames() and similar APIs. int _Py_NO_SANITIZE_THREAD PyUnstable_InterpreterFrame_GetLine(_PyInterpreterFrame *frame) { diff --git a/Python/import.c b/Python/import.c index 23b4bf51912..5914722c8f7 100644 --- a/Python/import.c +++ b/Python/import.c @@ -297,12 +297,32 @@ PyImport_GetModule(PyObject *name) mod = import_get_module(tstate, name); if (mod != NULL && mod != Py_None) { if (import_ensure_initialized(tstate->interp, mod, name) < 0) { + goto error; + } + /* Verify the module is still in sys.modules. Another thread may have + removed it (due to import failure) between our import_get_module() + call and the _initializing check in import_ensure_initialized(). */ + PyObject *mod_check = import_get_module(tstate, name); + if (mod_check != mod) { + Py_XDECREF(mod_check); + if (_PyErr_Occurred(tstate)) { + goto error; + } + /* The module was removed or replaced. Return NULL to report + "not found" rather than trying to keep up with racing + modifications to sys.modules; returning the new value would + require looping to redo the ensure_initialized check. */ Py_DECREF(mod); - remove_importlib_frames(tstate); return NULL; } + Py_DECREF(mod_check); } return mod; + +error: + Py_DECREF(mod); + remove_importlib_frames(tstate); + return NULL; } /* Get the module object corresponding to a module name. @@ -2127,13 +2147,29 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, } main_finally: + if (rc < 0) { + _Py_ext_module_loader_result_apply_error(&res, name_buf); + } + /* Switch back to the subinterpreter. */ if (switched) { + // gh-144601: The exception object can't be transferred across + // interpreters. Instead, we print out an unraisable exception, and + // then raise a different exception for the calling interpreter. + if (rc < 0) { + assert(PyErr_Occurred()); + PyErr_FormatUnraisable("Exception while importing from subinterpreter"); + } assert(main_tstate != tstate); switch_back_from_main_interpreter(tstate, main_tstate, mod); /* Any module we got from the init function will have to be * reloaded in the subinterpreter. */ mod = NULL; + if (rc < 0) { + PyErr_SetString(PyExc_ImportError, + "failed to import from subinterpreter due to exception"); + goto error; + } } /*****************************************************************/ @@ -2142,7 +2178,6 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, /* Finally we handle the error return from _PyImport_RunModInitFunc(). */ if (rc < 0) { - _Py_ext_module_loader_result_apply_error(&res, name_buf); goto error; } @@ -3813,6 +3848,27 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, if (import_ensure_initialized(tstate->interp, mod, abs_name) < 0) { goto error; } + /* Verify the module is still in sys.modules. Another thread may have + removed it (due to import failure) between our import_get_module() + call and the _initializing check in import_ensure_initialized(). + If removed, we retry the import to preserve normal semantics: the + caller gets the exception from the actual import failure rather + than a synthetic error. */ + PyObject *mod_check = import_get_module(tstate, abs_name); + if (mod_check != mod) { + Py_XDECREF(mod_check); + if (_PyErr_Occurred(tstate)) { + goto error; + } + Py_DECREF(mod); + mod = import_find_and_load(tstate, abs_name); + if (mod == NULL) { + goto error; + } + } + else { + Py_DECREF(mod_check); + } } else { Py_XDECREF(mod); diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 8feb259a63a..40136054221 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -620,6 +620,12 @@ _PyPerfTrampoline_AfterFork_Child(void) int was_active = _PyIsPerfTrampolineActive(); _PyPerfTrampoline_Fini(); if (was_active) { + // After fork, Fini may leave the old code watcher registered + // if trampolined code objects from the parent still exist + // (trampoline_refcount > 0). Clear it unconditionally before + // Init registers a new one, to prevent two watchers sharing + // the same globals and double-decrementing trampoline_refcount. + perf_trampoline_reset_state(); _PyPerfTrampoline_Init(1); } } diff --git a/Python/preconfig.c b/Python/preconfig.c index 5b26c75de8b..7b4168c4667 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -584,7 +584,7 @@ _Py_get_xoption(const PyWideStringList *xoptions, const wchar_t *name) for (Py_ssize_t i=0; i < xoptions->length; i++) { const wchar_t *option = xoptions->items[i]; size_t len; - wchar_t *sep = wcschr(option, L'='); + const wchar_t *sep = wcschr(option, L'='); if (sep != NULL) { len = (sep - option); } @@ -615,7 +615,7 @@ preconfig_init_utf8_mode(PyPreConfig *config, const _PyPreCmdline *cmdline) const wchar_t *xopt; xopt = _Py_get_xoption(&cmdline->xoptions, L"utf8"); if (xopt) { - wchar_t *sep = wcschr(xopt, L'='); + const wchar_t *sep = wcschr(xopt, L'='); if (sep) { xopt = sep + 1; if (wcscmp(xopt, L"1") == 0) { diff --git a/Python/pythonrun.c b/Python/pythonrun.c index e005a509f6f..91c81ac0d8d 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1151,6 +1151,7 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) "traceback", "_print_exception_bltin"); if (print_exception_fn == NULL || !PyCallable_Check(print_exception_fn)) { + Py_XDECREF(print_exception_fn); goto fallback; } diff --git a/Python/remote_debug.h b/Python/remote_debug.h index ed213859a8a..df1225f2eda 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -133,6 +133,31 @@ typedef struct { Py_ssize_t page_size; } proc_handle_t; +// Forward declaration for use in validation function +static int +_Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst); + +// Optional callback to validate a candidate section address found during +// memory map searches. Returns 1 if the address is valid, 0 to skip it. +// This allows callers to filter out duplicate/stale mappings (e.g. from +// ctypes dlopen) whose sections were never initialized. +typedef int (*section_validator_t)(proc_handle_t *handle, uintptr_t address); + +// Validate that a candidate address starts with _Py_Debug_Cookie. +static int +_Py_RemoteDebug_ValidatePyRuntimeCookie(proc_handle_t *handle, uintptr_t address) +{ + if (address == 0) { + return 0; + } + char buf[sizeof(_Py_Debug_Cookie) - 1]; + if (_Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(buf), buf) != 0) { + PyErr_Clear(); + return 0; + } + return memcmp(buf, _Py_Debug_Cookie, sizeof(buf)) == 0; +} + static void _Py_RemoteDebug_FreePageCache(proc_handle_t *handle) { @@ -490,7 +515,8 @@ pid_to_task(pid_t pid) } static uintptr_t -search_map_for_section(proc_handle_t *handle, const char* secname, const char* substr) { +search_map_for_section(proc_handle_t *handle, const char* secname, const char* substr, + section_validator_t validator) { mach_vm_address_t address = 0; mach_vm_size_t size = 0; mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); @@ -542,7 +568,9 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s if (strncmp(filename, substr, strlen(substr)) == 0) { uintptr_t result = search_section_in_file( secname, map_filename, address, size, proc_ref); - if (result != 0) { + if (result != 0 + && (validator == NULL || validator(handle, result))) + { return result; } } @@ -659,7 +687,8 @@ search_elf_file_for_section( } static uintptr_t -search_linux_map_for_section(proc_handle_t *handle, const char* secname, const char* substr) +search_linux_map_for_section(proc_handle_t *handle, const char* secname, const char* substr, + section_validator_t validator) { char maps_file_path[64]; sprintf(maps_file_path, "/proc/%d/maps", handle->pid); @@ -734,9 +763,12 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c if (strstr(filename, substr)) { retval = search_elf_file_for_section(handle, secname, start, path); - if (retval) { + if (retval + && (validator == NULL || validator(handle, retval))) + { break; } + retval = 0; } } @@ -832,7 +864,8 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* static uintptr_t -search_windows_map_for_section(proc_handle_t* handle, const char* secname, const wchar_t* substr) { +search_windows_map_for_section(proc_handle_t* handle, const char* secname, const wchar_t* substr, + section_validator_t validator) { HANDLE hProcSnap; do { hProcSnap = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE, handle->pid); @@ -855,8 +888,11 @@ search_windows_map_for_section(proc_handle_t* handle, const char* secname, const for (BOOL hasModule = Module32FirstW(hProcSnap, &moduleEntry); hasModule; hasModule = Module32NextW(hProcSnap, &moduleEntry)) { // Look for either python executable or DLL if (wcsstr(moduleEntry.szModule, substr)) { - runtime_addr = analyze_pe(moduleEntry.szExePath, moduleEntry.modBaseAddr, secname); - if (runtime_addr != NULL) { + void *candidate = analyze_pe(moduleEntry.szExePath, moduleEntry.modBaseAddr, secname); + if (candidate != NULL + && (validator == NULL || validator(handle, (uintptr_t)candidate))) + { + runtime_addr = candidate; break; } } @@ -877,7 +913,8 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle) #ifdef MS_WINDOWS // On Windows, search for 'python' in executable or DLL - address = search_windows_map_for_section(handle, "PyRuntime", L"python"); + address = search_windows_map_for_section(handle, "PyRuntime", L"python", + _Py_RemoteDebug_ValidatePyRuntimeCookie); if (address == 0) { // Error out: 'python' substring covers both executable and DLL PyObject *exc = PyErr_GetRaisedException(); @@ -888,7 +925,8 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle) } #elif defined(__linux__) // On Linux, search for 'python' in executable or DLL - address = search_linux_map_for_section(handle, "PyRuntime", "python"); + address = search_linux_map_for_section(handle, "PyRuntime", "python", + _Py_RemoteDebug_ValidatePyRuntimeCookie); if (address == 0) { // Error out: 'python' substring covers both executable and DLL PyObject *exc = PyErr_GetRaisedException(); @@ -902,7 +940,8 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle) const char* candidates[] = {"libpython", "python", "Python", NULL}; for (const char** candidate = candidates; *candidate; candidate++) { PyErr_Clear(); - address = search_map_for_section(handle, "PyRuntime", *candidate); + address = search_map_for_section(handle, "PyRuntime", *candidate, + _Py_RemoteDebug_ValidatePyRuntimeCookie); if (address != 0) { break; } @@ -1095,6 +1134,7 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, if (entry->data == NULL) { entry->data = PyMem_RawMalloc(page_size); if (entry->data == NULL) { + PyErr_NoMemory(); _set_debug_exception_cause(PyExc_MemoryError, "Cannot allocate %zu bytes for page cache entry " "during read from PID %d at address 0x%lx", @@ -1104,7 +1144,7 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, } if (_Py_RemoteDebug_ReadRemoteMemory(handle, page_base, page_size, entry->data) < 0) { - // Try to just copy the exact ammount as a fallback + // Try to just copy the exact amount as a fallback PyErr_Clear(); goto fallback; } diff --git a/Python/symtable.c b/Python/symtable.c index f633e281019..549ef131388 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -806,6 +806,8 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, PyObject *k, *v; Py_ssize_t pos = 0; int remove_dunder_class = 0; + int remove_dunder_classdict = 0; + int remove_dunder_cond_annotations = 0; while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { // skip comprehension parameter @@ -828,15 +830,27 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (existing == NULL && PyErr_Occurred()) { return 0; } - // __class__ is never allowed to be free through a class scope (see + // __class__, __classdict__ and __conditional_annotations__ are + // never allowed to be free through a class scope (see // drop_class_free) if (scope == FREE && ste->ste_type == ClassBlock && - _PyUnicode_EqualToASCIIString(k, "__class__")) { + (_PyUnicode_EqualToASCIIString(k, "__class__") || + _PyUnicode_EqualToASCIIString(k, "__classdict__") || + _PyUnicode_EqualToASCIIString(k, "__conditional_annotations__"))) { scope = GLOBAL_IMPLICIT; if (PySet_Discard(comp_free, k) < 0) { return 0; } - remove_dunder_class = 1; + + if (_PyUnicode_EqualToASCIIString(k, "__class__")) { + remove_dunder_class = 1; + } + else if (_PyUnicode_EqualToASCIIString(k, "__conditional_annotations__")) { + remove_dunder_cond_annotations = 1; + } + else { + remove_dunder_classdict = 1; + } } if (!existing) { // name does not exist in scope, copy from comprehension @@ -876,6 +890,12 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (remove_dunder_class && PyDict_DelItemString(comp->ste_symbols, "__class__") < 0) { return 0; } + if (remove_dunder_classdict && PyDict_DelItemString(comp->ste_symbols, "__classdict__") < 0) { + return 0; + } + if (remove_dunder_cond_annotations && PyDict_DelItemString(comp->ste_symbols, "__conditional_annotations__") < 0) { + return 0; + } return 1; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 545b130836e..c97042c99fa 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1759,7 +1759,7 @@ sys_getwindowsversion_impl(PyObject *module) PyObject *realVersion = _sys_getwindowsversion_from_kernel32(); if (!realVersion) { if (!PyErr_ExceptionMatches(PyExc_WindowsError)) { - return NULL; + goto error; } PyErr_Clear(); diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 8ae12e54576..4795068d0a0 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -224,13 +224,20 @@ tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) assert(PyStackRef_CodeCheck(pyframe->f_executable)); frame->filename = &_Py_STR(anon_unknown); - int lineno = PyUnstable_InterpreterFrame_GetLine(pyframe); + int lineno = -1; + PyCodeObject *code = _PyFrame_GetCode(pyframe); + // PyUnstable_InterpreterFrame_GetLine() cannot but used, since it uses + // a critical section which can trigger a deadlock. + int lasti = _PyFrame_SafeGetLasti(pyframe); + if (lasti >= 0) { + lineno = _PyCode_SafeAddr2Line(code, lasti); + } if (lineno < 0) { lineno = 0; } frame->lineno = (unsigned int)lineno; - PyObject *filename = filename = _PyFrame_GetCode(pyframe)->co_filename; + PyObject *filename = code->co_filename; if (filename == NULL) { #ifdef TRACE_DEBUG tracemalloc_error("failed to get the filename of the code object"); @@ -853,7 +860,8 @@ _PyTraceMalloc_Stop(void) TABLES_LOCK(); if (!tracemalloc_config.tracing) { - goto done; + TABLES_UNLOCK(); + return; } /* stop tracing Python memory allocations */ @@ -870,10 +878,12 @@ _PyTraceMalloc_Stop(void) raw_free(tracemalloc_traceback); tracemalloc_traceback = NULL; - (void)PyRefTracer_SetTracer(NULL, NULL); - -done: TABLES_UNLOCK(); + + // Call it after TABLES_UNLOCK() since it calls _PyEval_StopTheWorldAll() + // which would lead to a deadlock with TABLES_LOCK() which doesn't detach + // the thread state. + (void)PyRefTracer_SetTracer(NULL, NULL); } diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index 88b7856b9c7..090fd759224 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -225,16 +225,27 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: if file.parent == GITHUB_WORKFLOWS_PATH: if file.name in ("build.yml", "reusable-cifuzz.yml"): - run_tests = run_ci_fuzz = run_ci_fuzz_stdlib = True + run_tests = run_ci_fuzz = run_ci_fuzz_stdlib = run_windows_tests = True has_platform_specific_change = False + continue if file.name == "reusable-docs.yml": run_docs = True + continue + if file.name == "reusable-windows.yml": + run_tests = True + run_windows_tests = True + continue if file.name == "reusable-windows-msi.yml": run_windows_msi = True + continue if file.name == "reusable-macos.yml": + run_tests = True platforms_changed.add("macos") + continue if file.name == "reusable-wasi.yml": + run_tests = True platforms_changed.add("wasi") + continue if not doc_file and file not in RUN_TESTS_IGNORE: run_tests = True diff --git a/Tools/check-c-api-docs/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt index c23784d7e11..1477774a6d2 100644 --- a/Tools/check-c-api-docs/ignored_c_api.txt +++ b/Tools/check-c-api-docs/ignored_c_api.txt @@ -20,26 +20,12 @@ Py_UTF8Mode Py_HASH_EXTERNAL # modsupport.h PyABIInfo_FREETHREADING_AGNOSTIC -# moduleobject.h -PyModuleDef_Type # object.h Py_INVALID_SIZE -Py_TPFLAGS_HAVE_VERSION_TAG -Py_TPFLAGS_INLINE_VALUES -Py_TPFLAGS_IS_ABSTRACT # pyexpat.h PyExpat_CAPI_MAGIC PyExpat_CAPSULE_NAME # pyport.h -Py_ALIGNED -Py_ARITHMETIC_RIGHT_SHIFT -Py_CAN_START_THREADS -Py_FORCE_EXPANSION -Py_GCC_ATTRIBUTE -Py_LL -Py_SAFE_DOWNCAST -Py_ULL -Py_VA_COPY PYLONG_BITS_IN_DIGIT PY_DWORD_MAX PY_FORMAT_SIZE_T diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py index b815376b7ed..cc7d8575f5c 100644 --- a/Tools/ftscalingbench/ftscalingbench.py +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -201,6 +201,23 @@ def instantiate_dataclass(): for _ in range(1000 * WORK_SCALE): obj = MyDataClass(x=1, y=2, z=3) +@register_benchmark +def super_call(): + # TODO: super() on the same class from multiple threads still doesn't + # scale well, so use a class per-thread here for now. + class Base: + def method(self): + return 1 + + class Derived(Base): + def method(self): + return super().method() + + obj = Derived() + for _ in range(1000 * WORK_SCALE): + obj.method() + + def bench_one_thread(func): t0 = time.perf_counter_ns() func() diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index ab9840c1c52..86baf8a3a74 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -45,14 +45,15 @@ OPENSSL_OLD_VERSIONS = [ "1.1.1w", "3.1.8", + "3.2.6", ] OPENSSL_RECENT_VERSIONS = [ - "3.0.18", - "3.2.6", - "3.3.5", - "3.4.3", - "3.5.4", + "3.0.19", + "3.3.6", + "3.4.4", + "3.5.5", + "3.6.1", # See make_ssl_data.py for notes on adding a new version. ] diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py index d4cca68c3e3..6b1d7da26e2 100644 --- a/Tools/unicode/makeunicodedata.py +++ b/Tools/unicode/makeunicodedata.py @@ -99,18 +99,13 @@ CASED_MASK = 0x2000 EXTENDED_CASE_MASK = 0x4000 -# these ranges need to match unicodedata.c:is_unified_ideograph -cjk_ranges = [ - ('3400', '4DBF'), # CJK Ideograph Extension A CJK - ('4E00', '9FFF'), # CJK Ideograph - ('20000', '2A6DF'), # CJK Ideograph Extension B - ('2A700', '2B739'), # CJK Ideograph Extension C - ('2B740', '2B81D'), # CJK Ideograph Extension D - ('2B820', '2CEA1'), # CJK Ideograph Extension E - ('2CEB0', '2EBE0'), # CJK Ideograph Extension F - ('2EBF0', '2EE5D'), # CJK Ideograph Extension I - ('30000', '3134A'), # CJK Ideograph Extension G - ('31350', '323AF'), # CJK Ideograph Extension H +# Maps the range names in UnicodeData.txt to prefixes for +# derived names specified by rule NR2. +# Hangul should always be at index 0, since it uses special format. +derived_name_range_names = [ + ("Hangul Syllable", "HANGUL SYLLABLE "), + ("CJK Ideograph", "CJK UNIFIED IDEOGRAPH-"), + ("Tangut Ideograph", "TANGUT IDEOGRAPH-"), ] @@ -124,7 +119,7 @@ def maketables(trace=0): for version in old_versions: print("--- Reading", UNICODE_DATA % ("-"+version), "...") - old_unicode = UnicodeData(version, cjk_check=False) + old_unicode = UnicodeData(version, ideograph_check=False) print(len(list(filter(None, old_unicode.table))), "characters") merge_old_version(version, unicode, old_unicode) @@ -698,6 +693,23 @@ def makeunicodename(unicode, trace): fprint(' {%d, {%s}},' % (len(sequence), seq_str)) fprint('};') + fprint(dedent(""" + typedef struct { + Py_UCS4 first; + Py_UCS4 last; + int prefixid; + } derived_name_range; + """)) + + fprint('static const derived_name_range derived_name_ranges[] = {') + for name_range in unicode.derived_name_ranges: + fprint(' {0x%s, 0x%s, %d},' % name_range) + fprint('};') + + fprint('static const char * const derived_name_prefixes[] = {') + for _, prefix in derived_name_range_names: + fprint(' "%s",' % prefix) + fprint('};') def merge_old_version(version, new, old): # Changes to exclusion file not implemented yet @@ -905,14 +917,14 @@ def from_row(row: List[str]) -> UcdRecord: class UnicodeData: # table: List[Optional[UcdRecord]] # index is codepoint; None means unassigned - def __init__(self, version, cjk_check=True): + def __init__(self, version, ideograph_check=True): self.changed = [] table = [None] * 0x110000 for s in UcdFile(UNICODE_DATA, version): char = int(s[0], 16) table[char] = from_row(s) - cjk_ranges_found = [] + self.derived_name_ranges = [] # expand first-last ranges field = None @@ -926,15 +938,15 @@ def __init__(self, version, cjk_check=True): s.name = "" field = dataclasses.astuple(s)[:15] elif s.name[-5:] == "Last>": - if s.name.startswith("